From 32bc3e3bc65aae4b0e0e591c8b533ac53b7ca469 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Fri, 16 Oct 2026 23:58:05 -0500 Subject: [PATCH 01/13] state_exec: grammar spec, conformance cases and parse corpus (grammar P0) Declare the state_exec s-expression grammar in docs/STATE_EXEC_GRAMMAR.md (W3C EBNF, Unicode source rules, literal tables, and a table of every intentional change from the hand-written parser, tagged by phase P2-P4). The parser itself is unchanged; later phases build it from this grammar. state_exec_grammar_test tags each case with the phase it holds from: landed cases must hold, pending cases must still differ, the doc's EBNF must be closed, and every production needs a case. state_exec_corpus_test pins how the 597 DSL scripts the repo carries parse today (state_exec_corpus/, regenerated with gen_state_exec_corpus.py and CVC_UPDATE_GOLDENS=1). Also correct parser.h (no quasiquote) and the developer guide (false and #f read as nil, bytes literals exist). --- docs/STATE_EXEC_DEVELOPER_GUIDE.md | 8 +- docs/STATE_EXEC_GRAMMAR.md | 306 +++ inc/cvc/state/state_exec/parser.h | 6 +- src/cvc/tests/CMakeLists.txt | 20 + src/cvc/tests/gen_state_exec_corpus.py | 268 ++ .../tests/state_exec_corpus/.gitattributes | 2 + src/cvc/tests/state_exec_corpus/expected.txt | 1366 +++++++++ src/cvc/tests/state_exec_corpus/scripts.txt | 2437 +++++++++++++++++ src/cvc/tests/state_exec_corpus_test.cpp | 102 + src/cvc/tests/state_exec_grammar_dump.h | 240 ++ src/cvc/tests/state_exec_grammar_test.cpp | 284 ++ 11 files changed, 5035 insertions(+), 4 deletions(-) create mode 100644 docs/STATE_EXEC_GRAMMAR.md create mode 100644 src/cvc/tests/gen_state_exec_corpus.py create mode 100644 src/cvc/tests/state_exec_corpus/.gitattributes create mode 100644 src/cvc/tests/state_exec_corpus/expected.txt create mode 100644 src/cvc/tests/state_exec_corpus/scripts.txt create mode 100644 src/cvc/tests/state_exec_corpus_test.cpp create mode 100644 src/cvc/tests/state_exec_grammar_dump.h create mode 100644 src/cvc/tests/state_exec_grammar_test.cpp diff --git a/docs/STATE_EXEC_DEVELOPER_GUIDE.md b/docs/STATE_EXEC_DEVELOPER_GUIDE.md index 048aa8e28..1b3bb5002 100644 --- a/docs/STATE_EXEC_DEVELOPER_GUIDE.md +++ b/docs/STATE_EXEC_DEVELOPER_GUIDE.md @@ -81,6 +81,9 @@ auto results = sched.run(); ## 2. The DSL Language +The full syntax (literals, escapes, Unicode rules) is specified in +[STATE_EXEC_GRAMMAR.md](STATE_EXEC_GRAMMAR.md). + ### Types | Type | Literal examples | C++ variant tag | @@ -88,9 +91,10 @@ auto results = sched.run(); | Integer | `42`, `-7` | `int64_t` | | Float | `3.14`, `1e-5` | `double` | | String | `"hello"` | `std::string` | -| Boolean | `true`, `false` | `bool` | +| Bytes | `b"\x00\xff"` | `bytes_value` | +| Boolean | `true`, `#t` (`false` and `#f` read as nil) | `bool` | | Symbol | `x`, `my-func` | `symbol` | -| Nil | `nil` | `std::monostate` | +| Nil | `nil`, `false`, `#f` | `std::monostate` | | List | `(1 2 3)` | `list_ptr` | ### Special Forms diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md new file mode 100644 index 000000000..c979d0abb --- /dev/null +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -0,0 +1,306 @@ +# state_exec Grammar + +This is the formal grammar of the state_exec s-expression language: the text that +`cvc::state_exec::parse()` and `parse_all()` read (`inc/cvc/state/state_exec/parser.h`). Ariadne +`.ari` documents embed it in `init`, `on`, `bind` and `*_when` fields. + +**Status: target grammar.** The parser in `src/cvc/state/state_exec/parser.cpp` is still the +hand-written one. It moves onto this grammar in phases (P2 to P4, see +[Changes from the pre-grammar parser](#changes-from-the-pre-grammar-parser)). Until then it differs +from this document exactly where that table says. `state_exec_grammar_test` checks both sides: +cases for landed phases must hold, and cases for pending phases must still differ. + +From P2 on, the parser is built from this declaration. Each production below becomes a +[PEGTL](https://github.com/taocpp/PEGTL) rule with the same name in +`src/cvc/state/state_exec/grammar.h`, and `state_exec_grammar_test` checks that the names match. + +## Notation + +The grammar uses the [W3C EBNF notation](https://www.w3.org/TR/xml/#sec-notation): `::=` defines +a production, `|` separates alternatives, `?` `*` `+` are optional, zero-or-more and one-or-more, +`A - B` matches what `A` matches unless `B` also matches it, `'…'` and `"…"` are literal text, +`[…]` is a character class, and `#xN` is the Unicode code point N. + +The input is the sequence of Unicode scalar values decoded from UTF-8. Bytes that are not +well-formed UTF-8 match nothing, so they are a parse error. + +Choices are ordered and matching is greedy, as in a PEG. The prose sections below state the +few places where that matters. + +## Grammar + +```ebnf +/* Whole inputs. parse_all() reads a program; parse() reads a single_datum. */ +program ::= bom? atmosphere* (datum atmosphere*)* +single_datum ::= bom? atmosphere* (datum atmosphere*)? + +/* Data */ +datum ::= list | quoted | text_literal | bytes_literal | atom +list ::= '(' atmosphere* (datum atmosphere*)* ')' +quoted ::= "'" atmosphere* datum + +/* Source characters and atmosphere */ +bom ::= #xFEFF +scalar_value ::= [#x0-#xD7FF] | [#xE000-#x10FFFF] +source_char ::= scalar_value - (control | bidi_control | #xFEFF) +control ::= [#x0-#x8] | [#xE-#x1F] | [#x7F-#x84] | [#x86-#x9F] +bidi_control ::= #x61C | [#x202A-#x202E] | [#x2066-#x2069] +atmosphere ::= whitespace | line_comment +whitespace ::= [#x9-#xD] | #x20 | #x85 | #x200E | #x200F | #x2028 | #x2029 +other_space ::= #xA0 | #x1680 | [#x2000-#x200A] | #x202F | #x205F | #x3000 +line_comment ::= ';' (source_char - line_end)* +line_end ::= #xA | #xD + +/* Atoms: the longest run of constituents, then classified (see "Atoms") */ +atom ::= boolean | nil | number | symbol +token ::= constituent+ +constituent ::= source_char - (whitespace | other_space | delimiter | reserved) +delimiter ::= #x22 | #x27 | #x28 | #x29 | #x3B /* " ' ( ) ; */ +reserved ::= #x2C | #x5B | #x5C | #x5D | #x60 | #x7B | #x7C | #x7D /* , [ \ ] ` { | } */ +boolean ::= '#t' | 'true' +nil ::= '#f' | 'false' | 'nil' +number ::= integer | decimal_float | float_special +integer ::= sign? (dec_digits | radix_integer) +radix_integer ::= ('0x' | '0X' | '#x' | '#X') hex_digits + | ('0o' | '0O' | '#o' | '#O') oct_digits + | ('0b' | '0B' | '#b' | '#B') bin_digits +sign ::= '+' | '-' +dec_digits ::= [0-9] ('_'? [0-9])* +hex_digits ::= hex ('_'? hex)* +oct_digits ::= [0-7] ('_'? [0-7])* +bin_digits ::= [0-1] ('_'? [0-1])* +hex ::= [0-9A-Fa-f] +decimal_float ::= sign? ( dec_digits '.' dec_digits? exponent? + | '.' dec_digits exponent? + | dec_digits exponent ) +exponent ::= [eE] sign? dec_digits +float_special ::= '+inf.0' | '-inf.0' | '+nan.0' | '-nan.0' +radix_prefixed ::= sign? [0#] [xXoObB] constituent* +symbol ::= token - (boolean | nil | number | radix_prefixed) + +/* Text strings */ +text_literal ::= text_string | raw_text +text_string ::= '"' (string_char | text_escape)* '"' +string_char ::= source_char - (#x22 | #x5C) +text_escape ::= '\' (simple_escape | octal_escape | hex_escape | unicode_escape) +simple_escape ::= [abfnrtv] | #x22 | #x5C +octal_escape ::= [0-7] ([0-7] [0-7]?)? +hex_escape ::= 'x' hex hex +unicode_escape ::= 'u' hex hex hex hex + | 'u{' hex (hex (hex (hex (hex hex?)?)?)?)? '}' +raw_text ::= 'r' raw_body +raw_body ::= '"' raw_char* '"' + | hash_fence '"' raw_char* '"' hash_fence +hash_fence ::= '#'+ +raw_char ::= source_char + +/* Byte strings */ +bytes_literal ::= byte_string | raw_bytes +byte_string ::= 'b"' (string_char | byte_escape)* '"' +byte_escape ::= '\' (simple_escape | octal_escape | hex_escape) +raw_bytes ::= 'br' raw_body +``` + +## Source text + +- **Encoding.** Source is UTF-8. A malformed byte sequence, an overlong form or an encoded + surrogate is an error. One leading U+FEFF (a byte-order mark) is skipped; U+FEFF anywhere else + is an error. +- **Forbidden characters.** Outside of escapes, these may not appear anywhere, including strings + and comments: + - C0 controls other than TAB, LF, VT, FF and CR; + - DEL; + - C1 controls other than NEL; + - the bidirectional embedding, override and isolate controls U+202A–U+202E and + U+2066–U+2069, plus U+061C. This is the "Trojan Source" defence (CVE-2021-42574): these + characters can make source text display in a different order from the order it parses in. + Write them as `\u{202E}` and so on when a string needs one. +- **Whitespace** is Unicode Pattern_White_Space: TAB, LF, VT, FF, CR, space, NEL, U+200E, + U+200F, U+2028 and U+2029. The other Unicode spaces (`other_space`: NBSP, U+3000 and so on) are + allowed inside strings and comments, but between tokens they are an error rather than a silent + separator, because they look like a space and are not one. +- **Comments** run from `;` to the end of the line. LF and CR both end a line. +- **Positions.** `parse_error::line()` and `col()` are 1-based, and point at the character where + the error was found, such as the non-hex character in `"\x4"`. A new line starts after LF, + after CR, and once after CRLF. Columns count code points, not bytes. +- **Nesting.** At most 256 lists and quotes may be nested. Deeper input is an error, not a stack + overflow. The limit is sized for the 64 KiB default stack of a wasm build; real scripts nest far + less. + +## Data + +| Production | Value (`value_t` alternative) | +|---|---| +| `list` | `list_ptr` of the elements; `()` is an empty list | +| `quoted` | the two-element list `(quote datum)` | +| `text_string`, `raw_text` | `std::string` holding UTF-8 text | +| `byte_string`, `raw_bytes` | `bytes_value` | +| `boolean` | `true` | +| `nil` | nil (`std::monostate`). `#f` and `false` read as nil, not as `bool false` | +| `integer` | `int64_t` | +| `decimal_float`, `float_special` | `double` | +| `symbol` | `symbol` | + +`parse_all()` returns one value per top-level datum. `parse()` returns the one datum. It returns +nil for input that holds only atmosphere, and anything after the datum is an error. + +## Atoms + +An atom is the longest run of constituents (`token`). The whole token is then classified, in this +order: + +1. `#t` and `true` are `boolean`. `#f`, `false` and `nil` are `nil`. +2. A token that matches `number` exactly is a number. +3. A token that matches `radix_prefixed` but is not a valid `integer` is an error, for example + `0x1G`, `0b102`, `#xyz` or `0x`. A typo in a hex literal is reported, not read as a symbol. +4. Every other token is a `symbol`. That includes `+`, `-`, `...`, `->`, `1+`, `#foo`, `inf` and + `nan`. + +**Symbols** may use any character except whitespace, `other_space`, the delimiters `" ' ( ) ;`, +the reserved characters, controls and bidi controls. So `λ`, `→`, `café`, `日本語` and emoji +(including ZWJ sequences and variation selectors) are all symbol characters. Symbols are not +normalized, so a composed and a decomposed `é` are different symbols. That matches the raw-bytes +policy for strings and keys in UNICODE_SUPPORT_ROADMAP.md (SE-4). The grammar does not check +confusable or default-ignorable characters (U+200B and the like) in symbols. + +**Reserved characters** `` , [ \ ] ` { | } `` are errors outside strings and comments. They are +kept free for quasiquote (`` ` ``, `,`), dict and vector literals (`{}` and `[]`; a dict literal is +sketched in roadmap/STATE_BINARY_STREAMING.md) and quoted symbols (`|…|`). + +**Numbers.** + +| Form | Examples | Value | +|---|---|---| +| decimal integer | `42`, `-7`, `+5`, `007`, `1_000_000` | `int64_t` | +| hex, octal, binary integer | `0x1F`, `#x1f`, `-0x10`, `0o17`, `#o17`, `0b1011`, `#b1011`, `0xFFFF_FFFF` | `int64_t` | +| float | `1.5`, `1.`, `.5`, `1e3`, `-2.5E-3`, `6.022_140e23` | `double` | +| special float | `+inf.0`, `-inf.0`, `+nan.0`, `-nan.0` | `double` | + +- `_` may separate digits, one at a time, never at the start or end of a digit run. +- The sign goes before any radix prefix: `-0x10`, not `0x-10`. +- An integer must fit in `int64_t`, from -9223372036854775808 to 9223372036854775807. Overflow is + an error, not a silent conversion to `double`. +- Floats are read locale-independently and correctly rounded. A float too large for a `double` + (`1e999`) is an error. One too small rounds to zero or to a subnormal. +- There are no hex floats, rationals or complex numbers. + +## Strings + +A **text string** (`"…"`) is UTF-8 text. Any source character may appear in it except `"` and +`\`, including raw line breaks. A string can span lines. Escapes: + +| Escape | Meaning in a text string | Meaning in a byte string | +|---|---|---| +| `\n` `\t` `\r` | LF, TAB, CR | the same bytes | +| `\a` `\b` `\f` `\v` | BEL, BS, FF, VT | the same bytes | +| `\"` `\\` | `"` and `\` | the same bytes | +| `\N`, `\NN`, `\NNN` (octal, longest match) | code point 0–0o777; `\0` is NUL | byte 0–0o377; larger is an error | +| `\xNN` (exactly two hex digits) | code point U+0000–U+00FF | byte 0x00–0xFF | +| `\uXXXX` (exactly four) or `\u{X…}` (one to six) | that code point | not allowed | + +- Any other character after `\` is an error. +- A `\u` escape that names a surrogate (U+D800–U+DFFF) or a value above U+10FFFF is an error. +- Escaped code points are stored UTF-8 encoded. So `"\xe9"`, `"é"` and `"\351"` are all the + one-character string `é`. + +A **byte string** (`b"…"`) is a `bytes_value`. A non-ASCII source character contributes its +UTF-8 encoding, so `b"é"` is the two bytes C3 A9. + +A **raw string** (`r"…"`) or **raw byte string** (`br"…"`) processes no escapes: every character +between the quotes is taken as written, backslashes included. To put `"` in a raw string, fence +it with one or more `#`: `r#"say "hi""#`. The string ends at the first `"` followed by as many `#` +as opened it, so `r##"a"#b"##` is `a"#b`. + +**Prefixes.** `b`, `r` and `br` start a literal only at the start of a datum and only when +touching the quote, or the `#` fence of a raw string. `ab"x"` is the symbol `ab` followed by the +string `x`, and `r#foo` is a symbol. Once a datum starts with `b"`, `r"`, `br"` or a prefix, +hashes and `"`, it is a string literal, and leaving it unterminated is an error. + +## Changes from the pre-grammar parser + +Phase P2 replaces the hand-written lexer with the grammar, including the number syntax. P3 adds +the Unicode source rules and strict escapes. P4 adds octal escapes, the extra C escapes and raw +strings. Every row is a case in `state_exec_grammar_test`. + +| Input | Pre-grammar parser | Grammar | Phase | +|---|---|---|---| +| `a)` (a stray `)`) | the symbols `a` and `)` | error | P2 | +| `[a]`, `a,b`, `` `a ``, `{}`, `a\|b`, `a\b` | odd one-character or glued symbols | error (reserved) | P2 | +| `parse("1 2")` | `1`; the rest is ignored | error | P2 | +| `'` at end of input | `(quote nil)` | error | P2 | +| 257 nested lists | parsed | error | P2 | +| `; c` CR `1` | comment runs to LF, so no datum | `1` | P2 | +| `0x1F` | `31.0`, a double (via `strtod`) | `31` | P2 | +| `#x1F`, `0o17`, `0b101`, `1_000` | symbols | integers | P2 | +| `0xZZ`, `#bar`, `0x` | symbols | error | P2 | +| `+5` | `5.0`, a double | `5` | P2 | +| `9223372036854775808` | `9.2233720368547758e18`, a double | error | P2 | +| `inf`, `nan`, `-INF` | doubles | symbols | P2 | +| `+inf.0`, `-inf.0`, `+nan.0` | symbols | doubles | P2 | +| `1e999` | `+inf` | error | P2 | +| `1.5` with a `,`-decimal C locale set by the host | symbol (from reading `strtod`; not tested) | `1.5` | P2 | +| bytes that are not UTF-8 | kept in the symbol or string | error | P3 | +| U+FEFF at the start | a symbol before the first datum | skipped | P3 | +| NBSP between `a` and `b` | one symbol, `a b` with an NBSP | error | P3 | +| NEL or U+2028 between `a` and `b` | one symbol | two symbols | P3 | +| a raw bidi control or C0 control, in a string or comment | kept | error | P3 | +| `"\q"` | `\q`, backslash kept | error | P3 | +| `"\uD800"`, `"\u{110000}"` | U+FFFD | error | P3 | +| error columns | bytes | code points | P3 | +| `"\101"`, `"\012"` | `\101`; NUL then `12` | `A`; LF | P4 | +| `b"\377"`, `b"\400"` | `\377` kept; `\400` kept | the byte FF; error | P4 | +| `"\a\b\f\v"` | backslashes kept | BEL BS FF VT | P4 | +| `r"a\nb"`, `br"…"` | the symbol `r` or `br`, then a string | raw string, raw bytes | P4 | +| `r#"say "hi""#` | a symbol `r#`, strings and symbols | raw string `say "hi"` | P4 | + +Printing is not part of the grammar. `to_string` round-trip (escaping strings, shortest floats, +`+inf.0`) is phase P5. + +## Not in the grammar + +These were considered and left out. Each would need its own decision: + +- block comments (`#| … |#`) and datum comments (`#;`); +- quasiquote, unquote and unquote-splicing (the reserved `` ` `` and `,`); +- quoted symbols (`|…|`), dict literals (`{…}`) and vector literals (`[…]`); +- hex byte literals such as `#x"de ad be ef"`; +- `\N{NAME}` escapes, which need the Unicode name tables; +- line continuations inside strings; +- hex floats; +- NFC normalization of symbols or strings. + +## Tests and corpus + +- `src/cvc/tests/state_exec_grammar_test.cpp` checks the grammar itself: + - every EBNF production above is defined once, and every one except `program` and + `single_datum` is used; + - every production has at least one accept or reject case; + - each case holds once its phase has landed, and still differs while its phase is pending. + `kLandedPhase` in the test says which phases have landed. +- `src/cvc/tests/state_exec_corpus_test.cpp` parses the corpus: every DSL script the repo carries, + drawn from the `.ari` files, the docs' `lisp` blocks, the C++ test sources and the pycvc tests. + It compares each parse with `state_exec_corpus/expected.txt`, so a parser change cannot alter + how an existing script reads without that file changing in the same PR. + +To regenerate the corpus after adding scripts, run the extractor (it needs PyYAML), then rewrite +the expected parses: + +```sh +python src/cvc/tests/gen_state_exec_corpus.py +CVC_UPDATE_GOLDENS=1 build/bin/state_exec_corpus_test +``` + +The corpus held 597 scripts when it was made, and all of them parse under the pre-grammar +parser. Of the grammar's intentional changes, only the reserved `{` `}` touch any of them: a dict +literal sketched in roadmap/STATE_BINARY_STREAMING.md, which no parser has implemented. The 177 +s-expression scripts in txwtf's cfos tests, checked for the txwtf.cfos-to-state_exec port, touch +none. + +The expected parses use the test's own dump format (`state_exec_grammar_dump.h`), not +`to_string`. In that format: +- integers print in decimal; +- whole doubles print as `N.0`, other doubles in the shortest form that reads back exactly, and + the specials as `+inf.0`, `-inf.0` and `+nan.0`; +- strings print escaped, bytes as `b"…"`, and nil as `nil`; +- a symbol is wrapped in `|…|` when its bare text would read as something else; +- a parse error prints as `!error line:col`. diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index 2d3267648..8a96aec50 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -3,8 +3,10 @@ * @brief S-expression parser for the state_exec DSL. * * Converts source text into a tree of value_t nodes. Supports atoms - * (integers, floats, strings, symbols), lists, quote/quasiquote/unquote, - * and line comments. Reports parse errors with line/column information. + * (integers, floats, strings, byte strings, symbols), lists, quote ('x), and + * line comments. Reports parse errors with line/column information. The + * syntax is specified in docs/STATE_EXEC_GRAMMAR.md, which also lists where + * this parser still differs from it. */ #ifndef CVC_STATE_EXEC_PARSER_H #define CVC_STATE_EXEC_PARSER_H diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index 7517e5ce9..00cb4e9dd 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -65,6 +65,8 @@ add_executable(state_exec_evaluator_test state_exec_evaluator_test.cpp) add_executable(state_exec_stackless_evaluator_test state_exec_stackless_evaluator_test.cpp) add_executable(state_exec_codec_test state_exec_codec_test.cpp) add_executable(state_exec_unicode_test state_exec_unicode_test.cpp) +add_executable(state_exec_grammar_test state_exec_grammar_test.cpp) +add_executable(state_exec_corpus_test state_exec_corpus_test.cpp) add_executable(state_exec_async_test state_exec_async_test.cpp) add_executable(async_lane_test async_lane_test.cpp) add_executable(state_exec_scheduler_test state_exec_scheduler_test.cpp) @@ -326,6 +328,8 @@ list(APPEND TEST_TARGETS state_exec_stackless_evaluator_test state_exec_codec_test state_exec_unicode_test + state_exec_grammar_test + state_exec_corpus_test state_exec_async_test async_lane_test state_exec_scheduler_test @@ -1235,6 +1239,18 @@ target_link_libraries(state_exec_unicode_test GTest::gtest GTest::gtest_main ) +target_link_libraries(state_exec_grammar_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) +target_link_libraries(state_exec_corpus_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) target_link_libraries(state_exec_async_test PRIVATE @@ -1558,6 +1574,8 @@ target_compile_features(state_exec_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_stackless_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_codec_test PRIVATE cxx_std_20) target_compile_features(state_exec_unicode_test PRIVATE cxx_std_20) +target_compile_features(state_exec_grammar_test PRIVATE cxx_std_20) +target_compile_features(state_exec_corpus_test PRIVATE cxx_std_20) target_compile_features(state_exec_async_test PRIVATE cxx_std_20) target_compile_features(async_lane_test PRIVATE cxx_std_20) target_compile_features(state_exec_scheduler_test PRIVATE cxx_std_20) @@ -1966,6 +1984,8 @@ cvc_discover_tests(state_exec_evaluator_test) cvc_discover_tests(state_exec_stackless_evaluator_test) cvc_discover_tests(state_exec_codec_test) cvc_discover_tests(state_exec_unicode_test) +cvc_discover_tests(state_exec_grammar_test) +cvc_discover_tests(state_exec_corpus_test) cvc_discover_tests(state_exec_async_test) cvc_discover_tests(async_lane_test) cvc_discover_tests(state_exec_scheduler_test) diff --git a/src/cvc/tests/gen_state_exec_corpus.py b/src/cvc/tests/gen_state_exec_corpus.py new file mode 100644 index 000000000..ca7d54d8c --- /dev/null +++ b/src/cvc/tests/gen_state_exec_corpus.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Regenerate the state_exec parse corpus (state_exec_corpus/scripts.txt). + +The corpus is a frozen snapshot of the DSL source text the repo already carries, so the +grammar-driven parser (docs/STATE_EXEC_GRAMMAR.md) can be checked against how the +pre-grammar parser read every one of them. It collects: + +* every YAML scalar in a tracked ``*.ari`` file whose text starts with ``(`` (``init``, + ``on``, ``bind``, ``*_when`` and friends), +* every ``lisp``/``scheme`` fenced block in a tracked ``*.md`` file (unlabeled blocks are + skipped: the ones that start with ``(`` are ASCII diagrams), +* every C++ string literal (adjacent literals concatenated, escapes decoded) that starts + with ``(`` and ends with ``)`` in a tracked ``src/**/*.cpp``/``*.h`` file that mentions + ``state_exec`` or ``ariadne``. A one-line literal must start with ``(`` itself, which + skips messages like ``" (expected path:...)"``, and the parentheses must balance, which + skips prefixes a test completes at run time, +* every Python string constant of that shape in ``bindings/pycvc/test_pycvc_*.py``. + +Identical scripts are kept once, under the first source that has them. The expected parse +of each script is not computed here: ``state_exec_corpus_test`` writes +``state_exec_corpus/expected.txt`` when run with ``CVC_UPDATE_GOLDENS=1``. + +Usage (from anywhere in the checkout; needs PyYAML): + + python src/cvc/tests/gen_state_exec_corpus.py +""" + +from __future__ import annotations + +import ast +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +OUT = Path(__file__).resolve().parent / "state_exec_corpus" / "scripts.txt" + +# A corpus entry is "@@@ script " + the script's lines; no script line may start with +# "@@@ ", which the writer checks. +MARK = "@@@ " + + +def tracked(*patterns: str) -> list[str]: + out = subprocess.run( + ["git", "ls-files", "--", *patterns], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return sorted(p for p in out.splitlines() if p) + + +def balanced(text: str) -> bool: + """True if the parentheses outside strings and ; comments balance.""" + depth, i, n = 0, 0, len(text) + while i < n: + c = text[i] + if c == ";": + while i < n and text[i] != "\n": + i += 1 + elif c == '"': + i += 1 + while i < n and text[i] != '"': + i += 2 if text[i] == "\\" else 1 + elif c == "(": + depth += 1 + elif c == ")": + depth -= 1 + if depth < 0: + return False + i += 1 + return depth == 0 + + +def looks_like_program(text: str) -> bool: + # Only for literals in C++ and Python sources, where a fragment (a prefix the test + # completes at run time) is common: those are skipped as unbalanced. + s = text.strip() + if "\n" not in s and not text.startswith("("): + return False + return len(s) >= 3 and s.startswith("(") and s.endswith(")") and balanced(s) + + +# -- .ari ------------------------------------------------------------------------------------ + + +def ari_scripts(path: str): + # BaseLoader keeps every scalar a string, so the key `on:` is not read as YAML 1.1 `true`. + doc = yaml.load((ROOT / path).read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + + def walk(node, where): + if isinstance(node, dict): + for k, v in node.items(): + yield from walk(v, f"{where}.{k}" if where else str(k)) + elif isinstance(node, list): + for i, v in enumerate(node): + yield from walk(v, f"{where}[{i}]") + elif isinstance(node, str) and node.lstrip().startswith("("): + yield f"{path}:{where}", node + + yield from walk(doc, "") + + +# -- Markdown -------------------------------------------------------------------------------- + +FENCE = re.compile(r"^\s*(`{3,})\s*([\w+-]*)\s*$") + + +def md_scripts(path: str): + lines = (ROOT / path).read_text(encoding="utf-8").split("\n") + i = 0 + while i < len(lines): + m = FENCE.match(lines[i]) + if not m: + i += 1 + continue + ticks, lang = m.group(1), m.group(2) + j = i + 1 + while j < len(lines) and not lines[j].strip().startswith(ticks): + j += 1 + body = "\n".join(lines[i + 1 : j]) + if lang in ("lisp", "scheme"): + yield f"{path}:{i + 1}", body + i = j + 1 + + +# -- C++ ------------------------------------------------------------------------------------- + +CPP_TOKEN = re.compile( + r""" + (?P//[^\n]*|/\*.*?\*/) + | (?P(?:u8|u|U|L)?R"(?P[^()\\\s]{0,16})\((?P.*?)\)(?P=delim)") + | (?P(?:u8|u|U|L)?"(?P(?:[^"\\\n]|\\.)*)") + | (?P'(?:[^'\\\n]|\\.)*') + | (?P\s+) + | (?P.) + """, + re.DOTALL | re.VERBOSE, +) +CPP_ESC = re.compile( + r"\\(x[0-9A-Fa-f]+|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|[0-7]{1,3}|.)", re.DOTALL +) +SIMPLE = { + "n": "\n", + "t": "\t", + "r": "\r", + "a": "\a", + "b": "\b", + "f": "\f", + "v": "\v", + "\\": "\\", + '"': '"', + "'": "'", + "?": "?", +} + + +def cpp_unescape(body: str) -> str: + # Escapes become the bytes the compiler would put in a narrow (UTF-8) literal; the result is + # decoded as UTF-8 so a "\xC3\xA9" pair reads back as the character it encodes. + out = bytearray() + + def emit_cp(cp: int): + out.extend(chr(cp).encode("utf-8", "surrogatepass")) + + pos = 0 + for m in CPP_ESC.finditer(body): + out.extend(body[pos : m.start()].encode("utf-8")) + e = m.group(1) + if e[0] == "x": + out.append(int(e[1:], 16) & 0xFF) + elif e[0] in "uU": + emit_cp(int(e[1:], 16)) + elif e[0] in "01234567": + out.append(int(e, 8) & 0xFF) + else: + out.extend(SIMPLE.get(e, "\\" + e).encode("utf-8")) + pos = m.end() + out.extend(body[pos:].encode("utf-8")) + return out.decode("utf-8", "surrogateescape") + + +def cpp_scripts(path: str): + text = (ROOT / path).read_text(encoding="utf-8") + if "state_exec" not in text and "ariadne" not in text: + return + pending, start = None, 0 # concatenation of adjacent literals, and where it began + for m in CPP_TOKEN.finditer(text): + kind = m.lastgroup + if kind in ("raw", "str"): + piece = ( + m.group("rawbody") if kind == "raw" else cpp_unescape(m.group("body")) + ) + if pending is None: + pending, start = piece, m.start() + else: + pending += piece + elif kind in ("ws", "comment"): + continue + else: + if pending is not None and looks_like_program(pending): + yield f"{path}:{text.count(chr(10), 0, start) + 1}", pending + pending = None + if pending is not None and looks_like_program(pending): + yield f"{path}:{text.count(chr(10), 0, start) + 1}", pending + + +# -- Python ---------------------------------------------------------------------------------- + + +def py_scripts(path: str): + tree = ast.parse((ROOT / path).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and looks_like_program(node.value) + ): + yield f"{path}:{node.lineno}", node.value + + +# -- main ------------------------------------------------------------------------------------ + + +def main() -> int: + sources = [] + sources += [s for p in tracked("*.ari") for s in ari_scripts(p)] + sources += [s for p in tracked("*.md") for s in md_scripts(p)] + sources += [s for p in tracked("src/*.cpp", "src/*.h") for s in cpp_scripts(p)] + sources += [ + s for p in tracked("bindings/pycvc/test_pycvc_*.py") for s in py_scripts(p) + ] + + seen: set[str] = set() + entries = [] + for where, script in sources: + script = script.replace("\r\n", "\n") + if script in seen: + continue + seen.add(script) + if any(line.startswith(MARK) for line in script.split("\n")): + print( + f"{where}: a line starts with {MARK!r}; cannot store it", + file=sys.stderr, + ) + return 1 + entries.append((where, script)) + + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open("w", encoding="utf-8", newline="\n", errors="surrogateescape") as f: + f.write( + "# state_exec parse corpus. Generated by src/cvc/tests/gen_state_exec_corpus.py;\n" + ) + f.write( + "# do not edit by hand. Each entry is a '@@@ script ' line, then the script.\n" + ) + for where, script in entries: + f.write(f"{MARK}script {where}\n{script}\n") + print(f"{len(entries)} scripts -> {OUT.relative_to(ROOT)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cvc/tests/state_exec_corpus/.gitattributes b/src/cvc/tests/state_exec_corpus/.gitattributes new file mode 100644 index 000000000..6b27b86eb --- /dev/null +++ b/src/cvc/tests/state_exec_corpus/.gitattributes @@ -0,0 +1,2 @@ +# Corpus files are compared byte for byte; keep them LF on every checkout. +* text eol=lf diff --git a/src/cvc/tests/state_exec_corpus/expected.txt b/src/cvc/tests/state_exec_corpus/expected.txt new file mode 100644 index 000000000..b198a8cff --- /dev/null +++ b/src/cvc/tests/state_exec_corpus/expected.txt @@ -0,0 +1,1366 @@ +# Expected parses of scripts.txt, in the grammar_dump format of +# state_exec_grammar_dump.h. Written by state_exec_corpus_test with +# CVC_UPDATE_GOLDENS=1; do not edit by hand. +@@@ script bindings/pycvc/demo.ari:init +(state-set "demo.agents" "128") +(state-set "demo.speed" "1.0") +(state-set "demo.wire" "false") +@@@ script bindings/pycvc/demo.ari:windows[0].children[5].bind +(if (> (int (state-get "demo.agents")) 256) "many" "few") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[9].on +(state-set "graphics.root.children.domain.fe.request" "mesh") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[9].disabled_when +(= (state-get "graphics.root.children.domain.fe.busy") "1") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[13].visible_when +(= (state-get "graphics.root.children.domain.fe.problem") "poisson") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[14].visible_when +(= (state-get "graphics.root.children.domain.fe.problem") "heat") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[17].on +(state-set "graphics.root.children.domain.fe.request" "solve") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[17].disabled_when +(or (= (state-get "graphics.root.children.domain.fe.busy") "1") (= (state-get "graphics.root.children.domain.fe.stats.tets") "0")) +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[18].on +(state-set "graphics.root.children.domain.fe.request" "stop") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[18].enabled_when +(= (state-get "graphics.root.children.domain.fe.stoppable") "1") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[10].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "smooth") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[10].disabled_when +(= (state-get "graphics.root.children.mesh.mesh_ops.busy") "1") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[14].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "decimate") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[21].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "repair") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[22].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "orient") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[23].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "reset") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[28].visible_when +(= (state-get "graphics.root.children.mesh.mesh_ops.color.mode") "geodesic") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[29].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "curvature") +@@@ script src/cvc/ariadne/components/sim_transport.ari:units.sim_transport.children[3].enabled_when +(= (state-get "sim.paused") "1") +@@@ script src/cvcGL/examples/bunny_shadow.ari:init +(state-set "show_bunny" "true") +@@@ script src/cvcGL/examples/bunny_shadow.ari:windows[3].children[3].on +(begin (state-set "lighting.key_intensity" "1.9") (state-set "lighting.key_azimuth" "-38") (state-set "lighting.key_elevation" "52") (state-set "lighting.key_cone" "34")) +@@@ script src/cvcGL/examples/hello.ari:init +(state-set "demo.agents" "256") +@@@ script src/cvcGL/examples/hello.ari:windows[0].children[3].visible_when +(> (int (state-get "demo.agents")) 256) +@@@ script src/cvcGL/examples/hello.ari:windows[0].children[4].bind +(if (> (int (state-get "demo.agents")) 256) "heavy" "ok") +@@@ script src/cvcGL/examples/nav_compute.ari:init +(begin (state-set "nav.status" "idle") (state-set "nav.result" "-")) +@@@ script src/cvcGL/examples/nav_compute.ari:windows[0].children[3].on +(begin (state-set "nav.status" "running") (nav-step-async) (state-set "nav.result" (msg-recv "nav.done")) (state-set "nav.status" "done")) +@@@ script src/cvcGL/examples/nav_convoy.ari:init +(begin (state-set "show.agents" "1") (state-set "show.grid" "1") (state-set "sim.paused" "0") (state-set "sim.speed" "1.0")) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:134 +(int 3.7) +(int "123") +(int #t) +(float 5) +(float "3.14") +(float #t) +(is-int 42) +(is-int 3.14) +(is-float 3.14) +(is-float 42) +(is-string "hi") +(is-string 42) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:158 +(begin (defun fact (n) (if (<= n 1) 1 (* n (fact (- n 1))))) (fact 10)) +(begin (set result (list)) (for x (list 1 2 3 4 5) (set result (append result (list (* x x))))) result) +(state-set "shared.counter" "0") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:190 +(set g (generator (lambda () (yield 1) (yield 2) (yield 3)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:200 +(range 5) +(range 2 8) +(range 0 10 3) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:208 +(set g (range 3)) +(next g) +(next g) +(next g) +(next g) +(gen-done? g) +(for x (range 5) (print x)) +(collect (range 5)) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:231 +(set evens (generator (lambda () (for x (range 10) (if (= (% x 2) 0) (yield x) nil))))) +(collect evens) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:241 +(set squares (generator (lambda () (for x (range 5) (yield (* x x)))))) +(collect squares) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:250 +(set taken (list)) +(for x (range 1000) (if (>= (length taken) 4) (break nil) (append taken x))) +taken +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:261 +(set running-sum (generator (lambda () (let ((total 0)) (for x (range 1 6) (begin (set total (+ total x)) (yield total))))))) +(collect running-sum) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:273 +(set counter (generator (lambda () (let ((n 0)) (while (< n 1000) (begin (yield n) (set n (+ n 1)))))))) +(for x counter (if (= x 5) (break x) nil)) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:288 +(while t (break)) +(while t (break 42)) +(for x (list 10 20 30 40) (if (> x 25) (break x) nil)) +(set sum 0) +(for i (list 1 2 3) (for j (list 10 20 30) (if (= j 20) (break nil) (set sum (+ sum j))))) +sum +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:448 +(set child-pid (spawn "(begin (state-set \"child.done\" \"yes\") 42)" "child-worker")) +(set cpid (fork)) +(self-pid) +(self-uid) +(ps) +(inspect 2) +(pause 2) +(resume 2) +(kill 2) +(memory-usage) +(memory-limit) +(time-elapsed) +(time-limit) +(step-count) +(message-count) +(message-limit) +(cluster-id) +(node-id) +(scheduler-stats) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:531 +(state-set "myapp.config.timeout" "30") +(state-get "myapp.config.timeout") +(state-exists "myapp.config.timeout") +(state-children "myapp.config") +(state-delete "myapp.old-data") +(state-data-set "myapp.blob" some-value) +(state-data-get "myapp.blob") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:554 +(state-expire "myapp.session.abc" 60) +(state-expire-at "myapp.session.abc" "2026-01-01T00:00:00Z") +(state-has-expiry "myapp.session.abc") +(state-is-expired "myapp.session.abc") +(state-clear-expiry "myapp.session.abc") +(state-sweep-expired "myapp.session") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:582 +(msg-send "myapp.events.updates" "hello from process 1") +(msg-send "myapp.events.data" "{\"key\": \"value\"}" "application/json") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:595 +(begin (set i 0) (while (< i 10) (begin (state-set (str-concat "queue.item." (str i)) (str (* i i))) (msg-send "queue.notify" (str i)) (set i (+ i 1)))) "producer-done") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:607 +(begin (set count 0) (while (< count 10) (if (state-exists (str-concat "queue.item." (str count))) (begin (set val (state-get (str-concat "queue.item." (str count)))) (state-set (str-concat "consumed." (str count)) val) (set count (+ count 1))) nil)) count) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:705 +(set msg (msg-recv "events.data")) +(get-attr msg "payload") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:723 +(if (> (msg-pending "events.data") 0) (set msg (msg-recv "events.data")) (print "no messages yet")) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:735 +(sleep 0.5) +(while #t (begin (if (> (msg-pending "work.queue") 0) (begin (set job (msg-recv "work.queue")) (process-job job)) (sleep 0.1)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:756 +(begin (set results (list)) (set done nil) (while (not done) (begin (set m (msg-recv "tasks.input")) (set payload (get-attr m "payload")) (if (= payload "stop") (set done #t) (set results (append results (list payload)))))) results) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1183 +(begin (set my-pid (self-pid)) (state-set (str-concat "__migrate_request." (str my-pid)) "target-node-B") my-pid) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1308 +(string.split "a,b,c" ",") +(string.join (list "a" "b" "c") ",") +(string.replace "hello" "l" "r") +(string.trim " hello ") +(string.upper "hello") +(string.lower "HELLO") +(string.starts-with "hello" "he") +(string.ends-with "hello" "lo") +(string.contains "hello" "ell") +(string.substring "hello" 1 3) +(string.char-at "hello" 0) +(string.length "hello") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1325 +(math.sqrt 16) +(math.abs -5) +(math.floor 3.7) +(math.ceil 3.2) +(math.round 3.5) +(math.pow 2 10) +(math.log 2.718) +(math.sin 0) +(math.cos 0) +(math.min 3 7 1 5) +(math.max 3 7 1 5) +(math.clamp 15 0 10) +(math.pi) +(math.e) +(math.random) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1345 +(collections.map (lambda (x) (* x x)) (list 1 2 3)) +(collections.filter (lambda (x) (> x 2)) (list 1 2 3 4)) +(collections.reduce (lambda (a b) (+ a b)) (list 1 2 3) 0) +(collections.zip (list 1 2) (list "a" "b")) +(collections.flatten (list (list 1 2) (list 3 4))) +(collections.sort (list 3 1 2)) +(collections.reverse (list 1 2 3)) +(collections.range 1 5) +(collections.unique (list 1 2 2 3 3)) +(collections.dict-keys (dict "a" 1 "b" 2)) +(collections.dict-values (dict "a" 1 "b" 2)) +(collections.dict-merge (dict "a" 1) (dict "b" 2)) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1445 +(set i 0) +(while (< i 5) (let ((i (+ i 1))) (print i))) +(set i 0) +(while (< i 5) (begin (print i) (set i (+ i 1)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1466 +(state-set "counter" (str 42)) +(set raw (state-get "counter")) +(set n (int raw)) +(if (> n 40) "big" "small") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1478 +(set g (range 3)) +(collect g) +(collect g) +(defun make-range () (range 3)) +(collect (make-range)) +(collect (make-range)) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1496 +(msg-recv "work.queue") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1526 +(set done nil) +(for i (range 10) (if done nil (for j (range 10) (if (= j 3) (begin (set done #t) (break nil)) nil)))) +@@@ script docs/STREAMING.md:107 +(set cam (stream-open (dict "id" "cam0" "codec" "rgba8" "w" 1280 "h" 720))) +(get-attr cam "ok") +(get-attr cam "token") +(get-attr cam "seq_channel") +(stream-info cam) +(stream-close cam) +@@@ script docs/STREAMING.md:162 +(gl-bind-stream "screen" (get-attr cam "token")) +@@@ script docs/STREAMING.md:499 +(set cam (stream-open (dict "id" "cam0" "codec" "rgba8" "w" 1280 "h" 720))) +(stream-capture-camera (get-attr cam "token") 0) +(gl-bind-stream "screen" (get-attr cam "token")) +@@@ script docs/roadmap/ARIADNE_CONFERENCE_ROADMAP.md:97 +(set room (room-join "conf.example.org:7000" "standup" (dict "name" "Joe"))) +(set cam (stream-open (dict "id" "self/cam" "codec" "rgba8" "w" 1280 "h" 720))) +(stream-capture-camera (get-attr cam "token") 0) +(room-publish room (dict "kind" "camera" "token" (get-attr cam "token"))) +(set mic (stream-open (dict "id" "self/mic" "codec" "f32" "rate" 48000 "ch" 1))) +(stream-capture-audio (get-attr mic "token") (dict "device" 0)) +(room-publish room (dict "kind" "audio" "token" (get-attr mic "token"))) +@@@ script docs/roadmap/CVCGL-UI-DSL-ROADMAP.md:4452 +(state-set "demo.n" 256) +(window "Controls" (slider-int "Agents" :bind "demo.n" :lo 1 :hi 512) (combo "Belief" :bind "demo.belief" :options (list "shared" "grouped" "private")) (button "Reset" :on "reset")) +(scene (geometry-node "mesh" :source "file://blocks.off" :visible-when (= (state-get "demo.show") "1"))) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:91 +(vol-read "mydata" "/path/to/brain.rawiv") +(set dims (vol-dims "mydata")) +(set bbox (vol-bbox "mydata")) +(vol-normalize "mydata" "normalized") +(vol-scale "normalized" 2.0 "scaled") +(iso "scaled" 0.5 "surface" (dict "method" "fastcontouring")) +(sdf "surface" "sdf-vol" (dict "dim" (list 128 128 128))) +(tet "sdf-vol" 0.0 "tet-mesh") +(vol-write "sdf-vol" "/path/to/output.rawiv") +(geom-write "tet-mesh" "/path/to/output.off") +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:288 +(iso "brain-vol" 128.0 "brain-surf" (dict "method" "fastcontouring" "improve" #t "compute-normals" #t)) +(sdf "brain-surf" "brain-sdf" (dict "dim" (list 256 256 256) "flip-normals" nil)) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:307 +(defun process-brain (input-file output-dir) (begin (vol-read "raw" input-file) (print (str "Loaded: " (vol-dims "raw"))) (vol-normalize "raw" "norm") (vol-clip "norm" 0.1 0.9 "clipped") (iso "clipped" 0.5 "surface" (dict "method" "fastcontouring" "improve" #t "compute-normals" #t)) (print (str "Surface: " (geom-num-tris "surface") " tris")) (tet "clipped" 0.5 "tet-mesh" (dict "improve" #t)) (print (str "Tet mesh: " (geom-num-tets "tet-mesh") " tets")) (geom-write "surface" (str output-dir "/brain-surface.off")) (geom-write "tet-mesh" (str output-dir "/brain-tet.off")) (dict "tris" (geom-num-tris "surface") "tets" (geom-num-tets "tet-mesh")))) +(process-brain "/data/brain.rawiv" "/output") +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:343 +(defun process-all (file-list output-dir) (for f file-list (begin (print (str "Processing: " f)) (process-brain f output-dir)))) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:353 +(defun distribute-pipeline (files) (for f files (begin (state-set (str "jobs." f ".status") "pending") (state-set (str "jobs." f ".input") f) (msg-send "work-queue" f)))) +(defun worker () (while #t (set job (msg-recv "work-queue")) (state-set (str "jobs." job ".status") "running") (process-brain (state-get (str "jobs." job ".input")) "/output") (state-set (str "jobs." job ".status") "done"))) +@@@ script docs/roadmap/STATE_BINARY_STREAMING.md:220 +(define s (stream-open |{:kind| "video_raw" :codec "rgba8" :w 1920 :h 1080 |}|)) +(stream-info s) +(gl-bind-stream node s) +(msg-recv "stream.evt") +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1101 +(state-set "sensors.cache.temperature" (read-sensor)) +(state-expire "sensors.cache.temperature" 60) +(if (state-is-expired "sensors.cache.temperature") (begin (state-sweep-expired "sensors.cache") (refresh-cache)) (state-get "sensors.cache.temperature")) +(state-expire-at "jobs.batch-42" "2025-06-15T00:00:00Z") +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1355 +(let ((pid (fork))) (if (= pid 0) (begin (state-set "results.child" (compute))) (begin (state-set "results.parent" (str "spawned child " pid))))) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1383 +(let ((pid (fork))) (if (= pid -1) (print "fork failed: resource limit") |...|)) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1470 +(require "string") +(split "hello world" " ") +(stdlib.string.split "hello world" " ") +(require "math" (quote (sqrt abs floor ceil))) +(sqrt 144) +(stdlib-modules) +(stdlib-list "string") +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1709 +(require "geometry") +(distance 0 0 3 4) +(require "util") +(repeatedly 5 (lambda (i) (print i))) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1763 +(let ((mesh (state-data-get "/scene/model/geometry"))) (if (data? mesh) (begin (print (data-type mesh)) (state-data-set "/output/result" mesh)))) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1776 +(state-data-set "/scene/model/transformed" transformed-mesh) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1781 +(data-type mesh) +(data? mesh) +(data? 42) +(state-data-type "/path") +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1853 +(state-set "/scene/camera/x" 10) +(state-set "/scene/camera/y" 20) +(state-set "/scene/camera/z" 30) +(state-batch (state-set "/scene/camera/x" 10) (state-set "/scene/camera/y" 20) (state-set "/scene/camera/z" 30)) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1876 +(state-lock "/scene/camera" (let ((x (state-get "/scene/camera/x")) (y (state-get "/scene/camera/y"))) (state-set "/scene/camera/distance" (sqrt (+ (* x x) (* y y)))))) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1093 +(+ 1 2 3) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1099 +(* 7 6) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1105 +(let ((x 10)) (+ x 5)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1111 +(let ((f (lambda (x y) (+ x y)))) (f 3 4)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1120 +(+ 100 200 300) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1206 +(let ((square (lambda (x) (* x x)))) (square 9)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1211 +(car (list 10 20 30)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1216 +(if (> 5 3) 99 0) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1221 +(let ((a 10) (b 20)) (+ a b)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1228 +(begin 1 2 3 4 5 6 7 8 9 10) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1238 +(let ((pi 3.14159) (r 5)) (* pi r r)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1881 +(no children) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1891 +(+ 1 (+ 2 (+ 3 (+ 4 (+ 5 6))))) +@@@ script src/cvc-cli/main.cpp:1554 +(no children) +@@@ script src/cvc/ariadne/ariadne.cpp:1256 +(no image) +@@@ script src/cvc/ariadne/ftxui_backend.cpp:89 +(empty) +@@@ script src/cvc/tests/ariadne_loader_test.cpp:535 +(state-set "paused" "true") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2256 +(> (int (state-get "n")) 5) +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2276 +(state-exists "ready") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2277 +(state-exists "busy") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2301 +(list "p" "q") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2402 +(state-exists "show") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:3688 +(int (state-get "n")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:80 +(closable) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:356 +(state-set "paused" (if (= (state-get "paused") "true") "false" "true")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:376 +(begin (state-set "agents" "64") (state-set "speed" "1.0")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:400 +(begin (msg-recv "async.done") (state-set "async.result" "resumed")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:428 +(state-set "async.value" (msg-recv "vchan")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:486 +(begin (nav-step-async) (state-set "nav.result" (msg-recv "nav.done"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:511 +(state-set "tick.flag" "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:547 +(begin (state-set "input.last_key" (get-attr event "key")) (state-set "input.kind" (get-attr event "kind"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:565 +(state-set "input.last_key" (get-attr event "key")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:583 +(begin (state-set "ptr.kind" (get-attr event "kind")) (state-set "ptr.btn" (get-attr event "button"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:606 +(state-set "widget.clicked" "yes") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:728 +(begin (state-set "ev.btn" (get-attr event "button")) (state-set "ev.has_x" (if (>= (get-attr event "x") 0) "yes" "no"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:780 +(host-bump) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:959 +(state-set "agents" "128") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:960 +(no error) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1033 +(while #t (+ 1 1)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1077 +(state-set "n" "10") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1121 +(= (state-get "show") "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1150 +(int 2) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1351 +(state-exists "never") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1384 +(state-set "x" 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1399 +(state-set "mode" "on") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1401 +(= (state-get "mode") "on") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1414 +(while #t 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1431 +(msg-recv "x") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1431 +(await 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1470 +(state-set "flag" "0") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1472 +(state-get "flag") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1474 +(state-get "never_set") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1490 +(foobar 1 2) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1541 +(is-string (str-concat "a" "b")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1543 +(is-list (append (list 1) (list 2))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1545 +(apply and (list #t)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1547 +(begin (print "x") #t) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1565 +(= (quote (1 2 3)) (quote (1 2 3))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1567 +(= (quote (1 2)) (quote (9 9))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1569 +(let ((xs (list 1 2 3))) (> (length xs) 2)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1587 +(begin (defclass B (init (self) (while 1 1))) (B)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1589 +(eval (quote #t)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1591 +(begin (defmacro m () #t) #t) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1606 +(begin (set d (list 0 0)) (set i 0) (while (< i 30) (begin (set d (list d d)) (set i (+ i 1)))) (state-data-set "bomb" d)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1614 +(is-null (state-data-get "bomb")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1626 +(state-set "n" "3") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1647 +(state-set "lock" "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1649 +(state-exists "lock") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1686 +(state-set "n" "7") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1688 +(+ (int (state-get "n")) 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1689 +(state-get "n") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1775 +(list "shared" "grouped" "private") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1799 +(state-set "hint" "dynamic") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1803 +(state-get "hint") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1847 +(state-set "n" "2") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1912 +(* 1e+19 1e+19) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1928 +(begin (defun f () #t) (f)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1930 +(f) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2287 +(begin (set r (msg-recv (http-get-async "http://ex/x"))) (state-set "r.status" (get-attr r "status")) (state-set "r.ok" (get-attr r "ok")) (state-set "r.error" (get-attr r "error")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2325 +(begin (set r (msg-recv (http-get-async "http://ex/down"))) (state-set "r.ok" (get-attr r "ok")) (state-set "r.error" (get-attr r "error"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2353 +(state-set "r.status" (get-attr (msg-recv (http-get-async "http://ex/slow")) "status")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2383 +(begin (set r (http-get "http://ex/x")) (state-set "r.status" (get-attr r "status")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2414 +(state-set "r.status" (get-attr (await (http-get-async "http://ex/x")) "status")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2449 +(begin (set r (fetch "mem://x")) (state-set "r.ok" (get-attr r "ok")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2481 +(state-data-set "r.body" (get-attr (await (fetch-async "mem://y")) "body")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2519 +(begin (set r (http-request "POST" "http://ex/api" (dict "headers" (dict "Authorization" "Bearer tok-123") "form" (dict "grip" 1 "risk" 2)))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2560 +(begin (set r (http-get "http://ex/x" (dict "query" (dict "q" "hello world" "n" 2)))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2592 +(begin (set r (http-request "PUT" "http://ex/raw" (dict "body" "raw-payload"))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2625 +(begin (set g (await (http-get-async "http://ex/src"))) (set r (await (http-request-async "PUT" "http://ex/dst" (dict "body" (get-attr g "body") "headers" (dict "Authorization" "Bearer tok"))))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2688 +(begin (state-set "r.t" (str (pool-double 21))) (state-set "r.a" (str (await (pool-double-async 10))))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2754 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok")) (state-set "t.token" (get-attr h "token")) (state-set "t.seq" (get-attr h "seq_channel")) (state-set "t.live" (get-attr (stream-info h) "live")) (state-set "t.closed" (get-attr (stream-close h) "ok")) (state-set "t.live2" (get-attr (stream-info h) "live"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2785 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.token" (get-attr h "token")) (state-set "t.seq" (get-attr h "seq_channel"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2814 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2843 +(begin (set a (stream-open (dict "id" "c0" "w" 4 "h" 2 "expected_subscribers" -1))) (state-set "t.a" (get-attr a "ok")) (set b (stream-open (dict "id" "c1" "w" 999999 "h" 2))) (state-set "t.b" (get-attr b "ok"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2869 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok")) (state-set "t.err" (get-attr h "error"))) +@@@ script src/cvc/tests/async_lane_test.cpp:498 +(+ (lane-double 21) (await (lane-double-async 10))) +@@@ script src/cvc/tests/async_lane_test.cpp:525 +(while t (apply-built (msg-recv "app#built.tracks"))) +@@@ script src/cvc/tests/async_lane_test.cpp:526 +(while t (apply-plain (msg-recv "built"))) +@@@ script src/cvc/tests/async_lane_test.cpp:621 +(while t (begin (tick) (await 0))) +@@@ script src/cvc/tests/async_lane_test.cpp:652 +(while t (sleep (msg-recv (sim-launch)))) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:125 +(begin (set big "") (set i 0) (while (< i 100) (begin (set big (str-concat big "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789")) (set i (+ i 1)))) (state-set "data.large_string" big) (length big)) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:151 +(begin (state-data-set "data.typed_obj" (list 1 2 3 4 5)) (set obj (state-data-get "data.typed_obj")) (is-null obj)) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:168 +(begin (set i 0) (while (< i 50) (begin (state-set "data.counter" (str i)) (set i (+ i 1)))) (state-get "data.counter")) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:803 +(begin (state-set "queue.item.0" "apple") (state-set "queue.item.1" "banana") (state-set "queue.item.2" "cherry") (state-set "queue.count" "3") (msg-send "queue.ready" "items-available") "producer-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:813 +(begin (set result (list)) (set i 0) (while (< i 3) (begin (set key (str-concat "queue.item." (str i))) (set val (state-get key)) (if (!= val nil) (set i (+ i 1)) nil))) (state-set "consumed.total" (str i)) i) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:843 +(begin (state-set "partial.0" (str (+ 1 2 3))) "map-0-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:849 +(begin (state-set "partial.1" (str (+ 4 5 6))) "map-1-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:855 +(begin (state-set "partial.2" (str (+ 7 8 9))) "map-2-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:861 +(begin (set s0 (state-get "partial.0")) (set s1 (state-get "partial.1")) (set s2 (state-get "partial.2")) (state-set "result.total" (str (+ (if (= s0 nil) 0 6) (if (= s1 nil) 0 15) (if (= s2 nil) 0 24)))) 45) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:928 +(begin (spawn "(* 10 10)" "w0") (spawn "(* 20 20)" "w1") (spawn "(* 30 30)" "w2") 3) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:947 +(begin (set total 0) (set i 0) (while (< i 5) (begin (set total (+ total (* i i))) (state-set "progress.current" (str i)) (state-set "progress.total" (str total)) (set i (+ i 1)))) total) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:977 +(begin (state-set "document.header.title" "My Report") (state-set "document.header.author" "Alice") (state-set "document.header.date" "2026-05-24") (msg-send "document.events" "header-complete") "header-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:986 +(begin (state-set "document.body.section1" "Introduction text...") (state-set "document.body.section2" "Methods text...") (state-set "document.body.section3" "Results text...") (msg-send "document.events" "body-complete") "body-done") +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1013 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (set result (fib 12)) (state-set "computation.fib12.result" (str result)) (state-set "computation.fib12.status" "complete") (msg-send "computation.notifications" "fib12-done") result) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1094 +(begin (while #t 1)) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1099 +(+ 1 2) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1110 +(+ 3 4) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1111 +(+ 5 6) +@@@ script src/cvc/tests/state_exec_async_test.cpp:138 +(- 10 3) +@@@ script src/cvc/tests/state_exec_async_test.cpp:140 +(* 4 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:144 +(/ 10 4) +@@@ script src/cvc/tests/state_exec_async_test.cpp:147 +(+ (* 2 3) (- 10 4)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:152 +(< 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:153 +(< 2 1) +@@@ script src/cvc/tests/state_exec_async_test.cpp:157 +(= 5 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:158 +(= 5 6) +@@@ script src/cvc/tests/state_exec_async_test.cpp:163 +(if t 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:165 +(if nil 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:167 +(begin 1 2 3) +@@@ script src/cvc/tests/state_exec_async_test.cpp:169 +(begin (set x 42) x) +@@@ script src/cvc/tests/state_exec_async_test.cpp:172 +(begin (set i 0) (set sum 0) (while (< i 5) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:179 +(begin (set sum 0) (for x (list 1 2 3 4 5) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:188 +(begin (set sq (lambda (x) (* x x))) (sq 5)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:192 +(begin (set make-adder (lambda (n) (lambda (x) (+ x n)))) (set add5 (make-adder 5)) (add5 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:200 +(begin (defun fact (n) (if (< n 2) 1 (* n (fact (- n 1))))) (fact 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:209 +(let ((x 10) (y 20)) (+ x y)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:214 +(begin (set f (lambda () (begin (return 42) 99))) (f)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:220 +(quote (1 2 3)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:229 +(begin (defmacro double-it (x) (* x 2)) (double-it 21)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:237 +(eval (quote (+ 1 2))) +@@@ script src/cvc/tests/state_exec_async_test.cpp:242 +(str-concat "hello" " " "world") +@@@ script src/cvc/tests/state_exec_async_test.cpp:248 +(car (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:249 +(car (cdr (list 1 2 3))) +@@@ script src/cvc/tests/state_exec_async_test.cpp:252 +(length (list 1 2 3 4 5)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:257 +(begin (defun fib (n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 15)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:268 +(await (+ 1 2)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:275 +(+ 10 20) +@@@ script src/cvc/tests/state_exec_async_test.cpp:395 +(+ 10 20 30) +@@@ script src/cvc/tests/state_exec_async_test.cpp:449 +(begin (set i 0) (set s 0) (while (< i 5) (begin (set s (+ s i)) (set i (+ i 1)))) s) +@@@ script src/cvc/tests/state_exec_async_test.cpp:469 +(begin (set x 10) (set y 20) (+ x y)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:470 +(begin (defun fact (n) (if (< n 2) 1 (* n (fact (- n 1))))) (fact 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:471 +(begin (set sum 0) (for x (list 1 2 3 4 5) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:472 +(let ((a 5) (b 10)) (* a b)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:498 +(+ 40 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:509 +(begin 1 2 3 4 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:537 +(begin 1 2 3 4 5 6 7 8) +@@@ script src/cvc/tests/state_exec_async_test.cpp:570 +(nap) +@@@ script src/cvc/tests/state_exec_async_test.cpp:645 +(begin (set r (await 42)) r) +@@@ script src/cvc/tests/state_exec_async_test.cpp:670 +(begin (set s 0) (set i 0) (while (< i 200) (begin (set s (+ s i)) (set i (+ i 1)))) s) +@@@ script src/cvc/tests/state_exec_async_test.cpp:738 +(+ 2 3) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:67 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 20)) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:90 +(begin (set sum 0) (set i 0) (while (< i 1000) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:116 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 15)) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:141 +(begin (set x 0) (set i 0) (while (< i 100) (begin (set x (+ x i)) (set i (+ i 1)))) x) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:179 +(begin (set lst (list)) (set i 0) (while (< i 500) (begin (set lst (append lst (list i))) (set i (+ i 1)))) lst) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:253 +(begin (defun even? (n) (if (= n 0) #t (odd? (- n 1)))) (defun odd? (n) (if (= n 0) nil (even? (- n 1)))) (even? 200)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:297 +(begin (set sum 0) (set i 0) (while (< i 100) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:320 +(+ 1 2 3 4 5 6 7 8 9 10) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:427 +(/ 10 3) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:495 +(let ((x 10)) (let ((y 20)) (+ x y))) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:527 +(cdr (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:530 +(length (list 1 2 3 4)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:535 +(begin (set d (dict)) (set-attr d "key" 42) (get-attr d "key")) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:544 +(begin (defun fib (n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)) +@@@ script src/cvc/tests/state_exec_coordinator_test.cpp:469 +(begin (+ 1 2) (+ 3 4) (+ 5 6)) +@@@ script src/cvc/tests/state_exec_coordinator_test.cpp:653 +(begin (+ 1 2) (+ 3 4)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:41 +(- 10 3 2) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:43 +(* 3 4) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:47 +(% 10 3) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:49 +(+ (* 3 4) (- 10 5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:59 +(= 42 42) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:60 +(= 1 2) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:69 +(if nil 1) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:73 +(begin) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:78 +(quote (+ 1 2)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:90 +(begin (set sum 0) (for x (list 1 2 3 4) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:99 +(begin (set add1 (lambda (x) (+ x 1))) (add1 10)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:114 +(begin (set sum-all (lambda (&rest args) (apply + args))) (sum-all 1 2 3)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:123 +(begin (defun double (x) (* x 2)) (double 21)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:130 +(begin (defun fact (n) (if (<= n 1) 1 (* n (fact (- n 1))))) (fact 5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:149 +(begin (defun early (x) (if (< x 0) (return -1) x)) (early -5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:172 +(root) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:188 +(car (cdr (list 10 20 30))) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:189 +(car (cons 5 (list 10 20))) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:192 +(length (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:197 +(begin (set obj (dict "x" 10 "y" 20)) (set-attr obj "x" 99) (get-attr obj "x")) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:215 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:49 +(begin (set m (lambda (self) 99)) (set superd (dict "m" m)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:57 +(begin (set superd (dict "mk" list)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (length (super obj mk 4 5))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:64 +(begin (set m (lambda (self a) (+ a 90))) (set superd (dict "m" m)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m 9)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:72 +(begin (set superd2 (dict "mk" list)) (set superd1 (dict "__super__" superd2)) (set clsd (dict "__super__" superd1)) (set obj (dict "__class__" clsd)) (length (super obj mk 1))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:79 +(begin (set superd (dict "m" 5)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:101 +(while t 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:120 +(+ 2 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:145 +(set a 1) +(set b 2) +(+ a b) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:150 +(1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:151 +(no-such-fn 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:152 +(begin (set v 7) (v 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:153 +(if t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:154 +(while t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:155 +(for x (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:156 +(for 1 (list 1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:157 +(for x 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:158 +(set a) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:159 +(set 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:160 +(quote 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:161 +(lambda (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:162 +(lambda (1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:163 +(lambda 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:164 +(let ((a 1))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:165 +(let 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:166 +(let ((a)) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:167 +(let ((1 2)) 3) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:168 +(defun f (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:169 +(defun 5 (x) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:170 +(defmacro m (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:171 +(defmacro 5 (x) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:172 +(defmacro m (1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:173 +(eval) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:174 +(super 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:175 +(defclass) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:176 +(defclass 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:180 +(begin (defun f () (begin (return) 9)) (f)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:184 +(begin (set f (lambda (x) x)) (f 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:185 +(begin (set f (lambda (&rest) 1)) (f 5)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:189 +(begin (set f (lambda () 42)) (f)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:192 +(lambda nil 42) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:196 +(begin (defun f () (return 5)) (+ (f) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:200 +(begin (defmacro pick (a b) b) (pick 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:204 +(begin (defmacro m2 (x) (set y x) (+ y 1)) (m2 5)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:208 +(get-attr d "name") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:209 +(send d "speak") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:210 +(send d "add" 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:211 +(send d "finalize") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:215 +(begin (defclass Empty) (Empty 42)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:220 +(begin (defclass P1 (m (self) 7)) (defclass C1 (P1) (n (self) 8)) (send (C1) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:226 +(super d speak) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:227 +(super d "speak") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:228 +(super d add 3 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:237 +(super (dict "k" 1) m) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:238 +(super (Animal "a") speak) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:239 +(super d fly) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:243 +(begin (defclass KX (m (self) 7) 5 (5 1)) (send (KX) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:249 +(begin (set Q (lambda (x) x)) (defclass CQ Q (n (self) 4)) (send (CQ) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:252 +(begin (set Q2 (lambda (x) x)) (defclass CR (Q2) (n (self) 5)) (send (CR) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:286 +(set a 4) +(set b 5) +(+ a b) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:295 +(begin (set i 0) (while (< i 20) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:346 +(return) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:347 +(yield) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:353 +(defun f (1) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:365 +(begin (set v 7) (v)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:369 +(begin (set f (lambda (&rest xs) (length xs))) (f 1 2 3)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:378 +(begin (set f (lambda (x) (set y (* x 2)) (+ y 1))) (f 4)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:381 +(let () 42) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:384 +(begin (defun f (&rest xs) (length xs)) (f 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:392 +(for x (range 0) x) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:414 +(begin (defclass P2 (who (self) "parent")) (defclass C2 (P2)) (send (C2) "who")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:421 +(begin (defclass V (m (self &rest more) (length more))) (send (V) "m" 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:424 +(begin (defclass M (m (self) (set y 4) (+ y 2))) (send (M) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:428 +(begin (defclass K (m (self) 7) (5 1)) (send (K) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:441 +(defclass B (m (self 5) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:473 +(-) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:474 +(*) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:475 +(/) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:476 +(- 2.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:477 +(- 5.5 1 0.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:478 +(/ 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:479 +(+ 0.5 (list 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:480 +(+ (list 1.5) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:481 +(* (list 2 3)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:482 +(* 2.0 (list 3 4)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:483 +(% 7.5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:487 +(% 1 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:488 +(% 1.0 0.0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:489 +(/ 1 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:490 +(< nil 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:491 +(car) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:492 +(send 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:496 +(int (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:497 +(float (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:498 +(str-concat "a" 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:502 +(car (list)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:503 +(length (cdr (list))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:504 +(car 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:505 +(nth (list 1) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:506 +(nth (list 1) "x") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:507 +(set-nth (list 1) 5 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:508 +(del-nth (list 1) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:509 +(begin (set l (list 1 2 3)) (del-nth l 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:510 +(length (dict "a" 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:511 +(length t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:515 +(slice (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:516 +(length (slice (list 1 2 3) -1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:517 +(length (slice (list 1 2) 0 -1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:518 +(length (slice (list 1 2) 5 6)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:519 +(length (slice (list 1 2) 0 9)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:520 +(length (slice (list 1 2) 1 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:521 +(length (slice (list 1 2 3) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:525 +(dict "a") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:526 +(get-attr (dict 1 "x") "1") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:527 +(get-attr (dict) "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:528 +(get-attr 5 "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:529 +(get-attr (dict) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:530 +(del-attr (dict) "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:531 +(begin (set d (dict "k" 3)) (del-attr d "k")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:535 +(send (dict "a" 1) "m") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:536 +(begin (set o (dict "__class__" (dict "m" (lambda (s) 1)))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:539 +(begin (set o (dict "__class__" (dict "m" 5))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:541 +(begin (set o (dict "__class__" (dict "x" 1))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:546 +(begin (set superd (dict "mk" list)) (set clsd (dict "x" 1 "__super__" superd)) (set obj (dict "__class__" clsd)) (length (send obj "mk" 4 5))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:555 +(apply 5 (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:556 +(append 5 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:559 +(print 1 "a" 2.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:562 +(generator 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:563 +(next 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:564 +(gen-done? 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:565 +(collect 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:566 +(range 1 2 3 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:567 +(range "a") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:568 +(range "a" 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:569 +(range 1 "b") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:570 +(range 1 5 "c") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:571 +(range 1 5 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:576 +(collect (generator (lambda () (begin (yield 1) (yield 2))))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:585 +(and) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:586 +(and 1 nil 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:587 +(and 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:588 +(or nil nil) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:589 +(or nil 7) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:590 +(not nil) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:629 +(set c 1) +(+ c 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:756 +(+ 1 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:770 +(* 2 3) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:771 +(* 2 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:772 +(* 2 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:781 +(begin (set i 0) (while (< i 50) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:795 +(begin (set i 0) (while (< i 40) (set i (+ i 1)))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:808 +(set-attr box "sig" __signal__) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:809 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "sig")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:820 +(begin (set i 0) (while (< i 4) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:857 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "path")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:863 +(set-attr box "path" p) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:878 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:883 +(set-attr box "n" 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:883 +(set-attr box "n" 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:927 +(begin (set i 0) (while (< i 8) (set i (+ i 1))) (get-attr box "path")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:954 +(begin (set i 0) (while (< i 100) (set i (+ i 1)))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:987 +(+ 20 22) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:95 +(spawn "42" "child") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:152 +(self-uid) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:650 +(begin (state-set "mailbox.test" "ready") (msg-send "mailbox.test" "hello-world")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:664 +(begin (msg-send "test.path" "msg1") (msg-send "test.path" "msg2") (msg-send "test.path" "msg3") (message-count)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:687 +(begin (msg-send "test" "1") (msg-send "test" "2") (msg-send "test" "3") (msg-send "test" "4")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:706 +(begin (msg-send "test" "hello world!!") (msg-send "test" "this exceeds the byte cap") "done") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:725 +(begin (state-set "shared.item.0" "100") (state-set "shared.item.1" "200") (state-set "shared.item.2" "300") (state-set "shared.count" "3") "done") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:737 +(begin (state-get "shared.count")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:758 +(begin (state-set "parent.uid" (self-uid)) (set child-pid (spawn "(+ 1 2)" "child-worker")) child-pid) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:774 +(begin (set result (msg-send "events.log" "{\"level\":\"info\"}" "application/json")) (get-attr result "status")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:802 +(begin (msg-send "console.stdout" "Hello from DSL!") (msg-send "console.stdout" "Computing...") (set result (* 6 7)) (msg-send "console.stdout" (str-concat "Result: " (str result))) (msg-send "console.stderr" "Warning: example only") result) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:832 +(begin (set before (msg-pending "q.test")) (msg-send "q.test" "a") (msg-send "q.test" "b") (set after (msg-pending "q.test")) (list before after)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:855 +(begin (msg-send "drain.test" "first") (msg-send "drain.test" "second") (msg-send "drain.test" "third") "sent") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:873 +(begin (set m1 (msg-recv "drain.test")) (set m2 (msg-recv "drain.test")) (set m3 (msg-recv "drain.test")) (list (get-attr m1 "status") (get-attr m2 "status") (get-attr m3 "status"))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:959 +(+ 100 200) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1028 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1036 +(begin (defun fact (n) (if (<= n 1) 1 (* n (fact (- n 1))))) (fact 8)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1056 +(begin (set procs (ps)) (length procs)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1071 +(begin (set info (inspect (self-pid))) (get-attr info "pid")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1085 +(begin (set i 0) (set sum 0) (while (< i 10) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1101 +(begin (set total 0) (for x (list 10 20 30 40) (set total (+ total x))) total) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1114 +(begin (set double (lambda (x) (* x 2))) (double 21)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1125 +(let ((x 10) (y 20)) (+ x y)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1135 +(begin (set d (dict "name" "alice" "age" 30)) (get-attr d "name")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1157 +(begin (set stats (scheduler-stats)) (get-attr stats "total")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1212 +(begin (state-set "data.x" "42") (state-get "data.x")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1353 +(begin (set wid (state-watch "events.x" (lambda (p v) nil))) (state-unwatch wid)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1367 +(begin (state-watch "trigger.value" (lambda (path val) (state-set "got-event" "yes"))) (state-set "trigger.value" "hello") (set tries 0) (while (and (not (state-exists "got-event")) (< tries 100)) (set tries (+ tries 1))) (state-get "got-event")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1388 +(begin (state-watch "notify.path" (lambda (path val) (state-set "received-path" path))) (state-set "notify.path" "data") (set tries 0) (while (and (not (state-exists "received-path")) (< tries 100)) (set tries (+ tries 1))) (state-get "received-path")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1417 +(begin (state-set "consumer.log" "") (state-watch "queue.item" (lambda (path val) (state-set "consumer.log" (str-concat (state-get "consumer.log") (state-get path) ",")))) (state-set "queue.item" "apple") (state-set "queue.item" "banana") (state-set "queue.item" "cherry") (set tries 0) (while (and (not (= (state-get "consumer.log") "apple,banana,cherry,")) (< tries 300)) (set tries (+ tries 1))) (state-get "consumer.log")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1451 +(begin (state-set "stats.ticks" "") (state-watch "sensor.reading" (lambda (path val) (state-set "stats.ticks" (str-concat (state-get "stats.ticks") "|")))) (state-set "sensor.reading" "72") (state-set "sensor.reading" "73") (state-set "sensor.reading" "71") (state-set "sensor.reading" "74") (state-set "sensor.reading" "70") (set tries 0) (while (and (not (= (state-get "stats.ticks") "|||||")) (< tries 500)) (set tries (+ tries 1))) (state-get "stats.ticks")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1487 +(begin (state-watch "pipeline.input" (lambda (path val) (state-set "pipeline.stage1" (str-concat "processed:" (state-get path))))) (state-watch "pipeline.stage1" (lambda (path val) (state-set "pipeline.stage2" (str-concat "[" (state-get path) "]")))) (state-set "pipeline.input" "hello") (set tries 0) (while (and (not (state-exists "pipeline.stage2")) (< tries 500)) (set tries (+ tries 1))) (state-get "pipeline.stage2")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1525 +(begin (state-set "sampler.log" "") (set wid (state-watch "data.stream" (lambda (path val) (begin (set current-val (state-get path)) (set current-log (state-get "sampler.log")) (state-set "sampler.log" (str-concat current-log current-val ",")) (set updated (state-get "sampler.log")) (if (= updated (str-concat current-val ",")) nil (state-unwatch wid)))))) (state-set "data.stream" "alpha") (state-set "data.stream" "beta") (state-set "data.stream" "gamma") (state-set "data.stream" "delta") (set tries 0) (while (and (= (state-get "sampler.log") "") (< tries 300)) (set tries (+ tries 1))) (set tries 0) (while (< tries 100) (set tries (+ tries 1))) (state-get "sampler.log")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1577 +(begin (state-watch "jobs.request" (lambda (path val) (state-set "jobs.result" (str-concat "DONE:" (state-get path))))) (set i 0) (while (< i 500) (set i (+ i 1))) "worker-done") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1590 +(begin (state-set "jobs.request" "task-42") (set tries 0) (while (and (not (state-exists "jobs.result")) (< tries 500)) (set tries (+ tries 1))) (state-get "jobs.result")) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1650 +(begin (let ((i 0)) (while (< i 5) (begin (state-set (str-concat "queue.item." (str i)) (str (* i i))) (set i (+ i 1))))) (state-set "queue.count" "5") "produce-done") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1664 +(begin (set tries 0) (while (and (not (state-exists "queue.count")) (< tries 1000)) (set tries (+ tries 1))) (let ((total 0)) (for i (range 5) (set total (+ total 1))) total)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1691 +(begin (for i (range 10) (state-set (str-concat "batch." (str i)) (str (* i 10)))) (state-set "batch.done" "true") 10) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1703 +(begin (set tries 0) (while (and (not (state-exists "batch.done")) (< tries 2000)) (set tries (+ tries 1))) (let ((sum 0)) (for i (range 10) (set sum (+ sum (* i 10)))) sum)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1731 +(begin (for i (range 1 6) (state-set (str-concat "sq." (str i)) (str (* i i)))) (state-set "sq.ready" "5") "squares-done") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1743 +(begin (set tries 0) (while (and (not (state-exists "sq.ready")) (< tries 2000)) (set tries (+ tries 1))) (let ((g (generator (lambda () (for i (range 1 6) (let ((val (state-get (str-concat "sq." (str i))))) (yield (* 2 1))))))) (count 0)) (for x g (set count (+ count 1))) count)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1797 +(begin (for p (range 4) (let ((key (str-concat "data.producer." (str-concat (str p) ".done")))) (set tries 0) (while (and (not (state-exists key)) (< tries 5000)) (set tries (+ tries 1))))) (let ((total 0)) (for i (range 12) (set total (+ total 1))) total)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1828 +(begin (let ((squares (generator (lambda () (for i (range 1 8) (yield (* i i))))))) (let ((total 0)) (for v squares (set total (+ total v))) total))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1851 +(begin (let ((src (generator (lambda () (yield 10) (yield 20) (yield 30)))) (doubled (generator (lambda () (for x src (yield (* x 2))))))) (collect doubled))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1894 +(begin (set tries 0) (while (and (not (state-exists "status.2")) (< tries 5000)) (set tries (+ tries 1))) (let ((results (list))) (for i (range 3) (let ((val (state-get (str-concat "status." (str i))))) (append results val))) (length results))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1921 +(begin (let ((evens (generator (lambda () (for i (range 0 10) (if (= (% i 2) 0) (yield i))))))) (collect evens))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1946 +(begin (let ((counter (generator (lambda () (let ((n 0)) (while (< n 1000) (begin (yield n) (set n (+ n 1)))))))) (taken (list))) (for x counter (if (>= (length taken) 4) (break nil) (append taken x))) taken)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1976 +(begin (let ((keys (generator (lambda () (yield "name") (yield "age") (yield "role")))) (vals (generator (lambda () (yield "alice") (yield "30") (yield "engineer")))) (pairs (list))) (let ((k (next keys)) (v (next vals))) (while (not (is-null k)) (begin (append pairs (str-concat k "=" v)) (set k (next keys)) (set v (next vals))))) pairs)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2021 +(begin (set tries 0) (while (and (not (state-exists "worker.2.result")) (< tries 5000)) (set tries (+ tries 1))) (let ((total 0)) (for i (range 3) (set total (+ total 1))) total)) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2041 +(begin (let ((running-sum (generator (lambda () (let ((acc 0)) (for x (list 10 20 30 40 50) (begin (set acc (+ acc x)) (yield acc)))))))) (collect running-sum))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2073 +(begin (set received (list)) (set count 0) (while (< count 9) (begin (set msg (msg-recv "work.queue")) (append received (get-attr msg "status")) (set count (+ count 1)))) (set ones (generator (lambda () (for item received (yield 1))))) (length (collect ones))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2117 +(begin (set payloads (list)) (set n 0) (while (< n 8) (begin (set msg (msg-recv "data.in")) (append payloads (get-attr msg "status")) (set n (+ n 1)))) (set doubler (generator (lambda () (for p payloads (yield (str-concat p p)))))) (length (collect doubler))) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2161 +(begin (set msg (msg-recv "ch")) 42) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2170 +(begin (msg-send "ch" "hello") 99) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2376 +(msg-send "sig" "hi") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2382 +(msg-pending "sig") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2400 +(msg-recv "sig") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2409 +(msg-send "raw.chan" "x") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1526 +(self-pid) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1536 +(cluster-id) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1541 +(state-set "dsl.key" "dsl-value") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1542 +(state-get "dsl.key") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1547 +(state-set "dsl.exists" "yes") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1548 +(state-exists "dsl.exists") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1553 +(memory-limit) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1558 +(spawn "(+ 1 2)") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1585 +(string.upper "hello") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1590 +(math.sqrt 25) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1595 +(math.pi) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1600 +(string.length "hello world") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1605 +(collections.range 3) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:188 +(begin (set v1 (state-get "shared.x")) (set v2 (state-get "shared.y")) (set v3 (state-get "shared.z")) (str-concat v1 ":" v2 ":" v3)) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:231 +(begin (state-set "shared.x" "10") (state-set "shared.y" "20") (state-set "shared.z" "30") "written") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:325 +(begin (set m (msg-recv "oob.chan")) (get-attr m "payload")) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:394 +(begin (msg-send "oob.chan" "hello-from-A") "sent") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:489 +(begin (set reader (generator (lambda () (for i (range 5) (yield (state-get (str-concat "items.v" (str i)))))))) (set count 0) (set concat "") (for val reader (begin (set count (+ count 1)) (set concat (str-concat concat val ":")))) (state-set "result.consumer" (str-concat (str count) "|" concat)) count) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:553 +(begin (set val-gen (generator (lambda () (for i (range 5) (yield (str-concat "v" (str i))))))) (for val val-gen (begin (set idx (str-concat "items." val)) (state-set idx val))) (state-set "items.ready" "true") "produced") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:655 +(begin (set count 0) (for p (range 3) (for i (range 3) (begin (set key (str-concat "producer." (str p) ".item." (str i))) (if (state-exists key) (set count (+ count 1)) 0)))) count) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:807 +(begin (set result "") (set i 0) (while (< i 4) (begin (set m (msg-recv "pipe.data")) (set result (str-concat result (get-attr m "payload") ",")) (set i (+ i 1)))) (state-set "pipeline.result" result) result) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:892 +(begin (set payload-gen (generator (lambda () (for tag (list "aa" "bb" "cc" "dd") (yield tag))))) (for payload payload-gen (msg-send "pipe.data" payload)) "all-sent") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:988 +(begin (state-set "grpc.x" "alpha") (state-set "grpc.y" "beta") "written") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1012 +(begin (str-concat (state-get "grpc.x") ":" (state-get "grpc.y"))) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1064 +(begin (set m (msg-recv "grpc.chan")) (get-attr m "payload")) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1086 +(begin (msg-send "grpc.chan" "hello-grpc") "sent") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1147 +(begin (set squares (generator (lambda () (for i (range 5) (yield (* i i)))))) (let ((i 0)) (for sq squares (begin (state-set (str-concat "data.v" (str i)) (str sq)) (set i (+ i 1))))) (state-set "data.ready" "true") "produced") +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1179 +(begin (set reader (generator (lambda () (for i (range 5) (yield (int (state-get (str-concat "data.v" (str i))))))))) (set sum 0) (for val reader (set sum (+ sum val))) sum) +@@@ script src/cvc/tests/state_exec_parser_test.cpp:83 +(if (> x 0) "pos" "neg") +@@@ script src/cvc/tests/state_exec_parser_test.cpp:97 +(a (b (c (d)))) +@@@ script src/cvc/tests/state_exec_parser_test.cpp:143 +(+ 1 2) +@@@ script src/cvc/tests/state_exec_parser_test.cpp:152 +(defun f (x) x) +(f 42) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:177 +(- 100 50) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:280 +(begin 4 5 6) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:806 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 8)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:813 +(begin (defun fact (n) (if (<= n 1) 1 (* n (fact (- n 1))))) (fact 6)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:826 +(begin (let ((sum 0)) (for i (list 1 2 3 4 5) (set sum (+ sum i))) sum)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:838 +(begin (let ((i 0) (sum 0)) (while (< i 10) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:852 +(begin (let ((double (lambda (x) (* x 2)))) (double 21))) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1112 +(if #t 42 99) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1113 +(let ((x 10)) (* x x)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1113 +(length "hello") +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:157 +(begin (set i 0) (while (< i 1000) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:220 +(let ((g (generator (lambda () (yield 10) (yield 20) (yield 30))))) (list (next g) (next g) (next g))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:236 +(let ((g (generator (lambda () (yield 1))))) (list (next g) (next g))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:248 +(let ((g (generator (lambda () (yield (* 2 3)) (yield (+ 10 5)))))) (+ (next g) (next g))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:259 +(let ((r (range 0 5))) (+ (next r) (next r) (next r))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:267 +(collect (range 5)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:274 +(let ((total 0)) (for x (range 1 6) (set total (+ total x))) total) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:284 +(let ((accum (list))) (for x (generator (lambda () (yield "a") (yield "b") (yield "c"))) (append accum x)) (length accum)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:298 +(collect (generator (lambda () (let ((i 0)) (while (< i 4) (begin (yield i) (set i (+ i 1)))))))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:316 +(while t (break)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:322 +(while t (break 42)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:326 +(begin (set i 0) (while t (if (= i 5) (break i) (set i (+ i 1))))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:335 +(for x (list 1 2 3 4 5) (if (= x 3) (break x) nil)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:342 +(for x (range 1000) (if (= x 5) (break x) nil)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:349 +(begin (set sum 0) (for i (list 1 2 3) (for j (list 10 20 30) (if (= j 20) (break nil) (set sum (+ sum j))))) sum) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:360 +(begin (set i 0) (while t (begin (set i (+ i 1)) (if (= i 3) (break i) nil)))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:370 +(for x (list 1 2 3 4 5) (let ((doubled (* x 2))) (if (> doubled 6) (break doubled) nil))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:382 +(if #t 1 2) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:383 +(begin 1 2) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:384 +(quote 1) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:385 +(defclass C (init (self) 1)) +@@@ script src/cvc/tests/state_exec_types_test.cpp:72 +(1 2 3) +@@@ script src/cvc/tests/state_exec_types_test.cpp:83 +(1 ("a" "b")) +@@@ script src/cvcGL/test/cvcgl_media_audio.cpp:50 +(if (= (state-get "cap.done") "1") nil (begin (stream-capture-audio (get-attr (stream-open (dict "id" "mic" "codec" "f32" "rate" 48000 "ch" 1 "frames_per_chunk" 256)) "token") (dict "rate" 48000 "ch" 1 "frames" 256 "mix" (list (dict "tone" (dict "hz" 440)) (dict "tone" (dict "hz" 660))))) (state-set "cap.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_media_capture.cpp:52 +(if (= (state-get "cap.done") "1") nil (begin (stream-capture-camera (get-attr (stream-open (dict "id" "cam" "w" 64 "h" 48)) "token") (dict "synthetic" #t "w" 64 "h" 48 "hz" 240)) (state-set "cap.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_media_play.cpp:52 +(if (= (state-get "play.done") "1") nil (begin (stream-open (dict "id" "mic" "codec" "f32" "rate" 48000 "ch" 1 "frames_per_chunk" 256 "subscriber_depth" 8)) (stream-capture-audio "streams.mic" (dict "rate" 48000 "ch" 1 "frames" 256 "tone" (dict "hz" 440))) (stream-play-audio "streams.mic") (state-set "play.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:63 +(begin (set h (gl-bind-stream "screen" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "node" (get-attr h "node")) (state-set "token" (get-attr h "token")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:81 +(begin (set h (gl-bind-stream "screen" (dict "token" "demo.streams.cam0"))) (state-set "ok" (get-attr h "ok")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:96 +(begin (set h (gl-bind-stream "ghost" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:109 +(begin (set h (gl-bind-stream "screen" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:123 +(begin (set h (gl-bind-stream "screen")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script bindings/pycvc/test_pycvc_exec.py:49 +(state-set "dsl.out" "wrote-from-dsl") +@@@ script bindings/pycvc/test_pycvc_exec.py:123 +(capture #t) +@@@ script bindings/pycvc/test_pycvc_exec.py:147 +(capture (make-bytes)) +@@@ script bindings/pycvc/test_pycvc_exec.py:21 +(* 6 7) +@@@ script bindings/pycvc/test_pycvc_exec.py:22 +(begin (set x 5) x) +@@@ script bindings/pycvc/test_pycvc_exec.py:23 +(str-concat "ab" "cd") +@@@ script bindings/pycvc/test_pycvc_exec.py:25 +(> 1 2) +@@@ script bindings/pycvc/test_pycvc_exec.py:47 +(state-get "shared.x") +@@@ script bindings/pycvc/test_pycvc_exec.py:64 +(py-double 21) +@@@ script bindings/pycvc/test_pycvc_exec.py:65 +(py-add 40 2) +@@@ script bindings/pycvc/test_pycvc_exec.py:66 +(py-greet "cvc") +@@@ script bindings/pycvc/test_pycvc_exec.py:68 +(+ 1 (py-double 20)) +@@@ script bindings/pycvc/test_pycvc_exec.py:81 +(stash "py.k" "py.v") +@@@ script bindings/pycvc/test_pycvc_exec.py:86 +(fetch "py.k") +@@@ script bindings/pycvc/test_pycvc_exec.py:98 +(boom) +@@@ script bindings/pycvc/test_pycvc_exec.py:119 +(capture 5) +@@@ script bindings/pycvc/test_pycvc_exec.py:121 +(capture "text") +@@@ script bindings/pycvc/test_pycvc_exec_async.py:29 +(begin (adouble 21)) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:62 +(begin (slow 7)) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:78 +(begin (get-attr (boom 1) "__async_error__")) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:91 +(begin (inc (triple 4))) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:110 +(begin (spawn "(while t 1)") (state-set "r" (str (slow 7))) (state-get "r")) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:82 +(begin (idn 5)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:58 +(begin (spawn "(while t 1)") (state-set "r" (str (acount 6))) (state-get "r")) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:25 +(begin (adbl 21)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:40 +(begin (acount 10)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:96 +(begin (plus10 (inc (triple 4)))) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:128 +(begin (reenter 1)) +@@@ script bindings/pycvc/test_pycvc_integration.py:60 +(state-set "dsl.done" "yes") +@@@ script bindings/pycvc/test_pycvc_integration.py:58 +(+ 0 (verts)) diff --git a/src/cvc/tests/state_exec_corpus/scripts.txt b/src/cvc/tests/state_exec_corpus/scripts.txt new file mode 100644 index 000000000..cbdc30509 --- /dev/null +++ b/src/cvc/tests/state_exec_corpus/scripts.txt @@ -0,0 +1,2437 @@ +# state_exec parse corpus. Generated by src/cvc/tests/gen_state_exec_corpus.py; +# do not edit by hand. Each entry is a '@@@ script ' line, then the script. +@@@ script bindings/pycvc/demo.ari:init +(state-set "demo.agents" "128") +(state-set "demo.speed" "1.0") +(state-set "demo.wire" "false") + +@@@ script bindings/pycvc/demo.ari:windows[0].children[5].bind +(if (> (int (state-get "demo.agents")) 256) "many" "few") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[9].on +(state-set "graphics.root.children.domain.fe.request" "mesh") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[9].disabled_when +(= (state-get "graphics.root.children.domain.fe.busy") "1") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[13].visible_when +(= (state-get "graphics.root.children.domain.fe.problem") "poisson") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[14].visible_when +(= (state-get "graphics.root.children.domain.fe.problem") "heat") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[17].on +(state-set "graphics.root.children.domain.fe.request" "solve") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[17].disabled_when +(or (= (state-get "graphics.root.children.domain.fe.busy") "1") (= (state-get "graphics.root.children.domain.fe.stats.tets") "0")) +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[18].on +(state-set "graphics.root.children.domain.fe.request" "stop") +@@@ script src/cvc/ariadne/components/fe_controls.ari:units.fe_controls.children[18].enabled_when +(= (state-get "graphics.root.children.domain.fe.stoppable") "1") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[10].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "smooth") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[10].disabled_when +(= (state-get "graphics.root.children.mesh.mesh_ops.busy") "1") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[14].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "decimate") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[21].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "repair") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[22].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "orient") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[23].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "reset") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[28].visible_when +(= (state-get "graphics.root.children.mesh.mesh_ops.color.mode") "geodesic") +@@@ script src/cvc/ariadne/components/mesh_tools.ari:units.mesh_tools.children[29].on +(state-set "graphics.root.children.mesh.mesh_ops.request" "curvature") +@@@ script src/cvc/ariadne/components/sim_transport.ari:units.sim_transport.children[3].enabled_when +(= (state-get "sim.paused") "1") +@@@ script src/cvcGL/examples/bunny_shadow.ari:init +(state-set "show_bunny" "true") + +@@@ script src/cvcGL/examples/bunny_shadow.ari:windows[3].children[3].on +(begin + (state-set "lighting.key_intensity" "1.9") + (state-set "lighting.key_azimuth" "-38") + (state-set "lighting.key_elevation" "52") + (state-set "lighting.key_cone" "34")) + +@@@ script src/cvcGL/examples/hello.ari:init +(state-set "demo.agents" "256") + +@@@ script src/cvcGL/examples/hello.ari:windows[0].children[3].visible_when +(> (int (state-get "demo.agents")) 256) +@@@ script src/cvcGL/examples/hello.ari:windows[0].children[4].bind +(if (> (int (state-get "demo.agents")) 256) "heavy" "ok") +@@@ script src/cvcGL/examples/nav_compute.ari:init +(begin + (state-set "nav.status" "idle") + (state-set "nav.result" "-")) + +@@@ script src/cvcGL/examples/nav_compute.ari:windows[0].children[3].on +(begin + (state-set "nav.status" "running") + (nav-step-async) + (state-set "nav.result" (msg-recv "nav.done")) + (state-set "nav.status" "done")) + +@@@ script src/cvcGL/examples/nav_convoy.ari:init +(begin + (state-set "show.agents" "1") + (state-set "show.grid" "1") + (state-set "sim.paused" "0") + (state-set "sim.speed" "1.0")) + +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:134 +;; Convert to integer — truncates floats, parses strings +(int 3.7) ;; => 3 +(int "123") ;; => 123 +(int true) ;; => 1 + +;; Convert to float — widens integers, parses strings +(float 5) ;; => 5.0 +(float "3.14") ;; => 3.14 +(float true) ;; => 1.0 + +;; Type predicates +(is-int 42) ;; => true +(is-int 3.14) ;; => false +(is-float 3.14) ;; => true +(is-float 42) ;; => false +(is-string "hi") ;; => true +(is-string 42) ;; => false +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:158 +;; Factorial +(begin + (defun fact (n) + (if (<= n 1) 1 + (* n (fact (- n 1))))) + (fact 10)) +;; => 3628800 + +;; Map over a list using for-loop +(begin + (set result (list)) + (for x (list 1 2 3 4 5) + (set result (append result (list (* x x))))) + result) +;; => (1 4 9 16 25) + +;; Producer-consumer via state tree +;; Process 1 writes, Process 2 reads +(state-set "shared.counter" "0") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:190 +;; A generator that yields 1, 2, 3 +(set g (generator (lambda () + (yield 1) + (yield 2) + (yield 3)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:200 +(range 5) ;; yields 0, 1, 2, 3, 4 +(range 2 8) ;; yields 2, 3, 4, 5, 6, 7 +(range 0 10 3) ;; yields 0, 3, 6, 9 +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:208 +;; Pull one value at a time +(set g (range 3)) +(next g) ;; => 0 +(next g) ;; => 1 +(next g) ;; => 2 +(next g) ;; => nil (exhausted) + +;; Check if exhausted +(gen-done? g) ;; => true + +;; Iterate with for +(for x (range 5) + (print x)) ;; prints 0 1 2 3 4 + +;; Materialise all values into a list +(collect (range 5)) ;; => (0 1 2 3 4) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:231 +(set evens (generator (lambda () + (for x (range 10) + (if (= (% x 2) 0) + (yield x) nil))))) +(collect evens) ;; => (0 2 4 6 8) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:241 +(set squares (generator (lambda () + (for x (range 5) + (yield (* x x)))))) +(collect squares) ;; => (0 1 4 9 16) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:250 +(set taken (list)) +(for x (range 1000) + (if (>= (length taken) 4) + (break nil) + (append taken x))) +taken ;; => (0 1 2 3) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:261 +(set running-sum (generator (lambda () + (let ((total 0)) + (for x (range 1 6) + (begin + (set total (+ total x)) + (yield total))))))) +(collect running-sum) ;; => (1 3 6 10 15) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:273 +(set counter (generator (lambda () + (let ((n 0)) + (while (< n 1000) + (begin (yield n) + (set n (+ n 1)))))))) +;; Consume lazily with break +(for x counter + (if (= x 5) (break x) nil)) ;; => 5 +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:288 +;; Exit with no value (loop evaluates to nil) +(while t (break)) + +;; Exit with a value +(while t (break 42)) ;; => 42 + +;; Find first match +(for x (list 10 20 30 40) + (if (> x 25) (break x) nil)) ;; => 30 + +;; Nested loops — break exits only the innermost +(set sum 0) +(for i (list 1 2 3) + (for j (list 10 20 30) + (if (= j 20) (break nil) ;; exits inner for + (set sum (+ sum j))))) +sum ;; => 30 (10 + 10 + 10) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:448 +;; Spawn a child process +(set child-pid (spawn "(begin (state-set \"child.done\" \"yes\") 42)" "child-worker")) + +;; Fork the current process +(set cpid (fork)) + +;; Query own identity +(self-pid) ;; => 1 +(self-uid) ;; => "alice" + +;; List all processes +(ps) ;; => list of dicts [{pid: 1, name: "...", status: "running", ...}, ...] + +;; Inspect a specific process +(inspect 2) ;; => dict with full process_info + +;; Control other processes +(pause 2) +(resume 2) +(kill 2) + +;; Query resource usage +(memory-usage) (memory-limit) +(time-elapsed) (time-limit) +(step-count) +(message-count) (message-limit) + +;; System identity +(cluster-id) ;; => "cluster-1" +(node-id) ;; => "node-A" +(scheduler-stats) ;; => {total: N, running: N, ...} +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:531 +;; Write a value +(state-set "myapp.config.timeout" "30") + +;; Read it back +(state-get "myapp.config.timeout") ;; => "30" + +;; Check existence +(state-exists "myapp.config.timeout") ;; => true + +;; List children of a node +(state-children "myapp.config") ;; => ("timeout") + +;; Delete a node +(state-delete "myapp.old-data") + +;; Store/retrieve binary data objects +(state-data-set "myapp.blob" some-value) +(state-data-get "myapp.blob") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:554 +;; Expire a node 60 seconds from now +(state-expire "myapp.session.abc" 60) + +;; Expire at an absolute ISO 8601 time +(state-expire-at "myapp.session.abc" "2026-01-01T00:00:00Z") + +;; Query expiry status +(state-has-expiry "myapp.session.abc") ;; => true +(state-is-expired "myapp.session.abc") ;; => false (not yet) + +;; Clear expiry +(state-clear-expiry "myapp.session.abc") + +;; Sweep all expired nodes under a path (returns count removed) +(state-sweep-expired "myapp.session") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:582 +;; Send a plain-text message to a state tree path +(msg-send "myapp.events.updates" "hello from process 1") + +;; Send with an explicit content type +(msg-send "myapp.events.data" "{\"key\": \"value\"}" "application/json") + +;; msg-send returns a dict: {status: "delivered", path: "myapp.events.updates"} +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:595 +(begin + (set i 0) + (while (< i 10) + (begin + (state-set (str-concat "queue.item." (str i)) (str (* i i))) + (msg-send "queue.notify" (str i)) + (set i (+ i 1)))) + "producer-done") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:607 +(begin + ;; Poll for items (cooperative check) + (set count 0) + (while (< count 10) + (if (state-exists (str-concat "queue.item." (str count))) + (begin + (set val (state-get (str-concat "queue.item." (str count)))) + (state-set (str-concat "consumed." (str count)) val) + (set count (+ count 1))) + nil)) + count) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:705 +;; Block until a message arrives on "events.data" +(set msg (msg-recv "events.data")) + +;; msg is a dict — extract the payload +(get-attr msg "payload") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:723 +;; Non-blocking check +(if (> (msg-pending "events.data") 0) + (set msg (msg-recv "events.data")) + (print "no messages yet")) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:735 +;; Sleep for half a second +(sleep 0.5) + +;; Polling loop with sleep +(while true + (begin + (if (> (msg-pending "work.queue") 0) + (begin + (set job (msg-recv "work.queue")) + (process-job job)) + (sleep 0.1)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:756 +(begin + (set results (list)) + (set done false) + (while (not done) + (begin + (set m (msg-recv "tasks.input")) + (set payload (get-attr m "payload")) + (if (= payload "stop") + (set done true) + (set results (append results (list payload)))))) + results) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1183 +;; A process writing a migration request for itself +(begin + (set my-pid (self-pid)) + (state-set + (str-concat "__migrate_request." (str my-pid)) + "target-node-B") + ;; The coordinator or a management process can poll for + ;; __migrate_request.* entries and call coord.migrate() + my-pid) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1308 +(string.split "a,b,c" ",") ;; => ("a" "b" "c") +(string.join (list "a" "b" "c") ",") ;; => "a,b,c" +(string.replace "hello" "l" "r") ;; => "herro" +(string.trim " hello ") ;; => "hello" +(string.upper "hello") ;; => "HELLO" +(string.lower "HELLO") ;; => "hello" +(string.starts-with "hello" "he") ;; => true +(string.ends-with "hello" "lo") ;; => true +(string.contains "hello" "ell") ;; => true +(string.substring "hello" 1 3) ;; => "el" +(string.char-at "hello" 0) ;; => "h" +(string.length "hello") ;; => 5 +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1325 +(math.sqrt 16) ;; => 4.0 +(math.abs -5) ;; => 5 +(math.floor 3.7) ;; => 3.0 +(math.ceil 3.2) ;; => 4.0 +(math.round 3.5) ;; => 4.0 +(math.pow 2 10) ;; => 1024.0 +(math.log 2.718) ;; => ~1.0 +(math.sin 0) ;; => 0.0 +(math.cos 0) ;; => 1.0 +(math.min 3 7 1 5) ;; => 1 +(math.max 3 7 1 5) ;; => 7 +(math.clamp 15 0 10) ;; => 10 +(math.pi) ;; => 3.14159... +(math.e) ;; => 2.71828... +(math.random) ;; => random double in [0,1) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1345 +(collections.map (lambda (x) (* x x)) (list 1 2 3)) ;; => (1 4 9) +(collections.filter (lambda (x) (> x 2)) (list 1 2 3 4)) ;; => (3 4) +(collections.reduce (lambda (a b) (+ a b)) (list 1 2 3) 0) ;; => 6 +(collections.zip (list 1 2) (list "a" "b")) ;; => ((1 "a") (2 "b")) +(collections.flatten (list (list 1 2) (list 3 4))) ;; => (1 2 3 4) +(collections.sort (list 3 1 2)) ;; => (1 2 3) +(collections.reverse (list 1 2 3)) ;; => (3 2 1) +(collections.range 1 5) ;; => (1 2 3 4) +(collections.unique (list 1 2 2 3 3)) ;; => (1 2 3) +(collections.dict-keys (dict "a" 1 "b" 2)) ;; => ("a" "b") +(collections.dict-values (dict "a" 1 "b" 2)) ;; => (1 2) +(collections.dict-merge (dict "a" 1) (dict "b" 2)) ;; => {a: 1, b: 2} +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1445 +;; WRONG — i is scoped to the let body; the outer i never changes +(set i 0) +(while (< i 5) + (let ((i (+ i 1))) + (print i))) ;; infinite loop! + +;; RIGHT — use set for loop-carried state +(set i 0) +(while (< i 5) + (begin + (print i) + (set i (+ i 1)))) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1466 +(state-set "counter" (str 42)) +(set raw (state-get "counter")) ;; raw is "42" (string) +(set n (int raw)) ;; n is 42 (integer) +(if (> n 40) "big" "small") ;; => "big" +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1478 +(set g (range 3)) +(collect g) ;; => (0 1 2) +(collect g) ;; => () — already exhausted + +;; To reuse, wrap in a factory +(defun make-range () (range 3)) +(collect (make-range)) ;; => (0 1 2) +(collect (make-range)) ;; => (0 1 2) +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1496 +;; DEADLOCK — nothing else is running to send a message +(msg-recv "work.queue") +@@@ script docs/STATE_EXEC_DEVELOPER_GUIDE.md:1526 +(set done false) +(for i (range 10) + (if done nil + (for j (range 10) + (if (= j 3) + (begin (set done true) (break nil)) + nil)))) +@@@ script docs/STREAMING.md:107 +; open a video stream scoped to THIS document; returns a handle dict +(set cam (stream-open (dict "id" "cam0" "codec" "rgba8" "w" 1280 "h" 720))) +(get-attr cam "ok") ; => #t +(get-attr cam "token") ; => the canonical token to hand a sink +(get-attr cam "seq_channel") ; => a channel that ticks the latest frame seq + +(stream-info cam) ; => { ok live id token seq_channel evt_channel published subscribers } +(stream-close cam) ; => { ok #t } (also closed at document teardown) +@@@ script docs/STREAMING.md:162 +(gl-bind-stream "screen" (get-attr cam "token")) ; => { ok #t node token } +@@@ script docs/STREAMING.md:499 +; init: or a Start button's on: program +(set cam (stream-open (dict "id" "cam0" "codec" "rgba8" "w" 1280 "h" 720))) +; (Phase 3b) begin capture from the default camera into the stream: +(stream-capture-camera (get-attr cam "token") 0) +; bind it onto the scene's "screen" node: +(gl-bind-stream "screen" (get-attr cam "token")) +@@@ script docs/roadmap/ARIADNE_CONFERENCE_ROADMAP.md:97 +; ── join: server endpoint + room id; returns a room handle; mode comes from the server ── +(set room (room-join "conf.example.org:7000" "standup" (dict "name" "Joe"))) + +; ── publish my camera + mic into the room ── +(set cam (stream-open (dict "id" "self/cam" "codec" "rgba8" "w" 1280 "h" 720))) +(stream-capture-camera (get-attr cam "token") 0) ; §4 capture verb +(room-publish room (dict "kind" "camera" "token" (get-attr cam "token"))) + +(set mic (stream-open (dict "id" "self/mic" "codec" "f32" "rate" 48000 "ch" 1))) +(stream-capture-audio (get-attr mic "token") (dict "device" 0)) +(room-publish room (dict "kind" "audio" "token" (get-attr mic "token"))) + +; ── UI: a self tile + one tile per remote participant (roster-driven) ── +; scene / widget tree uses `repeat` over (room-roster room); each tile binds a +; (room-subscribe room PID "camera") proxy token into a `stream_view`, and +; (stream-play-audio (room-subscribe room PID "audio")) plays their voice. + +; ── controls: pure-.ari actions, no C++ ── +; button "Mute" on: (room-set-flag room (dict "mic" (not (room-flag room "self" "mic")))) +; button "Camera" on: (room-set-flag room (dict "cam" ...)) +; button "Leave" on: (room-leave room) +@@@ script docs/roadmap/CVCGL-UI-DSL-ROADMAP.md:4452 +(state-set "demo.n" 256) ; plain state, set before it is referenced +(window "Controls" + (slider-int "Agents" :bind "demo.n" :lo 1 :hi 512) + (combo "Belief" :bind "demo.belief" :options (list "shared" "grouped" "private")) + (button "Reset" :on "reset")) +(scene + (geometry-node "mesh" :source "file://blocks.off" + :visible-when (= (state-get "demo.show") "1"))) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:91 +;; Load a volume from disk into the state tree +(vol-read "mydata" "/path/to/brain.rawiv") + +;; Query properties +(set dims (vol-dims "mydata")) ;; => (256 256 256) +(set bbox (vol-bbox "mydata")) ;; => (0.0 0.0 0.0 1.0 1.0 1.0) + +;; Volume arithmetic (result stored at new path) +(vol-normalize "mydata" "normalized") +(vol-scale "normalized" 2.0 "scaled") + +;; Extract isosurface → geometry +(iso "scaled" 0.5 "surface" (dict "method" "fastcontouring")) + +;; Compute SDF from geometry → new volume +(sdf "surface" "sdf-vol" (dict "dim" (list 128 128 128))) + +;; Tetrahedralize +(tet "sdf-vol" 0.0 "tet-mesh") + +;; Write results +(vol-write "sdf-vol" "/path/to/output.rawiv") +(geom-write "tet-mesh" "/path/to/output.off") +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:288 +(iso "brain-vol" 128.0 "brain-surf" + (dict "method" "fastcontouring" + "improve" #t + "compute-normals" #t)) + +(sdf "brain-surf" "brain-sdf" + (dict "dim" (list 256 256 256) + "flip-normals" #f)) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:307 +(defun process-brain (input-file output-dir) + (begin + ;; Load + (vol-read "raw" input-file) + (print (str "Loaded: " (vol-dims "raw"))) + + ;; Pre-process + (vol-normalize "raw" "norm") + (vol-clip "norm" 0.1 0.9 "clipped") + + ;; Extract isosurface + (iso "clipped" 0.5 "surface" + (dict "method" "fastcontouring" + "improve" #t + "compute-normals" #t)) + (print (str "Surface: " (geom-num-tris "surface") " tris")) + + ;; Tetrahedralize for FEM + (tet "clipped" 0.5 "tet-mesh" + (dict "improve" #t)) + (print (str "Tet mesh: " (geom-num-tets "tet-mesh") " tets")) + + ;; Write results + (geom-write "surface" (str output-dir "/brain-surface.off")) + (geom-write "tet-mesh" (str output-dir "/brain-tet.off")) + + ;; Return summary + (dict "tris" (geom-num-tris "surface") + "tets" (geom-num-tets "tet-mesh")))) + +(process-brain "/data/brain.rawiv" "/output") +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:343 +(defun process-all (file-list output-dir) + (for f file-list + (begin + (print (str "Processing: " f)) + (process-brain f output-dir)))) +@@@ script docs/roadmap/DSL-GEOMETRY-VOLUME-ROADMAP.md:353 +;; Coordinator splits work across nodes +(defun distribute-pipeline (files) + (for f files + (begin + (state-set (str "jobs." f ".status") "pending") + (state-set (str "jobs." f ".input") f) + (msg-send "work-queue" f)))) + +;; Worker process +(defun worker () + (while #t + (set job (msg-recv "work-queue")) + (state-set (str "jobs." job ".status") "running") + (process-brain + (state-get (str "jobs." job ".input")) + "/output") + (state-set (str "jobs." job ".status") "done"))) +@@@ script docs/roadmap/STATE_BINARY_STREAMING.md:220 +(define s (stream-open {:kind "video_raw" :codec "rgba8" :w 1920 :h 1080})) +(stream-info s) ; descriptor + stats dict from the state tree +(gl-bind-stream node s) ; HOST action wires the C++ consumer to a GeometryNode texture +(msg-recv "stream.evt") ; lifecycle events (small dicts via post_message) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1101 +;; Expire a cache entry after 60 seconds +(state-set "sensors.cache.temperature" (read-sensor)) +(state-expire "sensors.cache.temperature" 60) + +;; Check before reading +(if (state-is-expired "sensors.cache.temperature") + (begin + (state-sweep-expired "sensors.cache") + (refresh-cache)) + (state-get "sensors.cache.temperature")) + +;; Set absolute deadline +(state-expire-at "jobs.batch-42" "2025-06-15T00:00:00Z") +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1355 +(let ((pid (fork))) + (if (= pid 0) + (begin + ;; child: do work + (state-set "results.child" (compute))) + (begin + ;; parent: pid is the child's PID + (state-set "results.parent" (str "spawned child " pid))))) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1383 +(let ((pid (fork))) + (if (= pid -1) + (print "fork failed: resource limit") + ...)) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1470 +;; Import a module into the local environment +(require "string") +(split "hello world" " ") ;; → ("hello" "world") + +;; Or call with fully qualified path +(stdlib.string.split "hello world" " ") + +;; Import specific symbols +(require "math" '(sqrt abs floor ceil)) +(sqrt 144) ;; → 12 + +;; List available modules +(stdlib-modules) ;; → ("string" "math" "io" "collections") + +;; List functions in a module +(stdlib-list "string") ;; → ("split" "join" "replace" ...) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1709 +(require "geometry") +(distance 0 0 3 4) ;; → 5.0 + +(require "util") +(repeatedly 5 (lambda (i) (print i))) ;; prints 0-4 +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1763 + (let ((mesh (state-data-get "/scene/model/geometry"))) + (if (data? mesh) + (begin + (print (data-type mesh)) ;; e.g. "CVC::Geometry" + (state-data-set "/output/result" mesh)))) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1776 + (state-data-set "/scene/model/transformed" transformed-mesh) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1781 + (data-type mesh) ;; → "CVC::Geometry" or nil + (data? mesh) ;; → t + (data? 42) ;; → nil + (state-data-type "/path") ;; → type name of data() on that node +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1853 +;; Without batching: handleStateChanged fires 3 times, possibly spawning +;; 3 handler threads. +(state-set "/scene/camera/x" 10) +(state-set "/scene/camera/y" 20) +(state-set "/scene/camera/z" 30) + +;; With batching: handleStateChanged fires once per unique child, after +;; all three writes complete. If the camera object batches internally, +;; this collapses to a single flush. +(state-batch + (state-set "/scene/camera/x" 10) + (state-set "/scene/camera/y" 20) + (state-set "/scene/camera/z" 30)) +@@@ script docs/roadmap/STATE_EXEC_PORTING_PLAN.md:1876 +(state-lock "/scene/camera" + (let ((x (state-get "/scene/camera/x")) + (y (state-get "/scene/camera/y"))) + ;; x and y are consistent — no other writer can interleave + (state-set "/scene/camera/distance" + (sqrt (+ (* x x) (* y y)))))) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1093 +(+ 1 2 3) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1099 +(* 7 6) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1105 +(let ((x 10)) (+ x 5)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1111 +(let ((f (lambda (x y) (+ x y)))) (f 3 4)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1120 +(+ 100 200 300) + +@@@ script src/cvc-cli/cvc_cli_test.cpp:1206 +(let ((square (lambda (x) (* x x)))) (square 9)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1211 +(car (list 10 20 30)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1216 +(if (> 5 3) 99 0) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1221 +(let ((a 10) (b 20)) (+ a b)) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1228 +(begin 1 2 3 4 5 6 7 8 9 10) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1238 +(let ((pi 3.14159) (r 5)) (* pi r r)) + +@@@ script src/cvc-cli/cvc_cli_test.cpp:1881 +(no children) +@@@ script src/cvc-cli/cvc_cli_test.cpp:1891 +(+ 1 (+ 2 (+ 3 (+ 4 (+ 5 6))))) +@@@ script src/cvc-cli/main.cpp:1554 +(no children) + +@@@ script src/cvc/ariadne/ariadne.cpp:1256 +(no image) +@@@ script src/cvc/ariadne/ftxui_backend.cpp:89 +(empty) +@@@ script src/cvc/tests/ariadne_loader_test.cpp:535 +(state-set "paused" "true") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2256 +(> (int (state-get "n")) 5) +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2276 +(state-exists "ready") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2277 +(state-exists "busy") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2301 +(list "p" "q") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:2402 +(state-exists "show") +@@@ script src/cvc/tests/ariadne_loader_test.cpp:3688 +(int (state-get "n")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:80 +(closable) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:356 +(state-set "paused" (if (= (state-get "paused") "true") "false" "true")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:376 +(begin (state-set "agents" "64") (state-set "speed" "1.0")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:400 +(begin (msg-recv "async.done") (state-set "async.result" "resumed")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:428 +(state-set "async.value" (msg-recv "vchan")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:486 +(begin (nav-step-async) (state-set "nav.result" (msg-recv "nav.done"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:511 +(state-set "tick.flag" "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:547 +(begin (state-set "input.last_key" (get-attr event "key")) (state-set "input.kind" (get-attr event "kind"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:565 +(state-set "input.last_key" (get-attr event "key")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:583 +(begin (state-set "ptr.kind" (get-attr event "kind")) (state-set "ptr.btn" (get-attr event "button"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:606 +(state-set "widget.clicked" "yes") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:728 +(begin (state-set "ev.btn" (get-attr event "button")) (state-set "ev.has_x" (if (>= (get-attr event "x") 0) "yes" "no"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:780 +(host-bump) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:959 +(state-set "agents" "128") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:960 +(no error) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1033 +(while true (+ 1 1)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1077 +(state-set "n" "10") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1121 +(= (state-get "show") "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1150 +(int 2) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1351 +(state-exists "never") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1384 +(state-set "x" 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1399 +(state-set "mode" "on") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1401 +(= (state-get "mode") "on") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1414 +(while true 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1431 +(msg-recv "x") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1431 +(await 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1470 +(state-set "flag" "0") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1472 +(state-get "flag") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1474 +(state-get "never_set") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1490 +(foobar 1 2) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1541 +(is-string (str-concat "a" "b")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1543 +(is-list (append (list 1) (list 2))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1545 +(apply and (list #t)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1547 +(begin (print "x") #t) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1565 +(= (quote (1 2 3)) (quote (1 2 3))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1567 +(= (quote (1 2)) (quote (9 9))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1569 +(let ((xs (list 1 2 3))) (> (length xs) 2)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1587 +(begin (defclass B (init (self) (while 1 1))) (B)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1589 +(eval (quote #t)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1591 +(begin (defmacro m () #t) #t) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1606 +(begin (set d (list 0 0)) (set i 0) (while (< i 30) (begin (set d (list d d)) (set i (+ i 1)))) (state-data-set "bomb" d)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1614 +(is-null (state-data-get "bomb")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1626 +(state-set "n" "3") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1647 +(state-set "lock" "1") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1649 +(state-exists "lock") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1686 +(state-set "n" "7") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1688 +(+ (int (state-get "n")) 1) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1689 +(state-get "n") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1775 +(list "shared" "grouped" "private") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1799 +(state-set "hint" "dynamic") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1803 +(state-get "hint") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1847 +(state-set "n" "2") +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1912 +(* 1e19 1e19) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1928 +(begin (defun f () #t) (f)) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:1930 +(f) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2287 +(begin (set r (msg-recv (http-get-async "http://ex/x"))) (state-set "r.status" (get-attr r "status")) (state-set "r.ok" (get-attr r "ok")) (state-set "r.error" (get-attr r "error")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2325 +(begin (set r (msg-recv (http-get-async "http://ex/down"))) (state-set "r.ok" (get-attr r "ok")) (state-set "r.error" (get-attr r "error"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2353 +(state-set "r.status" (get-attr (msg-recv (http-get-async "http://ex/slow")) "status")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2383 +(begin (set r (http-get "http://ex/x")) (state-set "r.status" (get-attr r "status")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2414 +(state-set "r.status" (get-attr (await (http-get-async "http://ex/x")) "status")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2449 +(begin (set r (fetch "mem://x")) (state-set "r.ok" (get-attr r "ok")) (state-data-set "r.body" (get-attr r "body"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2481 +(state-data-set "r.body" (get-attr (await (fetch-async "mem://y")) "body")) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2519 +(begin (set r (http-request "POST" "http://ex/api" (dict "headers" (dict "Authorization" "Bearer tok-123") "form" (dict "grip" 1 "risk" 2)))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2560 +(begin (set r (http-get "http://ex/x" (dict "query" (dict "q" "hello world" "n" 2)))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2592 +(begin (set r (http-request "PUT" "http://ex/raw" (dict "body" "raw-payload"))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2625 +(begin (set g (await (http-get-async "http://ex/src"))) (set r (await (http-request-async "PUT" "http://ex/dst" (dict "body" (get-attr g "body") "headers" (dict "Authorization" "Bearer tok"))))) (state-set "r.status" (get-attr r "status"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2688 +(begin (state-set "r.t" (str (pool-double 21))) (state-set "r.a" (str (await (pool-double-async 10))))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2754 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok")) (state-set "t.token" (get-attr h "token")) (state-set "t.seq" (get-attr h "seq_channel")) (state-set "t.live" (get-attr (stream-info h) "live")) (state-set "t.closed" (get-attr (stream-close h) "ok")) (state-set "t.live2" (get-attr (stream-info h) "live"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2785 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.token" (get-attr h "token")) (state-set "t.seq" (get-attr h "seq_channel"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2814 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2843 +(begin (set a (stream-open (dict "id" "c0" "w" 4 "h" 2 "expected_subscribers" -1))) (state-set "t.a" (get-attr a "ok")) (set b (stream-open (dict "id" "c1" "w" 999999 "h" 2))) (state-set "t.b" (get-attr b "ok"))) +@@@ script src/cvc/tests/ariadne_runtime_test.cpp:2869 +(begin (set h (stream-open (dict "id" "cam0" "w" 4 "h" 2))) (state-set "t.ok" (get-attr h "ok")) (state-set "t.err" (get-attr h "error"))) +@@@ script src/cvc/tests/async_lane_test.cpp:498 +(+ (lane-double 21) (await (lane-double-async 10))) +@@@ script src/cvc/tests/async_lane_test.cpp:525 +(while t (apply-built (msg-recv "app#built.tracks"))) +@@@ script src/cvc/tests/async_lane_test.cpp:526 +(while t (apply-plain (msg-recv "built"))) +@@@ script src/cvc/tests/async_lane_test.cpp:621 +(while t (begin (tick) (await 0))) +@@@ script src/cvc/tests/async_lane_test.cpp:652 +(while t (sleep (msg-recv (sim-launch)))) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:125 + + (begin + (set big "") + (set i 0) + (while (< i 100) + (begin + (set big (str-concat big "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789")) + (set i (+ i 1)))) + (state-set "data.large_string" big) + (length big)) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:151 + + (begin + (state-data-set "data.typed_obj" (list 1 2 3 4 5)) + (set obj (state-data-get "data.typed_obj")) + (is-null obj)) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:168 + + (begin + (set i 0) + (while (< i 50) + (begin + (state-set "data.counter" (str i)) + (set i (+ i 1)))) + (state-get "data.counter")) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:803 + + (begin + (state-set "queue.item.0" "apple") + (state-set "queue.item.1" "banana") + (state-set "queue.item.2" "cherry") + (state-set "queue.count" "3") + (msg-send "queue.ready" "items-available") + "producer-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:813 + + (begin + (set result (list)) + (set i 0) + (while (< i 3) + (begin + (set key (str-concat "queue.item." (str i))) + (set val (state-get key)) + (if (!= val nil) + (set i (+ i 1)) + nil))) + (state-set "consumed.total" (str i)) + i) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:843 + + (begin + (state-set "partial.0" (str (+ 1 2 3))) + "map-0-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:849 + + (begin + (state-set "partial.1" (str (+ 4 5 6))) + "map-1-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:855 + + (begin + (state-set "partial.2" (str (+ 7 8 9))) + "map-2-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:861 + + (begin + (set s0 (state-get "partial.0")) + (set s1 (state-get "partial.1")) + (set s2 (state-get "partial.2")) + (state-set "result.total" + (str (+ (if (= s0 nil) 0 6) + (if (= s1 nil) 0 15) + (if (= s2 nil) 0 24)))) + 45) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:928 +(begin (spawn "(* 10 10)" "w0") (spawn "(* 20 20)" "w1") (spawn "(* 30 30)" "w2") 3) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:947 + + (begin + (set total 0) + (set i 0) + (while (< i 5) + (begin + (set total (+ total (* i i))) + (state-set "progress.current" (str i)) + (state-set "progress.total" (str total)) + (set i (+ i 1)))) + total) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:977 + + (begin + (state-set "document.header.title" "My Report") + (state-set "document.header.author" "Alice") + (state-set "document.header.date" "2026-05-24") + (msg-send "document.events" "header-complete") + "header-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:986 + + (begin + (state-set "document.body.section1" "Introduction text...") + (state-set "document.body.section2" "Methods text...") + (state-set "document.body.section3" "Results text...") + (msg-send "document.events" "body-complete") + "body-done") + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1013 + + (begin + ;; Compute Fibonacci(12) = 144 + (defun fib (n) + (if (<= n 1) n + (+ (fib (- n 1)) (fib (- n 2))))) + (set result (fib 12)) + + ;; Write result to state tree + (state-set "computation.fib12.result" (str result)) + (state-set "computation.fib12.status" "complete") + + ;; Notify via messaging + (msg-send "computation.notifications" "fib12-done") + + result) + +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1094 +(begin (while true 1)) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1099 +(+ 1 2) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1110 +(+ 3 4) +@@@ script src/cvc/tests/state_exec_advanced_integration_test.cpp:1111 +(+ 5 6) +@@@ script src/cvc/tests/state_exec_async_test.cpp:138 +(- 10 3) +@@@ script src/cvc/tests/state_exec_async_test.cpp:140 +(* 4 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:144 +(/ 10 4) +@@@ script src/cvc/tests/state_exec_async_test.cpp:147 +(+ (* 2 3) (- 10 4)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:152 +(< 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:153 +(< 2 1) +@@@ script src/cvc/tests/state_exec_async_test.cpp:157 +(= 5 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:158 +(= 5 6) +@@@ script src/cvc/tests/state_exec_async_test.cpp:163 +(if t 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:165 +(if nil 1 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:167 +(begin 1 2 3) +@@@ script src/cvc/tests/state_exec_async_test.cpp:169 +(begin (set x 42) x) +@@@ script src/cvc/tests/state_exec_async_test.cpp:172 +(begin (set i 0) (set sum 0) (while (< i 5) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:179 +(begin (set sum 0) (for x (list 1 2 3 4 5) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:188 +(begin (set sq (lambda (x) (* x x))) (sq 5)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:192 +(begin (set make-adder (lambda (n) (lambda (x) (+ x n)))) (set add5 (make-adder 5)) (add5 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:200 +(begin (defun fact (n) (if (< n 2) 1 (* n (fact (- n 1))))) (fact 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:209 +(let ((x 10) (y 20)) (+ x y)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:214 +(begin (set f (lambda () (begin (return 42) 99))) (f)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:220 +(quote (1 2 3)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:229 +(begin (defmacro double-it (x) (* x 2)) (double-it 21)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:237 +(eval (quote (+ 1 2))) +@@@ script src/cvc/tests/state_exec_async_test.cpp:242 +(str-concat "hello" " " "world") +@@@ script src/cvc/tests/state_exec_async_test.cpp:248 +(car (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:249 +(car (cdr (list 1 2 3))) +@@@ script src/cvc/tests/state_exec_async_test.cpp:252 +(length (list 1 2 3 4 5)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:257 +(begin (defun fib (n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 15)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:268 +(await (+ 1 2)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:275 +(+ 10 20) +@@@ script src/cvc/tests/state_exec_async_test.cpp:395 +(+ 10 20 30) +@@@ script src/cvc/tests/state_exec_async_test.cpp:449 +(begin (set i 0) (set s 0) (while (< i 5) (begin (set s (+ s i)) (set i (+ i 1)))) s) +@@@ script src/cvc/tests/state_exec_async_test.cpp:469 +(begin (set x 10) (set y 20) (+ x y)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:470 +(begin (defun fact (n) (if (< n 2) 1 (* n (fact (- n 1))))) (fact 10)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:471 +(begin (set sum 0) (for x (list 1 2 3 4 5) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_async_test.cpp:472 +(let ((a 5) (b 10)) (* a b)) +@@@ script src/cvc/tests/state_exec_async_test.cpp:498 +(+ 40 2) +@@@ script src/cvc/tests/state_exec_async_test.cpp:509 +(begin 1 2 3 4 5) +@@@ script src/cvc/tests/state_exec_async_test.cpp:537 +(begin 1 2 3 4 5 6 7 8) +@@@ script src/cvc/tests/state_exec_async_test.cpp:570 +(nap) +@@@ script src/cvc/tests/state_exec_async_test.cpp:645 +(begin (set r (await 42)) r) +@@@ script src/cvc/tests/state_exec_async_test.cpp:670 +(begin (set s 0) (set i 0) (while (< i 200) (begin (set s (+ s i)) (set i (+ i 1)))) s) +@@@ script src/cvc/tests/state_exec_async_test.cpp:738 +(+ 2 3) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:67 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 20)) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:90 +(begin (set sum 0) (set i 0) (while (< i 1000) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:116 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 15)) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:141 +(begin (set x 0) (set i 0) (while (< i 100) (begin (set x (+ x i)) (set i (+ i 1)))) x) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:179 +(begin (set lst (list)) (set i 0) (while (< i 500) (begin (set lst (append lst (list i))) (set i (+ i 1)))) lst) +@@@ script src/cvc/tests/state_exec_bench_test.cpp:253 +(begin (defun even? (n) (if (= n 0) #t (odd? (- n 1)))) (defun odd? (n) (if (= n 0) #f (even? (- n 1)))) (even? 200)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:297 +(begin (set sum 0) (set i 0) (while (< i 100) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:320 +(+ 1 2 3 4 5 6 7 8 9 10) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:427 +(/ 10 3) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:495 +(let ((x 10)) (let ((y 20)) (+ x y))) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:527 +(cdr (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:530 +(length (list 1 2 3 4)) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:535 +(begin (set d (dict)) (set-attr d "key" 42) (get-attr d "key")) +@@@ script src/cvc/tests/state_exec_codec_test.cpp:544 +(begin (defun fib (n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)) +@@@ script src/cvc/tests/state_exec_coordinator_test.cpp:469 +(begin (+ 1 2) (+ 3 4) (+ 5 6)) +@@@ script src/cvc/tests/state_exec_coordinator_test.cpp:653 +(begin (+ 1 2) (+ 3 4)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:41 +(- 10 3 2) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:43 +(* 3 4) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:47 +(% 10 3) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:49 +(+ (* 3 4) (- 10 5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:59 +(= 42 42) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:60 +(= 1 2) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:69 +(if nil 1) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:73 +(begin) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:78 +(quote (+ 1 2)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:90 +(begin (set sum 0) (for x (list 1 2 3 4) (set sum (+ sum x))) sum) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:99 +(begin (set add1 (lambda (x) (+ x 1))) (add1 10)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:114 +(begin (set sum-all (lambda (&rest args) (apply + args))) (sum-all 1 2 3)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:123 +(begin (defun double (x) (* x 2)) (double 21)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:130 +(begin (defun fact (n) (if (<= n 1) 1 (* n (fact (- n 1))))) (fact 5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:149 +(begin (defun early (x) (if (< x 0) (return -1) x)) (early -5)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:172 +(root) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:188 +(car (cdr (list 10 20 30))) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:189 +(car (cons 5 (list 10 20))) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:192 +(length (list 1 2 3)) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:197 +(begin (set obj (dict "x" 10 "y" 20)) (set-attr obj "x" 99) (get-attr obj "x")) +@@@ script src/cvc/tests/state_exec_evaluator_test.cpp:215 +(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:49 +(begin (set m (lambda (self) 99)) (set superd (dict "m" m)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:57 +(begin (set superd (dict "mk" list)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (length (super obj mk 4 5))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:64 +(begin (set m (lambda (self a) (+ a 90))) (set superd (dict "m" m)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m 9)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:72 +(begin (set superd2 (dict "mk" list)) (set superd1 (dict "__super__" superd2)) (set clsd (dict "__super__" superd1)) (set obj (dict "__class__" clsd)) (length (super obj mk 1))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:79 +(begin (set superd (dict "m" 5)) (set clsd (dict "__super__" superd)) (set obj (dict "__class__" clsd)) (super obj m)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:101 +(while t 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:120 +(+ 2 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:145 +(set a 1) (set b 2) (+ a b) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:150 +(1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:151 +(no-such-fn 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:152 +(begin (set v 7) (v 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:153 +(if t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:154 +(while t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:155 +(for x (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:156 +(for 1 (list 1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:157 +(for x 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:158 +(set a) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:159 +(set 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:160 +(quote 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:161 +(lambda (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:162 +(lambda (1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:163 +(lambda 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:164 +(let ((a 1))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:165 +(let 5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:166 +(let ((a)) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:167 +(let ((1 2)) 3) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:168 +(defun f (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:169 +(defun 5 (x) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:170 +(defmacro m (x)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:171 +(defmacro 5 (x) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:172 +(defmacro m (1) 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:173 +(eval) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:174 +(super 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:175 +(defclass) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:176 +(defclass 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:180 +(begin (defun f () (begin (return) 9)) (f)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:184 +(begin (set f (lambda (x) x)) (f 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:185 +(begin (set f (lambda (&rest) 1)) (f 5)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:189 +(begin (set f (lambda () 42)) (f)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:192 +(lambda nil 42) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:196 +(begin (defun f () (return 5)) (+ (f) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:200 +(begin (defmacro pick (a b) b) (pick 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:204 +(begin (defmacro m2 (x) (set y x) (+ y 1)) (m2 5)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:208 +(get-attr d "name") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:209 +(send d "speak") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:210 +(send d "add" 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:211 +(send d "finalize") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:215 +(begin (defclass Empty) (Empty 42)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:220 +(begin (defclass P1 (m (self) 7)) (defclass C1 (P1) (n (self) 8)) (send (C1) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:226 +(super d speak) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:227 +(super d "speak") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:228 +(super d add 3 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:237 +(super (dict "k" 1) m) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:238 +(super (Animal "a") speak) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:239 +(super d fly) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:243 +(begin (defclass KX (m (self) 7) 5 (5 1)) (send (KX) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:249 +(begin (set Q (lambda (x) x)) (defclass CQ Q (n (self) 4)) (send (CQ) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:252 +(begin (set Q2 (lambda (x) x)) (defclass CR (Q2) (n (self) 5)) (send (CR) "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:286 +(set a 4) (set b 5) (+ a b) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:295 +(begin (set i 0) (while (< i 20) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:346 +(return) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:347 +(yield) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:353 +(defun f (1) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:365 +(begin (set v 7) (v)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:369 +(begin (set f (lambda (&rest xs) (length xs))) (f 1 2 3)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:378 +(begin (set f (lambda (x) (set y (* x 2)) (+ y 1))) (f 4)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:381 +(let () 42) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:384 +(begin (defun f (&rest xs) (length xs)) (f 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:392 +(for x (range 0) x) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:414 +(begin (defclass P2 (who (self) "parent")) (defclass C2 (P2)) (send (C2) "who")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:421 +(begin (defclass V (m (self &rest more) (length more))) (send (V) "m" 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:424 +(begin (defclass M (m (self) (set y 4) (+ y 2))) (send (M) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:428 +(begin (defclass K (m (self) 7) (5 1)) (send (K) "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:441 +(defclass B (m (self 5) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:473 +(-) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:474 +(*) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:475 +(/) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:476 +(- 2.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:477 +(- 5.5 1 0.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:478 +(/ 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:479 +(+ 0.5 (list 1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:480 +(+ (list 1.5) 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:481 +(* (list 2 3)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:482 +(* 2.0 (list 3 4)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:483 +(% 7.5 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:487 +(% 1 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:488 +(% 1.0 0.0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:489 +(/ 1 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:490 +(< nil 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:491 +(car) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:492 +(send 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:496 +(int (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:497 +(float (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:498 +(str-concat "a" 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:502 +(car (list)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:503 +(length (cdr (list))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:504 +(car 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:505 +(nth (list 1) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:506 +(nth (list 1) "x") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:507 +(set-nth (list 1) 5 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:508 +(del-nth (list 1) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:509 +(begin (set l (list 1 2 3)) (del-nth l 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:510 +(length (dict "a" 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:511 +(length t) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:515 +(slice (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:516 +(length (slice (list 1 2 3) -1 2)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:517 +(length (slice (list 1 2) 0 -1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:518 +(length (slice (list 1 2) 5 6)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:519 +(length (slice (list 1 2) 0 9)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:520 +(length (slice (list 1 2) 1 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:521 +(length (slice (list 1 2 3) 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:525 +(dict "a") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:526 +(get-attr (dict 1 "x") "1") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:527 +(get-attr (dict) "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:528 +(get-attr 5 "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:529 +(get-attr (dict) 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:530 +(del-attr (dict) "k") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:531 +(begin (set d (dict "k" 3)) (del-attr d "k")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:535 +(send (dict "a" 1) "m") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:536 +(begin (set o (dict "__class__" (dict "m" (lambda (s) 1)))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:539 +(begin (set o (dict "__class__" (dict "m" 5))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:541 +(begin (set o (dict "__class__" (dict "x" 1))) (send o "m")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:546 +(begin (set superd (dict "mk" list)) (set clsd (dict "x" 1 "__super__" superd)) (set obj (dict "__class__" clsd)) (length (send obj "mk" 4 5))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:555 +(apply 5 (list 1)) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:556 +(append 5 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:559 +(print 1 "a" 2.5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:562 +(generator 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:563 +(next 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:564 +(gen-done? 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:565 +(collect 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:566 +(range 1 2 3 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:567 +(range "a") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:568 +(range "a" 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:569 +(range 1 "b") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:570 +(range 1 5 "c") +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:571 +(range 1 5 0) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:576 +(collect (generator (lambda () (begin (yield 1) (yield 2))))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:585 +(and) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:586 +(and 1 nil 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:587 +(and 1 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:588 +(or nil nil) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:589 +(or nil 7) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:590 +(not nil) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:629 +(set c 1) (+ c 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:756 +(+ 1 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:770 +(* 2 3) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:771 +(* 2 4) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:772 +(* 2 5) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:781 +(begin (set i 0) (while (< i 50) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:795 +(begin (set i 0) (while (< i 40) (set i (+ i 1)))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:808 +(set-attr box "sig" __signal__) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:809 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "sig")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:820 +(begin (set i 0) (while (< i 4) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:857 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "path")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:863 +(set-attr box "path" p) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:878 +(begin (set i 0) (while (< i 6) (set i (+ i 1))) (get-attr box "n")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:883 +(set-attr box "n" 1) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:883 +(set-attr box "n" 2) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:927 +(begin (set i 0) (while (< i 8) (set i (+ i 1))) (get-attr box "path")) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:954 +(begin (set i 0) (while (< i 100) (set i (+ i 1)))) +@@@ script src/cvc/tests/state_exec_gaps_test.cpp:987 +(+ 20 22) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:95 +(spawn "42" "child") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:152 +(self-uid) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:650 + + (begin + (state-set "mailbox.test" "ready") + (msg-send "mailbox.test" "hello-world")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:664 + + (begin + (msg-send "test.path" "msg1") + (msg-send "test.path" "msg2") + (msg-send "test.path" "msg3") + (message-count)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:687 + + (begin + (msg-send "test" "1") + (msg-send "test" "2") + (msg-send "test" "3") + (msg-send "test" "4")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:706 + + (begin + (msg-send "test" "hello world!!") + (msg-send "test" "this exceeds the byte cap") + "done") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:725 + + (begin + (state-set "shared.item.0" "100") + (state-set "shared.item.1" "200") + (state-set "shared.item.2" "300") + (state-set "shared.count" "3") + "done") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:737 + + (begin + (state-get "shared.count")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:758 +(begin (state-set "parent.uid" (self-uid)) (set child-pid (spawn "(+ 1 2)" "child-worker")) child-pid) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:774 + + (begin + (set result (msg-send "events.log" "{\"level\":\"info\"}" "application/json")) + (get-attr result "status")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:802 + + (begin + (msg-send "console.stdout" "Hello from DSL!") + (msg-send "console.stdout" "Computing...") + (set result (* 6 7)) + (msg-send "console.stdout" (str-concat "Result: " (str result))) + (msg-send "console.stderr" "Warning: example only") + result) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:832 + + (begin + (set before (msg-pending "q.test")) + (msg-send "q.test" "a") + (msg-send "q.test" "b") + (set after (msg-pending "q.test")) + (list before after)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:855 + + (begin + (msg-send "drain.test" "first") + (msg-send "drain.test" "second") + (msg-send "drain.test" "third") + "sent") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:873 + + (begin + (set m1 (msg-recv "drain.test")) + (set m2 (msg-recv "drain.test")) + (set m3 (msg-recv "drain.test")) + (list (get-attr m1 "status") + (get-attr m2 "status") + (get-attr m3 "status"))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:959 +(+ 100 200) +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1028 + + (begin + (defun fib (n) + (if (<= n 1) n + (+ (fib (- n 1)) (fib (- n 2))))) + (fib 10)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1036 + + (begin + (defun fact (n) + (if (<= n 1) 1 + (* n (fact (- n 1))))) + (fact 8)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1056 + + (begin + (set procs (ps)) + (length procs)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1071 + + (begin + (set info (inspect (self-pid))) + (get-attr info "pid")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1085 + + (begin + (set i 0) + (set sum 0) + (while (< i 10) + (begin + (set sum (+ sum i)) + (set i (+ i 1)))) + sum) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1101 + + (begin + (set total 0) + (for x (list 10 20 30 40) + (set total (+ total x))) + total) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1114 + + (begin + (set double (lambda (x) (* x 2))) + (double 21)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1125 + + (let ((x 10) (y 20)) + (+ x y)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1135 + + (begin + (set d (dict "name" "alice" "age" 30)) + (get-attr d "name")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1157 + + (begin + (set stats (scheduler-stats)) + (get-attr stats "total")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1212 + + (begin + (state-set "data.x" "42") + (state-get "data.x")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1353 + + (begin + (set wid (state-watch "events.x" (lambda (p v) nil))) + (state-unwatch wid)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1367 + + (begin + (state-watch "trigger.value" + (lambda (path val) + (state-set "got-event" "yes"))) + ;; Trigger the watch + (state-set "trigger.value" "hello") + ;; Spin briefly — the handler fires at the next step boundary + (set tries 0) + (while (and (not (state-exists "got-event")) (< tries 100)) + (set tries (+ tries 1))) + (state-get "got-event")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1388 + + (begin + (state-watch "notify.path" + (lambda (path val) + (state-set "received-path" path))) + (state-set "notify.path" "data") + (set tries 0) + (while (and (not (state-exists "received-path")) (< tries 100)) + (set tries (+ tries 1))) + (state-get "received-path")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1417 + + (begin + ;; Consumer: watch for new items, append each to a log + (state-set "consumer.log" "") + (state-watch "queue.item" + (lambda (path val) + (state-set "consumer.log" + (str-concat (state-get "consumer.log") (state-get path) ",")))) + + ;; Produce 3 items — each triggers the handler reactively + (state-set "queue.item" "apple") + (state-set "queue.item" "banana") + (state-set "queue.item" "cherry") + + ;; Spin to let all handlers dispatch + (set tries 0) + (while (and (not (= (state-get "consumer.log") "apple,banana,cherry,")) + (< tries 300)) + (set tries (+ tries 1))) + + (state-get "consumer.log")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1451 + + (begin + (state-set "stats.ticks" "") + (state-watch "sensor.reading" + (lambda (path val) + (state-set "stats.ticks" + (str-concat (state-get "stats.ticks") "|")))) + + ;; 5 sensor readings + (state-set "sensor.reading" "72") + (state-set "sensor.reading" "73") + (state-set "sensor.reading" "71") + (state-set "sensor.reading" "74") + (state-set "sensor.reading" "70") + + (set tries 0) + (while (and (not (= (state-get "stats.ticks") "|||||")) + (< tries 500)) + (set tries (+ tries 1))) + + (state-get "stats.ticks")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1487 + + (begin + ;; Stage 1: prefix the input value + (state-watch "pipeline.input" + (lambda (path val) + (state-set "pipeline.stage1" + (str-concat "processed:" (state-get path))))) + + ;; Stage 2: wrap stage1 output + (state-watch "pipeline.stage1" + (lambda (path val) + (state-set "pipeline.stage2" + (str-concat "[" (state-get path) "]")))) + + ;; Kick off the pipeline + (state-set "pipeline.input" "hello") + + ;; Wait for full cascade + (set tries 0) + (while (and (not (state-exists "pipeline.stage2")) + (< tries 500)) + (set tries (+ tries 1))) + + ;; input="hello" -> stage1="processed:hello" -> stage2="[processed:hello]" + (state-get "pipeline.stage2")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1525 + + (begin + (state-set "sampler.log" "") + + ;; Store the watch ID so the handler can unsubscribe + (set wid + (state-watch "data.stream" + (lambda (path val) + (begin + (set current-val (state-get path)) + (set current-log (state-get "sampler.log")) + (state-set "sampler.log" + (str-concat current-log current-val ",")) + ;; After seeing 2 items (log has 2 commas), unsubscribe + (set updated (state-get "sampler.log")) + (if (= updated (str-concat current-val ",")) + nil + (state-unwatch wid)))))) + + ;; Produce 4 events — only the first 2 should be processed + (state-set "data.stream" "alpha") + (state-set "data.stream" "beta") + (state-set "data.stream" "gamma") + (state-set "data.stream" "delta") + + ;; Wait for handler to finish + (set tries 0) + (while (and (= (state-get "sampler.log") "") + (< tries 300)) + (set tries (+ tries 1))) + + ;; Small extra wait so second event can complete + (set tries 0) + (while (< tries 100) (set tries (+ tries 1))) + + (state-get "sampler.log")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1577 + + (begin + (state-watch "jobs.request" + (lambda (path val) + (state-set "jobs.result" + (str-concat "DONE:" (state-get path))))) + ;; Stay alive to process the request + (set i 0) + (while (< i 500) (set i (+ i 1))) + "worker-done") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1590 + + (begin + (state-set "jobs.request" "task-42") + (set tries 0) + (while (and (not (state-exists "jobs.result")) + (< tries 500)) + (set tries (+ tries 1))) + (state-get "jobs.result")) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1650 + + (begin + (let ((i 0)) + (while (< i 5) + (begin + (state-set (str-concat "queue.item." (str i)) (str (* i i))) + (set i (+ i 1))))) + (state-set "queue.count" "5") + "produce-done") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1664 + + (begin + ;; Busy-wait for count + (set tries 0) + (while (and (not (state-exists "queue.count")) (< tries 1000)) + (set tries (+ tries 1))) + ;; Read all items using a range generator + (let ((total 0)) + (for i (range 5) + (set total (+ total 1))) + total)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1691 + + (begin + (for i (range 10) + (state-set (str-concat "batch." (str i)) (str (* i 10)))) + (state-set "batch.done" "true") + 10) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1703 + + (begin + ;; Wait for producer to signal completion + (set tries 0) + (while (and (not (state-exists "batch.done")) (< tries 2000)) + (set tries (+ tries 1))) + ;; Collect the sum using range generator + (let ((sum 0)) + (for i (range 10) + (set sum (+ sum (* i 10)))) + sum)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1731 + + (begin + (for i (range 1 6) + (state-set (str-concat "sq." (str i)) (str (* i i)))) + (state-set "sq.ready" "5") + "squares-done") + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1743 + + (begin + ;; Wait for data + (set tries 0) + (while (and (not (state-exists "sq.ready")) (< tries 2000)) + (set tries (+ tries 1))) + ;; Generator that yields doubled square values + (let ((g (generator (lambda () + (for i (range 1 6) + (let ((val (state-get (str-concat "sq." (str i))))) + (yield (* 2 1))))))) + (count 0)) + (for x g + (set count (+ count 1))) + count)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1797 + + (begin + ;; Wait for all 4 producers + (for p (range 4) + (let ((key (str-concat "data.producer." (str-concat (str p) ".done")))) + (set tries 0) + (while (and (not (state-exists key)) (< tries 5000)) + (set tries (+ tries 1))))) + ;; Count all 12 values using a range generator + (let ((total 0)) + (for i (range 12) + (set total (+ total 1))) + total)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1828 + + (begin + ;; Source: range generator + (let ((squares (generator (lambda () + (for i (range 1 8) + (yield (* i i))))))) + ;; Consume and sum + (let ((total 0)) + (for v squares + (set total (+ total v))) + total))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1851 + + (begin + (let ((src (generator (lambda () + (yield 10) + (yield 20) + (yield 30)))) + (doubled (generator (lambda () + (for x src + (yield (* x 2))))))) + (collect doubled))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1894 + + (begin + ;; Wait for all 3 status keys + (set tries 0) + (while (and (not (state-exists "status.2")) (< tries 5000)) + (set tries (+ tries 1))) + ;; Generator-based scan + (let ((results (list))) + (for i (range 3) + (let ((val (state-get (str-concat "status." (str i))))) + (append results val))) + (length results))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1921 + + (begin + (let ((evens (generator (lambda () + (for i (range 0 10) + (if (= (% i 2) 0) + (yield i))))))) + (collect evens))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1946 + + (begin + (let ((counter (generator (lambda () + (let ((n 0)) + (while (< n 1000) + (begin (yield n) + (set n (+ n 1)))))))) + (taken (list))) + (for x counter + (if (>= (length taken) 4) + (break nil) + (append taken x))) + taken)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:1976 + + (begin + (let ((keys (generator (lambda () + (yield "name") + (yield "age") + (yield "role")))) + (vals (generator (lambda () + (yield "alice") + (yield "30") + (yield "engineer")))) + (pairs (list))) + ;; Manual zip: advance both, build pairs + (let ((k (next keys)) + (v (next vals))) + (while (not (is-null k)) + (begin + (append pairs (str-concat k "=" v)) + (set k (next keys)) + (set v (next vals))))) + pairs)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2021 + + (begin + (set tries 0) + (while (and (not (state-exists "worker.2.result")) (< tries 5000)) + (set tries (+ tries 1))) + (let ((total 0)) + (for i (range 3) + (set total (+ total 1))) + total)) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2041 + + (begin + (let ((running-sum (generator (lambda () + (let ((acc 0)) + (for x (list 10 20 30 40 50) + (begin + (set acc (+ acc x)) + (yield acc)))))))) + (collect running-sum))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2073 + + (begin + (set received (list)) + (set count 0) + (while (< count 9) + (begin + (set msg (msg-recv "work.queue")) + (append received (get-attr msg "status")) + (set count (+ count 1)))) + ;; Generator that maps each status to 1 + (set ones (generator (lambda () + (for item received + (yield 1))))) + ;; Collect into a list and take its length + (length (collect ones))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2117 + + (begin + (set payloads (list)) + (set n 0) + (while (< n 8) + (begin + (set msg (msg-recv "data.in")) + (append payloads (get-attr msg "status")) + (set n (+ n 1)))) + ;; Build a generator that doubles each collected payload string + (set doubler (generator (lambda () + (for p payloads + (yield (str-concat p p)))))) + ;; Collect the doubled strings and count them + (length (collect doubler))) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2161 + + (begin + (set msg (msg-recv "ch")) + 42) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2170 + + (begin + (msg-send "ch" "hello") + 99) + +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2376 +(msg-send "sig" "hi") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2382 +(msg-pending "sig") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2400 +(msg-recv "sig") +@@@ script src/cvc/tests/state_exec_integration_test.cpp:2409 +(msg-send "raw.chan" "x") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1526 +(self-pid) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1536 +(cluster-id) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1541 +(state-set "dsl.key" "dsl-value") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1542 +(state-get "dsl.key") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1547 +(state-set "dsl.exists" "yes") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1548 +(state-exists "dsl.exists") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1553 +(memory-limit) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1558 +(spawn "(+ 1 2)") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1585 +(string.upper "hello") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1590 +(math.sqrt 25) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1595 +(math.pi) +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1600 +(string.length "hello world") +@@@ script src/cvc/tests/state_exec_intrinsics_test.cpp:1605 +(collections.range 3) +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:188 + + (begin + (set v1 (state-get "shared.x")) + (set v2 (state-get "shared.y")) + (set v3 (state-get "shared.z")) + (str-concat v1 ":" v2 ":" v3)) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:231 + + (begin + (state-set "shared.x" "10") + (state-set "shared.y" "20") + (state-set "shared.z" "30") + "written") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:325 + + (begin + (set m (msg-recv "oob.chan")) + (get-attr m "payload")) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:394 + + (begin + (msg-send "oob.chan" "hello-from-A") + "sent") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:489 + + (begin + (set reader (generator (lambda () + (for i (range 5) + (yield (state-get (str-concat "items.v" (str i)))))))) + (set count 0) + (set concat "") + (for val reader + (begin + (set count (+ count 1)) + (set concat (str-concat concat val ":")))) + (state-set "result.consumer" (str-concat (str count) "|" concat)) + count) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:553 + + (begin + (set val-gen (generator (lambda () + (for i (range 5) + (yield (str-concat "v" (str i))))))) + (for val val-gen + (begin + (set idx (str-concat "items." val)) + (state-set idx val))) + (state-set "items.ready" "true") + "produced") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:655 + + (begin + (set count 0) + (for p (range 3) + (for i (range 3) + (begin + (set key (str-concat "producer." (str p) ".item." (str i))) + (if (state-exists key) + (set count (+ count 1)) + 0)))) + count) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:807 + + (begin + (set result "") + (set i 0) + (while (< i 4) + (begin + (set m (msg-recv "pipe.data")) + (set result (str-concat result (get-attr m "payload") ",")) + (set i (+ i 1)))) + (state-set "pipeline.result" result) + result) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:892 + + (begin + (set payload-gen (generator (lambda () + (for tag (list "aa" "bb" "cc" "dd") + (yield tag))))) + (for payload payload-gen + (msg-send "pipe.data" payload)) + "all-sent") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:988 + + (begin + (state-set "grpc.x" "alpha") + (state-set "grpc.y" "beta") + "written") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1012 + + (begin + (str-concat (state-get "grpc.x") ":" (state-get "grpc.y"))) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1064 + + (begin + (set m (msg-recv "grpc.chan")) + (get-attr m "payload")) + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1086 + + (begin + (msg-send "grpc.chan" "hello-grpc") + "sent") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1147 + + (begin + (set squares (generator (lambda () + (for i (range 5) + (yield (* i i)))))) + (let ((i 0)) + (for sq squares + (begin + (state-set (str-concat "data.v" (str i)) (str sq)) + (set i (+ i 1))))) + (state-set "data.ready" "true") + "produced") + +@@@ script src/cvc/tests/state_exec_multiprocess_test.cpp:1179 + + (begin + (set reader (generator (lambda () + (for i (range 5) + (yield (int (state-get (str-concat "data.v" (str i))))))))) + (set sum 0) + (for val reader + (set sum (+ sum val))) + sum) + +@@@ script src/cvc/tests/state_exec_parser_test.cpp:83 +(if (> x 0) "pos" "neg") +@@@ script src/cvc/tests/state_exec_parser_test.cpp:97 +(a (b (c (d)))) +@@@ script src/cvc/tests/state_exec_parser_test.cpp:143 +(+ 1 ; add one +2) +@@@ script src/cvc/tests/state_exec_parser_test.cpp:152 +(defun f (x) x) (f 42) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:177 +(- 100 50) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:280 +(begin 4 5 6) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:806 + + (begin + (defun fib (n) + (if (<= n 1) n + (+ (fib (- n 1)) (fib (- n 2))))) + (fib 8)) + +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:813 + + (begin + (defun fact (n) + (if (<= n 1) 1 + (* n (fact (- n 1))))) + (fact 6)) + +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:826 + + (begin + (let ((sum 0)) + (for i (list 1 2 3 4 5) + (set sum (+ sum i))) + sum)) + +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:838 + + (begin + (let ((i 0) (sum 0)) + (while (< i 10) + (begin + (set sum (+ sum i)) + (set i (+ i 1)))) + sum)) + +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:852 + + (begin + (let ((double (lambda (x) (* x 2)))) + (double 21))) + +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1112 +(if #t 42 99) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1113 +(let ((x 10)) (* x x)) +@@@ script src/cvc/tests/state_exec_scheduler_test.cpp:1113 +(length "hello") +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:157 +(begin (set i 0) (while (< i 1000) (set i (+ i 1))) i) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:220 + + (let ((g (generator (lambda () + (yield 10) + (yield 20) + (yield 30))))) + (list (next g) (next g) (next g))) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:236 + + (let ((g (generator (lambda () (yield 1))))) + (list (next g) (next g))) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:248 + + (let ((g (generator (lambda () + (yield (* 2 3)) + (yield (+ 10 5)))))) + (+ (next g) (next g))) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:259 + + (let ((r (range 0 5))) + (+ (next r) (next r) (next r))) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:267 +(collect (range 5)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:274 + + (let ((total 0)) + (for x (range 1 6) + (set total (+ total x))) + total) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:284 + + (let ((accum (list))) + (for x (generator (lambda () + (yield "a") + (yield "b") + (yield "c"))) + (append accum x)) + (length accum)) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:298 + + (collect (generator (lambda () + (let ((i 0)) + (while (< i 4) + (begin (yield i) + (set i (+ i 1)))))))) + +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:316 +(while t (break)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:322 +(while t (break 42)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:326 +(begin (set i 0) (while t (if (= i 5) (break i) (set i (+ i 1))))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:335 +(for x (list 1 2 3 4 5) (if (= x 3) (break x) nil)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:342 +(for x (range 1000) (if (= x 5) (break x) nil)) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:349 +(begin (set sum 0) (for i (list 1 2 3) (for j (list 10 20 30) (if (= j 20) (break nil) (set sum (+ sum j))))) sum) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:360 +(begin (set i 0) (while t (begin (set i (+ i 1)) (if (= i 3) (break i) nil)))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:370 +(for x (list 1 2 3 4 5) (let ((doubled (* x 2))) (if (> doubled 6) (break doubled) nil))) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:382 +(if #t 1 2) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:383 +(begin 1 2) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:384 +(quote 1) +@@@ script src/cvc/tests/state_exec_stackless_evaluator_test.cpp:385 +(defclass C (init (self) 1)) +@@@ script src/cvc/tests/state_exec_types_test.cpp:72 +(1 2 3) +@@@ script src/cvc/tests/state_exec_types_test.cpp:83 +(1 ("a" "b")) +@@@ script src/cvcGL/test/cvcgl_media_audio.cpp:50 +(if (= (state-get "cap.done") "1") #f (begin (stream-capture-audio (get-attr (stream-open (dict "id" "mic" "codec" "f32" "rate" 48000 "ch" 1 "frames_per_chunk" 256)) "token") (dict "rate" 48000 "ch" 1 "frames" 256 "mix" (list (dict "tone" (dict "hz" 440)) (dict "tone" (dict "hz" 660))))) (state-set "cap.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_media_capture.cpp:52 +(if (= (state-get "cap.done") "1") #f (begin (stream-capture-camera (get-attr (stream-open (dict "id" "cam" "w" 64 "h" 48)) "token") (dict "synthetic" #t "w" 64 "h" 48 "hz" 240)) (state-set "cap.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_media_play.cpp:52 +(if (= (state-get "play.done") "1") #f (begin (stream-open (dict "id" "mic" "codec" "f32" "rate" 48000 "ch" 1 "frames_per_chunk" 256 "subscriber_depth" 8)) (stream-capture-audio "streams.mic" (dict "rate" 48000 "ch" 1 "frames" 256 "tone" (dict "hz" 440))) (stream-play-audio "streams.mic") (state-set "play.done" "1"))) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:63 +(begin (set h (gl-bind-stream "screen" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "node" (get-attr h "node")) (state-set "token" (get-attr h "token")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:81 +(begin (set h (gl-bind-stream "screen" (dict "token" "demo.streams.cam0"))) (state-set "ok" (get-attr h "ok")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:96 +(begin (set h (gl-bind-stream "ghost" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:109 +(begin (set h (gl-bind-stream "screen" "streams.cam0")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script src/cvcGL/test/cvcgl_stream_bind.cpp:123 +(begin (set h (gl-bind-stream "screen")) (state-set "ok" (get-attr h "ok")) (state-set "err" (get-attr h "error")) (state-set "done" "1")) +@@@ script bindings/pycvc/test_pycvc_exec.py:49 +(state-set "dsl.out" "wrote-from-dsl") +@@@ script bindings/pycvc/test_pycvc_exec.py:123 +(capture #t) +@@@ script bindings/pycvc/test_pycvc_exec.py:147 +(capture (make-bytes)) +@@@ script bindings/pycvc/test_pycvc_exec.py:21 +(* 6 7) +@@@ script bindings/pycvc/test_pycvc_exec.py:22 +(begin (set x 5) x) +@@@ script bindings/pycvc/test_pycvc_exec.py:23 +(str-concat "ab" "cd") +@@@ script bindings/pycvc/test_pycvc_exec.py:25 +(> 1 2) +@@@ script bindings/pycvc/test_pycvc_exec.py:47 +(state-get "shared.x") +@@@ script bindings/pycvc/test_pycvc_exec.py:64 +(py-double 21) +@@@ script bindings/pycvc/test_pycvc_exec.py:65 +(py-add 40 2) +@@@ script bindings/pycvc/test_pycvc_exec.py:66 +(py-greet "cvc") +@@@ script bindings/pycvc/test_pycvc_exec.py:68 +(+ 1 (py-double 20)) +@@@ script bindings/pycvc/test_pycvc_exec.py:81 +(stash "py.k" "py.v") +@@@ script bindings/pycvc/test_pycvc_exec.py:86 +(fetch "py.k") +@@@ script bindings/pycvc/test_pycvc_exec.py:98 +(boom) +@@@ script bindings/pycvc/test_pycvc_exec.py:119 +(capture 5) +@@@ script bindings/pycvc/test_pycvc_exec.py:121 +(capture "text") +@@@ script bindings/pycvc/test_pycvc_exec_async.py:29 +(begin (adouble 21)) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:62 +(begin (slow 7)) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:78 +(begin (get-attr (boom 1) "__async_error__")) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:91 +(begin (inc (triple 4))) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:110 +(begin (spawn "(while t 1)") (state-set "r" (str (slow 7))) (state-get "r")) +@@@ script bindings/pycvc/test_pycvc_exec_async.py:82 +(begin (idn 5)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:58 +(begin (spawn "(while t 1)") (state-set "r" (str (acount 6))) (state-get "r")) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:25 +(begin (adbl 21)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:40 +(begin (acount 10)) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:96 +(begin (plus10 (inc (triple 4)))) +@@@ script bindings/pycvc/test_pycvc_exec_coro.py:128 +(begin (reenter 1)) +@@@ script bindings/pycvc/test_pycvc_integration.py:60 +(state-set "dsl.done" "yes") +@@@ script bindings/pycvc/test_pycvc_integration.py:58 +(+ 0 (verts)) diff --git a/src/cvc/tests/state_exec_corpus_test.cpp b/src/cvc/tests/state_exec_corpus_test.cpp new file mode 100644 index 000000000..2ed2fffe1 --- /dev/null +++ b/src/cvc/tests/state_exec_corpus_test.cpp @@ -0,0 +1,102 @@ +// Parses every DSL script the repo carries (state_exec_corpus/scripts.txt, a frozen snapshot made +// by gen_state_exec_corpus.py) and compares each parse with state_exec_corpus/expected.txt. The +// grammar work (docs/STATE_EXEC_GRAMMAR.md) replaces the parser; this pins how every existing +// script reads, so a change to that is visible in the same PR as an expected.txt diff. +// +// Regenerate expected.txt after an intended change: CVC_UPDATE_GOLDENS=1 state_exec_corpus_test + +#include "state_exec_grammar_dump.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace dump = cvc::state_exec::grammar_dump; + +namespace { + +const std::string kMark = "@@@ script "; + +fs::path corpus_dir() { return fs::path(__FILE__).parent_path() / "state_exec_corpus"; } + +std::string read_file(const fs::path &p) { + std::ifstream in(p, std::ios::binary); + std::stringstream ss; + ss << in.rdbuf(); + std::string s = ss.str(); + // The files are LF-only (.gitattributes); undo a CRLF checkout anyway. + for (auto i = s.find("\r\n"); i != std::string::npos; i = s.find("\r\n", i)) + s.erase(i, 1); + return s; +} + +// Split "@@@ script \n\n" entries into (source, body) pairs. A body is everything up +// to the next entry's mark, minus the one '\n' that ends it. +std::vector> read_entries(const fs::path &p) { + const std::string text = read_file(p); + std::vector> out; + // The mark only counts at the start of a line (the header comment mentions it). + std::size_t pos = text.compare(0, kMark.size(), kMark) == 0 ? 0 : text.find("\n" + kMark); + if (pos != std::string::npos && text[pos] == '\n') + ++pos; + while (pos != std::string::npos) { + const std::size_t eol = text.find('\n', pos); + std::string source = text.substr(pos + kMark.size(), eol - pos - kMark.size()); + std::size_t next = text.find("\n" + kMark, eol); + std::string body = + text.substr(eol + 1, (next == std::string::npos ? text.size() : next + 1) - (eol + 1)); + if (!body.empty() && body.back() == '\n') + body.pop_back(); + out.emplace_back(std::move(source), std::move(body)); + pos = next == std::string::npos ? next : next + 1; + } + return out; +} + +} // namespace + +TEST(StateExecCorpus, EveryScriptParsesAsRecorded) { + const auto scripts = read_entries(corpus_dir() / "scripts.txt"); + ASSERT_GT(scripts.size(), 100u) << "corpus missing: " << corpus_dir() / "scripts.txt"; + + std::vector actual; + std::size_t errors = 0; + for (const auto &[source, script] : scripts) { + actual.push_back(dump::dump_all(script)); + errors += actual.back().rfind("!error", 0) == 0; + } + std::cout << scripts.size() << " scripts, " << errors << " parse errors\n"; + + const fs::path expected_path = corpus_dir() / "expected.txt"; + if (const char *update = std::getenv("CVC_UPDATE_GOLDENS"); update && *update == '1') { + std::ofstream out(expected_path, std::ios::binary); + out << "# Expected parses of scripts.txt, in the grammar_dump format of\n" + "# state_exec_grammar_dump.h. Written by state_exec_corpus_test with\n" + "# CVC_UPDATE_GOLDENS=1; do not edit by hand.\n"; + for (std::size_t i = 0; i < scripts.size(); ++i) + out << kMark << scripts[i].first << '\n' << actual[i] << '\n'; + GTEST_SKIP() << "wrote " << expected_path; + } + + const auto expected = read_entries(expected_path); + ASSERT_EQ(expected.size(), scripts.size()) + << expected_path << " is stale; regenerate it with CVC_UPDATE_GOLDENS=1"; + int shown = 0; + for (std::size_t i = 0; i < scripts.size(); ++i) { + ASSERT_EQ(expected[i].first, scripts[i].first) + << expected_path << " is stale; regenerate it with CVC_UPDATE_GOLDENS=1"; + if (actual[i] != expected[i].second && ++shown <= 20) + ADD_FAILURE() << scripts[i].first << "\n--- script\n" + << scripts[i].second << "\n--- expected\n" + << expected[i].second << "\n--- actual\n" + << actual[i]; + } + EXPECT_EQ(shown, 0) << shown << " scripts parse differently from " << expected_path; +} diff --git a/src/cvc/tests/state_exec_grammar_dump.h b/src/cvc/tests/state_exec_grammar_dump.h new file mode 100644 index 000000000..908fb6dc0 --- /dev/null +++ b/src/cvc/tests/state_exec_grammar_dump.h @@ -0,0 +1,240 @@ +/** + * @file state_exec_grammar_dump.h + * @brief Exact, type-tagged rendering of parser output for the grammar tests. + * + * state_exec_grammar_test and state_exec_corpus_test compare parses through this dump, not + * through to_string(): to_string is lossy (unescaped strings, 6-digit floats) and changes in + * phase P5 of docs/STATE_EXEC_GRAMMAR.md, while a golden must mean the same thing before and + * after every phase. The format: + * + * nil / #t / #f nil and the bools + * 42, -7 int64 + * 1.5, 31.0, 1e+20 double: whole numbers below 1e16 as N.0, others in the shortest + * form that reads back exactly; always with '.', 'e' or a special + * +inf.0 -inf.0 +nan.0 + * "..." string: \\ \" \n \t \r, other ASCII controls as \xNN, invalid UTF-8 + * bytes as \xNN (always >= \x80), invisible or control code points as + * \u{X}, everything else verbatim + * b"..." bytes: printable ASCII verbatim, the rest \xNN + * foo, |1+| symbol; |...| (same escapes, plus \|) when the bare text would read + * as something else + * (a b) list + * + * dump_all() joins top-level values with '\n' and renders a parse_error as "!error line:col". + */ +#ifndef CVC_STATE_EXEC_GRAMMAR_DUMP_H +#define CVC_STATE_EXEC_GRAMMAR_DUMP_H + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cvc::state_exec::grammar_dump { + +/// Code points a dump writes as \u{X}: C1 controls, bidi controls, U+FEFF, and the +/// whitespace that is not ASCII (so an NBSP in a golden is visible). +inline bool invisible(char32_t cp) { + return (cp >= 0x80 && cp <= 0x9F) || cp == 0x61C || cp == 0xA0 || cp == 0x1680 || + (cp >= 0x2000 && cp <= 0x200F) || (cp >= 0x2028 && cp <= 0x202F) || cp == 0x205F || + (cp >= 0x2066 && cp <= 0x2069) || cp == 0x3000 || cp == 0xFEFF; +} + +inline void hex_byte(unsigned char b, std::string &out) { + static const char kHex[] = "0123456789abcdef"; + out += "\\x"; + out += kHex[b >> 4]; + out += kHex[b & 0x0F]; +} + +/// Escape UTF-8 text for a string or |symbol| dump; `quote` is the closing delimiter. +inline void text(std::string_view s, char quote, std::string &out) { + std::size_t pos = 0; + while (pos < s.size()) { + char32_t cp; + std::size_t len; + if (!utf8::decode(s, pos, cp, len)) { + hex_byte(static_cast(s[pos]), out); + pos += 1; + continue; + } + if (cp == '\\' || cp == static_cast(quote)) { + out += '\\'; + out += static_cast(cp); + } else if (cp == '\n') { + out += "\\n"; + } else if (cp == '\t') { + out += "\\t"; + } else if (cp == '\r') { + out += "\\r"; + } else if (cp < 0x20 || cp == 0x7F) { + hex_byte(static_cast(cp), out); + } else if (invisible(cp)) { + char buf[16]; + std::snprintf(buf, sizeof buf, "\\u{%X}", static_cast(cp)); + out += buf; + } else { + out.append(s.substr(pos, len)); + } + pos += len; + } +} + +/// Shortest decimal form that reads back to exactly `d`, marked as a double. +inline std::string number(double d) { + if (std::isnan(d)) + return "+nan.0"; + if (std::isinf(d)) + return d > 0 ? "+inf.0" : "-inf.0"; + char buf[40]; + if (d == std::trunc(d) && std::fabs(d) < 1e16) { // whole numbers read as 1000.0, not 1e+03 + std::snprintf(buf, sizeof buf, "%.1f", d); + return buf; + } + for (int prec = 1; prec <= 17; ++prec) { + std::snprintf(buf, sizeof buf, "%.*g", prec, d); + if (std::strtod(buf, nullptr) == d) + break; + } + std::string s(buf); + if (s.find_first_of(".e") == std::string::npos) + s += ".0"; + return s; +} + +/// True when a symbol's bare name would read back as something other than that symbol. +inline bool needs_bars(std::string_view name) { + if (name.empty() || name == "nil" || name == "true" || name == "false" || name[0] == '#') + return true; + const char c0 = name[0]; + if (c0 >= '0' && c0 <= '9') + return true; + if ((c0 == '+' || c0 == '-' || c0 == '.') && name.size() > 1) { + const char c1 = name[1]; + if ((c1 >= '0' && c1 <= '9') || c1 == '.' || c1 == 'i' || c1 == 'n') + return true; + } + std::size_t pos = 0; + while (pos < name.size()) { + char32_t cp; + std::size_t len; + if (!utf8::decode(name, pos, cp, len) || cp <= 0x20 || cp == 0x7F || invisible(cp)) + return true; + switch (cp) { + case '(': + case ')': + case '"': + case '\'': + case ';': + case ',': + case '[': + case ']': + case '\\': + case '`': + case '{': + case '|': + case '}': + return true; + default: + break; + } + pos += len; + } + return false; +} + +inline void value(const value_t &v, std::string &out) { + std::visit( + [&](auto &&arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + out += "nil"; + } else if constexpr (std::is_same_v) { + out += arg ? "#t" : "#f"; + } else if constexpr (std::is_same_v) { + out += std::to_string(arg); + } else if constexpr (std::is_same_v) { + out += number(arg); + } else if constexpr (std::is_same_v) { + out += '"'; + text(arg, '"', out); + out += '"'; + } else if constexpr (std::is_same_v) { + out += "b\""; + for (unsigned char b : arg.data) { + if (b == '"' || b == '\\') { + out += '\\'; + out += static_cast(b); + } else if (b >= 0x20 && b < 0x7F) { + out += static_cast(b); + } else { + hex_byte(b, out); + } + } + out += '"'; + } else if constexpr (std::is_same_v) { + if (needs_bars(arg.name)) { + out += '|'; + text(arg.name, '|', out); + out += '|'; + } else { + out += arg.name; + } + } else if constexpr (std::is_same_v) { + out += '('; + if (arg) { + for (std::size_t i = 0; i < arg->size(); ++i) { + if (i > 0) + out += ' '; + value((*arg)[i], out); + } + } + out += ')'; + } else { + out += "#<" + v.type_name() + ">"; // the parser never produces these + } + }, + v.v); +} + +inline std::string value(const value_t &v) { + std::string out; + value(v, out); + return out; +} + +inline std::string error(const parse_error &e) { + return "!error " + std::to_string(e.line()) + ":" + std::to_string(e.col()); +} + +/// parse_all(src), one dumped value per line. +inline std::string dump_all(const std::string &src) { + try { + std::string out; + for (const value_t &v : parse_all(src)) { + if (!out.empty()) + out += '\n'; + value(v, out); + } + return out; + } catch (const parse_error &e) { + return error(e); + } +} + +/// parse(src), dumped. +inline std::string dump_one(const std::string &src) { + try { + return value(parse(src)); + } catch (const parse_error &e) { + return error(e); + } +} + +} // namespace cvc::state_exec::grammar_dump + +#endif // CVC_STATE_EXEC_GRAMMAR_DUMP_H diff --git a/src/cvc/tests/state_exec_grammar_test.cpp b/src/cvc/tests/state_exec_grammar_test.cpp new file mode 100644 index 000000000..f8b0ca1a2 --- /dev/null +++ b/src/cvc/tests/state_exec_grammar_test.cpp @@ -0,0 +1,284 @@ +// Conformance tests for the state_exec grammar (docs/STATE_EXEC_GRAMMAR.md). +// +// Every case names the EBNF productions it exercises and the phase from which it holds: +// 0 holds for the pre-grammar (hand-written) parser too; +// N holds once phase PN of the grammar work lands (P2 = parser built from the grammar, +// P3 = Unicode source rules + strict escapes, P4 = octal escapes, C escapes, raw strings). +// Cases for landed phases (<= kLandedPhase) must hold. Cases for pending phases must still +// DIFFER, so the table stays an exact list of what each phase changes; when a phase lands, bump +// kLandedPhase in the same PR. The doc's EBNF is checked for closure, and every production must +// have a case. +// +// Inputs and expectations are byte-exact: non-ASCII is written as \xNN escapes (MSVC builds +// without /utf-8), split with "" where a hex letter follows an escape. + +#include "state_exec_grammar_dump.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace dump = cvc::state_exec::grammar_dump; + +namespace { + +constexpr int kLandedPhase = 0; + +enum class mode { all, one }; // parse_all() or parse() + +struct grammar_case { + int phase; + std::string rules; // space-separated EBNF production names + mode how; + std::string input; + std::string expect; // grammar_dump output; "!error" matches any error position +}; + +std::string nest(int depth) { return std::string(depth, '(') + std::string(depth, ')'); } + +const std::vector &cases() { + static const std::vector kCases = { + // --- holds today ---------------------------------------------------------------------- + {0, "program", mode::all, "", ""}, + {0, "program atmosphere", mode::all, "1 2\n3", "1\n2\n3"}, + {0, "single_datum", mode::one, "", "nil"}, + {0, "single_datum line_comment", mode::one, " ; note\n 42 ", "42"}, + {0, "datum list", mode::all, "(a (b c) ())", "(a (b c) ())"}, + {0, "list", mode::all, nest(256), nest(256)}, + {0, "list", mode::all, "(a", "!error"}, + {0, "quoted", mode::all, "'x ' (1 2) ''a", "(quote x)\n(quote (1 2))\n(quote (quote a))"}, + {0, "line_comment line_end", mode::all, "; c\n(a ; c2\n b)", "(a b)"}, + {0, "whitespace", mode::all, "\t\n\v\f\r a", "a"}, + {0, "text_literal text_string string_char", mode::all, "\"hi there\"", "\"hi there\""}, + {0, "string_char", mode::all, "\"a\nb\"", R"("a\nb")"}, + {0, "text_string", mode::all, "\"abc", "!error"}, + {0, "text_escape simple_escape", mode::all, R"("\n\t\r\\\"")", R"("\n\t\r\\\"")"}, + {0, "hex_escape hex", mode::all, R"("\x41\xe9")", "\"A\xC3\xA9\""}, + {0, "hex_escape", mode::all, R"("\xZ1")", "!error"}, + {0, "unicode_escape", mode::all, R"("é\u{1F600}")", "\"\xC3\xA9\xF0\x9F\x98\x80\""}, + {0, "unicode_escape", mode::all, R"("\u12")", "!error"}, + {0, "unicode_escape", mode::all, R"("\u{}")", "!error"}, + {0, "unicode_escape", mode::all, R"("\u{1234567}")", "!error"}, + {0, "octal_escape", mode::all, R"("\0")", R"("\x00")"}, + {0, "bytes_literal byte_string byte_escape", mode::all, R"(b"ab\x00\xff\n")", + R"(b"ab\x00\xff\x0a")"}, + {0, "byte_string", mode::all, "b\"\xC3\xA9\"", R"(b"\xc3\xa9")"}, + {0, "byte_string symbol", mode::all, "b (b) b\"\"", "b\n(b)\nb\"\""}, + {0, "boolean", mode::all, "#t true", "#t\n#t"}, + {0, "nil", mode::all, "#f false nil", "nil\nnil\nnil"}, + {0, "number integer dec_digits sign", mode::all, "42 -7 007 -9223372036854775808", + "42\n-7\n7\n-9223372036854775808"}, + {0, "decimal_float exponent", mode::all, "1.5 .5 1. 1e3 -2.5E-3", + "1.5\n0.5\n1.0\n1000.0\n-0.0025"}, + {0, "atom token constituent symbol", mode::all, + "foo-bar? -> + - caf\xC3\xA9 \xCE\xBB \xE6\x97\xA5\xE6\x9C\xAC a.b:c", + "foo-bar?\n->\n+\n-\ncaf\xC3\xA9\n\xCE\xBB\n\xE6\x97\xA5\xE6\x9C\xAC\na.b:c"}, + {0, "symbol", mode::all, "1+ ... #foo", "|1+|\n|...|\n|#foo|"}, + {0, "delimiter constituent", mode::all, "a\"b\"(c)'d;e", "a\n\"b\"\n(c)\n(quote d)"}, + {0, "raw_text symbol", mode::all, "r#foo", "r#foo"}, + + // --- P2: parser built from the grammar, numbers included ----------------------------- + {2, "program", mode::all, "a)", "!error"}, + {2, "reserved", mode::all, "[a]", "!error"}, + {2, "reserved", mode::all, "a,b", "!error"}, + {2, "reserved", mode::all, "`a", "!error"}, + {2, "reserved", mode::all, "{}", "!error"}, + {2, "reserved", mode::all, "a|b", "!error"}, + {2, "reserved", mode::all, "a\\b", "!error"}, + {2, "single_datum", mode::one, "1 2", "!error"}, + {2, "quoted", mode::all, "'", "!error"}, + {2, "list", mode::all, nest(257), "!error"}, + {2, "line_end line_comment", mode::all, "; c\r1", "1"}, + {2, "radix_integer hex_digits", mode::all, "0x1F 0X1f #x1F #X1f 0xFF_FF", + "31\n31\n31\n31\n65535"}, + {2, "radix_integer oct_digits bin_digits sign", mode::all, "0o17 #o17 0b101 #b101 -0x10 +0b1", + "15\n15\n5\n5\n-16\n1"}, + {2, "radix_prefixed", mode::all, "0xZZ", "!error"}, + {2, "radix_prefixed", mode::all, "#bar", "!error"}, + {2, "radix_prefixed", mode::all, "0x", "!error"}, + {2, "integer sign", mode::all, "+5", "5"}, + {2, "dec_digits", mode::all, "1_000", "1000"}, + {2, "integer", mode::all, "9223372036854775808", "!error"}, + {2, "float_special", mode::all, "+inf.0 -inf.0 +nan.0 -nan.0", + "+inf.0\n-inf.0\n+nan.0\n+nan.0"}, + {2, "symbol", mode::all, "inf nan infinity -INF", "inf\nnan\ninfinity\n-INF"}, + {2, "decimal_float", mode::all, "1e999", "!error"}, + {2, "decimal_float dec_digits", mode::all, "6.022_140e23", "6.02214e+23"}, + + // --- P3: Unicode source rules, strict escapes ---------------------------------------- + {3, "scalar_value", mode::all, + "a\xff" + "b", + "!error"}, + {3, "bom", mode::all, "\xEF\xBB\xBF(a)", "(a)"}, + {3, "other_space constituent", mode::all, + "a\xC2\xA0" + "b", + "!error"}, + {3, "whitespace", mode::all, + "a\xC2\x85" + "b a\xE2\x80\xA8" + "b", + "a\nb\na\nb"}, + {3, "source_char bidi_control", mode::all, + "\"a\xE2\x80\xAE" + "b\"", + "!error"}, + {3, "bidi_control line_comment", mode::all, "; \xE2\x81\xA6\n1", "!error"}, + {3, "control", mode::all, + "\"a\x01" + "b\"", + "!error"}, + {3, "text_escape", mode::all, R"("\q")", "!error"}, + {3, "unicode_escape", mode::all, R"("\uD800")", "!error"}, + {3, "unicode_escape", mode::all, R"("\u{110000}")", "!error"}, + // The error is reported at the offending '"', counted in code points: 1:6, not 1:8. + {3, "hex_escape", mode::all, "\"\xC3\xA9\\x4\"", "!error 1:6"}, + + // --- P4: octal escapes, C escapes, raw strings --------------------------------------- + {4, "octal_escape", mode::all, R"("\101\012")", R"("A\n")"}, + {4, "byte_escape octal_escape", mode::all, R"(b"\377")", R"(b"\xff")"}, + {4, "byte_escape", mode::all, R"(b"\400")", "!error"}, + {4, "simple_escape", mode::all, R"("\a\b\f\v")", R"("\x07\x08\x0c\x0b")"}, + {4, "raw_text raw_body raw_char", mode::all, R"(r"a\nb")", R"("a\\nb")"}, + {4, "hash_fence", mode::all, R"(r#"say "hi""#)", R"("say \"hi\"")"}, + {4, "hash_fence", mode::all, R"(r##"a"#b"##)", R"("a\"#b")"}, + {4, "raw_bytes", mode::all, R"(br"\x00")", R"(b"\\x00")"}, + {4, "raw_body", mode::all, R"(r#"abc")", "!error"}, + }; + return kCases; +} + +std::string run(const grammar_case &c) { + return c.how == mode::all ? dump::dump_all(c.input) : dump::dump_one(c.input); +} + +bool matches(const std::string &actual, const std::string &expect) { + if (expect == "!error") + return actual.rfind("!error", 0) == 0; + return actual == expect; +} + +std::string describe(const grammar_case &c) { + return "[P" + std::to_string(c.phase) + " " + c.rules + "] input: " + c.input; +} + +fs::path grammar_doc() { + // src/cvc/tests/ -> repo root -> docs/ + return fs::path(__FILE__).parent_path().parent_path().parent_path().parent_path() / "docs" / + "STATE_EXEC_GRAMMAR.md"; +} + +struct ebnf { + std::map defined; // production -> times defined + std::map> // production -> names its body references + uses; +}; + +// Read the ```ebnf block of the grammar doc: `name ::= body` lines, continuation lines, and +// /* comments */. Names inside quotes, character classes and #xN code points are not references. +ebnf read_ebnf(const fs::path &doc) { + std::ifstream in(doc); + std::stringstream ss; + ss << in.rdbuf(); + const std::string text = ss.str(); + const auto open = text.find("```ebnf\n"); + ebnf g; + if (open == std::string::npos) + return g; + const auto close = text.find("```", open + 8); + std::string body = text.substr(open + 8, close - open - 8); + + for (auto c = body.find("/*"); c != std::string::npos; c = body.find("/*")) // strip comments + body.erase(c, body.find("*/", c) + 2 - c); + + auto is_name_char = [](char ch) { return (ch >= 'a' && ch <= 'z') || ch == '_'; }; + std::istringstream lines(body); + std::string line, current; + while (std::getline(lines, line)) { + std::size_t i = 0; + const auto def = line.find("::="); + if (def != std::string::npos) { + std::size_t end = line.find_first_of(" \t"); + current = line.substr(0, end); + ++g.defined[current]; + i = def + 3; + } + while (i < line.size()) { + const char ch = line[i]; + if (ch == '\'' || ch == '"') { + i = line.find(ch, i + 1) + 1; + } else if (ch == '[') { + i = line.find(']', i + 1) + 1; + } else if (ch == '#' && i + 1 < line.size() && line[i + 1] == 'x') { + i += 2; + while (i < line.size() && std::isxdigit(static_cast(line[i]))) + ++i; + } else if (is_name_char(ch)) { + std::size_t j = i; + while (j < line.size() && is_name_char(line[j])) + ++j; + g.uses[current].insert(line.substr(i, j - i)); + i = j; + } else { + ++i; + } + } + } + return g; +} + +} // namespace + +TEST(StateExecGrammar, CasesForLandedPhasesHold) { + for (const grammar_case &c : cases()) { + if (c.phase > kLandedPhase) + continue; + EXPECT_PRED2(matches, run(c), c.expect) << describe(c); + } +} + +TEST(StateExecGrammar, CasesForPendingPhasesStillDiffer) { + for (const grammar_case &c : cases()) { + if (c.phase <= kLandedPhase) + continue; + EXPECT_FALSE(matches(run(c), c.expect)) + << describe(c) << "\nalready holds; set its phase to the one that made it hold"; + } +} + +TEST(StateExecGrammar, DocGrammarIsClosed) { + const ebnf g = read_ebnf(grammar_doc()); + ASSERT_GT(g.defined.size(), 40u) << "no ```ebnf block found in " << grammar_doc(); + std::set referenced; + for (const auto &[name, names] : g.uses) + referenced.insert(names.begin(), names.end()); + for (const auto &[name, count] : g.defined) { + EXPECT_EQ(count, 1) << name << " is defined more than once"; + if (name != "program" && name != "single_datum") + EXPECT_TRUE(referenced.count(name)) << name << " is defined but never used"; + } + for (const std::string &name : referenced) + EXPECT_TRUE(g.defined.count(name)) << name << " is used but never defined"; +} + +TEST(StateExecGrammar, EveryProductionHasACase) { + const ebnf g = read_ebnf(grammar_doc()); + std::set covered; + for (const grammar_case &c : cases()) { + std::istringstream names(c.rules); + for (std::string n; names >> n;) { + EXPECT_TRUE(g.defined.count(n)) << "case names unknown production '" << n << "'"; + covered.insert(n); + } + } + for (const auto &[name, count] : g.defined) + EXPECT_TRUE(covered.count(name)) << name << " has no case"; +} From 1381d6de5121acb49d155668c645203fa714c726 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 00:00:28 -0500 Subject: [PATCH 02/13] state_exec grammar: make the nesting limit configurable (default 256) Spec only: parse_options::max_depth on parse()/parse_all()/parser and execute_options::max_parse_depth for submitted scripts, both from P2. Records why the limit exists (the reader recurses on the native stack; the evaluators are stackless) and the corpus's real depths (max 13, median 2). --- docs/STATE_EXEC_GRAMMAR.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index c979d0abb..cef1095a9 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -123,9 +123,20 @@ raw_bytes ::= 'br' raw_body - **Positions.** `parse_error::line()` and `col()` are 1-based, and point at the character where the error was found, such as the non-hex character in `"\x4"`. A new line starts after LF, after CR, and once after CRLF. Columns count code points, not bytes. -- **Nesting.** At most 256 lists and quotes may be nested. Deeper input is an error, not a stack - overflow. The limit is sized for the 64 KiB default stack of a wasm build; real scripts nest far - less. +- **Nesting.** Lists and quotes may nest at most `max_depth` levels, 256 by default. + - `(a (b (c)))` is three levels, and `'x` counts as one, since it reads as `(quote x)`. + - The parser recurses on the native stack once per level. So deeper input is a `parse_error`, + not a stack overflow, even when the script comes from an untrusted source (a message, an HTTP + body, the multiprocess wire). The evaluators are stackless, so this limit is only about + reading. + - The default fits the 64 KiB default stack of a wasm build. Real scripts nest far less: the + deepest in the corpus is 13 levels, and the median is 2. + - The limit is configurable, from P2 on: + - `parse_options::max_depth`, an optional argument to `parse()`, `parse_all()` and `parser`; + - `execute_options::max_parse_depth`, for scripts submitted to a scheduler or coordinator, + next to `max_steps` and `max_memory`. 0 means the default. + + A host that raises it must give the parsing thread a big enough stack. ## Data @@ -228,7 +239,7 @@ strings. Every row is a case in `state_exec_grammar_test`. | `[a]`, `a,b`, `` `a ``, `{}`, `a\|b`, `a\b` | odd one-character or glued symbols | error (reserved) | P2 | | `parse("1 2")` | `1`; the rest is ignored | error | P2 | | `'` at end of input | `(quote nil)` | error | P2 | -| 257 nested lists | parsed | error | P2 | +| 257 nested lists | parsed (no limit) | error at the default `max_depth` | P2 | | `; c` CR `1` | comment runs to LF, so no datum | `1` | P2 | | `0x1F` | `31.0`, a double (via `strtod`) | `31` | P2 | | `#x1F`, `0o17`, `0b101`, `1_000` | symbols | integers | P2 | From c6486e242477963759968ce06f49b83dd3904cac Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 11:07:21 -0500 Subject: [PATCH 03/13] state_exec grammar: parser settings live on the state tree Spec only. max_depth resolves from state_exec.schedulers..parser.max_depth, then state_exec.defaults.parser.max_depth, then 256, matching scheduler::load_settings(); parse_options remains for callers without a tree. Drops the execute_options::max_parse_depth idea so the tree is the single source. --- docs/STATE_EXEC_GRAMMAR.md | 39 +++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index cef1095a9..77cf67060 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -131,12 +131,41 @@ raw_bytes ::= 'br' raw_body reading. - The default fits the 64 KiB default stack of a wasm build. Real scripts nest far less: the deepest in the corpus is 13 levels, and the median is 2. - - The limit is configurable, from P2 on: - - `parse_options::max_depth`, an optional argument to `parse()`, `parse_all()` and `parser`; - - `execute_options::max_parse_depth`, for scripts submitted to a scheduler or coordinator, - next to `max_steps` and `max_memory`. 0 means the default. + - The limit is a parser setting. Like state_exec's other settings, it lives on the state tree; + see [Parser settings](#parser-settings). A host that raises it must give the parsing thread a + big enough stack. - A host that raises it must give the parsing thread a big enough stack. +## Parser settings + +**Status:** not implemented yet. Today's parser has no nesting limit and reads no settings. + +The parser's settings live on the state tree, next to the scheduler's (`state_exec.defaults.*` +and `state_exec.schedulers..*`, see `scheduler::load_settings()`). Each setting resolves in +this order: + +1. `state_exec.schedulers..parser.`, for a scheduler that has an id; +2. `state_exec.defaults.parser.`, for everything on that tree; +3. the built-in default. + +A value that is missing, empty, not a positive integer or out of range is skipped, and the next +one in the order applies. + +| Key | Meaning | Default | +|---|---|---| +| `max_depth` | how many lists and quotes may nest | 256 | + +Who reads them: + +- **`scheduler`** resolves them in `load_settings()`, like `max_pending_messages`. It publishes + the effective values under `state_exec.schedulers..parser.*` and uses them for every + script submitted to it afterwards. +- **`async_scheduler`** has no settings loader. It reads `state_exec.defaults.parser.*` from its + watch root each time a script is submitted. +- **Ariadne** reads `state_exec.defaults.parser.*` from the app's tree when it compiles a + predicate. +- **Code without a state tree** (a standalone evaluator, the `.ari` channel lint) uses the + built-in defaults, or passes `parse_options` to `parse()` and `parse_all()` explicitly. + `parse_options_from_state()` does the resolution above for any host that has a tree. ## Data From e221ac7c62086966362351dd925b55ab2b204c41 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 11:24:34 -0500 Subject: [PATCH 04/13] state_exec: parser settings on the state tree, with a nesting limit (grammar P1b) The parser now stops at parser.max_depth nested lists and quotes (default 256) with a parse_error, instead of recursing until the native stack overflows. Like the scheduler's other settings, the limit lives on the state tree: state_exec.schedulers..parser.max_depth, then state_exec.defaults.parser.max_depth, then 256; a missing, malformed or zero value is skipped. - parser: parse_options (max_depth) on parse()/parse_all()/parser, a depth guard on lists and quotes, and parse_options_from_state(). - scheduler: load_settings() resolves parser.* and publishes the effective value under state_exec.schedulers..parser.max_depth; execute() parses with it. - async_scheduler (the app's exec scheduler) reads the defaults key on every submit; Ariadne predicates read it from the app tree. - stackless evaluators: create_state(script, env, parse_options). The grammar test moves from a numeric kLandedPhase to named phases; the 257-deep case is P1b and holds now. Tests cover the default, a custom depth, quotes, 100000 open parens, the error position, tree resolution, bad values, and both schedulers. --- docs/STATE_EXEC_GRAMMAR.md | 12 +- .../state_exec/async_stackless_evaluator.h | 3 +- inc/cvc/state/state_exec/parser.h | 30 ++- inc/cvc/state/state_exec/scheduler.h | 9 + .../state/state_exec/stackless_evaluator.h | 7 +- src/cvc/ariadne/ariadne.cpp | 6 +- src/cvc/state/state_exec/async_scheduler.cpp | 5 +- .../state_exec/async_stackless_evaluator.cpp | 5 +- src/cvc/state/state_exec/parser.cpp | 62 +++++- src/cvc/state/state_exec/scheduler.cpp | 6 +- .../state/state_exec/stackless_evaluator.cpp | 5 +- src/cvc/tests/state_exec_grammar_test.cpp | 178 +++++++++--------- src/cvc/tests/state_exec_integration_test.cpp | 71 +++++++ src/cvc/tests/state_exec_parser_test.cpp | 48 +++++ 14 files changed, 333 insertions(+), 114 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index 77cf67060..38b0426aa 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -5,7 +5,8 @@ This is the formal grammar of the state_exec s-expression language: the text tha `.ari` documents embed it in `init`, `on`, `bind` and `*_when` fields. **Status: target grammar.** The parser in `src/cvc/state/state_exec/parser.cpp` is still the -hand-written one. It moves onto this grammar in phases (P2 to P4, see +hand-written one, now with the nesting limit and its [settings](#parser-settings) (P1b). It +moves onto this grammar in phases (P2 to P4, see [Changes from the pre-grammar parser](#changes-from-the-pre-grammar-parser)). Until then it differs from this document exactly where that table says. `state_exec_grammar_test` checks both sides: cases for landed phases must hold, and cases for pending phases must still differ. @@ -137,8 +138,6 @@ raw_bytes ::= 'br' raw_body ## Parser settings -**Status:** not implemented yet. Today's parser has no nesting limit and reads no settings. - The parser's settings live on the state tree, next to the scheduler's (`state_exec.defaults.*` and `state_exec.schedulers..*`, see `scheduler::load_settings()`). Each setting resolves in this order: @@ -258,7 +257,8 @@ hashes and `"`, it is a string literal, and leaving it unterminated is an error. ## Changes from the pre-grammar parser -Phase P2 replaces the hand-written lexer with the grammar, including the number syntax. P3 adds +Phase P1b adds the nesting limit and the parser settings to the hand-written parser. Phase P2 +replaces the hand-written lexer with the grammar, including the number syntax. P3 adds the Unicode source rules and strict escapes. P4 adds octal escapes, the extra C escapes and raw strings. Every row is a case in `state_exec_grammar_test`. @@ -268,7 +268,7 @@ strings. Every row is a case in `state_exec_grammar_test`. | `[a]`, `a,b`, `` `a ``, `{}`, `a\|b`, `a\b` | odd one-character or glued symbols | error (reserved) | P2 | | `parse("1 2")` | `1`; the rest is ignored | error | P2 | | `'` at end of input | `(quote nil)` | error | P2 | -| 257 nested lists | parsed (no limit) | error at the default `max_depth` | P2 | +| 257 nested lists | parsed (no limit) | error at the default `max_depth` | P1b | | `; c` CR `1` | comment runs to LF, so no datum | `1` | P2 | | `0x1F` | `31.0`, a double (via `strtod`) | `31` | P2 | | `#x1F`, `0o17`, `0b101`, `1_000` | symbols | integers | P2 | @@ -316,7 +316,7 @@ These were considered and left out. Each would need its own decision: `single_datum` is used; - every production has at least one accept or reject case; - each case holds once its phase has landed, and still differs while its phase is pending. - `kLandedPhase` in the test says which phases have landed. + `kLanded` in the test lists the phases that have landed. - `src/cvc/tests/state_exec_corpus_test.cpp` parses the corpus: every DSL script the repo carries, drawn from the `.ari` files, the docs' `lisp` blocks, the C++ test sources and the pycvc tests. It compares each parse with `state_exec_corpus/expected.txt`, so a parser change cannot alter diff --git a/inc/cvc/state/state_exec/async_stackless_evaluator.h b/inc/cvc/state/state_exec/async_stackless_evaluator.h index 5be65c1d9..4aee7c92d 100644 --- a/inc/cvc/state/state_exec/async_stackless_evaluator.h +++ b/inc/cvc/state/state_exec/async_stackless_evaluator.h @@ -36,7 +36,8 @@ class async_stackless_evaluator { /// Create a fresh evaluator_state (delegates to inner evaluator). evaluator_state create_state(const value_t &expr, environment_ptr env = nullptr); - evaluator_state create_state(const std::string &script, environment_ptr env = nullptr); + evaluator_state create_state(const std::string &script, environment_ptr env = nullptr, + const parse_options &popts = {}); /// Single step as a coroutine — yields after the step. task step(evaluator_state &state); diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index 8a96aec50..b231f1403 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -11,12 +11,17 @@ #ifndef CVC_STATE_EXEC_PARSER_H #define CVC_STATE_EXEC_PARSER_H +#include #include #include #include #include #include +namespace cvc { +class state; +} + namespace cvc::state_exec { /// Parse error with location information. @@ -33,20 +38,35 @@ class parse_error : public std::runtime_error { std::size_t _col; }; +/// Parser settings ("Parser settings" in docs/STATE_EXEC_GRAMMAR.md). Hosts keep them on the +/// state tree under `parser.*`; parse_options_from_state() reads them from there. +struct parse_options { + /// How many lists and quotes may nest (`'x` reads as `(quote x)`, one level). The parser + /// recurses on the native stack once per level, so deeper input is a parse_error rather than a + /// stack overflow. State-tree key: `parser.max_depth`. + std::size_t max_depth = 256; +}; + +/// Resolve parser settings from a state tree, per key: `state_exec.schedulers.. +/// parser.` (when scheduler_id is not empty), then `state_exec.defaults.parser.`, then +/// the parse_options default. A value that is missing, empty, not a positive integer or out of +/// range is skipped. +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {}); + /// Parse a single S-expression from the input string. /// Returns the parsed value_t. /// @throws parse_error on syntax errors. -value_t parse(const std::string &input); +value_t parse(const std::string &input, const parse_options &opts = {}); /// Parse all S-expressions from the input string. /// Returns a vector of parsed value_t values. /// @throws parse_error on syntax errors. -std::vector parse_all(const std::string &input); +std::vector parse_all(const std::string &input, const parse_options &opts = {}); /// Internal tokenizer/parser — exposed for testing. class parser { public: - explicit parser(std::string_view input); + explicit parser(std::string_view input, const parse_options &opts = {}); /// Parse one expression. Returns nil if at end-of-input. value_t parse_expr(); @@ -58,10 +78,14 @@ class parser { std::size_t col() const { return _col; } private: + class depth_guard; + std::string_view _input; + parse_options _opts; std::size_t _pos = 0; std::size_t _line = 1; std::size_t _col = 1; + std::size_t _depth = 0; // lists and quotes open around the current position char peek() const; char advance(); diff --git a/inc/cvc/state/state_exec/scheduler.h b/inc/cvc/state/state_exec/scheduler.h index 4bb9e5989..e69ecd20c 100644 --- a/inc/cvc/state/state_exec/scheduler.h +++ b/inc/cvc/state/state_exec/scheduler.h @@ -180,8 +180,16 @@ class scheduler : public scheduler_base { /// After resolution, effective values are published back to the /// per-scheduler subtree so they are visible in the state tree. /// Requires set_watch_root() and (for per-scheduler) set_id() first. + /// + /// Settings: `max_pending_messages`, and the parser's `parser.*` + /// (parse_options_from_state), which apply to every script submitted + /// afterwards. void load_settings(); + /// The parser settings execute() parses scripts with (defaults until + /// load_settings() runs). + const parse_options &parser_settings() const { return parse_options_; } + /// Well-known state tree paths for scheduler configuration. static constexpr const char *settings_root = "state_exec"; static constexpr const char *defaults_prefix = "state_exec.defaults"; @@ -190,6 +198,7 @@ class scheduler : public scheduler_base { private: std::string id_; scheduling_policy policy_; + parse_options parse_options_; int next_pid_ = 1; int rr_index_ = 0; bool running_ = false; diff --git a/inc/cvc/state/state_exec/stackless_evaluator.h b/inc/cvc/state/state_exec/stackless_evaluator.h index 3ce531dd2..17a6b1b08 100644 --- a/inc/cvc/state/state_exec/stackless_evaluator.h +++ b/inc/cvc/state/state_exec/stackless_evaluator.h @@ -15,6 +15,7 @@ #include #include #include +#include // parse_options #include #include #include @@ -92,8 +93,10 @@ class stackless_evaluator { /// Create a fresh evaluator_state from a parsed expression. evaluator_state create_state(const value_t &expr, environment_ptr env = nullptr); - /// Create a fresh evaluator_state from a script string. - evaluator_state create_state(const std::string &script, environment_ptr env = nullptr); + /// Create a fresh evaluator_state from a script string, parsed with `popts` (a scheduler passes + /// the parser settings it resolved from the state tree). + evaluator_state create_state(const std::string &script, environment_ptr env = nullptr, + const parse_options &popts = {}); /// Execute one evaluation step. Returns true when done. bool step(evaluator_state &state); diff --git a/src/cvc/ariadne/ariadne.cpp b/src/cvc/ariadne/ariadne.cpp index 0660e8b38..85657179c 100644 --- a/src/cvc/ariadne/ariadne.cpp +++ b/src/cvc/ariadne/ariadne.cpp @@ -374,6 +374,7 @@ class ReactiveEngine { proc_->pid = 1; proc_->status = se::process_status::ready; cvc::state &root = cvc::state::instance(app); + tree_ = &root; ctx_.sched = &sched_; ctx_.tracker = &tracker_; ctx_.proc = proc_; @@ -577,7 +578,7 @@ class ReactiveEngine { if (it != compiled_.end()) return it->second ? &*it->second : nullptr; try { - se::value_t v = se::parse(src); + se::value_t v = se::parse(src, se::parse_options_from_state(*tree_)); // parser.* settings auto ins = compiled_.emplace(src, std::move(v)); return &*ins.first->second; } catch (const se::parse_error &) { @@ -592,7 +593,8 @@ class ReactiveEngine { static constexpr std::size_t kMaxOptions = 1024; // cap on computed combo options se::time_budget frame_{kFrameBudgetSeconds}; // the current frame's predicate evals' time - std::string root_path_; // §12: the scope the ctx_ is currently chrooted to (set_root_path) + std::string root_path_; // §12: the scope the ctx_ is currently chrooted to (set_root_path) + cvc::state *tree_ = nullptr; // the app's state tree, for state_exec.defaults.parser.* se::scheduler sched_; se::memory_tracker tracker_; std::shared_ptr proc_; diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index 8895ed041..d76728284 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -37,7 +37,10 @@ int async_scheduler::execute(const std::string &script, const execute_options &o proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); - proc.state = evaluator_.create_state(script, opts.env); + // No load_settings() here: read the parser settings from the tree on every submission. + const parse_options popts = + watch_root_ ? parse_options_from_state(*watch_root_) : parse_options{}; + proc.state = evaluator_.create_state(script, opts.env, popts); proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; diff --git a/src/cvc/state/state_exec/async_stackless_evaluator.cpp b/src/cvc/state/state_exec/async_stackless_evaluator.cpp index 813283d2f..2c1007cf6 100644 --- a/src/cvc/state/state_exec/async_stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/async_stackless_evaluator.cpp @@ -28,8 +28,9 @@ evaluator_state async_stackless_evaluator::create_state(const value_t &expr, env } evaluator_state async_stackless_evaluator::create_state(const std::string &script, - environment_ptr env) { - return inner_.create_state(script, env); + environment_ptr env, + const parse_options &popts) { + return inner_.create_state(script, env, popts); } // --------------------------------------------------------------------------- diff --git a/src/cvc/state/state_exec/parser.cpp b/src/cvc/state/state_exec/parser.cpp index d69c2c8c0..3ad94118a 100644 --- a/src/cvc/state/state_exec/parser.cpp +++ b/src/cvc/state/state_exec/parser.cpp @@ -1,22 +1,25 @@ #include #include #include +#include #include +#include // defaults_prefix / sched_prefix #include // SE-2: \u/\x escapes -> UTF-8, and UTF-8 identifier bytes +#include #include namespace cvc::state_exec { // -- Free functions ---------------------------------------------------------- -value_t parse(const std::string &input) { - parser p(input); +value_t parse(const std::string &input, const parse_options &opts) { + parser p(input, opts); auto result = p.parse_expr(); return result; } -std::vector parse_all(const std::string &input) { - parser p(input); +std::vector parse_all(const std::string &input, const parse_options &opts) { + parser p(input, opts); std::vector results; while (!p.at_end()) { results.push_back(p.parse_expr()); @@ -24,9 +27,53 @@ std::vector parse_all(const std::string &input) { return results; } +// A positive integer setting at `path`, or nullopt when it is missing, empty, malformed, zero or +// out of range (so the caller falls through to the next place in the resolution order). +static std::optional read_positive_setting(cvc::state &root, const std::string &path) { + cvc::state *node = root.findDescendant(path); + if (!node) + return std::nullopt; + const std::string v = node->value(); + std::size_t n = 0; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + if (ec != std::errc{} || end != v.data() + v.size() || n == 0) + return std::nullopt; + return n; +} + +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id) { + parse_options opts; + const std::string key = ".parser.max_depth"; + if (auto v = read_positive_setting(root, std::string(scheduler::defaults_prefix) + key)) + opts.max_depth = *v; + if (!scheduler_id.empty()) + if (auto v = read_positive_setting(root, std::string(scheduler::sched_prefix) + "." + + scheduler_id + key)) + opts.max_depth = *v; + return opts; +} + // -- parser ------------------------------------------------------------------ -parser::parser(std::string_view input) : _input(input) {} +parser::parser(std::string_view input, const parse_options &opts) : _input(input), _opts(opts) {} + +// One level of list/quote nesting for the lifetime of the guard. The check comes before the +// increment, so a guard that throws leaves the depth unchanged. +class parser::depth_guard { +public: + explicit depth_guard(parser &p) : _p(p) { + if (_p._depth >= _p._opts.max_depth) + _p.error("nesting deeper than " + std::to_string(_p._opts.max_depth) + + " levels (state_exec setting parser.max_depth)"); + ++_p._depth; + } + ~depth_guard() { --_p._depth; } + depth_guard(const depth_guard &) = delete; + depth_guard &operator=(const depth_guard &) = delete; + +private: + parser &_p; +}; bool parser::at_end() const { // Skip whitespace conceptually @@ -88,10 +135,13 @@ value_t parser::parse_expr() { char c = peek(); - if (c == '(') + if (c == '(') { + depth_guard level(*this); return parse_list(); + } if (c == '\'') { // Quote: 'expr → (quote expr) + depth_guard level(*this); advance(); auto quoted = parse_expr(); return make_list({value_t(symbol{"quote"}), quoted}); diff --git a/src/cvc/state/state_exec/scheduler.cpp b/src/cvc/state/state_exec/scheduler.cpp index a62aacca0..28f2584c0 100644 --- a/src/cvc/state/state_exec/scheduler.cpp +++ b/src/cvc/state/state_exec/scheduler.cpp @@ -55,10 +55,14 @@ void scheduler::load_settings() { max_pending_messages = effective; + // --- parser.* (same resolution order; see parse_options_from_state) --- + parse_options_ = parse_options_from_state(*watch_root_, id_); + // Publish effective values to per-scheduler subtree if (!id_.empty()) { std::string base = std::string(sched_prefix) + "." + id_; (*watch_root_)(base + ".max_pending_messages").value(std::to_string(max_pending_messages)); + (*watch_root_)(base + ".parser.max_depth").value(std::to_string(parse_options_.max_depth)); (*watch_root_)(base + ".policy") .value(policy_ == scheduling_policy::round_robin ? "round_robin" : policy_ == scheduling_policy::priority ? "priority" @@ -89,7 +93,7 @@ int scheduler::execute(const std::string &script, const execute_options &opts) { proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); - proc.state = evaluator_.create_state(script, opts.env); + proc.state = evaluator_.create_state(script, opts.env, parse_options_); proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; diff --git a/src/cvc/state/state_exec/stackless_evaluator.cpp b/src/cvc/state/state_exec/stackless_evaluator.cpp index ec55b1e5e..f498e79bd 100644 --- a/src/cvc/state/state_exec/stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/stackless_evaluator.cpp @@ -39,8 +39,9 @@ evaluator_state stackless_evaluator::create_state(const value_t &expr, environme return st; } -evaluator_state stackless_evaluator::create_state(const std::string &script, environment_ptr env) { - auto exprs = parse_all(script); +evaluator_state stackless_evaluator::create_state(const std::string &script, environment_ptr env, + const parse_options &popts) { + auto exprs = parse_all(script, popts); if (exprs.empty()) { evaluator_state st; st.done = true; diff --git a/src/cvc/tests/state_exec_grammar_test.cpp b/src/cvc/tests/state_exec_grammar_test.cpp index f8b0ca1a2..7653b6ab4 100644 --- a/src/cvc/tests/state_exec_grammar_test.cpp +++ b/src/cvc/tests/state_exec_grammar_test.cpp @@ -1,13 +1,14 @@ // Conformance tests for the state_exec grammar (docs/STATE_EXEC_GRAMMAR.md). // // Every case names the EBNF productions it exercises and the phase from which it holds: -// 0 holds for the pre-grammar (hand-written) parser too; -// N holds once phase PN of the grammar work lands (P2 = parser built from the grammar, -// P3 = Unicode source rules + strict escapes, P4 = octal escapes, C escapes, raw strings). -// Cases for landed phases (<= kLandedPhase) must hold. Cases for pending phases must still -// DIFFER, so the table stays an exact list of what each phase changes; when a phase lands, bump -// kLandedPhase in the same PR. The doc's EBNF is checked for closure, and every production must -// have a case. +// P0 holds for the pre-grammar (hand-written) parser too; +// P1b parser settings on the state tree (the nesting limit); +// P2 parser built from the grammar, numbers included; +// P3 Unicode source rules + strict escapes; +// P4 octal escapes, C escapes, raw strings. +// Cases for landed phases (kLanded) must hold. Cases for pending phases must still DIFFER, so +// the table stays an exact list of what each phase changes; add a phase to kLanded in the PR that +// lands it. The doc's EBNF is checked for closure, and every production must have a case. // // Inputs and expectations are byte-exact: non-ASCII is written as \xNN escapes (MSVC builds // without /utf-8), split with "" where a hex letter follows an escape. @@ -29,12 +30,13 @@ namespace dump = cvc::state_exec::grammar_dump; namespace { -constexpr int kLandedPhase = 0; +// Phases whose cases must hold. Add a phase here in the PR that lands it. +const std::set kLanded = {"P0", "P1b"}; enum class mode { all, one }; // parse_all() or parse() struct grammar_case { - int phase; + std::string phase; // "P0", "P1b", "P2", ... std::string rules; // space-separated EBNF production names mode how; std::string input; @@ -46,112 +48,112 @@ std::string nest(int depth) { return std::string(depth, '(') + std::string(depth const std::vector &cases() { static const std::vector kCases = { // --- holds today ---------------------------------------------------------------------- - {0, "program", mode::all, "", ""}, - {0, "program atmosphere", mode::all, "1 2\n3", "1\n2\n3"}, - {0, "single_datum", mode::one, "", "nil"}, - {0, "single_datum line_comment", mode::one, " ; note\n 42 ", "42"}, - {0, "datum list", mode::all, "(a (b c) ())", "(a (b c) ())"}, - {0, "list", mode::all, nest(256), nest(256)}, - {0, "list", mode::all, "(a", "!error"}, - {0, "quoted", mode::all, "'x ' (1 2) ''a", "(quote x)\n(quote (1 2))\n(quote (quote a))"}, - {0, "line_comment line_end", mode::all, "; c\n(a ; c2\n b)", "(a b)"}, - {0, "whitespace", mode::all, "\t\n\v\f\r a", "a"}, - {0, "text_literal text_string string_char", mode::all, "\"hi there\"", "\"hi there\""}, - {0, "string_char", mode::all, "\"a\nb\"", R"("a\nb")"}, - {0, "text_string", mode::all, "\"abc", "!error"}, - {0, "text_escape simple_escape", mode::all, R"("\n\t\r\\\"")", R"("\n\t\r\\\"")"}, - {0, "hex_escape hex", mode::all, R"("\x41\xe9")", "\"A\xC3\xA9\""}, - {0, "hex_escape", mode::all, R"("\xZ1")", "!error"}, - {0, "unicode_escape", mode::all, R"("é\u{1F600}")", "\"\xC3\xA9\xF0\x9F\x98\x80\""}, - {0, "unicode_escape", mode::all, R"("\u12")", "!error"}, - {0, "unicode_escape", mode::all, R"("\u{}")", "!error"}, - {0, "unicode_escape", mode::all, R"("\u{1234567}")", "!error"}, - {0, "octal_escape", mode::all, R"("\0")", R"("\x00")"}, - {0, "bytes_literal byte_string byte_escape", mode::all, R"(b"ab\x00\xff\n")", + {"P0", "program", mode::all, "", ""}, + {"P0", "program atmosphere", mode::all, "1 2\n3", "1\n2\n3"}, + {"P0", "single_datum", mode::one, "", "nil"}, + {"P0", "single_datum line_comment", mode::one, " ; note\n 42 ", "42"}, + {"P0", "datum list", mode::all, "(a (b c) ())", "(a (b c) ())"}, + {"P0", "list", mode::all, nest(256), nest(256)}, + {"P0", "list", mode::all, "(a", "!error"}, + {"P0", "quoted", mode::all, "'x ' (1 2) ''a", "(quote x)\n(quote (1 2))\n(quote (quote a))"}, + {"P0", "line_comment line_end", mode::all, "; c\n(a ; c2\n b)", "(a b)"}, + {"P0", "whitespace", mode::all, "\t\n\v\f\r a", "a"}, + {"P0", "text_literal text_string string_char", mode::all, "\"hi there\"", "\"hi there\""}, + {"P0", "string_char", mode::all, "\"a\nb\"", R"("a\nb")"}, + {"P0", "text_string", mode::all, "\"abc", "!error"}, + {"P0", "text_escape simple_escape", mode::all, R"("\n\t\r\\\"")", R"("\n\t\r\\\"")"}, + {"P0", "hex_escape hex", mode::all, R"("\x41\xe9")", "\"A\xC3\xA9\""}, + {"P0", "hex_escape", mode::all, R"("\xZ1")", "!error"}, + {"P0", "unicode_escape", mode::all, R"("é\u{1F600}")", "\"\xC3\xA9\xF0\x9F\x98\x80\""}, + {"P0", "unicode_escape", mode::all, R"("\u12")", "!error"}, + {"P0", "unicode_escape", mode::all, R"("\u{}")", "!error"}, + {"P0", "unicode_escape", mode::all, R"("\u{1234567}")", "!error"}, + {"P0", "octal_escape", mode::all, R"("\0")", R"("\x00")"}, + {"P0", "bytes_literal byte_string byte_escape", mode::all, R"(b"ab\x00\xff\n")", R"(b"ab\x00\xff\x0a")"}, - {0, "byte_string", mode::all, "b\"\xC3\xA9\"", R"(b"\xc3\xa9")"}, - {0, "byte_string symbol", mode::all, "b (b) b\"\"", "b\n(b)\nb\"\""}, - {0, "boolean", mode::all, "#t true", "#t\n#t"}, - {0, "nil", mode::all, "#f false nil", "nil\nnil\nnil"}, - {0, "number integer dec_digits sign", mode::all, "42 -7 007 -9223372036854775808", + {"P0", "byte_string", mode::all, "b\"\xC3\xA9\"", R"(b"\xc3\xa9")"}, + {"P0", "byte_string symbol", mode::all, "b (b) b\"\"", "b\n(b)\nb\"\""}, + {"P0", "boolean", mode::all, "#t true", "#t\n#t"}, + {"P0", "nil", mode::all, "#f false nil", "nil\nnil\nnil"}, + {"P0", "number integer dec_digits sign", mode::all, "42 -7 007 -9223372036854775808", "42\n-7\n7\n-9223372036854775808"}, - {0, "decimal_float exponent", mode::all, "1.5 .5 1. 1e3 -2.5E-3", + {"P0", "decimal_float exponent", mode::all, "1.5 .5 1. 1e3 -2.5E-3", "1.5\n0.5\n1.0\n1000.0\n-0.0025"}, - {0, "atom token constituent symbol", mode::all, + {"P0", "atom token constituent symbol", mode::all, "foo-bar? -> + - caf\xC3\xA9 \xCE\xBB \xE6\x97\xA5\xE6\x9C\xAC a.b:c", "foo-bar?\n->\n+\n-\ncaf\xC3\xA9\n\xCE\xBB\n\xE6\x97\xA5\xE6\x9C\xAC\na.b:c"}, - {0, "symbol", mode::all, "1+ ... #foo", "|1+|\n|...|\n|#foo|"}, - {0, "delimiter constituent", mode::all, "a\"b\"(c)'d;e", "a\n\"b\"\n(c)\n(quote d)"}, - {0, "raw_text symbol", mode::all, "r#foo", "r#foo"}, + {"P0", "symbol", mode::all, "1+ ... #foo", "|1+|\n|...|\n|#foo|"}, + {"P0", "delimiter constituent", mode::all, "a\"b\"(c)'d;e", "a\n\"b\"\n(c)\n(quote d)"}, + {"P0", "raw_text symbol", mode::all, "r#foo", "r#foo"}, // --- P2: parser built from the grammar, numbers included ----------------------------- - {2, "program", mode::all, "a)", "!error"}, - {2, "reserved", mode::all, "[a]", "!error"}, - {2, "reserved", mode::all, "a,b", "!error"}, - {2, "reserved", mode::all, "`a", "!error"}, - {2, "reserved", mode::all, "{}", "!error"}, - {2, "reserved", mode::all, "a|b", "!error"}, - {2, "reserved", mode::all, "a\\b", "!error"}, - {2, "single_datum", mode::one, "1 2", "!error"}, - {2, "quoted", mode::all, "'", "!error"}, - {2, "list", mode::all, nest(257), "!error"}, - {2, "line_end line_comment", mode::all, "; c\r1", "1"}, - {2, "radix_integer hex_digits", mode::all, "0x1F 0X1f #x1F #X1f 0xFF_FF", + {"P2", "program", mode::all, "a)", "!error"}, + {"P2", "reserved", mode::all, "[a]", "!error"}, + {"P2", "reserved", mode::all, "a,b", "!error"}, + {"P2", "reserved", mode::all, "`a", "!error"}, + {"P2", "reserved", mode::all, "{}", "!error"}, + {"P2", "reserved", mode::all, "a|b", "!error"}, + {"P2", "reserved", mode::all, "a\\b", "!error"}, + {"P2", "single_datum", mode::one, "1 2", "!error"}, + {"P2", "quoted", mode::all, "'", "!error"}, + {"P1b", "list", mode::all, nest(257), "!error"}, + {"P2", "line_end line_comment", mode::all, "; c\r1", "1"}, + {"P2", "radix_integer hex_digits", mode::all, "0x1F 0X1f #x1F #X1f 0xFF_FF", "31\n31\n31\n31\n65535"}, - {2, "radix_integer oct_digits bin_digits sign", mode::all, "0o17 #o17 0b101 #b101 -0x10 +0b1", - "15\n15\n5\n5\n-16\n1"}, - {2, "radix_prefixed", mode::all, "0xZZ", "!error"}, - {2, "radix_prefixed", mode::all, "#bar", "!error"}, - {2, "radix_prefixed", mode::all, "0x", "!error"}, - {2, "integer sign", mode::all, "+5", "5"}, - {2, "dec_digits", mode::all, "1_000", "1000"}, - {2, "integer", mode::all, "9223372036854775808", "!error"}, - {2, "float_special", mode::all, "+inf.0 -inf.0 +nan.0 -nan.0", + {"P2", "radix_integer oct_digits bin_digits sign", mode::all, + "0o17 #o17 0b101 #b101 -0x10 +0b1", "15\n15\n5\n5\n-16\n1"}, + {"P2", "radix_prefixed", mode::all, "0xZZ", "!error"}, + {"P2", "radix_prefixed", mode::all, "#bar", "!error"}, + {"P2", "radix_prefixed", mode::all, "0x", "!error"}, + {"P2", "integer sign", mode::all, "+5", "5"}, + {"P2", "dec_digits", mode::all, "1_000", "1000"}, + {"P2", "integer", mode::all, "9223372036854775808", "!error"}, + {"P2", "float_special", mode::all, "+inf.0 -inf.0 +nan.0 -nan.0", "+inf.0\n-inf.0\n+nan.0\n+nan.0"}, - {2, "symbol", mode::all, "inf nan infinity -INF", "inf\nnan\ninfinity\n-INF"}, - {2, "decimal_float", mode::all, "1e999", "!error"}, - {2, "decimal_float dec_digits", mode::all, "6.022_140e23", "6.02214e+23"}, + {"P2", "symbol", mode::all, "inf nan infinity -INF", "inf\nnan\ninfinity\n-INF"}, + {"P2", "decimal_float", mode::all, "1e999", "!error"}, + {"P2", "decimal_float dec_digits", mode::all, "6.022_140e23", "6.02214e+23"}, // --- P3: Unicode source rules, strict escapes ---------------------------------------- - {3, "scalar_value", mode::all, + {"P3", "scalar_value", mode::all, "a\xff" "b", "!error"}, - {3, "bom", mode::all, "\xEF\xBB\xBF(a)", "(a)"}, - {3, "other_space constituent", mode::all, + {"P3", "bom", mode::all, "\xEF\xBB\xBF(a)", "(a)"}, + {"P3", "other_space constituent", mode::all, "a\xC2\xA0" "b", "!error"}, - {3, "whitespace", mode::all, + {"P3", "whitespace", mode::all, "a\xC2\x85" "b a\xE2\x80\xA8" "b", "a\nb\na\nb"}, - {3, "source_char bidi_control", mode::all, + {"P3", "source_char bidi_control", mode::all, "\"a\xE2\x80\xAE" "b\"", "!error"}, - {3, "bidi_control line_comment", mode::all, "; \xE2\x81\xA6\n1", "!error"}, - {3, "control", mode::all, + {"P3", "bidi_control line_comment", mode::all, "; \xE2\x81\xA6\n1", "!error"}, + {"P3", "control", mode::all, "\"a\x01" "b\"", "!error"}, - {3, "text_escape", mode::all, R"("\q")", "!error"}, - {3, "unicode_escape", mode::all, R"("\uD800")", "!error"}, - {3, "unicode_escape", mode::all, R"("\u{110000}")", "!error"}, + {"P3", "text_escape", mode::all, R"("\q")", "!error"}, + {"P3", "unicode_escape", mode::all, R"("\uD800")", "!error"}, + {"P3", "unicode_escape", mode::all, R"("\u{110000}")", "!error"}, // The error is reported at the offending '"', counted in code points: 1:6, not 1:8. - {3, "hex_escape", mode::all, "\"\xC3\xA9\\x4\"", "!error 1:6"}, + {"P3", "hex_escape", mode::all, "\"\xC3\xA9\\x4\"", "!error 1:6"}, // --- P4: octal escapes, C escapes, raw strings --------------------------------------- - {4, "octal_escape", mode::all, R"("\101\012")", R"("A\n")"}, - {4, "byte_escape octal_escape", mode::all, R"(b"\377")", R"(b"\xff")"}, - {4, "byte_escape", mode::all, R"(b"\400")", "!error"}, - {4, "simple_escape", mode::all, R"("\a\b\f\v")", R"("\x07\x08\x0c\x0b")"}, - {4, "raw_text raw_body raw_char", mode::all, R"(r"a\nb")", R"("a\\nb")"}, - {4, "hash_fence", mode::all, R"(r#"say "hi""#)", R"("say \"hi\"")"}, - {4, "hash_fence", mode::all, R"(r##"a"#b"##)", R"("a\"#b")"}, - {4, "raw_bytes", mode::all, R"(br"\x00")", R"(b"\\x00")"}, - {4, "raw_body", mode::all, R"(r#"abc")", "!error"}, + {"P4", "octal_escape", mode::all, R"("\101\012")", R"("A\n")"}, + {"P4", "byte_escape octal_escape", mode::all, R"(b"\377")", R"(b"\xff")"}, + {"P4", "byte_escape", mode::all, R"(b"\400")", "!error"}, + {"P4", "simple_escape", mode::all, R"("\a\b\f\v")", R"("\x07\x08\x0c\x0b")"}, + {"P4", "raw_text raw_body raw_char", mode::all, R"(r"a\nb")", R"("a\\nb")"}, + {"P4", "hash_fence", mode::all, R"(r#"say "hi""#)", R"("say \"hi\"")"}, + {"P4", "hash_fence", mode::all, R"(r##"a"#b"##)", R"("a\"#b")"}, + {"P4", "raw_bytes", mode::all, R"(br"\x00")", R"(b"\\x00")"}, + {"P4", "raw_body", mode::all, R"(r#"abc")", "!error"}, }; return kCases; } @@ -167,7 +169,7 @@ bool matches(const std::string &actual, const std::string &expect) { } std::string describe(const grammar_case &c) { - return "[P" + std::to_string(c.phase) + " " + c.rules + "] input: " + c.input; + return "[" + c.phase + " " + c.rules + "] input: " + c.input; } fs::path grammar_doc() { @@ -239,7 +241,7 @@ ebnf read_ebnf(const fs::path &doc) { TEST(StateExecGrammar, CasesForLandedPhasesHold) { for (const grammar_case &c : cases()) { - if (c.phase > kLandedPhase) + if (!kLanded.count(c.phase)) continue; EXPECT_PRED2(matches, run(c), c.expect) << describe(c); } @@ -247,7 +249,7 @@ TEST(StateExecGrammar, CasesForLandedPhasesHold) { TEST(StateExecGrammar, CasesForPendingPhasesStillDiffer) { for (const grammar_case &c : cases()) { - if (c.phase <= kLandedPhase) + if (kLanded.count(c.phase)) continue; EXPECT_FALSE(matches(run(c), c.expect)) << describe(c) << "\nalready holds; set its phase to the one that made it hold"; diff --git a/src/cvc/tests/state_exec_integration_test.cpp b/src/cvc/tests/state_exec_integration_test.cpp index 4e8f792c4..0fc33685e 100644 --- a/src/cvc/tests/state_exec_integration_test.cpp +++ b/src/cvc/tests/state_exec_integration_test.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -2332,6 +2333,76 @@ TEST_F(SchedulerSettingsTest, TwoSchedulersWithDifferentOverrides) { EXPECT_EQ(sched_fast.max_pending_messages, 100u); EXPECT_EQ(sched_heavy.max_pending_messages, 5000u); } + +// --- parser.* settings (docs/STATE_EXEC_GRAMMAR.md, "Parser settings") --- + +TEST_F(SchedulerSettingsTest, ParserSettingsResolveFromTree) { + auto &root = cvc::state::instance(app_ctx); + EXPECT_EQ(parse_options_from_state(root).max_depth, 256u); // nothing set: built-in default + + root("state_exec.defaults.parser.max_depth").value("64"); + EXPECT_EQ(parse_options_from_state(root).max_depth, 64u); + EXPECT_EQ(parse_options_from_state(root, "other").max_depth, 64u); + + root("state_exec.schedulers.deep.parser.max_depth").value("1024"); + EXPECT_EQ(parse_options_from_state(root, "deep").max_depth, 1024u); + EXPECT_EQ(parse_options_from_state(root).max_depth, + 64u); // no id: the per-scheduler key is unused +} + +TEST_F(SchedulerSettingsTest, BadParserSettingsAreSkipped) { + auto &root = cvc::state::instance(app_ctx); + for (const char *bad : {"", "abc", "0", "-5", "12x", "99999999999999999999999"}) { + root("state_exec.defaults.parser.max_depth").value(bad); + EXPECT_EQ(parse_options_from_state(root).max_depth, 256u) << "value: " << bad; + } + // A bad per-scheduler value falls through to the defaults key, not to the built-in default. + root("state_exec.defaults.parser.max_depth").value("32"); + root("state_exec.schedulers.s.parser.max_depth").value("nope"); + EXPECT_EQ(parse_options_from_state(root, "s").max_depth, 32u); +} + +TEST_F(SchedulerSettingsTest, SchedulerParsesWithTreeDepthAndPublishesIt) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.max_depth").value("8"); + root("state_exec.schedulers.strict.parser.max_depth").value("3"); + + scheduler sched; + sched.set_watch_root(&root); + sched.set_id("strict"); + EXPECT_EQ(sched.parser_settings().max_depth, 256u); // until load_settings() + sched.load_settings(); + EXPECT_EQ(sched.parser_settings().max_depth, 3u); + EXPECT_EQ(root("state_exec.schedulers.strict.parser.max_depth").value(), "3"); + + EXPECT_NO_THROW(sched.execute(std::string("(+ 1 (+ 1 (+ 1 1)))"))); // 3 levels + EXPECT_THROW(sched.execute(std::string("(+ 1 (+ 1 (+ 1 (+ 1 1))))")), parse_error); // 4 +} + +TEST_F(SchedulerSettingsTest, SchedulerWithoutIdUsesDefaultsKey) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.max_depth").value("2"); + + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + EXPECT_EQ(sched.parser_settings().max_depth, 2u); + EXPECT_THROW(sched.execute(std::string("(+ 1 (+ 1 (+ 1 1)))")), parse_error); +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerReadsParserSettingsOnEachSubmit) { + auto &root = cvc::state::instance(app_ctx); + async_scheduler sched; + sched.set_watch_root(&root); + const std::string three_deep = "(+ 1 (+ 1 (+ 1 1)))"; + + EXPECT_NO_THROW(sched.execute(three_deep)); + root("state_exec.defaults.parser.max_depth").value("2"); + EXPECT_THROW(sched.execute(three_deep), parse_error); + root("state_exec.defaults.parser.max_depth").value("16"); + EXPECT_NO_THROW(sched.execute(three_deep)); +} + // =========================================================================== // §12 message-channel scoping — end-to-end through the scheduler + chroot // =========================================================================== diff --git a/src/cvc/tests/state_exec_parser_test.cpp b/src/cvc/tests/state_exec_parser_test.cpp index 5db1c3308..6b490eb88 100644 --- a/src/cvc/tests/state_exec_parser_test.cpp +++ b/src/cvc/tests/state_exec_parser_test.cpp @@ -238,3 +238,51 @@ TEST(StateExecParserTest, FullProgram) { EXPECT_EQ(std::get((**call)[0].v).name, "fib"); EXPECT_EQ(std::get((**call)[1].v), 10); } + +// -- Nesting limit (parse_options::max_depth; state-tree key parser.max_depth) -- + +static std::string nested(std::size_t depth) { + return std::string(depth, '(') + std::string(depth, ')'); +} + +TEST(StateExecParserTest, NestingLimitDefaultsTo256) { + EXPECT_EQ(parse_options{}.max_depth, 256u); + EXPECT_NO_THROW(parse(nested(256))); + EXPECT_THROW(parse(nested(257)), parse_error); +} + +TEST(StateExecParserTest, NestingLimitIsConfigurable) { + parse_options opts; + opts.max_depth = 3; + EXPECT_NO_THROW(parse("(((1)))", opts)); + EXPECT_THROW(parse("((((1))))", opts), parse_error); + EXPECT_NO_THROW(parse("'''x", opts)); // each quote is one level: (quote (quote (quote x))) + EXPECT_THROW(parse("''''x", opts), parse_error); + EXPECT_THROW(parse_all("(a) ((((b))))", opts), parse_error); + + opts.max_depth = 1000; + EXPECT_NO_THROW(parse(nested(1000), opts)); +} + +TEST(StateExecParserTest, NestingLimitStopsRunawayInputBeforeTheStackDoes) { + // A hundred thousand open parens would recurse that deep; the limit stops it at 257. + EXPECT_THROW(parse(std::string(100000, '(')), parse_error); +} + +TEST(StateExecParserTest, NestingErrorPointsAtTheParenThatGoesTooDeep) { + parse_options opts; + opts.max_depth = 2; + try { + parse("(a\n ((b)))", opts); + FAIL() << "expected a parse_error"; + } catch (const parse_error &e) { + EXPECT_EQ(e.line(), 2u); + EXPECT_EQ(e.col(), 4u); + } +} + +TEST(StateExecParserTest, NestingDepthResetsBetweenTopLevelForms) { + parse_options opts; + opts.max_depth = 2; + EXPECT_EQ(parse_all("((a)) ((b)) ((c))", opts).size(), 3u); +} From 7a4b4b1868690294bfbd8582b1f153ea946345bd Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 13:02:07 -0500 Subject: [PATCH 05/13] state_exec: make the parser stackless (explicit frame stack, resumable read) Lists and quotes were parsed by native recursion, so input depth was bounded by the thread's stack. The parser now keeps open lists and quotes on a heap stack of frames and reads in a loop: native stack depth is constant however deeply the input nests (a test reads a million open parens and reports the unclosed list instead of crashing). The loop is a resumable read(out, budget): it reads up to `budget` tokens and returns datum / end / more, carrying open frames between calls, so a scheduler can parse a script a slice at a time. parse() and parse_all() call it with no budget. Output is unchanged, quirks included (a stray top-level `)` still reads as a symbol until P2): the conformance cases, the 597-script corpus and the evaluator suites all pass. parser.max_depth stays on the state tree; it now bounds parsed data for the code that still walks values recursively (list destruction, the snapshot codec, to_string) rather than the parser's own stack. parse_fib20_expr: about 3.4 us vs 3.1 us for the recursive parser (median of 6, MSVC Release, loaded machine). --- docs/STATE_EXEC_GRAMMAR.md | 16 +-- inc/cvc/state/state_exec/parser.h | 35 +++++-- src/cvc/state/state_exec/parser.cpp | 124 ++++++++++++----------- src/cvc/tests/state_exec_parser_test.cpp | 56 ++++++++++ 4 files changed, 161 insertions(+), 70 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index 38b0426aa..c9e2ce74b 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -126,15 +126,17 @@ raw_bytes ::= 'br' raw_body after CR, and once after CRLF. Columns count code points, not bytes. - **Nesting.** Lists and quotes may nest at most `max_depth` levels, 256 by default. - `(a (b (c)))` is three levels, and `'x` counts as one, since it reads as `(quote x)`. - - The parser recurses on the native stack once per level. So deeper input is a `parse_error`, - not a stack overflow, even when the script comes from an untrusted source (a message, an HTTP - body, the multiprocess wire). The evaluators are stackless, so this limit is only about - reading. + - The parser is stackless. Open lists and quotes live on a heap stack, not on the native + stack, so no input depth can overflow it. + - The limit is for what happens to the data afterwards. Some code still walks values on the + native stack: destroying a nested list, the snapshot codec, `to_string`. Deeper input is a + `parse_error`, so a script from an untrusted source (a message, an HTTP body, the + multiprocess wire) can't produce data those would crash on. - The default fits the 64 KiB default stack of a wasm build. Real scripts nest far less: the - deepest in the corpus is 13 levels, and the median is 2. + deepest in the corpus is 13 levels, and the median is 2. Once that downstream code is + iterative too, the default can rise. - The limit is a parser setting. Like state_exec's other settings, it lives on the state tree; - see [Parser settings](#parser-settings). A host that raises it must give the parsing thread a - big enough stack. + see [Parser settings](#parser-settings). ## Parser settings diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index b231f1403..102d447c1 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -41,9 +41,10 @@ class parse_error : public std::runtime_error { /// Parser settings ("Parser settings" in docs/STATE_EXEC_GRAMMAR.md). Hosts keep them on the /// state tree under `parser.*`; parse_options_from_state() reads them from there. struct parse_options { - /// How many lists and quotes may nest (`'x` reads as `(quote x)`, one level). The parser - /// recurses on the native stack once per level, so deeper input is a parse_error rather than a - /// stack overflow. State-tree key: `parser.max_depth`. + /// How many lists and quotes may nest (`'x` reads as `(quote x)`, one level); deeper input is + /// a parse_error. The parser itself is stackless, but code downstream still walks values + /// recursively (destroying a nested list, the snapshot codec, to_string), so this keeps parsed + /// data within what they can handle. State-tree key: `parser.max_depth`. std::size_t max_depth = 256; }; @@ -64,6 +65,11 @@ value_t parse(const std::string &input, const parse_options &opts = {}); std::vector parse_all(const std::string &input, const parse_options &opts = {}); /// Internal tokenizer/parser — exposed for testing. +/// +/// Stackless: lists and quotes are tracked on a heap stack of open frames, not by native +/// recursion, so the native stack depth is constant however deeply the input nests. Reading is +/// also resumable (read() with a token budget), which lets a scheduler parse a script a slice at +/// a time inside a process's steps. class parser { public: explicit parser(std::string_view input, const parse_options &opts = {}); @@ -71,6 +77,19 @@ class parser { /// Parse one expression. Returns nil if at end-of-input. value_t parse_expr(); + /// What read() stopped at. + enum class read_status { + datum, ///< a top-level datum is complete; it is in `out` + end, ///< only whitespace and comments remain; nothing is open + more, ///< the token budget ran out; call read() again to continue + }; + + /// Read until a top-level datum completes, the input ends, or `budget` tokens (parens, quotes, + /// atoms, strings) have been read; `budget` is decremented by the tokens read. Open lists and + /// quotes carry over between calls. + /// @throws parse_error on syntax errors, including nesting past parse_options::max_depth. + read_status read(value_t &out, std::size_t &budget); + /// True if all input has been consumed. bool at_end() const; @@ -78,22 +97,26 @@ class parser { std::size_t col() const { return _col; } private: - class depth_guard; + /// An open list (`(` read, `)` not yet) or quote (`'` read, its datum not yet complete). + struct frame { + bool quote = false; + std::vector elements; // a list's elements so far + }; std::string_view _input; parse_options _opts; std::size_t _pos = 0; std::size_t _line = 1; std::size_t _col = 1; - std::size_t _depth = 0; // lists and quotes open around the current position + std::vector _frames; // innermost last; its size is the current nesting depth char peek() const; char advance(); void skip_whitespace_and_comments(); [[noreturn]] void error(const std::string &msg); + void open_frame(bool quote); value_t parse_atom(); - value_t parse_list(); value_t parse_string(); value_t parse_bytes_string(); // SE-3: b"..." byte-string literal (\xNN = a raw byte, no \u) value_t parse_number_or_symbol(char first); diff --git a/src/cvc/state/state_exec/parser.cpp b/src/cvc/state/state_exec/parser.cpp index 3ad94118a..340003b07 100644 --- a/src/cvc/state/state_exec/parser.cpp +++ b/src/cvc/state/state_exec/parser.cpp @@ -55,25 +55,9 @@ parse_options parse_options_from_state(cvc::state &root, const std::string &sche // -- parser ------------------------------------------------------------------ -parser::parser(std::string_view input, const parse_options &opts) : _input(input), _opts(opts) {} - -// One level of list/quote nesting for the lifetime of the guard. The check comes before the -// increment, so a guard that throws leaves the depth unchanged. -class parser::depth_guard { -public: - explicit depth_guard(parser &p) : _p(p) { - if (_p._depth >= _p._opts.max_depth) - _p.error("nesting deeper than " + std::to_string(_p._opts.max_depth) + - " levels (state_exec setting parser.max_depth)"); - ++_p._depth; - } - ~depth_guard() { --_p._depth; } - depth_guard(const depth_guard &) = delete; - depth_guard &operator=(const depth_guard &) = delete; - -private: - parser &_p; -}; +parser::parser(std::string_view input, const parse_options &opts) : _input(input), _opts(opts) { + _frames.reserve(32); // one allocation covers real scripts (the corpus nests at most 13 deep) +} bool parser::at_end() const { // Skip whitespace conceptually @@ -129,49 +113,75 @@ void parser::skip_whitespace_and_comments() { void parser::error(const std::string &msg) { throw parse_error(msg, _line, _col); } value_t parser::parse_expr() { - skip_whitespace_and_comments(); - if (_pos >= _input.size()) - return nil_value; - - char c = peek(); - - if (c == '(') { - depth_guard level(*this); - return parse_list(); - } - if (c == '\'') { - // Quote: 'expr → (quote expr) - depth_guard level(*this); - advance(); - auto quoted = parse_expr(); - return make_list({value_t(symbol{"quote"}), quoted}); - } - if (c == '"') - return parse_string(); - // SE-3: a `b` immediately followed by `"` (no space) is a bytes literal b"...". A lone `b`, or - // `b` followed by anything else, stays an ordinary symbol. - if (c == 'b' && _pos + 1 < _input.size() && _input[_pos + 1] == '"') { - advance(); // consume the 'b' prefix - return parse_bytes_string(); - } - - return parse_atom(); + _frames.clear(); // a datum left half-read by an earlier parse_error is abandoned + std::size_t unlimited = static_cast(-1); + value_t out; + return read(out, unlimited) == read_status::datum ? out : nil_value; } -value_t parser::parse_list() { - advance(); // consume '(' - std::vector elements; +// Push an open list or quote. The depth check comes first, so the error points at the `(` or `'` +// that would go one level too deep. +void parser::open_frame(bool quote) { + if (_frames.size() >= _opts.max_depth) + error("nesting deeper than " + std::to_string(_opts.max_depth) + + " levels (state_exec setting parser.max_depth)"); + _frames.push_back(frame{quote, {}}); +} - skip_whitespace_and_comments(); - while (peek() != ')') { - if (_pos >= _input.size()) - error("unclosed parenthesis"); - elements.push_back(parse_expr()); +parser::read_status parser::read(value_t &out, std::size_t &budget) { + while (budget > 0) { skip_whitespace_and_comments(); - } - advance(); // consume ')' + value_t datum; + if (_pos >= _input.size()) { + if (_frames.empty()) + return read_status::end; + if (!_frames.back().quote) + error("unclosed parenthesis"); + datum = nil_value; // a quote at end of input quotes nil (the pre-grammar behavior) + } else { + --budget; + const char c = peek(); + if (c == '(') { + open_frame(false); + advance(); + continue; + } + if (c == '\'') { // 'expr -> (quote expr) + open_frame(true); + advance(); + continue; + } + if (c == ')' && !_frames.empty() && !_frames.back().quote) { + advance(); + datum = make_list(std::move(_frames.back().elements)); + _frames.pop_back(); + } else if (c == '"') { + datum = parse_string(); + } else if (c == 'b' && _pos + 1 < _input.size() && _input[_pos + 1] == '"') { + // SE-3: a `b` immediately followed by `"` (no space) is a bytes literal b"...". A lone + // `b`, or `b` followed by anything else, stays an ordinary symbol. + advance(); // consume the 'b' prefix + datum = parse_bytes_string(); + } else { + // Everything else is an atom, including a `)` with no open list around it (it reads as + // a symbol until the grammar's P2 makes it an error). + datum = parse_atom(); + } + } - return make_list(std::move(elements)); + // A complete datum closes every quote waiting on it, then joins the innermost open list, or + // is the result when nothing is open. + while (!_frames.empty() && _frames.back().quote) { + _frames.pop_back(); + datum = make_list({value_t(symbol{"quote"}), std::move(datum)}); + } + if (_frames.empty()) { + out = std::move(datum); + return read_status::datum; + } + _frames.back().elements.push_back(std::move(datum)); + } + return read_status::more; } value_t parser::parse_string() { diff --git a/src/cvc/tests/state_exec_parser_test.cpp b/src/cvc/tests/state_exec_parser_test.cpp index 6b490eb88..ce00c4fee 100644 --- a/src/cvc/tests/state_exec_parser_test.cpp +++ b/src/cvc/tests/state_exec_parser_test.cpp @@ -286,3 +286,59 @@ TEST(StateExecParserTest, NestingDepthResetsBetweenTopLevelForms) { opts.max_depth = 2; EXPECT_EQ(parse_all("((a)) ((b)) ((c))", opts).size(), 3u); } + +// -- Stackless, resumable reading -- + +TEST(StateExecParserTest, ParserIsStacklessAtAnyDepth) { + // A million open parens with the limit out of the way: a recursive parser would overflow the + // native stack long before reaching the end. This one reads them all and reports the unclosed + // list. (The input never closes, so no deeply nested value is built and later destroyed.) + parse_options opts; + opts.max_depth = 2'000'000; + try { + parse(std::string(1'000'000, '('), opts); + FAIL() << "expected a parse_error"; + } catch (const parse_error &e) { + EXPECT_STREQ(e.what(), "unclosed parenthesis"); + } +} + +TEST(StateExecParserTest, ReadResumesAcrossTokenBudgets) { + const std::string src = "(a (b c)) 'd \"e\" (f '(g))"; + std::vector whole; + for (const value_t &v : parse_all(src)) + whole.push_back(to_string(v)); + + parser p(src); + std::vector sliced; + int pauses = 0; + for (;;) { + std::size_t budget = 1; // one token per call + value_t v; + const auto status = p.read(v, budget); + if (status == parser::read_status::end) + break; + if (status == parser::read_status::more) { + ++pauses; + continue; + } + sliced.push_back(to_string(v)); + } + EXPECT_EQ(sliced, whole); + EXPECT_GT(pauses, 5); // the lists really were read across several calls +} + +TEST(StateExecParserTest, ReadCountsTokensAgainstTheBudget) { + parser p("(a b) c"); + std::size_t budget = 3; // '(' 'a' 'b' + value_t v; + EXPECT_EQ(p.read(v, budget), parser::read_status::more); + EXPECT_EQ(budget, 0u); + budget = 10; + EXPECT_EQ(p.read(v, budget), parser::read_status::datum); // ')' closes the list + EXPECT_EQ(to_string(v), "(a b)"); + EXPECT_EQ(budget, 9u); + EXPECT_EQ(p.read(v, budget), parser::read_status::datum); + EXPECT_EQ(to_string(v), "c"); + EXPECT_EQ(p.read(v, budget), parser::read_status::end); +} From c040bddf065de397cad62cd301836a851aeb0803 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 13:22:57 -0500 Subject: [PATCH 06/13] state_exec: read submitted scripts inside the process, a slice per step The stackless parser's state is an input position plus a heap stack of open frames, so it can pause anywhere. New setting parser.step_tokens (on the state tree, resolved like parser.max_depth; default 0) lets a scheduler parse a submitted script inside the new process instead of at submit: each scheduler step reads N tokens, and the step that reaches the end sets up evaluation exactly as parsing up front would have. - Reading steps are process steps: they count against max_steps and max_time and interleave with other processes. - A syntax error surfaces like a runtime error: the async scheduler kills the process with it as the exit error; the sync scheduler throws it from step()/run(). With step_tokens = 0, execute() still throws. - Signals and watch events wait until the script is read (their handlers run on the evaluation stack, which doesn't exist yet). - The snapshot codec stores a mid-read process's source and settings and re-reads it after a restore. Adds parse_options::step_tokens, script_reader (owns its source and drives parser::read() a budget at a time) and evaluator_state::reading; stackless_evaluator::step() reads until the script is read. Tests cover the reader, the evaluator, the codec and both schedulers (results, syntax errors, max_steps, deferred signals). --- docs/STATE_EXEC_GRAMMAR.md | 27 ++++- inc/cvc/state/state_exec/parser.h | 32 ++++++ .../state/state_exec/stackless_evaluator.h | 13 ++- src/cvc/state/state_exec/async_scheduler.cpp | 9 +- src/cvc/state/state_exec/parser.cpp | 49 +++++++--- src/cvc/state/state_exec/scheduler.cpp | 10 +- .../state/state_exec/stackless_evaluator.cpp | 42 ++++++-- .../state/state_exec/state_value_codec.cpp | 27 +++++ src/cvc/tests/state_exec_codec_test.cpp | 31 ++++++ src/cvc/tests/state_exec_integration_test.cpp | 98 +++++++++++++++++++ src/cvc/tests/state_exec_parser_test.cpp | 29 ++++++ .../state_exec_stackless_evaluator_test.cpp | 40 ++++++++ 12 files changed, 383 insertions(+), 24 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index c9e2ce74b..8a5835397 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -148,12 +148,13 @@ this order: 2. `state_exec.defaults.parser.`, for everything on that tree; 3. the built-in default. -A value that is missing, empty, not a positive integer or out of range is skipped, and the next -one in the order applies. +A value that is missing, empty, not a non-negative integer or out of range is skipped, and the +next one in the order applies. `max_depth` must also be positive. | Key | Meaning | Default | |---|---|---| | `max_depth` | how many lists and quotes may nest | 256 | +| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0 | Who reads them: @@ -168,6 +169,28 @@ Who reads them: built-in defaults, or passes `parse_options` to `parse()` and `parse_all()` explicitly. `parse_options_from_state()` does the resolution above for any host that has a tree. +### Reading inside the scheduler + +The parser is stackless and resumable. Its whole state is an input position plus a heap stack +of open lists and quotes, and `parser::read(out, budget)` stops after `budget` tokens. So a +scheduler can parse a script the same way it evaluates one: a slice per step. With +`parser.step_tokens` = N: + +- **Submitting.** `execute(script)` returns at once. The new process starts in a reading phase. + Each of its scheduler steps reads N more tokens (parens, quotes, atoms, strings). The step + after the last token sets up evaluation, exactly as parsing at submit would have, and + evaluation steps follow. +- **Budgets.** Reading steps are process steps. They count against `max_steps` and `max_time`, + and they interleave with other processes. A huge script costs its own process's budget instead + of stalling the scheduler before the process exists. +- **Errors.** A syntax error surfaces where a runtime error would. The async scheduler kills the + process, with the parse error as its exit error. The sync scheduler throws it from `step()` or + `run()`. With step_tokens = 0, `execute()` throws it, as before. +- **Signals and watch events** wait until the script is read, because their handlers run on the + evaluation stack. +- **Snapshots.** A process snapshotted while reading stores its source and parser settings. After + a restore it reads the script again from the start. + ## Data | Production | Value (`value_t` alternative) | diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index 102d447c1..a3d4ab5a3 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -46,6 +46,13 @@ struct parse_options { /// recursively (destroying a nested list, the snapshot codec, to_string), so this keeps parsed /// data within what they can handle. State-tree key: `parser.max_depth`. std::size_t max_depth = 256; + + /// How a scheduler parses a submitted script. 0 parses all of it when it is submitted, so + /// execute() throws a parse_error. N > 0 parses it inside the new process, N tokens per + /// scheduler step, before evaluation starts. Those steps count against the process's max_steps + /// and max_time, and a syntax error then surfaces like a runtime error. State-tree key: + /// `parser.step_tokens`. + std::size_t step_tokens = 0; }; /// Resolve parser settings from a state tree, per key: `state_exec.schedulers.. @@ -122,6 +129,31 @@ class parser { value_t parse_number_or_symbol(char first); }; +/// Reads a whole script a slice at a time. A scheduler drives it from a process's steps when +/// parse_options::step_tokens is set. It owns its copy of the source, which its parser views, so +/// it can be neither copied nor moved; hold it by pointer. +class script_reader { +public: + script_reader(std::string source, const parse_options &opts); + script_reader(const script_reader &) = delete; + script_reader &operator=(const script_reader &) = delete; + + /// Read up to `budget` more tokens. Returns true once the whole script has been read; forms() + /// then holds every top-level datum, as parse_all() would return them. + /// @throws parse_error on syntax errors. + bool step(std::size_t budget); + + std::vector &forms() { return _forms; } + const std::string &source() const { return _source; } + const parse_options &options() const { return _opts; } + +private: + std::string _source; // declared before _parser, which views it + parse_options _opts; + parser _parser; + std::vector _forms; +}; + } // namespace cvc::state_exec #endif // CVC_STATE_EXEC_PARSER_H diff --git a/inc/cvc/state/state_exec/stackless_evaluator.h b/inc/cvc/state/state_exec/stackless_evaluator.h index 17a6b1b08..10902797c 100644 --- a/inc/cvc/state/state_exec/stackless_evaluator.h +++ b/inc/cvc/state/state_exec/stackless_evaluator.h @@ -74,6 +74,9 @@ struct evaluator_state { evaluator::macro_map user_macros; evaluation_stats stats; bool yielded = false; // Set by (yield val) — signals generator_next to stop + /// Set while the script is still being read, a slice per step (parse_options::step_tokens); + /// step() reads instead of evaluating until the whole script is read. + std::shared_ptr reading; }; /// Synchronous stackless evaluator (Tier 2 — local contexts only). @@ -94,7 +97,9 @@ class stackless_evaluator { evaluator_state create_state(const value_t &expr, environment_ptr env = nullptr); /// Create a fresh evaluator_state from a script string, parsed with `popts` (a scheduler passes - /// the parser settings it resolved from the state tree). + /// the parser settings it resolved from the state tree). With popts.step_tokens > 0 nothing is + /// parsed yet: the state reads the script popts.step_tokens tokens per step() first, and a + /// syntax error is thrown from step() rather than from here. evaluator_state create_state(const std::string &script, environment_ptr env = nullptr, const parse_options &popts = {}); @@ -137,6 +142,12 @@ class stackless_evaluator { private: void check_interrupted(); + // A state that evaluates `forms` as a script: done when empty, the form itself when one, a + // (begin ...) of them otherwise. + evaluator_state create_program_state(std::vector forms, environment_ptr env); + // One step of a state that is still reading its script (evaluator_state::reading). + bool step_read(evaluator_state &state); + // Frame management void pop_frame(evaluator_state &state, value_t result); void push_frame(evaluator_state &state, value_t expr, environment_ptr env); diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index d76728284..1f7e6b31d 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -139,12 +139,17 @@ void async_scheduler::execute_process_step(process &proc) { } } reset_guard{current_pid_, current_proc_}; - if (!proc.pending_signals.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + // A process still reading its script (parser.step_tokens) takes signals and watch events once + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. + const bool can_interrupt = + !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + + if (!proc.pending_signals.empty() && can_interrupt) { handle_signal(proc); return; } - if (!proc.pending_watch_events.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_watch_events.empty() && can_interrupt) { handle_watch_event(proc); return; } diff --git a/src/cvc/state/state_exec/parser.cpp b/src/cvc/state/state_exec/parser.cpp index 340003b07..8f3964a2b 100644 --- a/src/cvc/state/state_exec/parser.cpp +++ b/src/cvc/state/state_exec/parser.cpp @@ -27,29 +27,36 @@ std::vector parse_all(const std::string &input, const parse_options &op return results; } -// A positive integer setting at `path`, or nullopt when it is missing, empty, malformed, zero or -// out of range (so the caller falls through to the next place in the resolution order). -static std::optional read_positive_setting(cvc::state &root, const std::string &path) { +// A non-negative integer setting at `path` (positive unless `allow_zero`), or nullopt when it is +// missing, empty, malformed or out of range, so the caller falls through to the next place in +// the resolution order. +static std::optional read_size_setting(cvc::state &root, const std::string &path, + bool allow_zero) { cvc::state *node = root.findDescendant(path); if (!node) return std::nullopt; const std::string v = node->value(); std::size_t n = 0; const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); - if (ec != std::errc{} || end != v.data() + v.size() || n == 0) + if (ec != std::errc{} || end != v.data() + v.size() || v.empty() || (n == 0 && !allow_zero)) return std::nullopt; return n; } parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id) { parse_options opts; - const std::string key = ".parser.max_depth"; - if (auto v = read_positive_setting(root, std::string(scheduler::defaults_prefix) + key)) - opts.max_depth = *v; - if (!scheduler_id.empty()) - if (auto v = read_positive_setting(root, std::string(scheduler::sched_prefix) + "." + - scheduler_id + key)) - opts.max_depth = *v; + const auto resolve = [&](const char *key, bool allow_zero, std::size_t &out) { + const std::string suffix = std::string(".parser.") + key; + if (auto v = + read_size_setting(root, std::string(scheduler::defaults_prefix) + suffix, allow_zero)) + out = *v; + if (!scheduler_id.empty()) + if (auto v = read_size_setting( + root, std::string(scheduler::sched_prefix) + "." + scheduler_id + suffix, allow_zero)) + out = *v; + }; + resolve("max_depth", /*allow_zero=*/false, opts.max_depth); + resolve("step_tokens", /*allow_zero=*/true, opts.step_tokens); // 0 = parse at submit return opts; } @@ -184,6 +191,26 @@ parser::read_status parser::read(value_t &out, std::size_t &budget) { return read_status::more; } +// -- script_reader ------------------------------------------------------------- + +script_reader::script_reader(std::string source, const parse_options &opts) + : _source(std::move(source)), _opts(opts), _parser(_source, opts) {} + +bool script_reader::step(std::size_t budget) { + for (;;) { + value_t form; + switch (_parser.read(form, budget)) { + case parser::read_status::datum: + _forms.push_back(std::move(form)); + break; + case parser::read_status::end: + return true; + case parser::read_status::more: + return false; + } + } +} + value_t parser::parse_string() { advance(); // consume opening '"' std::string result; diff --git a/src/cvc/state/state_exec/scheduler.cpp b/src/cvc/state/state_exec/scheduler.cpp index 28f2584c0..6d1d09872 100644 --- a/src/cvc/state/state_exec/scheduler.cpp +++ b/src/cvc/state/state_exec/scheduler.cpp @@ -63,6 +63,7 @@ void scheduler::load_settings() { std::string base = std::string(sched_prefix) + "." + id_; (*watch_root_)(base + ".max_pending_messages").value(std::to_string(max_pending_messages)); (*watch_root_)(base + ".parser.max_depth").value(std::to_string(parse_options_.max_depth)); + (*watch_root_)(base + ".parser.step_tokens").value(std::to_string(parse_options_.step_tokens)); (*watch_root_)(base + ".policy") .value(policy_ == scheduling_policy::round_robin ? "round_robin" : policy_ == scheduling_policy::priority ? "priority" @@ -195,14 +196,19 @@ void scheduler::execute_process_step(process &proc) { } } reset_guard{current_pid_, current_proc_}; + // A process still reading its script (parser.step_tokens) takes signals and watch events once + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. + const bool can_interrupt = + !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + // 1. Handle pending signals first - if (!proc.pending_signals.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_signals.empty() && can_interrupt) { handle_signal(proc); return; // Signal setup counts as a step } // 1b. Handle pending watch events - if (!proc.pending_watch_events.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_watch_events.empty() && can_interrupt) { handle_watch_event(proc); return; } diff --git a/src/cvc/state/state_exec/stackless_evaluator.cpp b/src/cvc/state/state_exec/stackless_evaluator.cpp index f498e79bd..2aa10014b 100644 --- a/src/cvc/state/state_exec/stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/stackless_evaluator.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace cvc::state_exec { @@ -41,23 +42,52 @@ evaluator_state stackless_evaluator::create_state(const value_t &expr, environme evaluator_state stackless_evaluator::create_state(const std::string &script, environment_ptr env, const parse_options &popts) { - auto exprs = parse_all(script, popts); - if (exprs.empty()) { + if (popts.step_tokens > 0) { + // Read inside the process: step_read() parses a slice per step, then sets up evaluation. + evaluator_state st; + st.global_env = env ? env : environment::extend(global_env_); + st.reading = std::make_shared(script, popts); + return st; + } + return create_program_state(parse_all(script, popts), env); +} + +evaluator_state stackless_evaluator::create_program_state(std::vector forms, + environment_ptr env) { + if (forms.empty()) { evaluator_state st; st.done = true; return st; } - if (exprs.size() == 1) - return create_state(exprs[0], env); + if (forms.size() == 1) + return create_state(forms[0], env); std::vector begin_exprs; - begin_exprs.reserve(1 + exprs.size()); + begin_exprs.reserve(1 + forms.size()); begin_exprs.push_back(value_t{symbol{"begin"}}); - begin_exprs.insert(begin_exprs.end(), exprs.begin(), exprs.end()); + begin_exprs.insert(begin_exprs.end(), std::make_move_iterator(forms.begin()), + std::make_move_iterator(forms.end())); return create_state(make_list(std::move(begin_exprs)), env); } +// Read the next slice of the script. Once all of it is read, the state becomes exactly what +// create_state(script) would have built, keeping the step count and timing of the reading. A +// parse_error propagates like any error from step(): the sync scheduler rethrows it to its +// caller, and the async scheduler kills the process with it. +bool stackless_evaluator::step_read(evaluator_state &state) { + state.stats.increment_step(); + script_reader &reader = *state.reading; + if (!reader.step(reader.options().step_tokens)) + return false; + evaluator_state program = create_program_state(std::move(reader.forms()), state.global_env); + program.stats = std::move(state.stats); + state = std::move(program); // also releases the reader + return state.done; +} + bool stackless_evaluator::step(evaluator_state &state) { check_interrupted(); + if (state.reading) + return step_read(state); if (state.done || state.stack.empty()) { state.done = true; return true; diff --git a/src/cvc/state/state_exec/state_value_codec.cpp b/src/cvc/state/state_exec/state_value_codec.cpp index 6d5c9c5bf..0555ff7dc 100644 --- a/src/cvc/state/state_exec/state_value_codec.cpp +++ b/src/cvc/state/state_exec/state_value_codec.cpp @@ -5,6 +5,7 @@ cvc::state subtrees. */ +#include #include #include #include @@ -552,6 +553,15 @@ void encode_evaluator_state(cvc::state &node, const evaluator_state &es) { } } + // A script still being read (parser.step_tokens): keep its source and parser settings; the + // restored state reads it again from the start. + if (es.reading) { + auto &r = node("reading"); + r("source").value(es.reading->source()); + r("max_depth").value(std::to_string(es.reading->options().max_depth)); + r("step_tokens").value(std::to_string(es.reading->options().step_tokens)); + } + // Stats snapshot node("step_count").value(std::to_string(es.stats.get_step_count())); } @@ -600,6 +610,23 @@ evaluator_state decode_evaluator_state(cvc::state &node) { } } + // A script still being read restarts its reading from the beginning. + if (has_child(node, "reading")) { + auto &r = node("reading"); + parse_options opts; + const auto size_of = [&](const char *key, std::size_t fallback) { + const std::string v = r(key).value(); + std::size_t n = fallback; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + return ec == std::errc{} && end == v.data() + v.size() ? n : fallback; + }; + opts.max_depth = size_of("max_depth", opts.max_depth); + opts.step_tokens = size_of("step_tokens", 1); // a reading state always has a positive slice + if (opts.step_tokens == 0) + opts.step_tokens = 1; + es.reading = std::make_shared(r("source").value(), opts); + } + return es; } diff --git a/src/cvc/tests/state_exec_codec_test.cpp b/src/cvc/tests/state_exec_codec_test.cpp index 277c27d05..20fea8d41 100644 --- a/src/cvc/tests/state_exec_codec_test.cpp +++ b/src/cvc/tests/state_exec_codec_test.cpp @@ -291,6 +291,37 @@ TEST_F(EvaluatorStateCodecTest, CompletedState) { EXPECT_TRUE(restored.stack.empty()); } +TEST_F(EvaluatorStateCodecTest, StateStillReadingItsScript) { + // parser.step_tokens: a process snapshotted mid-read keeps its source and settings, and the + // restored state reads the script again and evaluates it. + parse_options popts; + popts.step_tokens = 2; + popts.max_depth = 64; + auto state = ev->create_state(std::string("(set x 20) (+ x 22)"), nullptr, popts); + ev->step(state); // part way through reading + ASSERT_TRUE(state.reading); + + auto &node = ctx.root()("__es_reading__"); + encode_evaluator_state(node, state); + auto restored = decode_evaluator_state(node); + + ASSERT_TRUE(restored.reading); + EXPECT_EQ(restored.reading->source(), "(set x 20) (+ x 22)"); + EXPECT_EQ(restored.reading->options().step_tokens, 2u); + EXPECT_EQ(restored.reading->options().max_depth, 64u); + + // Native builtins aren't serialized; put them back on the restored environment's root, as + // PauseResumeProducesCorrectResult does. Nothing is on the frame stack yet. + ASSERT_TRUE(restored.global_env); + environment_ptr root_env = restored.global_env; + while (root_env->outer) + root_env = root_env->outer; + const auto builtins_env = builtins::make_default_environment(); // must outlive the loop + for (auto &[name, val] : builtins_env->bindings) + root_env->set(name, val); + EXPECT_EQ(std::get(ev->run(restored).v), 42); +} + TEST_F(EvaluatorStateCodecTest, PausedState) { // Run a program partially (10 steps of a loop) auto state = diff --git a/src/cvc/tests/state_exec_integration_test.cpp b/src/cvc/tests/state_exec_integration_test.cpp index 0fc33685e..b3235c3a6 100644 --- a/src/cvc/tests/state_exec_integration_test.cpp +++ b/src/cvc/tests/state_exec_integration_test.cpp @@ -2403,6 +2403,104 @@ TEST_F(SchedulerSettingsTest, AsyncSchedulerReadsParserSettingsOnEachSubmit) { EXPECT_NO_THROW(sched.execute(three_deep)); } +// --- parser.step_tokens: reading a script inside the process, a slice per step --- + +TEST_F(SchedulerSettingsTest, StepTokensResolveFromTreeAndAllowZero) { + auto &root = cvc::state::instance(app_ctx); + EXPECT_EQ(parse_options_from_state(root).step_tokens, 0u); // built-in: parse at submit + root("state_exec.defaults.parser.step_tokens").value("64"); + EXPECT_EQ(parse_options_from_state(root).step_tokens, 64u); + root("state_exec.schedulers.eager.parser.step_tokens").value("0"); // 0 is a real setting here + EXPECT_EQ(parse_options_from_state(root, "eager").step_tokens, 0u); + root("state_exec.schedulers.eager.parser.step_tokens").value("-1"); // bad: falls through + EXPECT_EQ(parse_options_from_state(root, "eager").step_tokens, 64u); +} + +TEST_F(SchedulerSettingsTest, SyncSchedulerEvaluatesAScriptReadInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("2"); + scheduler sched; + sched.set_watch_root(&root); + sched.set_id("reader"); + sched.load_settings(); + EXPECT_EQ(sched.parser_settings().step_tokens, 2u); + EXPECT_EQ(root("state_exec.schedulers.reader.parser.step_tokens").value(), "2"); + + const int pid = sched.execute(std::string("(set a 40) (+ a 2)")); + auto results = sched.run(); + EXPECT_EQ(std::get(results[pid].v), 42); +} + +TEST_F(SchedulerSettingsTest, SyncSchedulerThrowsSyntaxErrorsFromRunWhenReadingInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("2"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + EXPECT_NO_THROW(sched.execute(std::string("(+ 1 2) (+ 3"))); // nothing is parsed yet + EXPECT_THROW(sched.run(), parse_error); // the process reads it +} + +TEST_F(SchedulerSettingsTest, ReadingStepsCountAgainstMaxSteps) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("1"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + std::string big = "(list"; + for (int i = 0; i < 100; ++i) + big += " " + std::to_string(i); + big += ")"; + execute_options opts; + opts.max_steps = 5; // far fewer than the 103 tokens to read + const int pid = sched.execute(big, opts); + sched.run(); + auto info = sched.get_process_info(pid); + ASSERT_TRUE(info); + EXPECT_EQ(info->status, process_status::killed); + EXPECT_EQ(info->step_count, 5u); +} + +TEST_F(SchedulerSettingsTest, SignalsWaitUntilTheScriptIsRead) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("1"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + execute_options opts; + opts.signal_handlers["SIGUSR1"] = make_list({value_t{symbol{"begin"}}, value_t{int64_t(7)}}); + const int pid = sched.execute(std::string("(begin 1 2 (+ 40 2))"), opts); + sched.step(); // reading has begun + EXPECT_TRUE(sched.send_signal(pid, "SIGUSR1")); + auto results = sched.run(); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::terminated); + EXPECT_EQ(std::get(results[pid].v), 42); +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerKillsAProcessWhoseScriptDoesNotParse) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("4"); + async_scheduler sched; + sched.set_watch_root(&root); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1 2) (+ 3"))); + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + const auto err = sched.get_exit_error(pid); + ASSERT_TRUE(err); + EXPECT_NE(err->find("unclosed parenthesis"), std::string::npos) << *err; +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerEvaluatesAScriptReadInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("3"); + async_scheduler sched; + sched.set_watch_root(&root); + const int pid = sched.execute(std::string("(set a 40) (+ a 2)")); + auto results = sched.sync_run(1000, 1.0); + EXPECT_EQ(std::get(results[pid].v), 42); +} + // =========================================================================== // §12 message-channel scoping — end-to-end through the scheduler + chroot // =========================================================================== diff --git a/src/cvc/tests/state_exec_parser_test.cpp b/src/cvc/tests/state_exec_parser_test.cpp index ce00c4fee..888bc8b86 100644 --- a/src/cvc/tests/state_exec_parser_test.cpp +++ b/src/cvc/tests/state_exec_parser_test.cpp @@ -328,6 +328,35 @@ TEST(StateExecParserTest, ReadResumesAcrossTokenBudgets) { EXPECT_GT(pauses, 5); // the lists really were read across several calls } +TEST(StateExecParserTest, ScriptReaderReadsAWholeScriptInSlices) { + const std::string src = "(define x 1) ; a comment\n(+ x '(2 3)) \"s\""; + std::vector whole; + for (const value_t &v : parse_all(src)) + whole.push_back(to_string(v)); + + script_reader reader(src, parse_options{}); + int slices = 1; + while (!reader.step(2)) // two tokens per slice + ++slices; + std::vector sliced; + for (const value_t &v : reader.forms()) + sliced.push_back(to_string(v)); + EXPECT_EQ(sliced, whole); + EXPECT_GT(slices, 5); + EXPECT_EQ(reader.source(), src); // owns its copy +} + +TEST(StateExecParserTest, ScriptReaderReportsSyntaxErrorsWhenItReachesThem) { + script_reader reader("(a) (b", parse_options{}); + EXPECT_FALSE(reader.step(2)); // '(' 'a' + EXPECT_THROW( + { + while (!reader.step(2)) { + } + }, + parse_error); +} + TEST(StateExecParserTest, ReadCountsTokensAgainstTheBudget) { parser p("(a b) c"); std::size_t budget = 3; // '(' 'a' 'b' diff --git a/src/cvc/tests/state_exec_stackless_evaluator_test.cpp b/src/cvc/tests/state_exec_stackless_evaluator_test.cpp index 906f59c63..3728ce8ed 100644 --- a/src/cvc/tests/state_exec_stackless_evaluator_test.cpp +++ b/src/cvc/tests/state_exec_stackless_evaluator_test.cpp @@ -416,3 +416,43 @@ TEST_F(StacklessEvaluatorTest, DeadlineIsNotSpentWhileTheThreadDoesNotRun) { auto st = napper.create_state(parse("(nap)")); EXPECT_EQ(std::get(napper.run(st, std::nullopt, /*timeout_sec=*/0.02).v), 7); } + +// ─── Reading inside the steps (parse_options::step_tokens) ───────────────── + +TEST_F(StacklessEvaluatorTest, SlicedReadingEvaluatesLikeParsingUpFront) { + const std::string script = "(defun sq (x) (* x x)) (set a (sq 7)) (+ a 1)"; + auto eager = ev->create_state(script); + ASSERT_FALSE(eager.reading); + const value_t expected = ev->run(eager); + + parse_options popts; + popts.step_tokens = 3; + auto sliced = ev->create_state(script, nullptr, popts); + ASSERT_TRUE(sliced.reading); // nothing parsed yet + EXPECT_TRUE(sliced.stack.empty()); + const value_t got = ev->run(sliced); + EXPECT_TRUE(sliced.done); + EXPECT_FALSE(sliced.reading); + EXPECT_EQ(to_string(got), to_string(expected)); + EXPECT_EQ(std::get(got.v), 50); + // The reading steps are steps: 25 tokens at 3 per step take 9 (the ninth reads the last token, + // finds the end of the script and sets up evaluation). + EXPECT_EQ(sliced.stats.get_step_count(), eager.stats.get_step_count() + 9); +} + +TEST_F(StacklessEvaluatorTest, SlicedReadingThrowsSyntaxErrorsFromStep) { + parse_options popts; + popts.step_tokens = 2; + evaluator_state st; + ASSERT_NO_THROW(st = ev->create_state(std::string("(+ 1 2) (+ 3"), nullptr, popts)); + EXPECT_THROW(ev->run(st), parse_error); +} + +TEST_F(StacklessEvaluatorTest, SlicedReadingOfAnEmptyScriptFinishesWithNil) { + parse_options popts; + popts.step_tokens = 4; + auto st = ev->create_state(std::string(" ; nothing here\n"), nullptr, popts); + EXPECT_TRUE(ev->step(st)); + EXPECT_TRUE(st.done); + EXPECT_TRUE(st.result.is_nil()); +} From 97c2a921054a60013ce831a4fd99cc25c11b84dc Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 13:32:09 -0500 Subject: [PATCH 07/13] state_exec: the app's exec scheduler reads scripts in steps (1024 tokens) app::exec_scheduler(), which runs Ariadne's actions and residents, now defaults parser.step_tokens to 1024, so it parses submitted scripts inside their processes. async_scheduler gains set_parser_defaults(): the built-in values under its watch root's state_exec.defaults.parser.* keys. A plain async_scheduler (pycvc's Exec) and the sync scheduler stay at 0, so they still report syntax errors at submit. Why 1024: every script the repo carries reads in one slice (the largest is 159 tokens, .ari scripts at most 24), so a normal action pays one extra step of its 5000; a slice costs about 80 us at the parser's measured ~80 ns/token, so a pathologically large script is spread over steps and frames instead of stalling a drain. Ariadne now says why a process died: a killed action's warning carries its exit error, and the per-frame sweep reports a killed resident (a syntax error in its body, a runtime error, its per-step cap) once, instead of letting it vanish silently. --- docs/STATE_EXEC_GRAMMAR.md | 18 +++++++++-- inc/cvc/state/state_exec/async_scheduler.h | 8 +++++ inc/cvc/state/state_exec/parser.h | 7 +++-- src/cvc/ariadne/ariadne.cpp | 18 ++++++++++- src/cvc/core/app.cpp | 9 ++++++ src/cvc/state/state_exec/async_scheduler.cpp | 5 +-- src/cvc/state/state_exec/parser.cpp | 5 +-- src/cvc/tests/ariadne_runtime_test.cpp | 23 +++++++++++++- src/cvc/tests/state_exec_integration_test.cpp | 31 +++++++++++++++++++ 9 files changed, 112 insertions(+), 12 deletions(-) diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index 8a5835397..30605c60d 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -154,7 +154,7 @@ next one in the order applies. `max_depth` must also be positive. | Key | Meaning | Default | |---|---|---| | `max_depth` | how many lists and quotes may nest | 256 | -| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0 | +| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler | Who reads them: @@ -162,7 +162,17 @@ Who reads them: the effective values under `state_exec.schedulers..parser.*` and uses them for every script submitted to it afterwards. - **`async_scheduler`** has no settings loader. It reads `state_exec.defaults.parser.*` from its - watch root each time a script is submitted. + watch root each time a script is submitted. Keys that aren't set fall back to the scheduler's + own defaults (`set_parser_defaults()`), which are the built-in ones unless the host changes + them. + - **The app's exec scheduler** (`app::exec_scheduler()`, which runs Ariadne's actions and + residents) sets `step_tokens` = 1024, so it reads scripts inside their processes. + - **Why 1024:** every script in the corpus fits in one slice (the largest is 159 tokens, and + `.ari` scripts are at most 24). So a normal action pays one extra step out of its 5000. A + slice costs about 80 µs at the parser's measured ~80 ns per token, so a pathologically large + script is spread over steps and frames instead of stalling a drain. + - pycvc's `Exec` keeps a private `async_scheduler` and stays at 0, so its `run()` still raises + on a syntax error straight away. - **Ariadne** reads `state_exec.defaults.parser.*` from the app's tree when it compiles a predicate. - **Code without a state tree** (a standalone evaluator, the `.ari` channel lint) uses the @@ -185,7 +195,9 @@ scheduler can parse a script the same way it evaluates one: a slice per step. Wi of stalling the scheduler before the process exists. - **Errors.** A syntax error surfaces where a runtime error would. The async scheduler kills the process, with the parse error as its exit error. The sync scheduler throws it from `step()` or - `run()`. With step_tokens = 0, `execute()` throws it, as before. + `run()`. With step_tokens = 0, `execute()` throws it, as before. Ariadne reports a killed + action or resident with its exit error ("ari: on: action program did not complete: …", + "ari: resident [...] stopped: …"). - **Signals and watch events** wait until the script is read, because their handlers run on the evaluation stack. - **Snapshots.** A process snapshotted while reading stores its source and parser settings. After diff --git a/inc/cvc/state/state_exec/async_scheduler.h b/inc/cvc/state/state_exec/async_scheduler.h index 750c61d68..bb440d3de 100644 --- a/inc/cvc/state/state_exec/async_scheduler.h +++ b/inc/cvc/state/state_exec/async_scheduler.h @@ -163,6 +163,13 @@ class async_scheduler : public scheduler_base { void set_watch_root(cvc::state *root) override { watch_root_ = root; } + /// This scheduler's built-in parser settings: what a submitted script is parsed with when the + /// watch root's `state_exec.defaults.parser.*` keys don't say otherwise. parse_options{} (parse + /// at submit) unless the host sets it; the app's exec scheduler reads in steps + /// (app::exec_scheduler()). + void set_parser_defaults(const parse_options &defaults) { parser_defaults_ = defaults; } + const parse_options &parser_defaults() const { return parser_defaults_; } + private: scheduling_policy policy_; int next_pid_ = 1; @@ -180,6 +187,7 @@ class async_scheduler : public scheduler_base { async_stackless_evaluator evaluator_; memory_tracker mem_tracker_; cvc::state *watch_root_ = nullptr; + parse_options parser_defaults_; std::unordered_map processes_; diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index a3d4ab5a3..5267c0c3d 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -57,9 +57,10 @@ struct parse_options { /// Resolve parser settings from a state tree, per key: `state_exec.schedulers.. /// parser.` (when scheduler_id is not empty), then `state_exec.defaults.parser.`, then -/// the parse_options default. A value that is missing, empty, not a positive integer or out of -/// range is skipped. -parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {}); +/// `base` (a host's built-in defaults; parse_options{} unless it has its own). A value that is +/// missing, empty, malformed or out of range is skipped; max_depth must also be positive. +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {}, + const parse_options &base = {}); /// Parse a single S-expression from the input string. /// Returns the parsed value_t. diff --git a/src/cvc/ariadne/ariadne.cpp b/src/cvc/ariadne/ariadne.cpp index 85657179c..dba3f7fc9 100644 --- a/src/cvc/ariadne/ariadne.cpp +++ b/src/cvc/ariadne/ariadne.cpp @@ -1467,12 +1467,28 @@ void Runtime::Impl::pump_and_sweep_actions() { if (!info || info->status == se::process_status::terminated) { it = live_actions_.erase(it); // normal completion (or already reaped from the process map) } else if (info->status == se::process_status::killed) { - warn_once("ari: on: action program did not complete (a runtime error or resource limit)"); + // The exit error says why: a syntax error (scripts are read inside the process), a runtime + // error or a resource limit. + warn_once("ari: on: action program did not complete: " + + sched.get_exit_error(it->first).value_or("killed")); it = live_actions_.erase(it); } else { ++it; // ready/running/waiting — still in flight (e.g. suspended on await/msg-recv/sleep) } } + // A resident never finishes on its own, so one that was killed (a syntax error in its body, + // read inside the process; a runtime error; its per-step cap) is reported, once per reason. + const auto report_killed_resident = [&](int pid, const std::string &channel) { + if (pid < 0) + return; + auto info = sched.get_process_info(pid); + if (info && info->status == se::process_status::killed) + warn_once("ari: resident [" + channel + + "] stopped: " + sched.get_exit_error(pid).value_or("killed")); + }; + report_killed_resident(tick_resident_pid_, tick_channel()); + report_killed_resident(key_resident_pid_, key_channel()); + report_killed_resident(pointer_resident_pid_, pointer_channel()); } void Runtime::Impl::ensure_resident(const std::string &channel, const std::string &body, diff --git a/src/cvc/core/app.cpp b/src/cvc/core/app.cpp index d776c16bf..7df1bb94f 100644 --- a/src/cvc/core/app.cpp +++ b/src/cvc/core/app.cpp @@ -1022,6 +1022,15 @@ cvc::state_exec::async_scheduler &app::exec_scheduler() { // Wire it to this app's state root so state-watches (state-watch/on_change:) // resolve against the app tree. Processes are chrooted per-owner on submit. _execScheduler->set_watch_root(&root()); + // Read submitted scripts inside their processes, 1024 tokens per step (docs/ + // STATE_EXEC_GRAMMAR.md, "Reading inside the scheduler"). Every script the repo carries fits + // in one slice (the largest is 159 tokens; .ari scripts are at most 24), so a normal + // action costs one extra step out of its 5000. A slice costs about 80 us at the parser's + // measured ~80 ns per token, so a pathologically large script is spread over steps and frames + // instead of stalling a drain. state_exec.defaults.parser.step_tokens overrides it. + cvc::state_exec::parse_options parser_defaults; + parser_defaults.step_tokens = 1024; + _execScheduler->set_parser_defaults(parser_defaults); } return *_execScheduler; } diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index 1f7e6b31d..4de30dd99 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -37,9 +37,10 @@ int async_scheduler::execute(const std::string &script, const execute_options &o proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); - // No load_settings() here: read the parser settings from the tree on every submission. + // No load_settings() here: read the parser settings from the tree on every submission, over + // this scheduler's own defaults (set_parser_defaults). const parse_options popts = - watch_root_ ? parse_options_from_state(*watch_root_) : parse_options{}; + watch_root_ ? parse_options_from_state(*watch_root_, {}, parser_defaults_) : parser_defaults_; proc.state = evaluator_.create_state(script, opts.env, popts); proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); diff --git a/src/cvc/state/state_exec/parser.cpp b/src/cvc/state/state_exec/parser.cpp index 8f3964a2b..dd024c405 100644 --- a/src/cvc/state/state_exec/parser.cpp +++ b/src/cvc/state/state_exec/parser.cpp @@ -43,8 +43,9 @@ static std::optional read_size_setting(cvc::state &root, const std: return n; } -parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id) { - parse_options opts; +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id, + const parse_options &base) { + parse_options opts = base; const auto resolve = [&](const char *key, bool allow_zero, std::size_t &out) { const std::string suffix = std::string(".parser.") + key; if (auto v = diff --git a/src/cvc/tests/ariadne_runtime_test.cpp b/src/cvc/tests/ariadne_runtime_test.cpp index 970158b3a..1ae4c670d 100644 --- a/src/cvc/tests/ariadne_runtime_test.cpp +++ b/src/cvc/tests/ariadne_runtime_test.cpp @@ -16,7 +16,7 @@ #include // §13.8 (fetch uri): register a custom scheme handler #include #include -#include // launch_pool_task generalization test +#include // launch_pool_task generalization test #include // item 2: full type for app.computePool().parallel_for #include // §13.8: swap the transport for a fake in the http-get test #include @@ -520,6 +520,27 @@ TEST(AriadneResident, TickProgramRunsOncePerDrain) { EXPECT_TRUE(rt.take_reactive_warnings().empty()); } +// The app's exec scheduler reads a submitted script inside its process (parser.step_tokens), so a +// resident whose body doesn't parse is killed by the syntax error once it is read, and the sweep +// reports it with the parse error rather than letting it vanish. +TEST(AriadneResident, ResidentWithASyntaxErrorIsReported) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + rt.set_tick_program("(state-set \"tick.flag\" \"1\""); // unclosed + rt.render(); + rt.drain(); + rt.drain(); + const std::vector warns = rt.take_reactive_warnings(); + bool reported = false; + for (const std::string &w : warns) + reported = reported || (w.find("stopped") != std::string::npos && + w.find("unclosed parenthesis") != std::string::npos); + EXPECT_TRUE(reported) << "warnings: " << ::testing::PrintToString(warns); + EXPECT_NE(cvc::state::instance(app)("tick.flag").value(), "1"); // the body never ran +} + // set_tick_program("") clears the resident: after clearing, a drain no longer re-fires the body. TEST(AriadneResident, ClearingTickProgramStopsIt) { cvc::app app; diff --git a/src/cvc/tests/state_exec_integration_test.cpp b/src/cvc/tests/state_exec_integration_test.cpp index b3235c3a6..4ae0e0808 100644 --- a/src/cvc/tests/state_exec_integration_test.cpp +++ b/src/cvc/tests/state_exec_integration_test.cpp @@ -2491,6 +2491,37 @@ TEST_F(SchedulerSettingsTest, AsyncSchedulerKillsAProcessWhoseScriptDoesNotParse EXPECT_NE(err->find("unclosed parenthesis"), std::string::npos) << *err; } +TEST_F(SchedulerSettingsTest, AsyncSchedulerParserDefaultsSitUnderTheTree) { + auto &root = cvc::state::instance(app_ctx); + async_scheduler sched; + sched.set_watch_root(&root); + EXPECT_EQ(sched.parser_defaults().step_tokens, 0u); // a plain async_scheduler parses at submit + EXPECT_THROW(sched.execute(std::string("(+ 1")), parse_error); + + parse_options defaults; + defaults.step_tokens = 1024; + sched.set_parser_defaults(defaults); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1"))); // now read inside the process + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + + root("state_exec.defaults.parser.step_tokens").value("0"); // the tree overrides the defaults + EXPECT_THROW(sched.execute(std::string("(+ 1")), parse_error); +} + +TEST_F(SchedulerSettingsTest, AppExecSchedulerReadsScriptsInSteps) { + auto &sched = app_ctx.exec_scheduler(); + EXPECT_EQ(sched.parser_defaults().step_tokens, 1024u); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1 (+ 2"))); + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + const auto err = sched.get_exit_error(pid); + ASSERT_TRUE(err); + EXPECT_NE(err->find("unclosed parenthesis"), std::string::npos) << *err; +} + TEST_F(SchedulerSettingsTest, AsyncSchedulerEvaluatesAScriptReadInSlices) { auto &root = cvc::state::instance(app_ctx); root("state_exec.defaults.parser.step_tokens").value("3"); From 9ed8702fa4e497c5d464fe4b8f620f2b3c79f752 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 13:41:38 -0500 Subject: [PATCH 08/13] pycvc: give Exec the same state_exec defaults as the app's exec scheduler pycvc's Exec runs programs on a private async_scheduler that already watches the app's state tree, but its built-in parser defaults were plain parse_options{} while app::exec_scheduler() read scripts in 1024-token steps. Both now take async_scheduler::host_parser_defaults(), one definition (with the reasoning for 1024) for every libcvc host, and the app tree's state_exec.defaults.parser.* keys override both. Exec.run() still raises a RuntimeError with the parse error's message: from the killed process's exit error when the script is read in steps, or from execute() when the tree sets step_tokens to 0. Tests: test_pycvc_exec.py checks the parse error's message and that max_depth / step_tokens on the app tree configure Exec; the integration test pins the app scheduler to host_parser_defaults(). --- bindings/pycvc/pycvc_exec.cpp | 9 +++- bindings/pycvc/test_pycvc_exec.py | 42 +++++++++++++++++++ docs/STATE_EXEC_GRAMMAR.md | 15 ++++--- inc/cvc/state/state_exec/async_scheduler.h | 16 ++++++- src/cvc/core/app.cpp | 12 ++---- src/cvc/tests/state_exec_integration_test.cpp | 5 ++- 6 files changed, 81 insertions(+), 18 deletions(-) diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 27ea4d652..441dfb940 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -378,6 +378,10 @@ Exec::Exec(const std::shared_ptr &app) : impl_(std::make_sharedcomputePool(); impl_->sched.set_watch_root(impl_->root); + // The same state_exec defaults as the app's exec scheduler, so a program behaves the same from + // Python as from Ariadne: scripts are read inside the process (host_parser_defaults), and the + // app tree's state_exec.defaults.parser.* keys override them for both. + impl_->sched.set_parser_defaults(async_scheduler::host_parser_defaults()); impl_->host_proc = make_process(); impl_->host_proc->pid = 0; @@ -554,7 +558,10 @@ std::string Exec::run(const std::string &src) { execute_options opts; opts.name = "pycvc"; opts.env = impl_->env; - const int pid = sched.execute(src, opts); // may throw on parse error (nothing to clean up yet) + // A syntax error is read inside the process (parser.step_tokens) and surfaces below as the + // killed process's exit error; with step_tokens = 0 on the tree, execute() throws it here + // instead (nothing to clean up yet). Either way run() raises with the parse error's message. + const int pid = sched.execute(src, opts); // Drive the private scheduler to completion. A purely synchronous program finishes in the first // slice; a program that parks (await / an async fn) yields, and the pump waits for a compute-pool diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index dff90e373..eb627bcbf 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -36,6 +36,48 @@ def test_parse_error_raises(): raise AssertionError("a parse error must raise") +def _run_error(ex, src): + """run(src) must raise; return the message.""" + try: + ex.run(src) + except Exception as e: # noqa: BLE001 - pycvc surfaces C++ errors as RuntimeError + return str(e) + raise AssertionError(f"{src!r} must raise") + + +def test_parse_error_names_the_problem(): + # Exec uses the same state_exec defaults as the app's exec scheduler: the script + # is read inside its process, so a syntax error comes back as the process's exit + # error. run() raises with the parse error's own message, as it did when the + # script was parsed at submit. + app = pycvc.make_app() + ex = pycvc.Exec(app) + assert "unclosed parenthesis" in _run_error(ex, "(begin (set x") + + +def test_parser_settings_come_from_the_app_tree(): + # Exec's scheduler watches the app's state tree, so state_exec.defaults.parser.* + # configures it exactly as it configures the app's exec scheduler + # (docs/STATE_EXEC_GRAMMAR.md). + app = pycvc.make_app() + ex = pycvc.Exec(app) + deep = "(+ 1 (+ 2 (+ 3 (+ 4 5))))" # four levels of nesting + assert ex.run(deep) == "15" + + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "3") + assert "nesting deeper than 3" in _run_error(ex, deep) + assert ex.run("(+ 1 (+ 2 3))") == "6" # three levels still parse + + # Parsing at submit (step_tokens 0) gives the same results and the same errors. + pycvc.state_set(app, "state_exec.defaults.parser.step_tokens", "0") + assert "nesting deeper than 3" in _run_error(ex, deep) + assert "unclosed parenthesis" in _run_error(ex, "(begin (set x") + assert ex.run("(+ 1 (+ 2 3))") == "6" + + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "256") + assert ex.run(deep) == "15" + + # ── DSL <-> state <-> Python round trips (one shared app) ─────────────── diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index 30605c60d..c6045b560 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -154,7 +154,7 @@ next one in the order applies. `max_depth` must also be positive. | Key | Meaning | Default | |---|---|---| | `max_depth` | how many lists and quotes may nest | 256 | -| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler | +| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler and pycvc's `Exec` | Who reads them: @@ -165,14 +165,19 @@ Who reads them: watch root each time a script is submitted. Keys that aren't set fall back to the scheduler's own defaults (`set_parser_defaults()`), which are the built-in ones unless the host changes them. - - **The app's exec scheduler** (`app::exec_scheduler()`, which runs Ariadne's actions and - residents) sets `step_tokens` = 1024, so it reads scripts inside their processes. + - **libcvc's own hosts all use `async_scheduler::host_parser_defaults()`** (`step_tokens` = + 1024), so a script behaves the same in each: + - the app's exec scheduler (`app::exec_scheduler()`, which runs Ariadne's actions and + residents); + - pycvc's `Exec`, on its private scheduler. That scheduler watches the same app tree, so the + same keys override both. - **Why 1024:** every script in the corpus fits in one slice (the largest is 159 tokens, and `.ari` scripts are at most 24). So a normal action pays one extra step out of its 5000. A slice costs about 80 µs at the parser's measured ~80 ns per token, so a pathologically large script is spread over steps and frames instead of stalling a drain. - - pycvc's `Exec` keeps a private `async_scheduler` and stays at 0, so its `run()` still raises - on a syntax error straight away. + - pycvc's `Exec.run()` raises a `RuntimeError` carrying the parse error either way: from the + killed process's exit error, or straight from `execute()` when the tree sets + `step_tokens` = 0. - **Ariadne** reads `state_exec.defaults.parser.*` from the app's tree when it compiles a predicate. - **Code without a state tree** (a standalone evaluator, the `.ari` channel lint) uses the diff --git a/inc/cvc/state/state_exec/async_scheduler.h b/inc/cvc/state/state_exec/async_scheduler.h index bb440d3de..8af85f86f 100644 --- a/inc/cvc/state/state_exec/async_scheduler.h +++ b/inc/cvc/state/state_exec/async_scheduler.h @@ -165,11 +165,23 @@ class async_scheduler : public scheduler_base { /// This scheduler's built-in parser settings: what a submitted script is parsed with when the /// watch root's `state_exec.defaults.parser.*` keys don't say otherwise. parse_options{} (parse - /// at submit) unless the host sets it; the app's exec scheduler reads in steps - /// (app::exec_scheduler()). + /// at submit) unless the host sets it. void set_parser_defaults(const parse_options &defaults) { parser_defaults_ = defaults; } const parse_options &parser_defaults() const { return parser_defaults_; } + /// The parser defaults libcvc's own hosts give their async schedulers, so a script behaves the + /// same in each: app::exec_scheduler() (Ariadne) and pycvc's Exec. Scripts are read inside + /// their process, 1024 tokens per step (docs/STATE_EXEC_GRAMMAR.md, "Reading inside the + /// scheduler"). Every script the repo carries fits in one slice (the largest is 159 tokens; + /// .ari scripts are at most 24), so a normal action pays one extra step of its 5000. A slice + /// costs about 80 us at the parser's measured ~80 ns per token, so a pathologically large + /// script is spread over steps instead of stalling a drain. + static parse_options host_parser_defaults() { + parse_options defaults; + defaults.step_tokens = 1024; + return defaults; + } + private: scheduling_policy policy_; int next_pid_ = 1; diff --git a/src/cvc/core/app.cpp b/src/cvc/core/app.cpp index 7df1bb94f..5af747911 100644 --- a/src/cvc/core/app.cpp +++ b/src/cvc/core/app.cpp @@ -1022,15 +1022,9 @@ cvc::state_exec::async_scheduler &app::exec_scheduler() { // Wire it to this app's state root so state-watches (state-watch/on_change:) // resolve against the app tree. Processes are chrooted per-owner on submit. _execScheduler->set_watch_root(&root()); - // Read submitted scripts inside their processes, 1024 tokens per step (docs/ - // STATE_EXEC_GRAMMAR.md, "Reading inside the scheduler"). Every script the repo carries fits - // in one slice (the largest is 159 tokens; .ari scripts are at most 24), so a normal - // action costs one extra step out of its 5000. A slice costs about 80 us at the parser's - // measured ~80 ns per token, so a pathologically large script is spread over steps and frames - // instead of stalling a drain. state_exec.defaults.parser.step_tokens overrides it. - cvc::state_exec::parse_options parser_defaults; - parser_defaults.step_tokens = 1024; - _execScheduler->set_parser_defaults(parser_defaults); + // Read submitted scripts inside their processes, like pycvc's Exec (see host_parser_defaults + // for why 1024 tokens a step). state_exec.defaults.parser.* on this tree overrides it. + _execScheduler->set_parser_defaults(cvc::state_exec::async_scheduler::host_parser_defaults()); } return *_execScheduler; } diff --git a/src/cvc/tests/state_exec_integration_test.cpp b/src/cvc/tests/state_exec_integration_test.cpp index 4ae0e0808..561f38ca7 100644 --- a/src/cvc/tests/state_exec_integration_test.cpp +++ b/src/cvc/tests/state_exec_integration_test.cpp @@ -2512,7 +2512,10 @@ TEST_F(SchedulerSettingsTest, AsyncSchedulerParserDefaultsSitUnderTheTree) { TEST_F(SchedulerSettingsTest, AppExecSchedulerReadsScriptsInSteps) { auto &sched = app_ctx.exec_scheduler(); - EXPECT_EQ(sched.parser_defaults().step_tokens, 1024u); + EXPECT_EQ(async_scheduler::host_parser_defaults().step_tokens, 1024u); + EXPECT_EQ(sched.parser_defaults().step_tokens, + async_scheduler::host_parser_defaults().step_tokens); // shared with pycvc's Exec + EXPECT_EQ(sched.parser_defaults().max_depth, async_scheduler::host_parser_defaults().max_depth); int pid = -1; ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1 (+ 2"))); sched.sync_run(1000, 1.0); From 753abb17fe1098c6a19707a960c768f4124bf43c Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 14:07:25 -0500 Subject: [PATCH 09/13] pycvc: expose the state_exec parser-settings surface under its C++ names pycvc's state_exec surface was only Exec; the parser settings these PRs add had no Python counterpart, so the defaults weren't even observable. In pycvc's direct-wrap style (the real C++ types, no facade): - pycvc.parse_options: the real cvc::state_exec::parse_options struct (max_depth, step_tokens), C++ defaults included. - pycvc.async_scheduler.host_parser_defaults(): the real static, declared to SWIG on its own (async_scheduler isn't wrapped yet, so no constructor). - pycvc.parse_options_from_state(app, scheduler_id="", base=...): the C++ resolver, taking the app where C++ takes the root (pycvc's convention). - Exec.parser_defaults() / Exec.set_parser_defaults(): the same calls as the async_scheduler Exec wraps. test_pycvc_exec.py covers each against the C++ behaviour; the README gains a state_exec section. Built locally (SWIG 4.4.1, Python 3.11): exec, exec_async, exec_coro and ari tests, 26 passed. --- bindings/pycvc/README.md | 24 +++++++++++++++ bindings/pycvc/pycvc.i | 26 ++++++++++++++++ bindings/pycvc/pycvc_exec.cpp | 16 ++++++++++ bindings/pycvc/pycvc_exec.h | 15 ++++++++++ bindings/pycvc/test_pycvc_exec.py | 49 +++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+) diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index c77d513de..2d3952d22 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -184,3 +184,27 @@ Every widget reads/writes `cvc::state`, so the same value is drivable from a script, a config file or a replicated peer. Raw custom-widget ImGui (free-form windows) is a separate, deferred design (a curated subset compiled inside cvcGL). Contract test: `test_pycvc_gl_imgui.py`. + +## state_exec DSL (`Exec`) and parser settings + +`pycvc.Exec(app)` runs state_exec programs over the app's state tree. The parser +settings are the C++ surface, under the C++ names (see +`docs/STATE_EXEC_GRAMMAR.md`, "Parser settings"): + +```python +import pycvc +app = pycvc.make_app() +ex = pycvc.Exec(app) +ex.run("(+ 1 2)") # "3" + +pycvc.parse_options() # max_depth 256, step_tokens 0 +pycvc.async_scheduler.host_parser_defaults() # step_tokens 1024 +ex.parser_defaults() # == host_parser_defaults() +pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "64") # tree overrides +pycvc.parse_options_from_state(app, "", ex.parser_defaults()) # what Exec uses +``` + +`Exec` starts from the same defaults as the app's exec scheduler (Ariadne), and +the same `state_exec.defaults.parser.*` keys override both. A syntax error +raises a `RuntimeError` with the parse error's message. Contract test: +`test_pycvc_exec.py`. diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 535391b20..404a59e87 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1437,6 +1437,32 @@ class AsyncStateObserver(state_observer): self._workers = [] %} +// ── state_exec parser settings (docs/STATE_EXEC_GRAMMAR.md, "Parser settings") ── +// DIRECT WRAP, under the C++ names: the real cvc::state_exec::parse_options +// struct and async_scheduler::host_parser_defaults(). Only these members are +// declared to SWIG; the rest of parser.h / async_scheduler.h (value_t, the +// parser, the scheduler itself) is not wrapped yet, so async_scheduler has no +// Python constructor. parse_options_from_state and Exec's parser_defaults / +// set_parser_defaults are in pycvc_exec.h. +%{ +#include +#include +%} +namespace cvc { +namespace state_exec { +struct parse_options { + size_t max_depth; // lists and quotes that may nest (default 256) + size_t step_tokens; // 0: parse at submit; N: read N tokens per process step +}; +%nodefaultctor async_scheduler; +%nodefaultdtor async_scheduler; +class async_scheduler { +public: + static parse_options host_parser_defaults(); +}; +} +} + // ── Phase 4: DSL execution + Python-authored DSL functions ────────────── // pycvc.Exec(app) runs state_exec DSL programs over app's state root and lets a // Python callable be registered as a DSL function (register_fn). PyObject* is diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 441dfb940..94f5c8d28 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -648,4 +648,20 @@ std::string Exec::run(const std::string &src) { return render_result(*result); } +cvc::state_exec::parse_options Exec::parser_defaults() const { + return impl_->sched.parser_defaults(); +} + +void Exec::set_parser_defaults(const cvc::state_exec::parse_options &defaults) { + impl_->sched.set_parser_defaults(defaults); +} + +cvc::state_exec::parse_options +parse_options_from_state(const std::shared_ptr &app, const std::string &scheduler_id, + const cvc::state_exec::parse_options &base) { + if (!app) + throw std::invalid_argument("pycvc.parse_options_from_state: null app handle"); + return cvc::state_exec::parse_options_from_state(cvc::state::instance(*app), scheduler_id, base); +} + } // namespace pycvc diff --git a/bindings/pycvc/pycvc_exec.h b/bindings/pycvc/pycvc_exec.h index 192de81d5..55c3c9482 100644 --- a/bindings/pycvc/pycvc_exec.h +++ b/bindings/pycvc/pycvc_exec.h @@ -7,6 +7,7 @@ // registered here becomes callable from DSL source like any builtin. #pragma once +#include // parse_options #include #include @@ -20,6 +21,13 @@ typedef _object PyObject; namespace pycvc { +// cvc::state_exec::parse_options_from_state on `app`'s state root (pycvc passes the app where +// C++ passes the root): per key, state_exec.schedulers..parser. (when +// scheduler_id is not empty), then state_exec.defaults.parser., then `base`. +cvc::state_exec::parse_options parse_options_from_state( + const std::shared_ptr &app, const std::string &scheduler_id = "", + const cvc::state_exec::parse_options &base = cvc::state_exec::parse_options()); + class Exec { public: // Build a DSL environment + scheduler over `app`'s state root. @@ -86,6 +94,13 @@ class Exec { // (each owns its own scheduler, so they do not interfere). std::string run(const std::string &src); + // The parser settings this Exec's scheduler parses programs with when the app tree's + // state_exec.defaults.parser.* keys don't say otherwise; the same calls as + // async_scheduler::parser_defaults() / set_parser_defaults(). They start as + // async_scheduler::host_parser_defaults(), as the app's exec scheduler's do. + cvc::state_exec::parse_options parser_defaults() const; + void set_parser_defaults(const cvc::state_exec::parse_options &defaults); + private: struct ExecImpl; std::shared_ptr impl_; diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index eb627bcbf..6d932392b 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -78,6 +78,55 @@ def test_parser_settings_come_from_the_app_tree(): assert ex.run(deep) == "15" +# ── state_exec parser settings: the C++ surface, under the C++ names ──── + + +def test_parse_options_is_the_cpp_struct(): + o = pycvc.parse_options() + assert (o.max_depth, o.step_tokens) == (256, 0) # C++ defaults + o.max_depth = 32 + o.step_tokens = 8 + assert (o.max_depth, o.step_tokens) == (32, 8) + + +def test_exec_starts_from_the_host_parser_defaults(): + # The same defaults as the app's exec scheduler: + # async_scheduler::host_parser_defaults(). + host = pycvc.async_scheduler.host_parser_defaults() + assert (host.max_depth, host.step_tokens) == (256, 1024) + ex = pycvc.Exec(pycvc.make_app()) + d = ex.parser_defaults() + assert (d.max_depth, d.step_tokens) == (host.max_depth, host.step_tokens) + + +def test_parse_options_from_state_resolves_like_cpp(): + app = pycvc.make_app() + # Nothing set: the C++ default. + assert pycvc.parse_options_from_state(app).step_tokens == 0 + # What Exec reads with: its own defaults under the tree. + ex = pycvc.Exec(app) + effective = pycvc.parse_options_from_state(app, "", ex.parser_defaults()) + assert effective.step_tokens == 1024 + + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "40") + pycvc.state_set(app, "state_exec.schedulers.s1.parser.max_depth", "50") + assert pycvc.parse_options_from_state(app).max_depth == 40 + assert pycvc.parse_options_from_state(app, "s1").max_depth == 50 + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "nope") # bad: skipped + assert pycvc.parse_options_from_state(app).max_depth == 256 + + +def test_set_parser_defaults_changes_how_exec_parses(): + ex = pycvc.Exec(pycvc.make_app()) + o = pycvc.parse_options() + o.max_depth = 2 + ex.set_parser_defaults(o) + assert ex.parser_defaults().max_depth == 2 + assert ex.parser_defaults().step_tokens == 0 + assert "nesting deeper than 2" in _run_error(ex, "(+ 1 (+ 2 (+ 3 4)))") + assert ex.run("(+ 1 (+ 2 3))") == "6" + + # ── DSL <-> state <-> Python round trips (one shared app) ─────────────── From 4d3bc6be68a2bee5a3cf9eb657b73e51be18d2e7 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 14:22:54 -0500 Subject: [PATCH 10/13] state_exec: free, print, compare, copy and snapshot deep values without native recursion The parser's nesting limit (parser.max_depth, default 256) exists because the reader recursed; it can rise only once nothing else recurses once per level of a value. Freeing, printing, comparing, copying, macro-expanding and snapshotting a value all did. - Release: value_tag and environment destructors drop the last owner of a list, dict, closure, generator or enclosing scope through detail::release_nested. The first release on a thread runs a loop over a local queue (8 owners inline, the rest on the heap); owners dropped while it frees something are queued, not freed inline. Dropping an owner that is not the last is still a plain decrement. - environment::lookup and set_existing are loops. - to_string, values_equal, deep_copy (state-data-get) and the stackless evaluator's macro substitute keep their work on heap stacks. The 1000-level caps of to_string, values_equal and deep_copy are gone: they did not protect small stacks (to_string overflowed a 256 KiB stack at 800 levels) and would have become functional limits once max_depth rises. to_string's 64 KiB length cap stays. - Snapshot codec: encode_value and decode_value are iterative. cvc::state itself recurses once per tree level on every write (fullName() and the childChanged climb to the root), so a list or dict more than 8 levels below where its stretch of nesting began goes into a __spill__ table under the encode root, referenced by __type__ "spill_ref" and the entry's index. Values at most 8 levels deep encode exactly as before. A value that contains itself throws instead of overflowing; decode rejects bad or reused references. __len__ is written once per container, not once per element. New state_exec_deep_data_test builds values 100000 levels deep at runtime and drops, prints, compares, macro-expands and snapshots them (10000 levels for the codec) on a 256 KiB thread. state_exec_bench_test gains ValueDropThroughput. --- inc/cvc/state/state_exec/types.h | 46 ++- src/cvc/state/state_exec/intrinsics.cpp | 86 ++-- .../state/state_exec/stackless_evaluator.cpp | 35 +- .../state/state_exec/state_value_codec.cpp | 240 ++++++++--- src/cvc/state/state_exec/types.cpp | 235 +++++++---- src/cvc/tests/CMakeLists.txt | 11 + src/cvc/tests/state_exec_bench_test.cpp | 54 +++ src/cvc/tests/state_exec_deep_data_test.cpp | 390 ++++++++++++++++++ src/cvc/tests/state_exec_intrinsics_test.cpp | 26 +- 9 files changed, 941 insertions(+), 182 deletions(-) create mode 100644 src/cvc/tests/state_exec_deep_data_test.cpp diff --git a/inc/cvc/state/state_exec/types.h b/inc/cvc/state/state_exec/types.h index 9a6b34d53..0ce3556cb 100644 --- a/inc/cvc/state/state_exec/types.h +++ b/inc/cvc/state/state_exec/types.h @@ -71,6 +71,24 @@ struct bytes_value { bool operator==(const bytes_value &o) const { return data == o.data; } }; +namespace detail { +/// Drop one owner of a nested value's storage (a list, dict, closure, generator or +/// environment) without recursing once per nesting level. Freeing a 100000-deep list through +/// plain shared_ptr destructors takes a native stack frame per level and overflows the stack. +/// Here the first release on a thread runs a loop: owners dropped while it frees something +/// (that thing's elements, bindings, enclosing scope) are queued and freed one at a time by the +/// loop, so the stack stays a few frames deep whatever the depth. +void release_nested(std::shared_ptr owner) noexcept; + +/// Drop `owner` through release_nested when it is the last one; dropping any other owner only +/// decrements a count. (Should another thread drop the last but one meanwhile, this drop frees +/// the storage inline, and its elements start releases of their own: a few frames, still.) +template void drop_owner(std::shared_ptr &owner) noexcept { + if (owner && owner.use_count() == 1) + release_nested(std::move(owner)); +} +} // namespace detail + /// The DSL value variant. All runtime values are represented as value_t. /// /// Variants: @@ -114,6 +132,23 @@ struct value_tag { value_tag(generator_ptr g) : v(std::move(g)) {} value_tag(bytes_value b) : v(std::move(b)) {} + value_tag(const value_tag &) = default; + value_tag(value_tag &&) = default; + value_tag &operator=(const value_tag &) = default; + value_tag &operator=(value_tag &&) = default; + + /// Nested storage is freed through detail::drop_owner, never recursively. + ~value_tag() { + if (auto *l = std::get_if(&v)) + detail::drop_owner(*l); + else if (auto *d = std::get_if(&v)) + detail::drop_owner(*d); + else if (auto *c = std::get_if(&v)) + detail::drop_owner(*c); + else if (auto *g = std::get_if(&v)) + detail::drop_owner(*g); + } + /// Check if the value is nil (monostate). bool is_nil() const { return std::holds_alternative(v); } @@ -145,6 +180,9 @@ struct environment { std::unordered_map bindings; std::shared_ptr outer; + /// Drops `outer` through detail::drop_owner, so a long scope chain is not freed recursively. + ~environment(); + /// Look up a name, walking the scope chain. value_t *lookup(const std::string &name); const value_t *lookup(const std::string &name) const; @@ -189,13 +227,15 @@ inline const value_t true_value{true}; inline const value_t false_value{false}; /// Convert a value_t to a human-readable string representation. Bounded and cycle-safe: -/// output is length- and depth-capped (a shared/deep/cyclic structure is truncated with an -/// ellipsis rather than expanding exponentially or overflowing the stack). +/// output is length-capped (a shared/cyclic structure is truncated with an ellipsis rather +/// than expanding exponentially), and the walk is iterative, so depth alone neither truncates +/// it nor overflows the stack. std::string to_string(const value_t &val); /// Check structural equality of two values. Memoized on pointer-pairs, so a shared or /// cyclic structure is compared in time proportional to its PHYSICAL size (distinct node -/// pairs), never its exponential logical unfolding. +/// pairs), never its exponential logical unfolding. Iterative, so any depth compares; it polls +/// the evaluation deadline and throws std::runtime_error once that is spent. bool values_equal(const value_t &a, const value_t &b); // -- time budgets ------------------------------------------------------------- diff --git a/src/cvc/state/state_exec/intrinsics.cpp b/src/cvc/state/state_exec/intrinsics.cpp index 1de778b3b..43ca8ce6c 100644 --- a/src/cvc/state/state_exec/intrinsics.cpp +++ b/src/cvc/state/state_exec/intrinsics.cpp @@ -92,51 +92,51 @@ std::string coerce_state_string(const value_t &v) { // not alias node-resident storage. Bounded and cycle-safe, mirroring values_equal / to_string // (it is the third structural walker and must not be the weak link): it MEMOIZES on the source // node pointer, so a shared-pointer DAG is copied in PHYSICAL time (the copy preserves the same -// sharing, disjoint from the stored original) and a cycle terminates; it caps recursion depth -// (stack safety on a deeply-nested value); and it polls the evaluation deadline so a huge -// payload aborts at the time budget instead of running to completion in one native step. -constexpr int kDeepCopyMaxDepth = 1000; - -value_t deep_copy_impl(const value_t &v, std::unordered_map &memo, - int depth) { - if (eval_deadline_expired()) - throw std::runtime_error("state-data-get: evaluation exceeded time budget"); - if (depth > kDeepCopyMaxDepth) - throw std::runtime_error("state-data-get: value nested too deeply to copy"); - if (auto *l = std::get_if(&v.v)) { - if (!*l) - return v; - auto found = memo.find(static_cast(l->get())); - if (found != memo.end()) - return found->second; // shared node / cycle already copied -> reuse (bounds the walk) - auto out = std::make_shared>(); - value_t result{out}; - memo.emplace(static_cast(l->get()), result); // register BEFORE recursing - out->reserve((*l)->size()); - for (const auto &e : **l) - out->push_back(deep_copy_impl(e, memo, depth + 1)); - return result; - } - if (auto *dp = std::get_if(&v.v)) { - if (!*dp) - return v; - auto found = memo.find(static_cast(dp->get())); - if (found != memo.end()) - return found->second; - auto out = std::make_shared>>(); - value_t result{out}; - memo.emplace(static_cast(dp->get()), result); - out->reserve((*dp)->size()); - for (const auto &kv : **dp) - out->emplace_back(kv.first, deep_copy_impl(kv.second, memo, depth + 1)); - return result; - } - return v; // scalar / symbol (opaque callables are refused at state-data-set) -} - +// sharing, disjoint from the stored original) and a cycle terminates; it keeps the elements +// still to copy on a heap stack, so any depth copies without native recursion; and it polls +// the evaluation deadline so a huge payload aborts at the time budget instead of running to +// completion in one native step. value_t deep_copy(const value_t &v) { std::unordered_map memo; - return deep_copy_impl(v, memo, 0); + value_t result; + // (source, destination) pairs: each destination is a slot of a copy already allocated at its + // final size, so it stays put while the copy fills in. + std::vector> pending{{&v, &result}}; + while (!pending.empty()) { + if (eval_deadline_expired()) + throw std::runtime_error("state-data-get: evaluation exceeded time budget"); + auto [src, dst] = pending.back(); + pending.pop_back(); + if (auto *l = std::get_if(&src->v); l && *l) { + auto found = memo.find(static_cast(l->get())); + if (found != memo.end()) { + *dst = found->second; // shared node / cycle already copied -> reuse (bounds the walk) + continue; + } + auto out = std::make_shared>((*l)->size()); + *dst = value_t{out}; + memo.emplace(static_cast(l->get()), *dst); // register BEFORE descending + for (std::size_t i = (*l)->size(); i-- > 0;) + pending.emplace_back(&(**l)[i], &(*out)[i]); + } else if (auto *dp = std::get_if(&src->v); dp && *dp) { + auto found = memo.find(static_cast(dp->get())); + if (found != memo.end()) { + *dst = found->second; + continue; + } + auto out = std::make_shared>>(); + out->reserve((*dp)->size()); + for (const auto &kv : **dp) + out->emplace_back(kv.first, value_t{}); + *dst = value_t{out}; + memo.emplace(static_cast(dp->get()), *dst); + for (std::size_t i = (*dp)->size(); i-- > 0;) + pending.emplace_back(&(**dp)[i].second, &(*out)[i].second); + } else { + *dst = *src; // scalar / symbol / null (opaque callables are refused at state-data-set) + } + } + return result; } void require_root(const intrinsics_context *ctx, const char *name) { diff --git a/src/cvc/state/state_exec/stackless_evaluator.cpp b/src/cvc/state/state_exec/stackless_evaluator.cpp index 2aa10014b..a1483a6a3 100644 --- a/src/cvc/state/state_exec/stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/stackless_evaluator.cpp @@ -1112,22 +1112,27 @@ void stackless_evaluator::handle_defclass(evaluator_state &state, eval_frame &fr value_t stackless_evaluator::substitute(const value_t &tmpl, const std::unordered_map &subst) { - if (auto *sym = std::get_if(&tmpl.v)) { - auto it = subst.find(sym->name); - if (it != subst.end()) - return it->second; - return tmpl; - } - if (auto *lp = std::get_if(&tmpl.v)) { - if (!*lp) - return tmpl; - auto result = std::make_shared>(); - result->reserve((*lp)->size()); - for (auto &e : **lp) - result->push_back(substitute(e, subst)); - return value_t{std::move(result)}; + // Like the evaluator, the copy keeps its work on the heap, not the native stack, so a + // template of any depth expands. (source, destination) pairs: each destination is a slot of + // a list copy allocated at its final size, so it stays put while the copy fills in. + value_t result; + std::vector> pending{{&tmpl, &result}}; + while (!pending.empty()) { + auto [src, dst] = pending.back(); + pending.pop_back(); + if (auto *sym = std::get_if(&src->v)) { + auto it = subst.find(sym->name); + *dst = it != subst.end() ? it->second : *src; + } else if (auto *lp = std::get_if(&src->v); lp && *lp) { + auto copy = std::make_shared>((*lp)->size()); + *dst = value_t{copy}; + for (std::size_t i = (*lp)->size(); i-- > 0;) + pending.emplace_back(&(**lp)[i], &(*copy)[i]); + } else { + *dst = *src; + } } - return tmpl; + return result; } } // namespace cvc::state_exec diff --git a/src/cvc/state/state_exec/state_value_codec.cpp b/src/cvc/state/state_exec/state_value_codec.cpp index 0555ff7dc..1106f07b4 100644 --- a/src/cvc/state/state_exec/state_value_codec.cpp +++ b/src/cvc/state/state_exec/state_value_codec.cpp @@ -10,8 +10,11 @@ #include #include #include +#include #include #include +#include +#include namespace cvc::state_exec { @@ -95,13 +98,113 @@ std::string b64_decode(const std::string &in) { return out; } +// Nested lists and dicts are written and read with explicit stacks, not native recursion, and +// the state subtree they make is kept shallow too: cvc::state recurses once per TREE level on +// every write (fullName() builds the path parent by parent, childChanged() climbs to the root), +// so a list nested 100000 deep must not become a subtree 200000 deep. A list or dict +// kSpillDepth levels below where its stretch of nesting began is written into the __spill__ +// table under the node encode_value was called on, as an entry of its own, and the place it +// would have gone holds a reference: __type__ "spill_ref", value the entry's index. Its own +// nesting starts again from that entry. Values at most kSpillDepth levels deep are written +// exactly as before, with no table. +// +// 8 levels make at most ~30 tree levels per stretch (a dict level is 3: __entries__, the +// index, __val__). A write that deep, under a snapshot root ~5 levels down, fits a 64 KiB +// stack (wasm's); with 16, the dicts' ~55 did not. +constexpr std::size_t kSpillDepth = 8; +const char *SPILL_KEY = "__spill__"; + +// The non-null list or dict a value holds, or null. +const void *container_of(const value_t &v) { + if (auto *l = std::get_if(&v.v)) + return l->get(); + if (auto *d = std::get_if(&v.v)) + return d->get(); + return nullptr; +} + +// Write a value that is not a non-null list or dict. +void encode_leaf(cvc::state &node, const value_t &val); + +// Read a value whose __type__ is `type`, which is not "list", "dict" or "spill_ref". +value_t decode_leaf(cvc::state &node, const std::string &type); + } // namespace // --------------------------------------------------------------------------- // encode_value // --------------------------------------------------------------------------- -void encode_value(cvc::state &node, const value_t &val) { +void encode_value(cvc::state &root, const value_t &val) { + // A list or dict whose elements are being written. + struct frame { + const value_t *val; + cvc::state *items; // its __elems__ or __entries__ node + std::size_t next; // the next element to write + std::size_t size; // its length, as written to __len__ + std::size_t nest; // how many levels below its stretch's start it is + }; + std::vector stack; + std::unordered_set open; // the lists and dicts on `stack` + std::optional spill; + + // Write `v` into `node`: a leaf now, a list or dict by its type and a frame for its elements. + auto begin = [&](cvc::state &node, const value_t &v, std::size_t nest) { + const void *c = container_of(v); + if (!c) { + encode_leaf(node, v); + return; + } + if (!open.insert(c).second) + throw std::runtime_error("encode_value: the value contains itself"); + cvc::state *target = &node; + if (nest >= kSpillDepth) { + if (!spill) + spill.emplace(root(SPILL_KEY)); + set_type(node, "spill_ref"); + node.value(std::to_string(spill->size())); + target = &spill->push_back(); + nest = 0; + } + const bool is_list = std::holds_alternative(v.v); + set_type(*target, is_list ? "list" : "dict"); + cvc::state &items = (*target)(is_list ? "__elems__" : "__entries__"); + // The length is known up front, so it is written once, not once per element as + // state_list::push_back would (every write signals each ancestor). An empty list or dict + // gets the "0" that state_list's constructor writes, as before. + const std::size_t size = + is_list ? std::get(v.v)->size() : std::get(v.v)->size(); + if (size > 0) + items("__len__").value(size); + else + (void)state_list(items); + stack.push_back({&v, &items, 0, size, nest}); + }; + + begin(root, val, 0); + while (!stack.empty()) { + frame &f = stack.back(); + if (f.next == f.size) { + open.erase(container_of(*f.val)); + stack.pop_back(); + continue; + } + const std::size_t i = f.next++; + const std::size_t nest = f.nest + 1; + cvc::state &slot = state_list(*f.items).at(i); + if (auto *l = std::get_if(&f.val->v)) { + begin(slot, (**l)[i], nest); // may push, invalidating f + } else { + const auto &[k, v] = (*std::get(f.val->v))[i]; + slot("__key__").value(k); + begin(slot("__val__"), v, nest); // may push, invalidating f + } + } +} + +namespace { + +void encode_leaf(cvc::state &node, const value_t &val) { std::visit( [&](auto &&arg) { using T = std::decay_t; @@ -127,25 +230,9 @@ void encode_value(cvc::state &node, const value_t &val) { set_type(node, "symbol"); node.value(arg.name); } else if constexpr (std::is_same_v) { - set_type(node, "list"); - if (arg) { - state_list sl(node("__elems__")); - for (auto &e : *arg) { - auto &elem = sl.push_back(); - encode_value(elem, e); - } - } + set_type(node, "list"); // null: a non-null list is encode_value's } else if constexpr (std::is_same_v) { - set_type(node, "dict"); - if (arg) { - auto &entries_node = node("__entries__"); - state_list sl(entries_node); - for (auto &[k, v] : *arg) { - auto &entry = sl.push_back(); - entry("__key__").value(k); - encode_value(entry("__val__"), v); - } - } + set_type(node, "dict"); // null: a non-null dict is encode_value's } else if constexpr (std::is_same_v) { set_type(node, "closure"); if (arg) { @@ -187,13 +274,96 @@ void encode_value(cvc::state &node, const value_t &val) { val.v); } +} // namespace + // --------------------------------------------------------------------------- // decode_value // --------------------------------------------------------------------------- -value_t decode_value(cvc::state &node) { - std::string type = get_type(node); +value_t decode_value(cvc::state &root) { + // A list or dict whose elements are being read. + struct frame { + cvc::state *items; // its __elems__ or __entries__ node + std::size_t next; // the next element to read + std::size_t size; + std::vector *list; + ordered_map *dict; + }; + std::vector stack; + std::optional spill; + std::vector spill_used; // each entry is referenced once; a second reference is corrupt + + // Read the value at `node` into `out`: a leaf now, a list or dict as an empty container that + // a frame fills. + auto begin = [&](cvc::state &node, value_t &out) { + cvc::state *src = &node; + std::string type = get_type(node); + if (type == "spill_ref") { + if (!spill) { + if (!has_child(root, SPILL_KEY)) + throw std::runtime_error("decode_value: spill_ref without a spill table"); + spill.emplace(root(SPILL_KEY)); + spill_used.assign(spill->size(), false); + } + const std::string ref = node.value(); + std::size_t i = 0; + const auto [end, ec] = std::from_chars(ref.data(), ref.data() + ref.size(), i); + if (ec != std::errc{} || end != ref.data() + ref.size() || i >= spill_used.size() || + spill_used[i]) + throw std::runtime_error("decode_value: bad spill_ref: " + ref); + spill_used[i] = true; + src = &spill->at(i); + type = get_type(*src); + if (type != "list" && type != "dict") + throw std::runtime_error("decode_value: spill entry " + ref + " is not a list or dict"); + } + if (type == "list") { + auto elems = std::make_shared>(); + out = value_t(elems); + if (has_child(*src, "__elems__")) { + cvc::state &items = (*src)("__elems__"); + const std::size_t n = state_list(items).size(); + elems->reserve(n); + stack.push_back({&items, 0, n, elems.get(), nullptr}); + } + } else if (type == "dict") { + auto entries = std::make_shared(); + out = value_t(entries); + if (has_child(*src, "__entries__")) { + cvc::state &items = (*src)("__entries__"); + const std::size_t n = state_list(items).size(); + entries->reserve(n); + stack.push_back({&items, 0, n, nullptr, entries.get()}); + } + } else { + out = decode_leaf(*src, type); + } + }; + + value_t result; + begin(root, result); + while (!stack.empty()) { + frame &f = stack.back(); + if (f.next == f.size) { + stack.pop_back(); + continue; + } + cvc::state &item = state_list(*f.items).at(f.next++); + if (f.list) { + value_t &slot = f.list->emplace_back(); + begin(item, slot); // may push, invalidating f + } else { + std::string key = item("__key__").value(); + value_t &slot = f.dict->emplace_back(std::move(key), value_t{}).second; + begin(item("__val__"), slot); // may push, invalidating f + } + } + return result; +} + +namespace { +value_t decode_leaf(cvc::state &node, const std::string &type) { if (type.empty() || type == "nil") return nil_value; @@ -215,32 +385,6 @@ value_t decode_value(cvc::state &node) { if (type == "symbol") return value_t(symbol{node.value()}); - if (type == "list") { - auto elems = std::make_shared>(); - if (has_child(node, "__elems__")) { - state_list sl(node("__elems__")); - elems->reserve(sl.size()); - for (size_t i = 0; i < sl.size(); ++i) - elems->push_back(decode_value(sl.at(i))); - } - return value_t(std::move(elems)); - } - - if (type == "dict") { - auto entries = std::make_shared>>(); - if (has_child(node, "__entries__")) { - state_list sl(node("__entries__")); - entries->reserve(sl.size()); - for (size_t i = 0; i < sl.size(); ++i) { - auto &entry = sl.at(i); - std::string key = entry("__key__").value(); - value_t val = decode_value(entry("__val__")); - entries->emplace_back(std::move(key), std::move(val)); - } - } - return value_t(std::move(entries)); - } - if (type == "closure") { auto cls = std::make_shared(); // Params @@ -280,6 +424,8 @@ value_t decode_value(cvc::state &node) { throw std::runtime_error("decode_value: unknown type: " + type); } +} // namespace + // --------------------------------------------------------------------------- // encode_environment // --------------------------------------------------------------------------- diff --git a/src/cvc/state/state_exec/types.cpp b/src/cvc/state/state_exec/types.cpp index 7369a0977..00865238c 100644 --- a/src/cvc/state/state_exec/types.cpp +++ b/src/cvc/state/state_exec/types.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -135,6 +136,77 @@ bool eval_deadline_expired() { return spent; } +// -- releasing nested values ------------------------------------------------- + +namespace { +// The owners a release has yet to free: the first few in uninitialized inline slots, the rest +// on the heap, so freeing an ordinary small value constructs and allocates nothing extra. +class release_queue { +public: + release_queue() = default; + release_queue(const release_queue &) = delete; + release_queue &operator=(const release_queue &) = delete; + ~release_queue() { + while (count_ > 0) + slot(--count_).~shared_ptr(); + } + + // Takes `owner`, unless out of memory: then it is left with the caller. + void push(std::shared_ptr &owner) noexcept { + if (count_ < kInline) { + new (&slot(count_++)) std::shared_ptr(std::move(owner)); + return; + } + try { + heap_.push_back(std::move(owner)); + } catch (...) { + // push_back left `owner` as it was. + } + } + bool pop(std::shared_ptr &out) noexcept { + if (!heap_.empty()) { + out = std::move(heap_.back()); + heap_.pop_back(); + return true; + } + if (count_ == 0) + return false; + std::shared_ptr &s = slot(--count_); + out = std::move(s); + s.~shared_ptr(); + return true; + } + +private: + static constexpr std::size_t kInline = 8; + std::shared_ptr &slot(std::size_t i) { + return *std::launder(reinterpret_cast *>(inline_) + i); + } + alignas(std::shared_ptr) unsigned char inline_[kInline * sizeof(std::shared_ptr)]; + std::size_t count_ = 0; + std::vector> heap_; +}; + +// The queue of the release running on this thread, if one is: release_nested() called while it +// frees something (from that thing's destructor) queues its owner there instead of freeing it +// inline. A raw pointer to the running release's local queue, so no thread-exit destructor is +// involved and nothing outlives the release. +thread_local release_queue *t_release_queue = nullptr; +} // namespace + +void detail::release_nested(std::shared_ptr owner) noexcept { + if (release_queue *queue = t_release_queue) { + queue->push(owner); // out of memory: `owner` is still here, and freed here, recursively + return; + } + release_queue queue; + t_release_queue = &queue; + owner.reset(); + for (std::shared_ptr next; queue.pop(next);) + next.reset(); // a no-op unless this was the last owner + t_release_queue = nullptr; +} + // -- value_tag --------------------------------------------------------------- std::string value_tag::type_name() const { @@ -175,35 +247,35 @@ std::string value_tag::type_name() const { // -- environment ------------------------------------------------------------- +environment::~environment() { detail::drop_owner(outer); } + value_t *environment::lookup(const std::string &name) { - auto it = bindings.find(name); - if (it != bindings.end()) - return &it->second; - if (outer) - return outer->lookup(name); + for (environment *e = this; e; e = e->outer.get()) { + auto it = e->bindings.find(name); + if (it != e->bindings.end()) + return &it->second; + } return nullptr; } const value_t *environment::lookup(const std::string &name) const { - auto it = bindings.find(name); - if (it != bindings.end()) - return &it->second; - if (outer) - return outer->lookup(name); + for (const environment *e = this; e; e = e->outer.get()) { + auto it = e->bindings.find(name); + if (it != e->bindings.end()) + return &it->second; + } return nullptr; } void environment::set(const std::string &name, value_t val) { bindings[name] = std::move(val); } void environment::set_existing(const std::string &name, value_t val) { - auto it = bindings.find(name); - if (it != bindings.end()) { - it->second = std::move(val); - return; - } - if (outer) { - outer->set_existing(name, std::move(val)); - return; + for (environment *e = this; e; e = e->outer.get()) { + auto it = e->bindings.find(name); + if (it != e->bindings.end()) { + it->second = std::move(val); + return; + } } // Not found anywhere — set in this scope bindings[name] = std::move(val); @@ -218,16 +290,26 @@ environment_ptr environment::extend(environment_ptr parent) { // -- to_string --------------------------------------------------------------- namespace { -// Bounds for to_string: cap total OUTPUT length (a shared/wide structure would otherwise -// expand exponentially) and recursion DEPTH (a deep or cyclic structure would otherwise -// overflow the C++ stack). Past either, emit an ellipsis and stop — enough for debug / -// coercion / error messages, and impossible to weaponise into a hang or OOM. +// to_string caps total OUTPUT length (a shared/wide structure would otherwise expand +// exponentially, a cyclic one forever); past it, it emits an ellipsis and stops -- enough for +// debug / coercion / error messages, and impossible to weaponise into a hang or OOM. Depth +// needs no cap of its own: the walk keeps its open lists and dicts on a heap stack, not the +// native one, and each level opened costs output, so the length cap bounds that stack too. constexpr std::size_t kToStringMaxLen = std::size_t{1} << 16; // 64 KiB -constexpr int kToStringMaxDepth = 1000; +constexpr const char *kEllipsis = "\xE2\x80\xA6"; // U+2026 HORIZONTAL ELLIPSIS -void to_string_impl(const value_t &val, std::string &out, int depth) { - if (out.size() >= kToStringMaxLen || depth > kToStringMaxDepth) { - out += "\xE2\x80\xA6"; // U+2026 HORIZONTAL ELLIPSIS +// A list or dict to_string has opened and not yet closed. +struct print_frame { + const std::vector *list = nullptr; + const ordered_map *dict = nullptr; + std::size_t next = 0; // the next element to print +}; + +// Print `val`; a non-empty list or dict is only opened (its bracket printed and a frame pushed +// for to_string's loop to print its elements and close it). +void print_value(const value_t &val, std::string &out, std::vector &open) { + if (out.size() >= kToStringMaxLen) { + out += kEllipsis; return; } std::visit( @@ -274,16 +356,7 @@ void to_string_impl(const value_t &val, std::string &out, int depth) { return; } out += '('; - for (std::size_t i = 0; i < arg->size(); ++i) { - if (i > 0) - out += ' '; - if (out.size() >= kToStringMaxLen) { - out += "\xE2\x80\xA6"; - break; - } - to_string_impl((*arg)[i], out, depth + 1); - } - out += ')'; + open.push_back({arg.get(), nullptr, 0}); } else if constexpr (std::is_same_v) out += ""; else if constexpr (std::is_same_v) { @@ -292,19 +365,7 @@ void to_string_impl(const value_t &val, std::string &out, int depth) { return; } out += '{'; - for (std::size_t i = 0; i < arg->size(); ++i) { - if (i > 0) - out += ", "; - if (out.size() >= kToStringMaxLen) { - out += "\xE2\x80\xA6"; - break; - } - out += '"'; - out += (*arg)[i].first; - out += "\": "; - to_string_impl((*arg)[i].second, out, depth + 1); - } - out += '}'; + open.push_back({nullptr, arg.get(), 0}); } else if constexpr (std::is_same_v) out += ""; else if constexpr (std::is_same_v) @@ -320,7 +381,35 @@ void to_string_impl(const value_t &val, std::string &out, int depth) { std::string to_string(const value_t &val) { std::string out; - to_string_impl(val, out, 0); + std::vector open; + print_value(val, out, open); + while (!open.empty()) { + print_frame &f = open.back(); + const std::size_t i = f.next++; + const bool is_list = f.list != nullptr; + if (i == (is_list ? f.list->size() : f.dict->size())) { + out += is_list ? ')' : '}'; + open.pop_back(); + continue; + } + if (i > 0) + out += is_list ? " " : ", "; + if (out.size() >= kToStringMaxLen) { + out += kEllipsis; + out += is_list ? ')' : '}'; + open.pop_back(); + continue; + } + if (is_list) { + print_value((*f.list)[i], out, open); // may push, invalidating f + } else { + const auto &[key, value] = (*f.dict)[i]; + out += '"'; + out += key; + out += "\": "; + print_value(value, out, open); // may push, invalidating f + } + } return out; } @@ -333,19 +422,13 @@ struct ptr_pair_hash { } }; using ptr_pair_set = std::unordered_set, ptr_pair_hash>; +using value_pairs = std::vector>; -constexpr int kValuesEqualMaxDepth = 1000; // guard the C++ stack on a deeply-nested compare - -// `seen` memoizes the (a,b) pointer-pairs already under comparison. Re-encountering a pair -// (a shared node reached by two paths, or a cycle) returns true co-inductively: this bounds -// work to the number of DISTINCT node-pairs (the physical size), never the exponential -// logical unfolding of a shared DAG, and terminates on cyclic structures instead of -// recursing forever. For finite acyclic values the result is identical to a naive compare. -// A depth cap + deadline poll keep even a deeply-nested or huge NON-shared structure bounded -// (memoization alone does not: distinct nodes are never re-encountered). -bool values_equal_impl(const value_t &a, const value_t &b, ptr_pair_set &seen, int depth) { - if (depth > kValuesEqualMaxDepth || eval_deadline_expired()) - throw std::runtime_error("values_equal: structure too deep or large to compare"); +// Whether a and b are equal as far as one level shows; the element pairs a list or dict pair +// still needs compared are pushed onto `pending`. `seen` memoizes the container pairs already +// under comparison: re-encountering one (a shared node reached by two paths, or a cycle) +// counts as equal co-inductively. +bool shallow_equal(const value_t &a, const value_t &b, ptr_pair_set &seen, value_pairs &pending) { if (a.v.index() != b.v.index()) return false; return std::visit( @@ -375,9 +458,8 @@ bool values_equal_impl(const value_t &a, const value_t &b, ptr_pair_set &seen, i static_cast(arg_b.get())}) .second) return true; // this pair is already being compared (shared/cyclic) -> equal - for (std::size_t i = 0; i < arg_a->size(); ++i) - if (!values_equal_impl((*arg_a)[i], (*arg_b)[i], seen, depth + 1)) - return false; + for (std::size_t i = arg_a->size(); i-- > 0;) + pending.emplace_back(&(*arg_a)[i], &(*arg_b)[i]); return true; } else if constexpr (std::is_same_v) return arg_a == arg_b; // identity comparison @@ -392,12 +474,11 @@ bool values_equal_impl(const value_t &a, const value_t &b, ptr_pair_set &seen, i static_cast(arg_b.get())}) .second) return true; - for (std::size_t i = 0; i < arg_a->size(); ++i) { + for (std::size_t i = 0; i < arg_a->size(); ++i) if ((*arg_a)[i].first != (*arg_b)[i].first) return false; - if (!values_equal_impl((*arg_a)[i].second, (*arg_b)[i].second, seen, depth + 1)) - return false; - } + for (std::size_t i = arg_a->size(); i-- > 0;) + pending.emplace_back(&(*arg_a)[i].second, &(*arg_b)[i].second); return true; } else if constexpr (std::is_same_v) return false; // functions are never equal @@ -414,9 +495,23 @@ bool values_equal_impl(const value_t &a, const value_t &b, ptr_pair_set &seen, i } } // namespace +// The memo bounds work to the number of DISTINCT node-pairs (the physical size), never the +// exponential logical unfolding of a shared DAG, and terminates on cyclic structures; for +// finite acyclic values the result is identical to a naive compare. The pairs still to compare +// wait on a heap stack, so depth costs no native stack. A deadline poll bounds a huge +// NON-shared structure (memoization alone does not: distinct nodes are never re-encountered). bool values_equal(const value_t &a, const value_t &b) { ptr_pair_set seen; - return values_equal_impl(a, b, seen, 0); + value_pairs pending{{&a, &b}}; + while (!pending.empty()) { + if (eval_deadline_expired()) + throw std::runtime_error("values_equal: structure too large to compare"); + const auto [x, y] = pending.back(); + pending.pop_back(); + if (!shallow_equal(*x, *y, seen, pending)) + return false; + } + return true; } } // namespace cvc::state_exec diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index 00cb4e9dd..3a59b1ee7 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -76,6 +76,7 @@ add_executable(state_exec_bench_test state_exec_bench_test.cpp) add_executable(state_exec_integration_test state_exec_integration_test.cpp) add_executable(state_exec_advanced_integration_test state_exec_advanced_integration_test.cpp) add_executable(state_exec_gaps_test state_exec_gaps_test.cpp) +add_executable(state_exec_deep_data_test state_exec_deep_data_test.cpp) if(NOT WIN32) add_executable(state_exec_multiprocess_test state_exec_multiprocess_test.cpp) endif() @@ -339,6 +340,7 @@ list(APPEND TEST_TARGETS state_exec_integration_test state_exec_advanced_integration_test state_exec_gaps_test + state_exec_deep_data_test ) if(NOT WIN32) list(APPEND TEST_TARGETS state_exec_multiprocess_test) @@ -1206,6 +1208,13 @@ target_link_libraries(state_exec_types_test GTest::gtest_main ) +target_link_libraries(state_exec_deep_data_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + target_link_libraries(state_exec_builtins_test PRIVATE cvc @@ -1569,6 +1578,7 @@ target_compile_features(state_cross_cluster_link_test PRIVATE cxx_std_17) target_compile_features(state_link_bench_test PRIVATE cxx_std_17) target_compile_features(state_exec_parser_test PRIVATE cxx_std_20) target_compile_features(state_exec_types_test PRIVATE cxx_std_20) +target_compile_features(state_exec_deep_data_test PRIVATE cxx_std_20) target_compile_features(state_exec_builtins_test PRIVATE cxx_std_20) target_compile_features(state_exec_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_stackless_evaluator_test PRIVATE cxx_std_20) @@ -1979,6 +1989,7 @@ cvc_discover_tests(state_cross_cluster_link_test) cvc_discover_tests(state_link_bench_test) cvc_discover_tests(state_exec_parser_test) cvc_discover_tests(state_exec_types_test) +cvc_discover_tests(state_exec_deep_data_test) cvc_discover_tests(state_exec_builtins_test) cvc_discover_tests(state_exec_evaluator_test) cvc_discover_tests(state_exec_stackless_evaluator_test) diff --git a/src/cvc/tests/state_exec_bench_test.cpp b/src/cvc/tests/state_exec_bench_test.cpp index f22725734..52b1fef8f 100644 --- a/src/cvc/tests/state_exec_bench_test.cpp +++ b/src/cvc/tests/state_exec_bench_test.cpp @@ -131,6 +131,60 @@ TEST_F(ExecBenchTest, EvalFibonacci) { report("eval_fib15", N, std::chrono::duration_cast(t1 - t0)); } +TEST_F(ExecBenchTest, ValueDropThroughput) { + if (!bench_enabled()) { + SUCCEED(); + return; + } + + // Building and dropping values: the release path every list/dict/closure takes. + constexpr int64_t N = 1000000; + auto t0 = hrclock::now(); + for (int64_t i = 0; i < N; ++i) { + value_t v = make_list({value_t{i}, make_list({value_t{i}}), make_dict({{"k", value_t{i}}})}); + (void)v; + } + auto t1 = hrclock::now(); + report("value_drop_small", N, std::chrono::duration_cast(t1 - t0)); + + // Copies dropped while another owner lives. + const value_t shared = make_list({value_t{int64_t{1}}, value_t{int64_t{2}}}); + std::vector copies(64); + t0 = hrclock::now(); + for (int64_t i = 0; i < N; ++i) + copies[static_cast(i % 64)] = shared; + t1 = hrclock::now(); + report("value_drop_shared_copy", N, std::chrono::duration_cast(t1 - t0)); + + // 1000-level lists, built and dropped. + constexpr int64_t kDepth = 1000, kReps = 1000; + t0 = hrclock::now(); + for (int64_t r = 0; r < kReps; ++r) { + value_t v = make_list({value_t{r}}); + for (int64_t i = 1; i < kDepth; ++i) + v = make_list({std::move(v)}); + } + t1 = hrclock::now(); + report("value_drop_deep1000", kDepth * kReps, std::chrono::duration_cast(t1 - t0)); + + // The same from the DSL: a list nested 2000 deep, dropped when the evaluator is. + const std::string src = "(begin " + " (set acc (list)) " + " (set i 0) " + " (while (< i 2000) " + " (begin (set acc (list i acc)) (set i (+ i 1)))) " + " i)"; + auto env = builtins::make_default_environment(); + constexpr int M = 50; + t0 = hrclock::now(); + for (int i = 0; i < M; ++i) { + stackless_evaluator ev(env); + ev.evaluate_script(src); + } + t1 = hrclock::now(); + report("eval_nest_list_2000", M, std::chrono::duration_cast(t1 - t0)); +} + TEST_F(ExecBenchTest, SchedulerThroughput) { if (!bench_enabled()) { SUCCEED(); diff --git a/src/cvc/tests/state_exec_deep_data_test.cpp b/src/cvc/tests/state_exec_deep_data_test.cpp new file mode 100644 index 000000000..98a46d6af --- /dev/null +++ b/src/cvc/tests/state_exec_deep_data_test.cpp @@ -0,0 +1,390 @@ +// Deeply nested state_exec values must not overflow the native stack. +// +// The parser's nesting limit (parser.max_depth on the state tree) exists because the reader +// recurses; it can rise only once nothing else does either. These tests build values 100000 +// levels deep at runtime (not through the parser) and drop, print, compare and snapshot them +// (the deep copy behind state-data-get is in state_exec_intrinsics_test). Each body runs on a +// thread with a small fixed stack (kStackKiB), so a walker that recurses once per level fails +// here on every platform, whatever the main thread's stack is. +// +// For probing, CVC_DEEP_DATA_DEPTH overrides the depths and CVC_DEEP_DATA_STACK_KIB the stack +// (0 runs the body on the calling thread). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cvc::state_exec; + +namespace { + +constexpr std::size_t kDepth = 100000; +// The snapshot codec writes cvc::state nodes, and each write signals every ancestor, so a level +// costs tens of microseconds: 10000 levels keep the codec tests to seconds, and are still far +// past the 441 levels the recursive codec survived on a 1 MiB stack (104 on 256 KiB). +constexpr std::size_t kCodecDepth = 10000; +constexpr std::size_t kStackKiB = 256; + +std::size_t env_size(const char *name, std::size_t fallback) { + const char *s = std::getenv(name); + return s && *s ? static_cast(std::strtoull(s, nullptr, 10)) : fallback; +} + +std::size_t depth() { return env_size("CVC_DEEP_DATA_DEPTH", kDepth); } +std::size_t codec_depth() { return env_size("CVC_DEEP_DATA_DEPTH", kCodecDepth); } + +// Run `body` on a thread with a kStackKiB stack and rethrow what it throws. +void on_small_stack(const std::function &body) { + const std::size_t kib = env_size("CVC_DEEP_DATA_STACK_KIB", kStackKiB); + if (kib == 0) { + body(); + return; + } + std::exception_ptr error; + boost::thread::attributes attrs; + attrs.set_stack_size(kib * 1024); + boost::thread t(attrs, [&] { + try { + body(); + } catch (...) { + error = std::current_exception(); + } + }); + t.join(); + if (error) + std::rethrow_exception(error); +} + +// (((... (leaf) ...))): `n` lists, the innermost holding `leaf`. +value_t nested_list(std::size_t n, value_t leaf = value_t{int64_t{1}}) { + value_t v = make_list({std::move(leaf)}); + for (std::size_t i = 1; i < n; ++i) + v = make_list({std::move(v)}); + return v; +} + +// {"k": {"k": ... {"k": 1} ...}}: `n` dicts. +value_t nested_dict(std::size_t n) { + value_t v = make_dict({{"k", value_t{int64_t{1}}}}); + for (std::size_t i = 1; i < n; ++i) + v = make_dict({{"k", std::move(v)}}); + return v; +} + +// Lists and dicts alternating, each level also holding a scalar sibling. +value_t nested_mixed(std::size_t n) { + value_t v = value_t{int64_t{0}}; + for (std::size_t i = 0; i < n; ++i) + v = i % 2 ? make_dict({{"i", value_t{int64_t(i)}}, {"v", std::move(v)}}) + : make_list({value_t{std::string("s")}, std::move(v)}); + return v; +} + +// The innermost list of nested_list(), as a weak pointer: expired once the value is freed. +std::weak_ptr> innermost(const value_t &v) { + const value_t *cur = &v; + list_ptr last; + while (auto *l = std::get_if(&cur->v)) { + last = *l; + if ((*l)->empty()) + break; + cur = &(*l)->front(); + } + return last; +} + +// How many lists deep a nested_list() is, walked iteratively. +std::size_t list_depth(const value_t &v) { + std::size_t n = 0; + const value_t *cur = &v; + while (auto *l = std::get_if(&cur->v)) { + ++n; + if (!*l || (*l)->empty()) + break; + cur = &(*l)->front(); + } + return n; +} + +// The paths of every node under `node`, relative to it (state::children() lists all +// descendants, by full name). +std::vector descendants(cvc::state &node) { + const std::size_t prefix = node.fullName().size() + 1; + std::vector paths; + for (const std::string &full : node.children()) + paths.push_back(full.substr(prefix)); + return paths; +} + +// The depth of the state subtree under `node`. +std::size_t state_depth(cvc::state &node) { + std::size_t deepest = 0; + for (const std::string &path : descendants(node)) + deepest = std::max(deepest, 1 + std::count(path.begin(), path.end(), '.')); + return deepest; +} + +// ---- drop ------------------------------------------------------------------ + +TEST(StateExecDeepData, DropList) { + on_small_stack([] { + value_t v = nested_list(depth()); + auto last = innermost(v); + v = value_t{}; + EXPECT_TRUE(last.expired()); + }); +} + +TEST(StateExecDeepData, DropListOutOfScope) { + on_small_stack([] { + std::weak_ptr> last; + { + value_t v = nested_list(depth()); + last = innermost(v); + } + EXPECT_TRUE(last.expired()); + }); +} + +TEST(StateExecDeepData, DropDict) { + on_small_stack([] { value_t v = nested_dict(depth()); }); +} + +TEST(StateExecDeepData, DropMixed) { + on_small_stack([] { value_t v = nested_mixed(depth()); }); +} + +TEST(StateExecDeepData, DropBareListPtr) { + // The last owner is a list_ptr, not a value_t: the vector's elements free themselves. + on_small_stack([] { + list_ptr l = std::get(nested_list(depth()).v); + l.reset(); + }); +} + +TEST(StateExecDeepData, DropKeepsSharedTail) { + on_small_stack([] { + const std::size_t n = depth(); + value_t tail = nested_list(n); + value_t a = make_list({tail}); + value_t b = make_list({tail}); + tail = value_t{}; + a = value_t{}; + EXPECT_EQ(list_depth(b), n + 1); // still whole: `a` only dropped its reference + auto last = innermost(b); + b = value_t{}; + EXPECT_TRUE(last.expired()); + }); +} + +TEST(StateExecDeepData, DropClosureChain) { + // closure -> env_snapshot -> binding -> closure -> ...: the chain a recursive function that + // returns a closure over its inner call's closure builds at runtime. + on_small_stack([] { + value_t v; + for (std::size_t i = 0; i < depth(); ++i) { + auto env = std::make_shared(); + env->set("next", std::move(v)); + auto c = std::make_shared(); + c->env_snapshot = std::move(env); + v = value_t{c}; + } + }); +} + +TEST(StateExecDeepData, DropEnvironmentChain) { + on_small_stack([] { + environment_ptr env; + for (std::size_t i = 0; i < depth(); ++i) + env = environment::extend(std::move(env)); + std::weak_ptr inner = env; + env.reset(); + EXPECT_TRUE(inner.expired()); + }); +} + +// ---- to_string ------------------------------------------------------------- + +TEST(StateExecDeepData, ToStringIsBounded) { + on_small_stack([] { + value_t v = nested_list(depth()); + const std::string s = to_string(v); + // Output is capped at 64 KiB (plus the closing parens of what was opened), never per + // level of depth. + EXPECT_LE(s.size(), std::size_t{2} * 65536 + 16); + EXPECT_EQ(s.front(), '('); + EXPECT_NE(s.find("\xE2\x80\xA6"), std::string::npos); + }); +} + +TEST(StateExecDeepData, ToStringPrintsDeepValueWithinLengthCap) { + // 5000 levels print in ~10 KB, under the length cap, so they print whole: depth alone + // never truncates. + on_small_stack([] { + const std::size_t n = 5000; + EXPECT_EQ(to_string(nested_list(n)), std::string(n, '(') + "1" + std::string(n, ')')); + }); +} + +TEST(StateExecDeepData, ToStringDeepDict) { + on_small_stack([] { + const std::string s = to_string(nested_dict(depth())); + EXPECT_LE(s.size(), std::size_t{3} * 65536); + EXPECT_EQ(s.rfind("{\"k\": {\"k\": ", 0), 0u); + }); +} + +// ---- values_equal ---------------------------------------------------------- + +TEST(StateExecDeepData, ValuesEqualDeepLists) { + on_small_stack([] { + const std::size_t n = depth(); + EXPECT_TRUE(values_equal(nested_list(n), nested_list(n))); + EXPECT_FALSE(values_equal(nested_list(n), nested_list(n, value_t{int64_t{2}}))); + EXPECT_FALSE(values_equal(nested_list(n), nested_list(n - 1))); + }); +} + +TEST(StateExecDeepData, ValuesEqualDeepDicts) { + on_small_stack([] { + const std::size_t n = depth(); + EXPECT_TRUE(values_equal(nested_dict(n), nested_dict(n))); + EXPECT_TRUE(values_equal(nested_mixed(n), nested_mixed(n))); + EXPECT_FALSE(values_equal(nested_mixed(n), nested_mixed(n - 1))); + }); +} + +// ---- macro expansion ------------------------------------------------------- + +TEST(StateExecDeepData, MacroExpandsDeepTemplate) { + // (begin (defmacro m (x) (quote (((... x ...))))) (m 7)), the template built at runtime. + on_small_stack([] { + const std::size_t n = depth(); + auto sym = [](const char *s) { return value_t{symbol{s}}; }; + value_t body = make_list({sym("quote"), nested_list(n, sym("x"))}); + value_t expr = make_list({sym("begin"), + make_list({sym("defmacro"), sym("m"), make_list({sym("x")}), body}), + make_list({sym("m"), value_t{int64_t{7}}})}); + stackless_evaluator ev(builtins::make_default_environment()); + value_t r = ev.evaluate(expr); + EXPECT_EQ(list_depth(r), n); + EXPECT_TRUE(values_equal(r, nested_list(n, value_t{int64_t{7}}))); + }); +} + +// ---- snapshot codec -------------------------------------------------------- + +class StateExecDeepDataCodec : public ::testing::Test { +protected: + cvc::app ctx; + + // A node as deep as a process snapshot's would be. + cvc::state &snapshot_node(const std::string &name) { + return ctx.root()("state_exec.processes.p1.snapshot." + name); + } + + // Round-trip `v` through `node`. The state tree written stays shallow, whatever the value's + // depth: cvc::state's own walks (fullName(), childChanged up to the root) recurse once per + // tree level. + void round_trip(cvc::state &node, const value_t &v) { + on_small_stack([&] { + encode_value(node, v); + EXPECT_TRUE(values_equal(v, decode_value(node))); + }); + EXPECT_LE(state_depth(node), 32u); + } +}; + +TEST_F(StateExecDeepDataCodec, ListRoundTrip) { + const std::size_t n = codec_depth(); + value_t v = nested_list(n); + round_trip(snapshot_node("deep_list"), v); + EXPECT_EQ(list_depth(decode_value(snapshot_node("deep_list"))), n); +} + +TEST_F(StateExecDeepDataCodec, DictRoundTrip) { + round_trip(snapshot_node("deep_dict"), nested_dict(codec_depth())); +} + +TEST_F(StateExecDeepDataCodec, MixedRoundTrip) { + round_trip(snapshot_node("deep_mixed"), nested_mixed(codec_depth())); +} + +TEST_F(StateExecDeepDataCodec, ShallowLayoutUnchanged) { + // Values shallower than the spill depth encode exactly as before (no __spill__ table), so + // snapshots of ordinary values are unchanged and old snapshots still decode. + auto &node = ctx.root()("shallow"); + encode_value(node, make_list({make_list({value_t{int64_t{7}}}), make_dict({{"a", value_t{}}})})); + EXPECT_EQ(node("__type__").value(), "list"); + EXPECT_EQ(node("__elems__.000000.__type__").value(), "list"); + EXPECT_EQ(node("__elems__.000000.__elems__.000000.__type__").value(), "int"); + EXPECT_EQ(node("__elems__.000000.__elems__.000000").value(), "7"); + EXPECT_EQ(node("__elems__.000001.__type__").value(), "dict"); + EXPECT_EQ(node("__elems__.000001.__entries__.000000.__key__").value(), "a"); + EXPECT_EQ(node("__elems__.000001.__entries__.000000.__val__.__type__").value(), "nil"); + EXPECT_EQ(node.findDescendant("__spill__"), nullptr); +} + +TEST_F(StateExecDeepDataCodec, CyclicValueThrows) { + auto l = std::make_shared>(); + l->push_back(value_t{int64_t{1}}); + l->push_back(value_t{l}); + EXPECT_THROW(encode_value(ctx.root()("cycle"), value_t{l}), std::runtime_error); + l->clear(); // break the cycle so the list is freed +} + +TEST_F(StateExecDeepDataCodec, SharedValueEncodesTwice) { + // A DAG (the same list reached twice) is not a cycle: it encodes, and decodes as two copies. + value_t inner = make_list({value_t{int64_t{3}}}); + value_t v = make_list({inner, inner}); + auto &node = ctx.root()("dag"); + encode_value(node, v); + EXPECT_TRUE(values_equal(v, decode_value(node))); +} + +TEST_F(StateExecDeepDataCodec, BadSpillReferenceThrows) { + auto &node = ctx.root()("bad_ref"); + encode_value(node, nested_list(200)); + ASSERT_NE(node.findDescendant("__spill__"), nullptr); + std::vector refs; + for (const std::string &path : descendants(node)) { + const std::string suffix = ".__type__"; + if (path.size() > suffix.size() && path.ends_with(suffix) && node(path).value() == "spill_ref") + refs.push_back(&node(path.substr(0, path.size() - suffix.size()))); + } + ASSERT_GE(refs.size(), 2u); + const std::string first = refs[0]->value(); + // Past the end of the table. + refs[0]->value("999999"); + EXPECT_THROW(decode_value(node), std::runtime_error); + // Not a number. + refs[0]->value("x"); + EXPECT_THROW(decode_value(node), std::runtime_error); + // Every reference to entry 0, which then holds a reference to itself: decoding must stop + // at the second use, not expand forever. + std::vector saved; + for (cvc::state *r : refs) { + saved.push_back(r->value()); + r->value("0"); + } + EXPECT_THROW(decode_value(node), std::runtime_error); + for (std::size_t i = 0; i < refs.size(); ++i) + refs[i]->value(saved[i]); + refs[0]->value(first); + EXPECT_TRUE(values_equal(decode_value(node), nested_list(200))); +} + +} // namespace diff --git a/src/cvc/tests/state_exec_intrinsics_test.cpp b/src/cvc/tests/state_exec_intrinsics_test.cpp index cd66db841..574e6e1e8 100644 --- a/src/cvc/tests/state_exec_intrinsics_test.cpp +++ b/src/cvc/tests/state_exec_intrinsics_test.cpp @@ -2,6 +2,7 @@ /// @brief Tests for Phase 5: Intrinsics, Resource Policy, Stdlib. #include +#include #include #include #include @@ -227,12 +228,29 @@ TEST_F(StateTreeIntrinsicsTest, StateDataGetOfSharedDagIsBoundedAndPreservesShar EXPECT_EQ(c0->get(), c1->get()); // sharing preserved -> the copy is a DAG, not a 2^40 tree } -TEST_F(StateTreeIntrinsicsTest, StateDataGetOfTooDeepValueThrows) { +TEST_F(StateTreeIntrinsicsTest, StateDataGetOfDeepValueCopies) { + // The deep copy keeps its work on the heap, so depth has no cap: a value 100000 levels deep + // copies on a 256 KiB stack, into a private copy equal to the original. value_t d{int64_t{0}}; - for (int i = 0; i < 2000; ++i) - d = make_list({d}); // depth 2000 > the deep-copy depth cap + for (int i = 0; i < 100000; ++i) + d = make_list({d}); call("state-data-set", {std::string("deep"), d}); - EXPECT_THROW(call("state-data-get", {std::string("deep")}), std::runtime_error); // stack-safe + value_t got; + std::string error; + boost::thread::attributes small_stack; + small_stack.set_stack_size(256 * 1024); + boost::thread t(small_stack, [&] { + try { + got = call("state-data-get", {std::string("deep")}); + } catch (const std::exception &e) { + error = e.what(); + } + }); + t.join(); + EXPECT_EQ(error, ""); + ASSERT_TRUE(std::holds_alternative(got.v)); + EXPECT_NE(std::get(got.v).get(), std::get(d.v).get()); + EXPECT_TRUE(values_equal(got, d)); } TEST_F(StateTreeIntrinsicsTest, StateSetCoercesScalarsToString) { From 6f11d9ea878a8d4a80e75e27aab6ab2f7f50e7dd Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 15:17:39 -0500 Subject: [PATCH 11/13] state_exec: deep-data work a slice per step, and the budget by kind Freeing, printing, comparing, copying and macro-expanding a value no longer recurse, but their work still grows with the value. Like reading a script (parser.step_tokens), a scheduler can now run it a slice per step, so it counts against max_steps and max_time and other processes run between the slices. Each process also reports what its budget went on: reading, evaluating or deep-data work. - data_job (new): the walks as resumable jobs -- compare, print, copy, concat, macro substitution. A job holds the lists and dicts it walks by owning pointer and reads them by index, so a value changed between two slices changes the answer, never memory safety. to_string, values_equal and the state-data-get copy are these walks run whole. - data.step_nodes on the state tree (data_options, data_options_from_state), resolved like parser.*. The sync scheduler resolves and publishes it in load_settings(); the async scheduler reads it on every submit over set_data_defaults(). The app's exec scheduler and pycvc's Exec use host_data_defaults(): 1024 nodes a step (a slice is 0.05-0.4 ms at the measured 45-380 ns a node). - With N > 0, =, !=, str, str-concat, print and state-data-get hand their walk to the evaluator (run_data_job), and macro expansion does the same. One slice runs in the call's step; whatever is left runs in data steps while the call's frame waits. - Freeing: last owners dropped during a step go to the process's release backlog, freed at the step's end with what is left of the slice. More than that is freed in data steps before evaluation resumes. - Signals and watch events wait while a walk is pending. Snapshots keep data_step_nodes; a walk in flight runs again after a restore. - Budget by kind: evaluation_stats counts steps by step_kind (eval, read, data), deep-data nodes, and the seconds the schedulers charge to each kind. process_info, inspect and the new (budget-usage) report them. - pycvc: data_options, async_scheduler.host_data_defaults(), data_options_from_state and Exec.data_defaults/set_data_defaults, under the C++ names. New state_exec_data_steps_test covers the jobs (sliced = whole, values changed between slices, sharing and cycles), the evaluator (native calls, macros, freeing, max_steps, snapshots) and both schedulers (settings, max_steps, interleaving, signals, budget-usage/inspect). state_exec_bench_test gains DataWalkThroughput. --- bindings/pycvc/pycvc.i | 6 + bindings/pycvc/pycvc_exec.cpp | 20 +- bindings/pycvc/pycvc_exec.h | 14 +- bindings/pycvc/test_pycvc_exec.py | 53 ++ docs/STATE_EXEC_GRAMMAR.md | 58 ++ inc/cvc/state/state_exec/async_scheduler.h | 25 + inc/cvc/state/state_exec/data_job.h | 161 +++++ inc/cvc/state/state_exec/evaluator.h | 71 +- inc/cvc/state/state_exec/parser.h | 7 +- inc/cvc/state/state_exec/scheduler.h | 10 +- inc/cvc/state/state_exec/scheduler_base.h | 22 + .../state/state_exec/stackless_evaluator.h | 29 +- inc/cvc/state/state_exec/types.h | 51 +- src/cvc/CMakeLists.txt | 2 + src/cvc/core/app.cpp | 3 + src/cvc/state/state_exec/async_scheduler.cpp | 15 +- src/cvc/state/state_exec/builtins.cpp | 27 +- src/cvc/state/state_exec/data_job.cpp | 612 ++++++++++++++++++ src/cvc/state/state_exec/intrinsics.cpp | 87 +-- src/cvc/state/state_exec/scheduler.cpp | 22 +- .../state/state_exec/stackless_evaluator.cpp | 161 +++-- .../state/state_exec/state_value_codec.cpp | 13 + src/cvc/state/state_exec/types.cpp | 270 ++------ src/cvc/tests/CMakeLists.txt | 11 + src/cvc/tests/state_exec_bench_test.cpp | 45 ++ src/cvc/tests/state_exec_data_steps_test.cpp | 432 +++++++++++++ 26 files changed, 1872 insertions(+), 355 deletions(-) create mode 100644 inc/cvc/state/state_exec/data_job.h create mode 100644 src/cvc/state/state_exec/data_job.cpp create mode 100644 src/cvc/tests/state_exec_data_steps_test.cpp diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 404a59e87..669b853e3 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1444,8 +1444,10 @@ class AsyncStateObserver(state_observer): // parser, the scheduler itself) is not wrapped yet, so async_scheduler has no // Python constructor. parse_options_from_state and Exec's parser_defaults / // set_parser_defaults are in pycvc_exec.h. +// The deep-data settings (data_options, "Deep data on the executor") are wrapped the same way. %{ #include +#include #include %} namespace cvc { @@ -1454,11 +1456,15 @@ struct parse_options { size_t max_depth; // lists and quotes that may nest (default 256) size_t step_tokens; // 0: parse at submit; N: read N tokens per process step }; +struct data_options { + size_t step_nodes; // 0: deep-data work in the step that needs it; N: N nodes per step +}; %nodefaultctor async_scheduler; %nodefaultdtor async_scheduler; class async_scheduler { public: static parse_options host_parser_defaults(); + static data_options host_data_defaults(); }; } } diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 94f5c8d28..404ab9394 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -379,9 +379,11 @@ Exec::Exec(const std::shared_ptr &app) : impl_(std::make_sharedcomputePool(); impl_->sched.set_watch_root(impl_->root); // The same state_exec defaults as the app's exec scheduler, so a program behaves the same from - // Python as from Ariadne: scripts are read inside the process (host_parser_defaults), and the - // app tree's state_exec.defaults.parser.* keys override them for both. + // Python as from Ariadne: scripts are read inside the process (host_parser_defaults) and deep + // data is handled a slice per step (host_data_defaults), and the app tree's + // state_exec.defaults.parser.* / data.* keys override them for both. impl_->sched.set_parser_defaults(async_scheduler::host_parser_defaults()); + impl_->sched.set_data_defaults(async_scheduler::host_data_defaults()); impl_->host_proc = make_process(); impl_->host_proc->pid = 0; @@ -656,6 +658,20 @@ void Exec::set_parser_defaults(const cvc::state_exec::parse_options &defaults) { impl_->sched.set_parser_defaults(defaults); } +cvc::state_exec::data_options Exec::data_defaults() const { return impl_->sched.data_defaults(); } + +void Exec::set_data_defaults(const cvc::state_exec::data_options &defaults) { + impl_->sched.set_data_defaults(defaults); +} + +cvc::state_exec::data_options data_options_from_state(const std::shared_ptr &app, + const std::string &scheduler_id, + const cvc::state_exec::data_options &base) { + if (!app) + throw std::invalid_argument("pycvc.data_options_from_state: null app handle"); + return cvc::state_exec::data_options_from_state(cvc::state::instance(*app), scheduler_id, base); +} + cvc::state_exec::parse_options parse_options_from_state(const std::shared_ptr &app, const std::string &scheduler_id, const cvc::state_exec::parse_options &base) { diff --git a/bindings/pycvc/pycvc_exec.h b/bindings/pycvc/pycvc_exec.h index 55c3c9482..c3210891b 100644 --- a/bindings/pycvc/pycvc_exec.h +++ b/bindings/pycvc/pycvc_exec.h @@ -7,7 +7,8 @@ // registered here becomes callable from DSL source like any builtin. #pragma once -#include // parse_options +#include // data_options +#include // parse_options #include #include @@ -28,6 +29,12 @@ cvc::state_exec::parse_options parse_options_from_state( const std::shared_ptr &app, const std::string &scheduler_id = "", const cvc::state_exec::parse_options &base = cvc::state_exec::parse_options()); +// cvc::state_exec::data_options_from_state on `app`'s state root, resolved like +// parse_options_from_state under state_exec.*.data.. +cvc::state_exec::data_options data_options_from_state( + const std::shared_ptr &app, const std::string &scheduler_id = "", + const cvc::state_exec::data_options &base = cvc::state_exec::data_options()); + class Exec { public: // Build a DSL environment + scheduler over `app`'s state root. @@ -101,6 +108,11 @@ class Exec { cvc::state_exec::parse_options parser_defaults() const; void set_parser_defaults(const cvc::state_exec::parse_options &defaults); + // The deep-data settings, likewise: async_scheduler::data_defaults() / set_data_defaults(), + // starting as async_scheduler::host_data_defaults(). + cvc::state_exec::data_options data_defaults() const; + void set_data_defaults(const cvc::state_exec::data_options &defaults); + private: struct ExecImpl; std::shared_ptr impl_; diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index 6d932392b..9bccaa5bf 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -127,6 +127,59 @@ def test_set_parser_defaults_changes_how_exec_parses(): assert ex.run("(+ 1 (+ 2 3))") == "6" +# ── state_exec deep-data settings (docs/STATE_EXEC_GRAMMAR.md, "Deep data on +# the executor"), under the C++ names ──────────────────────────────────── + + +def test_data_options_is_the_cpp_struct(): + o = pycvc.data_options() + assert o.step_nodes == 0 # C++ default: deep-data work runs whole + o.step_nodes = 64 + assert o.step_nodes == 64 + + +def test_exec_starts_from_the_host_data_defaults(): + # The same defaults as the app's exec scheduler: + # async_scheduler::host_data_defaults(). + host = pycvc.async_scheduler.host_data_defaults() + assert host.step_nodes == 1024 + ex = pycvc.Exec(pycvc.make_app()) + assert ex.data_defaults().step_nodes == host.step_nodes + + +def test_data_options_from_state_resolves_like_cpp(): + app = pycvc.make_app() + assert pycvc.data_options_from_state(app).step_nodes == 0 + ex = pycvc.Exec(app) + assert pycvc.data_options_from_state(app, "", ex.data_defaults()).step_nodes == 1024 + pycvc.state_set(app, "state_exec.defaults.data.step_nodes", "16") + pycvc.state_set(app, "state_exec.schedulers.s1.data.step_nodes", "8") + assert pycvc.data_options_from_state(app).step_nodes == 16 + assert pycvc.data_options_from_state(app, "s1").step_nodes == 8 + o = pycvc.data_options() + o.step_nodes = 0 + ex.set_data_defaults(o) + assert ex.data_defaults().step_nodes == 0 + + +def test_deep_data_gives_the_same_results_however_it_is_sliced(): + # Compare and print values nested 3000 deep: whole in one step, 1024 nodes a step (the host + # default) or 4 nodes a step. + app = pycvc.make_app() + ex = pycvc.Exec(app) + prog = ( + "(begin (set a 1) (set b 1) (set i 0)" + " (while (< i 3000) (begin (set a (list a)) (set b (list b)) (set i (+ i 1))))" + " (list (= a b) (length (str a))))" + ) + sliced = ex.run(prog) + pycvc.state_set(app, "state_exec.defaults.data.step_nodes", "0") + whole = ex.run(prog) + pycvc.state_set(app, "state_exec.defaults.data.step_nodes", "4") + assert ex.run(prog) == whole == sliced + assert whole.startswith("(#t ") + + # ── DSL <-> state <-> Python round trips (one shared app) ─────────────── diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index c6045b560..00df8b858 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -208,6 +208,64 @@ scheduler can parse a script the same way it evaluates one: a slice per step. Wi - **Snapshots.** A process snapshotted while reading stores its source and parser settings. After a restore it reads the script again from the start. +### Deep data on the executor + +Walking a nested value no longer recurses on the native stack: freeing, printing (`to_string`), +comparing (`values_equal`), copying (`state-data-get`), macro expansion and the snapshot codec +keep their work on the heap. That work is still proportional to the value's size, so a scheduler +can run it the way it reads a script: a slice per step, counted against the process's budget. + +The setting is `data.step_nodes`, resolved like the parser's (`state_exec.schedulers..data.step_nodes`, +then `state_exec.defaults.data.step_nodes`, then the scheduler's built-in default; +`data_options_from_state()`): + +| Key | Meaning | Default | +|---|---|---| +| `step_nodes` | 0: deep-data work runs whole, inside the step that needs it. N: a step does at most N nodes of it; the rest runs N per step in the steps after | 0; **1024** for the app's exec scheduler and pycvc's `Exec` | + +A node is an element visited: a list element or a dict entry walked, or an element freed. With +`step_nodes` = N: + +- **Native calls.** `=`, `!=`, `str`, `str-concat`, `print` and `state-data-get` hand their walk + to the evaluator (`run_data_job`). It does one slice inside the call's step; if the walk isn't + done, the call's frame waits and the following steps are **data steps** that each run N more + nodes. The step that finishes it completes the call. A walk under N nodes finishes in the + call's own step, so ordinary programs take no extra steps. +- **Macro expansion** works the same way, before the expansion is evaluated. +- **Freeing.** Last owners dropped during a step (a list, dict, closure, generator or scope) + go to the process's release backlog instead of being freed on the spot. The step frees up to + what is left of its N nodes at its end. If more is left, the next steps are data steps that + only free, and evaluation resumes once the backlog is empty. +- **Budgets.** Data steps are process steps. They count against `max_steps` and `max_time`, and + other processes run between them. +- **Signals and watch events** wait while a walk is pending, because the walk's call frame is + on top of the evaluation stack. +- **Snapshots** keep `step_nodes` but not a walk in flight: after a restore, the waiting call + or macro expansion runs again. +- **Values changed between slices.** A walk holds the lists and dicts it walks by owning + pointer and reads them by index, checking bounds at every node. A change between two slices + (by another process) changes the answer, never memory safety. + +Other deep walks run whole inside one step, as before, but count their nodes. These are +`collections.*` helpers and string coercion for `state-set`. + +Measured per node (MSVC Release, `state_exec_bench_test` `DataWalkThroughput`): ~45 ns to free, +~70 ns to print, ~250 ns to compare and ~380 ns to copy. So a 1024-node slice costs 0.05-0.4 ms. + +### Budget by kind + +Each process's steps are split by what they were spent on (`step_kind`): + +- **read**: reading the script (`parser.step_tokens`); +- **data**: deep-data steps (`data.step_nodes`); +- **eval**: evaluating code. + +`process_info` carries `read_steps`, `eval_steps`, `data_steps` (they add up to `step_count`), +`data_nodes` (deep-data nodes, whichever steps did them), and `read_time`, `eval_time`, +`data_time`. The times are the seconds of the steps a scheduler ran, charged to each step's kind. +`inspect` reports them for any pid, and `(budget-usage)` reports them, with `steps`, for the +running process. + ## Data | Production | Value (`value_t` alternative) | diff --git a/inc/cvc/state/state_exec/async_scheduler.h b/inc/cvc/state/state_exec/async_scheduler.h index 8af85f86f..9c2231a93 100644 --- a/inc/cvc/state/state_exec/async_scheduler.h +++ b/inc/cvc/state/state_exec/async_scheduler.h @@ -182,6 +182,30 @@ class async_scheduler : public scheduler_base { return defaults; } + /// This scheduler's built-in deep-data settings: what a submitted process runs with when the + /// watch root's `state_exec.defaults.data.*` keys don't say otherwise. data_options{} (no + /// slicing) unless the host sets it. + void set_data_defaults(const data_options &defaults) { data_defaults_ = defaults; } + const data_options &data_defaults() const { return data_defaults_; } + + /// The deep-data settings a process submitted now gets: the watch root's keys over + /// data_defaults(). Read on every submit, like the parser settings. + data_options data_settings() const { + return watch_root_ ? data_options_from_state(*watch_root_, {}, data_defaults_) : data_defaults_; + } + + /// The deep-data defaults libcvc's own hosts give their async schedulers (app::exec_scheduler() + /// and pycvc's Exec), next to host_parser_defaults(): deep-data work runs 1024 nodes per step + /// (docs/STATE_EXEC_GRAMMAR.md, "Deep data on the executor"). At the measured ~45 ns a node + /// to free, ~70 to print, ~250 to compare and ~380 to copy, a slice costs 0.05-0.4 ms, so a huge + /// value is spread over steps instead of stalling a drain, while the data of a normal program + /// (far under 1024 nodes a step) costs it no extra steps. + static data_options host_data_defaults() { + data_options defaults; + defaults.step_nodes = 1024; + return defaults; + } + private: scheduling_policy policy_; int next_pid_ = 1; @@ -200,6 +224,7 @@ class async_scheduler : public scheduler_base { memory_tracker mem_tracker_; cvc::state *watch_root_ = nullptr; parse_options parser_defaults_; + data_options data_defaults_; std::unordered_map processes_; diff --git a/inc/cvc/state/state_exec/data_job.h b/inc/cvc/state/state_exec/data_job.h new file mode 100644 index 000000000..06bcc0a1a --- /dev/null +++ b/inc/cvc/state/state_exec/data_job.h @@ -0,0 +1,161 @@ +/** + * @file data_job.h + * @brief Deep-data work (comparing, printing, copying, expanding nested values) in slices. + * + * A walk over a nested value costs one unit per node it visits. A data_job keeps its walk on the + * heap, so it can stop after any number of nodes and go on later. A scheduler runs a process's + * deep-data work a slice at a time (data.step_nodes on the state tree), as it reads a script a + * slice at a time (parser.step_tokens): the work counts against the process's max_steps and + * max_time, and other processes run between the slices. + */ +#ifndef CVC_STATE_EXEC_DATA_JOB_H +#define CVC_STATE_EXEC_DATA_JOB_H + +#include +#include +#include +#include +#include +#include +#include + +namespace cvc { +class state; +} + +namespace cvc::state_exec { + +struct evaluation_stats; + +/// A walk over a nested value, done a slice at a time. A job holds what it walks (the shared +/// lists and dicts, by owning pointer) and reads them by index, checking bounds at every node, +/// so a value changed between two slices (by another process) cannot leave it dangling; the +/// result then reflects each part as it was when visited. +class data_job { +public: + virtual ~data_job() = default; + + /// Visit at most `nodes` more nodes; 0 visits all that remain. True once the walk is done. + virtual bool step(std::size_t nodes) = 0; + + /// The walk's result, once step() has returned true. Call it once. + virtual value_t result() = 0; + + /// Nodes visited so far. + std::size_t nodes() const { return nodes_; } + +protected: + std::size_t nodes_ = 0; +}; + +/// Whether walking `v` visits any nodes: it is a non-empty list or dict. +bool is_compound(const value_t &v); + +/// values_equal(a, b), as a bool value. +std::unique_ptr make_equal_job(value_t a, value_t b); + +/// to_string(v), as a string value. +std::unique_ptr make_print_job(value_t v); + +/// `parts` printed one after another with `sep` between them, as a string value: a string as its +/// raw text, anything else as to_string() prints it. +std::unique_ptr make_concat_job(std::vector parts, std::string sep); + +/// A private copy of v's lists and dicts, keeping their sharing (a node reached twice is copied +/// once; a cycle is copied as a cycle). Anything else is shared, not copied. +std::unique_ptr make_copy_job(value_t v); + +/// A macro template with its parameters replaced: each symbol named in `subst` becomes its +/// value; lists are copied, everything else is kept. +std::unique_ptr make_substitute_job(value_t tmpl, + std::unordered_map subst); + +/// Run a job to completion and return its result. +value_t run_to_completion(data_job &job); + +/// Deep-data settings ("Deep data on the executor" in docs/STATE_EXEC_GRAMMAR.md). Hosts keep them +/// on the state tree under `data.*`; data_options_from_state() reads them from there. +struct data_options { + /// How many nodes of deep-data work a scheduler step may do. 0 does all of it in the step that + /// needs it. N > 0 does up to N there; what is left runs N per step in the steps after it + /// (the process waits, and those steps count against its max_steps and max_time). Freeing + /// counts too: up to N owners a step dropped are freed at its end, the rest in steps after. + /// State-tree key: `data.step_nodes`. + std::size_t step_nodes = 0; +}; + +/// Resolve data settings from a state tree, per key, like parse_options_from_state(): +/// `state_exec.schedulers..data.`, then `state_exec.defaults.data.`, then +/// `base`. A value that is missing, empty or malformed is skipped. +data_options data_options_from_state(cvc::state &root, const std::string &scheduler_id = {}, + const data_options &base = {}); + +/// Do a native function's deep-data work. Inside a stackless evaluator step that slices +/// (data.step_nodes > 0), called from the native function that step is applying, `job` does one +/// slice here; if that finishes it, `then(result)` is returned. If not, the job and `then` pass +/// to the evaluator, which runs a slice per step and completes the call with then's value once +/// the job is done; what this returns is then ignored, so the native function should return it +/// at once. Anywhere else the job runs to completion here. Without `then`, the job's result is +/// the call's. +value_t run_data_job(std::unique_ptr job, std::function then = {}); + +/// Count `nodes` units of deep-data work done in the current evaluator step (a walk run to +/// completion counts itself; the release backlog counts what it frees). +void count_data_nodes(std::size_t nodes); + +namespace detail { + +/// A native call's deep-data job left to run after the call returned (see run_data_job). +struct deferred_data_job { + std::unique_ptr job; + std::function then; +}; + +/// The data side of one stackless evaluator step, armed for the step's duration (nested evaluator +/// steps arm their own and restore this one). It counts the step's deep-data nodes into `stats` +/// and, when the step slices, routes the owners it drops into `backlog` (see release_backlog). +class data_step_scope { +public: + data_step_scope(std::size_t step_nodes, evaluation_stats &stats, release_backlog *backlog); + ~data_step_scope(); + data_step_scope(const data_step_scope &) = delete; + data_step_scope &operator=(const data_step_scope &) = delete; + + /// The slice: how many nodes the step may visit (0 = no slicing). + std::size_t step_nodes() const { return step_nodes_; } + /// Nodes counted in this step so far. + std::size_t nodes() const { return nodes_; } + /// What is left of the slice (never 0 while slicing, so a job always advances). + std::size_t remaining() const; + + /// Let run_data_job() defer a job while `allowed` (around the native call being applied). + void allow_defer(bool allowed) { may_defer_ = allowed; } + /// The job a native call deferred, if any. + std::unique_ptr take_deferred() { return std::move(deferred_); } + + /// Free backlog owners with what is left of the slice (all of them when not slicing). + void drain_backlog(); + + /// The scope armed on this thread, if any. + static data_step_scope *current(); + +private: + friend value_t cvc::state_exec::run_data_job(std::unique_ptr, + std::function); + friend void cvc::state_exec::count_data_nodes(std::size_t); + + std::size_t step_nodes_; + evaluation_stats &stats_; + release_backlog *backlog_; + std::size_t nodes_ = 0; + bool may_defer_ = false; + std::unique_ptr deferred_; + data_step_scope *outer_; + backlog_scope routing_; +}; + +} // namespace detail + +} // namespace cvc::state_exec + +#endif // CVC_STATE_EXEC_DATA_JOB_H diff --git a/inc/cvc/state/state_exec/evaluator.h b/inc/cvc/state/state_exec/evaluator.h index 16e2af8c4..b10171e9f 100644 --- a/inc/cvc/state/state_exec/evaluator.h +++ b/inc/cvc/state/state_exec/evaluator.h @@ -43,18 +43,26 @@ class return_exception : public std::exception { const char *what() const noexcept override { return "return"; } }; +/// What a step was spent on: evaluating code, reading the script a slice at a time +/// (parser.step_tokens), or deep-data work a slice at a time (data.step_nodes: comparing, +/// printing, copying or expanding a nested value, or freeing one). +enum class step_kind { eval = 0, read = 1, data = 2 }; + /// Thread-safe evaluation statistics. struct evaluation_stats { evaluation_stats() = default; // Move-only: transfer ownership of internals evaluation_stats(evaluation_stats &&o) noexcept - : start_time(o.start_time), end_time(o.end_time), - step_count(o.step_count.load(std::memory_order_relaxed)), complete(o.complete) {} + : start_time(o.start_time), end_time(o.end_time), complete(o.complete), + last_kind(o.last_kind) { + copy_counts(o); + } evaluation_stats &operator=(evaluation_stats &&o) noexcept { start_time = o.start_time; end_time = o.end_time; - step_count.store(o.step_count.load(std::memory_order_relaxed), std::memory_order_relaxed); complete = o.complete; + last_kind = o.last_kind; + copy_counts(o); return *this; } evaluation_stats(const evaluation_stats &) = delete; @@ -64,6 +72,11 @@ struct evaluation_stats { std::lock_guard lk(mu); start_time = std::chrono::steady_clock::now(); step_count.store(0, std::memory_order_relaxed); + for (std::size_t k = 0; k < kKinds; ++k) { + kind_steps[k].store(0, std::memory_order_relaxed); + kind_ns[k].store(0, std::memory_order_relaxed); + } + data_node_count.store(0, std::memory_order_relaxed); complete = false; } void mark_complete() { @@ -71,8 +84,42 @@ struct evaluation_stats { end_time = std::chrono::steady_clock::now(); complete = true; } - void increment_step() { step_count.fetch_add(1, std::memory_order_relaxed); } + /// Count a step, as spent on `kind`. + void increment_step(step_kind kind = step_kind::eval) { + step_count.fetch_add(1, std::memory_order_relaxed); + if (kind != step_kind::eval) // eval steps are the rest, so the common step costs no more + kind_steps[static_cast(kind)].fetch_add(1, std::memory_order_relaxed); + last_kind = kind; + } uint64_t get_step_count() const { return step_count.load(std::memory_order_relaxed); } + /// Steps spent on `kind`. + uint64_t steps(step_kind kind) const { + if (kind != step_kind::eval) + return kind_steps[static_cast(kind)].load(std::memory_order_relaxed); + const uint64_t other = steps(step_kind::read) + steps(step_kind::data); + const uint64_t all = get_step_count(); + return all > other ? all - other : 0; + } + /// The kind of the last step counted. + step_kind last_step_kind() const { return last_kind; } + + /// Count nodes of deep-data work (elements walked, owners freed), in whatever step did it. + void add_data_nodes(uint64_t nodes) { + data_node_count.fetch_add(nodes, std::memory_order_relaxed); + } + uint64_t data_nodes() const { return data_node_count.load(std::memory_order_relaxed); } + + /// A scheduler that times its steps charges each step's seconds here, to the step's kind. + void add_step_time(double seconds) { + if (seconds > 0.0) + kind_ns[static_cast(last_kind)].fetch_add(static_cast(seconds * 1e9), + std::memory_order_relaxed); + } + /// Seconds of scheduler-timed steps spent on `kind`. + double step_time(step_kind kind) const { + return 1e-9 * static_cast( + kind_ns[static_cast(kind)].load(std::memory_order_relaxed)); + } bool is_complete() const { std::lock_guard lk(mu); return complete; @@ -84,11 +131,27 @@ struct evaluation_stats { } private: + static constexpr std::size_t kKinds = 3; + void copy_counts(const evaluation_stats &o) { + step_count.store(o.step_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + for (std::size_t k = 0; k < kKinds; ++k) { + kind_steps[k].store(o.kind_steps[k].load(std::memory_order_relaxed), + std::memory_order_relaxed); + kind_ns[k].store(o.kind_ns[k].load(std::memory_order_relaxed), std::memory_order_relaxed); + } + data_node_count.store(o.data_node_count.load(std::memory_order_relaxed), + std::memory_order_relaxed); + } + mutable std::mutex mu; std::chrono::steady_clock::time_point start_time{}; std::chrono::steady_clock::time_point end_time{}; std::atomic step_count{0}; + std::atomic kind_steps[kKinds] = {}; + std::atomic kind_ns[kKinds] = {}; + std::atomic data_node_count{0}; bool complete{false}; + step_kind last_kind = step_kind::eval; }; /// Synchronous recursive evaluator (Tier 2 — local contexts only). diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index 5267c0c3d..617adfb33 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -42,9 +42,10 @@ class parse_error : public std::runtime_error { /// state tree under `parser.*`; parse_options_from_state() reads them from there. struct parse_options { /// How many lists and quotes may nest (`'x` reads as `(quote x)`, one level); deeper input is - /// a parse_error. The parser itself is stackless, but code downstream still walks values - /// recursively (destroying a nested list, the snapshot codec, to_string), so this keeps parsed - /// data within what they can handle. State-tree key: `parser.max_depth`. + /// a parse_error. The parser is stackless, and so are freeing, printing, comparing, copying, + /// macro-expanding and snapshotting a value; what still walks values recursively (the Ariadne + /// loader's channel lint, which stops at 256 levels, and pycvc's value conversion) is what this + /// keeps parsed data within. State-tree key: `parser.max_depth`. std::size_t max_depth = 256; /// How a scheduler parses a submitted script. 0 parses all of it when it is submitted, so diff --git a/inc/cvc/state/state_exec/scheduler.h b/inc/cvc/state/state_exec/scheduler.h index e69ecd20c..8aaeae9c5 100644 --- a/inc/cvc/state/state_exec/scheduler.h +++ b/inc/cvc/state/state_exec/scheduler.h @@ -181,8 +181,9 @@ class scheduler : public scheduler_base { /// per-scheduler subtree so they are visible in the state tree. /// Requires set_watch_root() and (for per-scheduler) set_id() first. /// - /// Settings: `max_pending_messages`, and the parser's `parser.*` - /// (parse_options_from_state), which apply to every script submitted + /// Settings: `max_pending_messages`, the parser's `parser.*` + /// (parse_options_from_state) and the deep-data `data.*` + /// (data_options_from_state), which apply to every process submitted /// afterwards. void load_settings(); @@ -190,6 +191,10 @@ class scheduler : public scheduler_base { /// load_settings() runs). const parse_options &parser_settings() const { return parse_options_; } + /// The deep-data settings execute() gives processes (defaults until + /// load_settings() runs). + const data_options &data_settings() const { return data_options_; } + /// Well-known state tree paths for scheduler configuration. static constexpr const char *settings_root = "state_exec"; static constexpr const char *defaults_prefix = "state_exec.defaults"; @@ -199,6 +204,7 @@ class scheduler : public scheduler_base { std::string id_; scheduling_policy policy_; parse_options parse_options_; + data_options data_options_; int next_pid_ = 1; int rr_index_ = 0; bool running_ = false; diff --git a/inc/cvc/state/state_exec/scheduler_base.h b/inc/cvc/state/state_exec/scheduler_base.h index 179d6c292..f17e6371a 100644 --- a/inc/cvc/state/state_exec/scheduler_base.h +++ b/inc/cvc/state/state_exec/scheduler_base.h @@ -72,6 +72,17 @@ struct process_info { std::string uid; std::string gid; uint64_t step_count; + // What the steps were spent on (step_kind): reading the script a slice at a time + // (parser.step_tokens), deep-data work a slice at a time (data.step_nodes), evaluating the + // rest. data_nodes counts the deep-data work in nodes, whichever steps did it; the times are + // seconds of scheduler-timed steps. + uint64_t read_steps = 0; + uint64_t eval_steps = 0; + uint64_t data_steps = 0; + uint64_t data_nodes = 0; + double read_time = 0.0; + double eval_time = 0.0; + double data_time = 0.0; double elapsed_time; uint64_t current_memory; uint64_t peak_memory; @@ -84,6 +95,17 @@ struct process_info { int parent_pid; }; +/// Fill `info`'s budget-by-kind fields from a process's evaluation stats. +inline void fill_budget_usage(process_info &info, const evaluation_stats &stats) { + info.read_steps = stats.steps(step_kind::read); + info.eval_steps = stats.steps(step_kind::eval); + info.data_steps = stats.steps(step_kind::data); + info.data_nodes = stats.data_nodes(); + info.read_time = stats.step_time(step_kind::read); + info.eval_time = stats.step_time(step_kind::eval); + info.data_time = stats.step_time(step_kind::data); +} + /// Aggregate scheduler statistics. struct scheduler_stats { int total_processes = 0; diff --git a/inc/cvc/state/state_exec/stackless_evaluator.h b/inc/cvc/state/state_exec/stackless_evaluator.h index 10902797c..bcd88d470 100644 --- a/inc/cvc/state/state_exec/stackless_evaluator.h +++ b/inc/cvc/state/state_exec/stackless_evaluator.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include // parse_options #include @@ -64,6 +65,16 @@ struct eval_frame { environment_ptr extra_env; }; +/// Deep-data work a step left unfinished (data_options::step_nodes): a native call's job (its +/// frame waits in the apply phase) or the expansion of a macro call (its frame waits to be +/// re-initialized with it). +struct pending_data_work { + enum class kind { native_call, macro_expansion }; + std::unique_ptr job; + std::function then; // native_call: turns the job's result into the call's + kind what = kind::native_call; +}; + /// Serializable evaluator state — the entire execution snapshot. struct evaluator_state { std::vector stack; @@ -77,6 +88,17 @@ struct evaluator_state { /// Set while the script is still being read, a slice per step (parse_options::step_tokens); /// step() reads instead of evaluating until the whole script is read. std::shared_ptr reading; + /// How many nodes of deep-data work a step may do (data_options::step_nodes; a scheduler sets + /// it). 0 does all of it in the step that needs it. + std::size_t data_step_nodes = 0; + /// Set while deep-data work runs a slice per step; step() runs it instead of evaluating until + /// it is done, then completes what started it. Not serialized: after a restore the waiting + /// call or macro expansion simply runs again. + std::shared_ptr data_work; + /// Last owners this state's steps dropped and have not freed yet (data_step_nodes > 0); each + /// step frees them with what is left of its slice, and steps that only free them follow + /// while any are left. + std::shared_ptr backlog; }; /// Synchronous stackless evaluator (Tier 2 — local contexts only). @@ -147,6 +169,12 @@ class stackless_evaluator { evaluator_state create_program_state(std::vector forms, environment_ptr env); // One step of a state that is still reading its script (evaluator_state::reading). bool step_read(evaluator_state &state); + // One step of deep-data work left unfinished (evaluator_state::data_work). + bool step_data(evaluator_state &state); + // Re-initialize the top frame to evaluate a macro call's expansion. + void begin_expansion(evaluator_state &state, value_t expansion); + // One evaluation step of the top frame. + void step_frame(evaluator_state &state); // Frame management void pop_frame(evaluator_state &state, value_t result); @@ -175,7 +203,6 @@ class stackless_evaluator { void call_closure(evaluator_state &state, eval_frame &frame, const closure_ptr &cls, const std::vector &args); void handle_defclass(evaluator_state &state, eval_frame &frame, const std::vector &args); - value_t substitute(const value_t &tmpl, const std::unordered_map &subst); environment_ptr global_env_; std::shared_ptr> diff --git a/inc/cvc/state/state_exec/types.h b/inc/cvc/state/state_exec/types.h index 0ce3556cb..b3e537069 100644 --- a/inc/cvc/state/state_exec/types.h +++ b/inc/cvc/state/state_exec/types.h @@ -77,16 +77,59 @@ namespace detail { /// plain shared_ptr destructors takes a native stack frame per level and overflows the stack. /// Here the first release on a thread runs a loop: owners dropped while it frees something /// (that thing's elements, bindings, enclosing scope) are queued and freed one at a time by the -/// loop, so the stack stays a few frames deep whatever the depth. -void release_nested(std::shared_ptr owner) noexcept; +/// loop, so the stack stays a few frames deep whatever the depth. `nodes` is what freeing it +/// costs as deep-data work: 1, plus a list's or dict's elements. +void release_nested(std::shared_ptr owner, std::size_t nodes = 1) noexcept; /// Drop `owner` through release_nested when it is the last one; dropping any other owner only /// decrements a count. (Should another thread drop the last but one meanwhile, this drop frees /// the storage inline, and its elements start releases of their own: a few frames, still.) template void drop_owner(std::shared_ptr &owner) noexcept { - if (owner && owner.use_count() == 1) - release_nested(std::move(owner)); + if (!owner || owner.use_count() != 1) + return; + std::size_t nodes = 1; + if constexpr (std::is_same_v> || + std::is_same_v>>) + nodes += owner->size(); + release_nested(std::move(owner), nodes); } + +/// Last owners dropped while a slicing process step runs (data.step_nodes > 0) and not freed +/// yet. While a backlog_scope routes drops here, release_nested() queues a last owner instead of +/// freeing it, and the step frees them a slice at a time: freeing a huge value then counts +/// against the process's budget and spreads over its steps instead of stalling one. (A single +/// owner is freed whole: a flat list of a million numbers is one free.) +class release_backlog { +public: + release_backlog() = default; + ~release_backlog(); // frees what is left, iteratively like any release + release_backlog(const release_backlog &) = delete; + release_backlog &operator=(const release_backlog &) = delete; + + bool empty() const { return owners_.empty(); } + + /// Free owners until their nodes reach `budget` (0: until none are left). Owners dropped by + /// that freeing join the backlog. Returns the nodes freed. + std::size_t drain(std::size_t budget); + +private: + friend void release_nested(std::shared_ptr, std::size_t) noexcept; + std::vector, std::size_t>> owners_; +}; + +/// RAII: while alive, last owners this thread drops join `backlog` instead of being freed on the +/// spot. A null backlog changes nothing (an outer scope, if any, keeps routing). Scopes nest. +class backlog_scope { +public: + explicit backlog_scope(release_backlog *backlog); + ~backlog_scope(); + backlog_scope(const backlog_scope &) = delete; + backlog_scope &operator=(const backlog_scope &) = delete; + +private: + release_backlog *outer_; + bool armed_; +}; } // namespace detail /// The DSL value variant. All runtime values are represented as value_t. diff --git a/src/cvc/CMakeLists.txt b/src/cvc/CMakeLists.txt index f3a8b322f..6a0b0c053 100644 --- a/src/cvc/CMakeLists.txt +++ b/src/cvc/CMakeLists.txt @@ -77,6 +77,7 @@ set(INCLUDE_FILES ../../inc/cvc/state/state_exec/async_evaluator.h ../../inc/cvc/state/state_exec/async_stackless_evaluator.h ../../inc/cvc/state/state_exec/state_value_codec.h + ../../inc/cvc/state/state_exec/data_job.h ../../inc/cvc/state/state_exec/process.h ../../inc/cvc/state/state_exec/memory_tracker.h ../../inc/cvc/state/state_exec/scheduler_base.h @@ -196,6 +197,7 @@ set(SOURCE_FILES state/state_exec/async_evaluator.cpp state/state_exec/async_stackless_evaluator.cpp state/state_exec/state_value_codec.cpp + state/state_exec/data_job.cpp state/state_exec/process.cpp state/state_exec/memory_tracker.cpp state/state_exec/scheduler.cpp diff --git a/src/cvc/core/app.cpp b/src/cvc/core/app.cpp index 5af747911..5e4cdde77 100644 --- a/src/cvc/core/app.cpp +++ b/src/cvc/core/app.cpp @@ -1025,6 +1025,9 @@ cvc::state_exec::async_scheduler &app::exec_scheduler() { // Read submitted scripts inside their processes, like pycvc's Exec (see host_parser_defaults // for why 1024 tokens a step). state_exec.defaults.parser.* on this tree overrides it. _execScheduler->set_parser_defaults(cvc::state_exec::async_scheduler::host_parser_defaults()); + // Deep-data work a slice per step too (host_data_defaults); state_exec.defaults.data.* + // overrides it. + _execScheduler->set_data_defaults(cvc::state_exec::async_scheduler::host_data_defaults()); } return *_execScheduler; } diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index 4de30dd99..9591c63f6 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -42,6 +42,7 @@ int async_scheduler::execute(const std::string &script, const execute_options &o const parse_options popts = watch_root_ ? parse_options_from_state(*watch_root_, {}, parser_defaults_) : parser_defaults_; proc.state = evaluator_.create_state(script, opts.env, popts); + proc.state.data_step_nodes = data_settings().step_nodes; proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; @@ -68,6 +69,7 @@ int async_scheduler::execute(const value_t &expr, const execute_options &opts) { proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); proc.state = evaluator_.create_state(expr, opts.env); + proc.state.data_step_nodes = data_settings().step_nodes; proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; @@ -141,9 +143,11 @@ void async_scheduler::execute_process_step(process &proc) { } reset_guard{current_pid_, current_proc_}; // A process still reading its script (parser.step_tokens) takes signals and watch events once - // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. - const bool can_interrupt = - !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. One + // in the middle of deep-data work (data.step_nodes) takes them once that is done: the work's + // call frame waits on top of the stack. + const bool can_interrupt = !proc.in_signal_handler && !proc.in_watch_handler && + !proc.state.reading && !proc.state.data_work; if (!proc.pending_signals.empty() && can_interrupt) { handle_signal(proc); @@ -157,6 +161,7 @@ void async_scheduler::execute_process_step(process &proc) { proc.status = process_status::running; proc.last_run_start = std::chrono::steady_clock::now(); + const auto step_start = proc.last_run_start; // Arm the cooperative deadline for this step so an uncapped nested evaluator or a // long-running builtin aborts at the process's remaining time budget too — the same @@ -202,6 +207,8 @@ void async_scheduler::execute_process_step(process &proc) { proc.accumulated_time += std::chrono::duration(now - proc.last_run_start).count(); if (charge_cpu) proc.charge_cpu_time(thread_cpu_seconds() - cpu0); + // Charge the step's time to what it was spent on (eval, read or data). + proc.state.stats.add_step_time(std::chrono::duration(now - step_start).count()); if ((proc.in_signal_handler || proc.in_watch_handler) && proc.state.done) { restore_from_signal(proc); @@ -706,6 +713,7 @@ int async_scheduler::fork(int pid) { child->state.done = parent.state.done; child->state.stack = parent.state.stack; child->state.user_macros = parent.state.user_macros; + child->state.data_step_nodes = parent.state.data_step_nodes; auto child_env = std::make_shared(); if (parent.state.global_env) { child_env->bindings = parent.state.global_env->bindings; @@ -786,6 +794,7 @@ process_info async_scheduler::make_info(const process &proc) const { info.uid = proc.uid; info.gid = proc.gid; info.step_count = proc.step_count(); + fill_budget_usage(info, proc.state.stats); info.elapsed_time = proc.elapsed_time(); info.current_memory = mem_tracker_.current_bytes(proc.pid); info.peak_memory = mem_tracker_.peak_bytes(proc.pid); diff --git a/src/cvc/state/state_exec/builtins.cpp b/src/cvc/state/state_exec/builtins.cpp index cd93f3da9..77250073f 100644 --- a/src/cvc/state/state_exec/builtins.cpp +++ b/src/cvc/state/state_exec/builtins.cpp @@ -1,5 +1,7 @@ +#include #include #include +#include #include #include // SE-1: (length "...") counts codepoints for a string #include @@ -274,12 +276,17 @@ value_t builtin_ge(std::span args) { value_t builtin_eq(std::span args) { expect_exact(args, 2, "="); - return value_t{values_equal(args[0], args[1])}; + if (!is_compound(args[0]) || !is_compound(args[1])) + return value_t{values_equal(args[0], args[1])}; + return run_data_job(make_equal_job(args[0], args[1])); } value_t builtin_ne(std::span args) { expect_exact(args, 2, "!="); - return value_t{!values_equal(args[0], args[1])}; + if (!is_compound(args[0]) || !is_compound(args[1])) + return value_t{!values_equal(args[0], args[1])}; + return run_data_job(make_equal_job(args[0], args[1]), + [](value_t equal) { return value_t{!std::get(equal.v)}; }); } // --------------------------------------------------------------------------- @@ -287,6 +294,8 @@ value_t builtin_ne(std::span args) { // --------------------------------------------------------------------------- value_t builtin_str_concat(std::span args) { + if (std::any_of(args.begin(), args.end(), is_compound)) + return run_data_job(make_concat_job({args.begin(), args.end()}, "")); std::string result; for (auto &a : args) { if (auto *s = std::get_if(&a.v)) @@ -299,6 +308,8 @@ value_t builtin_str_concat(std::span args) { value_t builtin_str(std::span args) { expect_exact(args, 1, "str"); + if (is_compound(args[0])) + return run_data_job(make_print_job(args[0])); return value_t{to_string(args[0])}; } @@ -625,6 +636,18 @@ value_t builtin_type_of(std::span args) { // --------------------------------------------------------------------------- value_t builtin_print(std::span args) { + if (std::any_of(args.begin(), args.end(), is_compound)) { + // The line goes out once all of it is printed. A string prints quoted here, as to_string + // prints it, so it goes in already printed. + std::vector parts; + parts.reserve(args.size()); + for (auto &a : args) + parts.push_back(std::holds_alternative(a.v) ? value_t{to_string(a)} : a); + return run_data_job(make_concat_job(std::move(parts), " "), [](value_t line) { + std::cout << std::get(line.v) << '\n'; + return nil_value; + }); + } for (size_t i = 0; i < args.size(); ++i) { if (i > 0) std::cout << ' '; diff --git a/src/cvc/state/state_exec/data_job.cpp b/src/cvc/state/state_exec/data_job.cpp new file mode 100644 index 000000000..0968e09a4 --- /dev/null +++ b/src/cvc/state/state_exec/data_job.cpp @@ -0,0 +1,612 @@ +/* + Copyright 2026 The University of Texas at Austin + + data_job — walks over nested values (compare, print, copy, macro expansion) that keep their + work on the heap and run a slice at a time. +*/ + +#include +#include +#include +#include +#include // evaluation_stats +#include +#include // the settings' key prefixes +#include +#include +#include +#include +#include +#include + +namespace cvc::state_exec { + +namespace { + +// A list or dict being walked, held by owning pointer and read by index. +struct cursor { + list_ptr list; + dict_ptr dict; + std::size_t next = 0; // the next element to visit + + std::size_t size() const { return list ? list->size() : dict->size(); } +}; + +// The cursor for `v` if it is a non-empty list or dict. +std::optional open_cursor(const value_t &v) { + if (auto *l = std::get_if(&v.v); l && *l && !(*l)->empty()) + return cursor{*l, nullptr, 0}; + if (auto *d = std::get_if(&v.v); d && *d && !(*d)->empty()) + return cursor{nullptr, *d, 0}; + return std::nullopt; +} + +// -- print ------------------------------------------------------------------- + +// to_string caps total OUTPUT length (a shared/wide structure would otherwise expand +// exponentially, a cyclic one forever); past it, it emits an ellipsis and stops -- enough for +// debug / coercion / error messages, and impossible to weaponise into a hang or OOM. Depth +// needs no cap of its own: the walk keeps its open lists and dicts on a heap stack, not the +// native one, and each level opened costs output, so the length cap bounds that stack too. +constexpr std::size_t kToStringMaxLen = std::size_t{1} << 16; // 64 KiB +constexpr const char *kEllipsis = "\xE2\x80\xA6"; // U+2026 HORIZONTAL ELLIPSIS + +class print_walk final : public data_job { +public: + explicit print_walk(value_t v) : root_(std::move(v)) {} + + bool step(std::size_t budget) override { + if (!started_) { + started_ = true; + print(root_); + } + for (std::size_t visited = 0; !open_.empty(); ++visited) { + if (budget != 0 && visited >= budget) + return false; + cursor &c = open_.back(); + const bool is_list = c.list != nullptr; + const std::size_t i = c.next++; + if (i >= c.size()) { + out_ += is_list ? ')' : '}'; + open_.pop_back(); + continue; + } + ++nodes_; + if (i > 0) + out_ += is_list ? " " : ", "; + if (out_.size() >= kToStringMaxLen) { + out_ += kEllipsis; + out_ += is_list ? ')' : '}'; + open_.pop_back(); + continue; + } + if (is_list) { + print((*c.list)[i]); // may push, invalidating c + } else { + const auto &[key, value] = (*c.dict)[i]; + out_ += '"'; + out_ += key; + out_ += "\": "; + print(value); // may push, invalidating c + } + } + return true; + } + + value_t result() override { return value_t{std::move(out_)}; } + std::string take_string() { return std::move(out_); } + +private: + // Print `val`; a non-empty list or dict is only opened (its bracket printed and a cursor + // pushed for step() to print its elements and close it). + void print(const value_t &val) { + if (out_.size() >= kToStringMaxLen) { + out_ += kEllipsis; + return; + } + std::visit( + [&](auto &&arg) { + using T = std::decay_t; + if constexpr (std::is_same_v) + out_ += "nil"; + else if constexpr (std::is_same_v) + out_ += arg ? "#t" : "#f"; + else if constexpr (std::is_same_v) + out_ += std::to_string(arg); + else if constexpr (std::is_same_v) { + std::ostringstream oss; + oss << arg; + out_ += oss.str(); + } else if constexpr (std::is_same_v) { + out_ += '"'; + out_ += arg; + out_ += '"'; + } else if constexpr (std::is_same_v) { + // b"..." — printable ASCII verbatim, everything else (incl. all bytes >= 0x80) + // hex-escaped as \xNN, so the rendering is always valid ASCII text and never dumps + // raw binary bytes. + static const char kHex[] = "0123456789abcdef"; + out_ += "b\""; + for (unsigned char b : arg.data) { + if (b == '"' || b == '\\') { + out_ += '\\'; + out_ += static_cast(b); + } else if (b >= 0x20 && b < 0x7F) { + out_ += static_cast(b); + } else { + out_ += "\\x"; + out_ += kHex[b >> 4]; + out_ += kHex[b & 0x0F]; + } + } + out_ += '"'; + } else if constexpr (std::is_same_v) + out_ += arg.name; + else if constexpr (std::is_same_v) { + if (!arg || arg->empty()) { + out_ += "()"; + return; + } + out_ += '('; + open_.push_back({arg, nullptr, 0}); + } else if constexpr (std::is_same_v) + out_ += ""; + else if constexpr (std::is_same_v) { + if (!arg || arg->empty()) { + out_ += "{}"; + return; + } + out_ += '{'; + open_.push_back({nullptr, arg, 0}); + } else if constexpr (std::is_same_v) + out_ += ""; + else if constexpr (std::is_same_v) + out_ += arg ? "type_name + ">" : ""; + else if constexpr (std::is_same_v) + out_ += arg && arg->exhausted ? "" : ""; + else + out_ += ""; + }, + val.v); + } + + value_t root_; + bool started_ = false; + std::string out_; + std::vector open_; +}; + +class concat_walk final : public data_job { +public: + concat_walk(std::vector parts, std::string sep) + : parts_(std::move(parts)), sep_(std::move(sep)) {} + + bool step(std::size_t budget) override { + std::size_t used = 0; + while (next_ < parts_.size()) { + if (!part_) { + if (next_ > 0) + out_ += sep_; + if (auto *s = std::get_if(&parts_[next_].v)) { + out_ += *s; + ++next_; + continue; + } + part_ = std::make_unique(parts_[next_]); + } + if (budget != 0 && used >= budget) + return false; + const std::size_t before = part_->nodes(); + const bool done = part_->step(budget == 0 ? 0 : budget - used); + used += std::max(1, part_->nodes() - before); + nodes_ += part_->nodes() - before; + if (!done) + return false; + out_ += part_->take_string(); + part_.reset(); + ++next_; + } + return true; + } + + value_t result() override { return value_t{std::move(out_)}; } + +private: + std::vector parts_; + std::string sep_; + std::size_t next_ = 0; + std::unique_ptr part_; // the part being printed + std::string out_; +}; + +// -- equal ------------------------------------------------------------------- + +struct ptr_pair_hash { + std::size_t operator()(const std::pair &p) const noexcept { + return std::hash()(p.first) * 1000003u ^ std::hash()(p.second); + } +}; +using ptr_pair_set = std::unordered_set, ptr_pair_hash>; + +// `seen_` memoizes the container pairs already under comparison: re-encountering one (a shared +// node reached by two paths, or a cycle) counts as equal co-inductively. That bounds the work to +// the DISTINCT node pairs (the physical size), never the exponential logical unfolding of a +// shared DAG, and ends on cyclic structures; for finite acyclic values the result is that of a +// naive compare. A deadline poll per node bounds a huge NON-shared structure (memoization alone +// does not: distinct nodes are never re-encountered). +class equal_walk final : public data_job { +public: + equal_walk(value_t a, value_t b) : a_(std::move(a)), b_(std::move(b)) {} + + bool step(std::size_t budget) override { + if (done_) + return true; + if (!started_) { + started_ = true; + if (!visit(a_, b_)) + return finish(false); + } + for (std::size_t visited = 0; !open_.empty(); ++visited) { + if (budget != 0 && visited >= budget) + return false; + auto &[ca, cb] = open_.back(); + const std::size_t n = ca.size(); + if (n != cb.size()) // changed between two slices + return finish(false); + const std::size_t i = ca.next++; + if (i >= n) { + open_.pop_back(); + continue; + } + ++nodes_; + if (eval_deadline_expired()) + throw std::runtime_error("values_equal: structure too large to compare"); + bool equal; + // Element references stay valid through visit(), which only pushes onto open_. + if (ca.list) { + const value_t &x = (*ca.list)[i]; + const value_t &y = (*cb.list)[i]; + equal = visit(x, y); // may push, invalidating ca and cb + } else { + const auto &[kx, x] = (*ca.dict)[i]; + const auto &[ky, y] = (*cb.dict)[i]; + equal = kx == ky && visit(x, y); // may push, invalidating ca and cb + } + if (!equal) + return finish(false); + } + return finish(true); + } + + value_t result() override { return value_t{equal_}; } + bool equal() const { return equal_; } + +private: + bool finish(bool equal) { + equal_ = equal; + done_ = true; + open_.clear(); + return true; + } + + // Whether x and y are equal as far as one level shows; a list or dict pair still to compare + // element by element gets a cursor pair. + bool visit(const value_t &x, const value_t &y) { + if (x.v.index() != y.v.index()) + return false; + return std::visit( + [&](auto &&arg_a) -> bool { + using T = std::decay_t; + auto &arg_b = std::get(y.v); + if constexpr (std::is_same_v) + return true; + else if constexpr (std::is_same_v || std::is_same_v) { + if (arg_a == arg_b) + return true; + if (!arg_a || !arg_b) + return false; + if (arg_a->size() != arg_b->size()) + return false; + if (!seen_) + seen_.emplace(); // not before the first pair: comparing scalars allocates nothing + if (!seen_ + ->insert({static_cast(arg_a.get()), + static_cast(arg_b.get())}) + .second) + return true; // this pair is already being compared (shared/cyclic) -> equal + if constexpr (std::is_same_v) + open_.push_back({cursor{arg_a, nullptr, 0}, cursor{arg_b, nullptr, 0}}); + else + open_.push_back({cursor{nullptr, arg_a, 0}, cursor{nullptr, arg_b, 0}}); + return true; + } else if constexpr (std::is_same_v) + return false; // functions are never equal + else + // bool, numbers, strings, symbols and bytes by value; closures, data objects and + // generators by identity + return arg_a == arg_b; + }, + x.v); + } + + value_t a_, b_; + bool started_ = false; + bool done_ = false; + bool equal_ = false; + std::optional seen_; + std::vector> open_; +}; + +// -- copy -------------------------------------------------------------------- + +// Memoized on the source node, so a shared-pointer DAG is copied in PHYSICAL time (the copy has +// the same sharing, disjoint from the original) and a cycle is copied as a cycle. A deadline +// poll per node aborts a huge payload at the time budget. +class copy_walk final : public data_job { +public: + explicit copy_walk(value_t v) : root_(std::move(v)) {} + + bool step(std::size_t budget) override { + if (!started_) { + started_ = true; + result_ = copy(root_); + } + for (std::size_t visited = 0; !open_.empty(); ++visited) { + if (budget != 0 && visited >= budget) + return false; + auto &[src, dst] = open_.back(); + const std::size_t i = src.next++; + if (i >= src.size()) { + open_.pop_back(); + continue; + } + ++nodes_; + if (eval_deadline_expired()) + throw std::runtime_error("state-data-get: evaluation exceeded time budget"); + if (src.list) { + list_ptr out = std::get(dst.v); + const value_t &e = (*src.list)[i]; + out->push_back(copy(e)); // may push, invalidating src and dst + } else { + dict_ptr out = std::get(dst.v); + const auto &[k, e] = (*src.dict)[i]; + out->emplace_back(k, copy(e)); // may push, invalidating src and dst + } + } + return true; + } + + value_t result() override { return std::move(result_); } + +private: + // The copy of `v`: a list or dict is a new empty one, filled by step() through its cursor. + value_t copy(const value_t &v) { + std::optional c = open_cursor(v); + if (!c) { + // empty or null lists and dicts still get their own (empty) copy + if (auto *l = std::get_if(&v.v); l && *l) + return make_list(); + if (auto *d = std::get_if(&v.v); d && *d) + return make_dict(); + return v; // scalar / symbol (opaque callables are refused at state-data-set) + } + const void *key = c->list ? static_cast(c->list.get()) + : static_cast(c->dict.get()); + if (auto found = memo_.find(key); found != memo_.end()) + return found->second; // shared node / cycle already copied -> reuse (bounds the walk) + value_t out; + if (c->list) { + auto l = std::make_shared>(); + l->reserve(c->list->size()); + out = value_t{l}; + } else { + auto d = std::make_shared(); + d->reserve(c->dict->size()); + out = value_t{d}; + } + memo_.emplace(key, out); // registered before its elements are copied + open_.emplace_back(std::move(*c), out); + return out; + } + + value_t root_; + bool started_ = false; + value_t result_; + std::unordered_map memo_; + std::vector> open_; +}; + +// -- substitute -------------------------------------------------------------- + +class substitute_walk final : public data_job { +public: + substitute_walk(value_t tmpl, std::unordered_map subst) + : root_(std::move(tmpl)), subst_(std::move(subst)) {} + + bool step(std::size_t budget) override { + if (!started_) { + started_ = true; + result_ = expand(root_); + } + for (std::size_t visited = 0; !open_.empty(); ++visited) { + if (budget != 0 && visited >= budget) + return false; + auto &[src, dst] = open_.back(); + const std::size_t i = src.next++; + if (i >= src.list->size()) { + open_.pop_back(); + continue; + } + ++nodes_; + list_ptr out = dst; + out->push_back(expand((*src.list)[i])); // may push, invalidating src and dst + } + return true; + } + + value_t result() override { return std::move(result_); } + +private: + value_t expand(const value_t &v) { + if (auto *sym = std::get_if(&v.v)) { + auto it = subst_.find(sym->name); + return it != subst_.end() ? it->second : v; + } + if (auto *l = std::get_if(&v.v); l && *l) { + auto out = std::make_shared>(); + out->reserve((*l)->size()); + open_.emplace_back(cursor{*l, nullptr, 0}, out); + return value_t{out}; + } + return v; + } + + value_t root_; + std::unordered_map subst_; + bool started_ = false; + value_t result_; + std::vector> open_; +}; + +// The data side of the evaluator step running on this thread, if any. +thread_local detail::data_step_scope *t_scope = nullptr; + +// Run `walk` to completion and count its nodes into the current step. +template Walk &finish_walk(Walk &walk) { + walk.step(0); + count_data_nodes(walk.nodes()); + return walk; +} + +std::optional read_size_setting(cvc::state &root, const std::string &path) { + cvc::state *node = root.findDescendant(path); + if (!node) + return std::nullopt; + const std::string v = node->value(); + std::size_t n = 0; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + if (ec != std::errc{} || end != v.data() + v.size() || v.empty()) + return std::nullopt; + return n; +} + +} // namespace + +// -- the walks as free functions ---------------------------------------------- + +std::string to_string(const value_t &val) { + print_walk walk(val); + return finish_walk(walk).take_string(); +} + +bool values_equal(const value_t &a, const value_t &b) { + equal_walk walk(a, b); + return finish_walk(walk).equal(); +} + +// -- jobs ---------------------------------------------------------------------- + +bool is_compound(const value_t &v) { + if (auto *l = std::get_if(&v.v)) + return *l && !(*l)->empty(); + if (auto *d = std::get_if(&v.v)) + return *d && !(*d)->empty(); + return false; +} + +std::unique_ptr make_concat_job(std::vector parts, std::string sep) { + return std::make_unique(std::move(parts), std::move(sep)); +} + +std::unique_ptr make_equal_job(value_t a, value_t b) { + return std::make_unique(std::move(a), std::move(b)); +} + +std::unique_ptr make_print_job(value_t v) { + return std::make_unique(std::move(v)); +} + +std::unique_ptr make_copy_job(value_t v) { + return std::make_unique(std::move(v)); +} + +std::unique_ptr make_substitute_job(value_t tmpl, + std::unordered_map subst) { + return std::make_unique(std::move(tmpl), std::move(subst)); +} + +value_t run_to_completion(data_job &job) { + const std::size_t before = job.nodes(); + job.step(0); + count_data_nodes(job.nodes() - before); + return job.result(); +} + +// -- settings ------------------------------------------------------------------ + +data_options data_options_from_state(cvc::state &root, const std::string &scheduler_id, + const data_options &base) { + data_options opts = base; + const auto resolve = [&](const char *key, std::size_t &out) { + const std::string suffix = std::string(".data.") + key; + if (auto v = read_size_setting(root, std::string(scheduler::defaults_prefix) + suffix)) + out = *v; + if (!scheduler_id.empty()) + if (auto v = read_size_setting(root, std::string(scheduler::sched_prefix) + "." + + scheduler_id + suffix)) + out = *v; + }; + resolve("step_nodes", opts.step_nodes); // 0 = no slicing + return opts; +} + +// -- running a native call's job ------------------------------------------------- + +value_t run_data_job(std::unique_ptr job, std::function then) { + detail::data_step_scope *scope = t_scope; + const bool may_defer = scope && scope->step_nodes_ > 0 && scope->may_defer_ && !scope->deferred_; + const std::size_t before = job->nodes(); + const bool done = job->step(may_defer ? scope->remaining() : 0); + count_data_nodes(job->nodes() - before); + if (!done) { + scope->deferred_ = std::make_unique( + detail::deferred_data_job{std::move(job), std::move(then)}); + return nil_value; + } + value_t r = job->result(); + return then ? then(std::move(r)) : r; +} + +void count_data_nodes(std::size_t nodes) { + if (nodes == 0) + return; + if (detail::data_step_scope *scope = t_scope) { + scope->nodes_ += nodes; + scope->stats_.add_data_nodes(nodes); + } +} + +// -- data_step_scope ------------------------------------------------------------- + +detail::data_step_scope::data_step_scope(std::size_t step_nodes, evaluation_stats &stats, + release_backlog *backlog) + : step_nodes_(step_nodes), stats_(stats), backlog_(step_nodes > 0 ? backlog : nullptr), + outer_(t_scope), routing_(backlog_) { + t_scope = this; +} + +detail::data_step_scope::~data_step_scope() { t_scope = outer_; } + +std::size_t detail::data_step_scope::remaining() const { + return step_nodes_ > nodes_ ? step_nodes_ - nodes_ : 1; +} + +void detail::data_step_scope::drain_backlog() { + if (!backlog_ || backlog_->empty() || nodes_ >= step_nodes_) + return; + count_data_nodes(backlog_->drain(step_nodes_ - nodes_)); +} + +detail::data_step_scope *detail::data_step_scope::current() { return t_scope; } + +} // namespace cvc::state_exec diff --git a/src/cvc/state/state_exec/intrinsics.cpp b/src/cvc/state/state_exec/intrinsics.cpp index 43ca8ce6c..de35c0652 100644 --- a/src/cvc/state/state_exec/intrinsics.cpp +++ b/src/cvc/state/state_exec/intrinsics.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -88,57 +89,6 @@ std::string coerce_state_string(const value_t &v) { return to_string(v); // list/dict/symbol/closure — readable fallback } -// Clone a value's compound structure (list/dict) so the caller gets a PRIVATE copy that does -// not alias node-resident storage. Bounded and cycle-safe, mirroring values_equal / to_string -// (it is the third structural walker and must not be the weak link): it MEMOIZES on the source -// node pointer, so a shared-pointer DAG is copied in PHYSICAL time (the copy preserves the same -// sharing, disjoint from the stored original) and a cycle terminates; it keeps the elements -// still to copy on a heap stack, so any depth copies without native recursion; and it polls -// the evaluation deadline so a huge payload aborts at the time budget instead of running to -// completion in one native step. -value_t deep_copy(const value_t &v) { - std::unordered_map memo; - value_t result; - // (source, destination) pairs: each destination is a slot of a copy already allocated at its - // final size, so it stays put while the copy fills in. - std::vector> pending{{&v, &result}}; - while (!pending.empty()) { - if (eval_deadline_expired()) - throw std::runtime_error("state-data-get: evaluation exceeded time budget"); - auto [src, dst] = pending.back(); - pending.pop_back(); - if (auto *l = std::get_if(&src->v); l && *l) { - auto found = memo.find(static_cast(l->get())); - if (found != memo.end()) { - *dst = found->second; // shared node / cycle already copied -> reuse (bounds the walk) - continue; - } - auto out = std::make_shared>((*l)->size()); - *dst = value_t{out}; - memo.emplace(static_cast(l->get()), *dst); // register BEFORE descending - for (std::size_t i = (*l)->size(); i-- > 0;) - pending.emplace_back(&(**l)[i], &(*out)[i]); - } else if (auto *dp = std::get_if(&src->v); dp && *dp) { - auto found = memo.find(static_cast(dp->get())); - if (found != memo.end()) { - *dst = found->second; - continue; - } - auto out = std::make_shared>>(); - out->reserve((*dp)->size()); - for (const auto &kv : **dp) - out->emplace_back(kv.first, value_t{}); - *dst = value_t{out}; - memo.emplace(static_cast(dp->get()), *dst); - for (std::size_t i = (*dp)->size(); i-- > 0;) - pending.emplace_back(&(**dp)[i].second, &(*out)[i].second); - } else { - *dst = *src; // scalar / symbol / null (opaque callables are refused at state-data-set) - } - } - return result; -} - void require_root(const intrinsics_context *ctx, const char *name) { if (!ctx->root) throw std::runtime_error(std::string(name) + ": no state root bound"); @@ -249,7 +199,7 @@ value_t intrinsic_state_data_get(intrinsics_context *ctx, std::span(&d)) - return deep_copy(*v); + return run_data_job(make_copy_job(*v)); // A RAW std::string blob on the data channel reads back as the DSL `bytes` type (opaque octets), // NOT an opaque data_object. That raw representation is what state-data-set stores for a `bytes` // value, what the `?data` URI channel writes (uri_state.cpp), and what the §13.9 HTTP cache parks @@ -461,6 +411,18 @@ value_t intrinsic_ps(intrinsics_context *ctx, std::span args) { return make_list(std::move(result)); } +// The budget-by-kind fields of `info`, as dict entries. +static void append_budget_usage(std::vector> &entries, + const process_info &info) { + entries.emplace_back("read_steps", value_t(static_cast(info.read_steps))); + entries.emplace_back("eval_steps", value_t(static_cast(info.eval_steps))); + entries.emplace_back("data_steps", value_t(static_cast(info.data_steps))); + entries.emplace_back("data_nodes", value_t(static_cast(info.data_nodes))); + entries.emplace_back("read_time", value_t(info.read_time)); + entries.emplace_back("eval_time", value_t(info.eval_time)); + entries.emplace_back("data_time", value_t(info.data_time)); +} + value_t intrinsic_inspect(intrinsics_context *ctx, std::span args) { expect_exact(args, 1, "inspect"); require_sched(ctx, "inspect"); @@ -483,6 +445,7 @@ value_t intrinsic_inspect(intrinsics_context *ctx, std::span args entries.emplace_back("uid", value_t(info->uid)); entries.emplace_back("gid", value_t(info->gid)); entries.emplace_back("steps", value_t(static_cast(info->step_count))); + append_budget_usage(entries, *info); entries.emplace_back("elapsed_time", value_t(info->elapsed_time)); entries.emplace_back("memory", value_t(static_cast(info->current_memory))); entries.emplace_back("peak_memory", value_t(static_cast(info->peak_memory))); @@ -556,6 +519,25 @@ value_t intrinsic_message_bytes_limit(intrinsics_context *ctx, std::span(ctx->proc->max_message_bytes)); } +// What the current process's budget went on: steps, seconds and deep-data nodes by kind (read, +// eval, data), plus the total steps -- the same fields inspect reports for any pid. +value_t intrinsic_budget_usage(intrinsics_context *ctx, std::span args) { + expect_exact(args, 0, "budget-usage"); + // The running process, as self-pid finds it: the scheduler's current one, else the context's. + process_info info{}; + if (ctx->sched && ctx->sched->current_pid() >= 0) { + if (auto running = ctx->sched->get_process_info(ctx->sched->current_pid())) + info = *running; + } else if (ctx->proc) { + info.step_count = ctx->proc->step_count(); + fill_budget_usage(info, ctx->proc->state.stats); + } + std::vector> entries; + entries.emplace_back("steps", value_t(static_cast(info.step_count))); + append_budget_usage(entries, info); + return make_dict(std::move(entries)); +} + value_t intrinsic_step_count(intrinsics_context *ctx, std::span args) { expect_exact(args, 0, "step-count"); if (!ctx->proc) @@ -872,6 +854,7 @@ void register_intrinsics(environment_ptr env, intrinsics_context *ctx) { reg("message-bytes", intrinsic_message_bytes); reg("message-bytes-limit", intrinsic_message_bytes_limit); reg("step-count", intrinsic_step_count); + reg("budget-usage", intrinsic_budget_usage); // System identity reg("cluster-id", intrinsic_cluster_id); diff --git a/src/cvc/state/state_exec/scheduler.cpp b/src/cvc/state/state_exec/scheduler.cpp index 6d1d09872..18c428ace 100644 --- a/src/cvc/state/state_exec/scheduler.cpp +++ b/src/cvc/state/state_exec/scheduler.cpp @@ -55,8 +55,9 @@ void scheduler::load_settings() { max_pending_messages = effective; - // --- parser.* (same resolution order; see parse_options_from_state) --- + // --- parser.* and data.* (same resolution order; see parse_options_from_state) --- parse_options_ = parse_options_from_state(*watch_root_, id_); + data_options_ = data_options_from_state(*watch_root_, id_); // Publish effective values to per-scheduler subtree if (!id_.empty()) { @@ -64,6 +65,7 @@ void scheduler::load_settings() { (*watch_root_)(base + ".max_pending_messages").value(std::to_string(max_pending_messages)); (*watch_root_)(base + ".parser.max_depth").value(std::to_string(parse_options_.max_depth)); (*watch_root_)(base + ".parser.step_tokens").value(std::to_string(parse_options_.step_tokens)); + (*watch_root_)(base + ".data.step_nodes").value(std::to_string(data_options_.step_nodes)); (*watch_root_)(base + ".policy") .value(policy_ == scheduling_policy::round_robin ? "round_robin" : policy_ == scheduling_policy::priority ? "priority" @@ -95,6 +97,7 @@ int scheduler::execute(const std::string &script, const execute_options &opts) { proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); proc.state = evaluator_.create_state(script, opts.env, parse_options_); + proc.state.data_step_nodes = data_options_.step_nodes; proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; @@ -120,6 +123,7 @@ int scheduler::execute(const value_t &expr, const execute_options &opts) { proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); proc.state = evaluator_.create_state(expr, opts.env); + proc.state.data_step_nodes = data_options_.step_nodes; proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); return pid; @@ -197,9 +201,11 @@ void scheduler::execute_process_step(process &proc) { } reset_guard{current_pid_, current_proc_}; // A process still reading its script (parser.step_tokens) takes signals and watch events once - // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. - const bool can_interrupt = - !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. One + // in the middle of deep-data work (data.step_nodes) takes them once that is done: the work's + // call frame waits on top of the stack. + const bool can_interrupt = !proc.in_signal_handler && !proc.in_watch_handler && + !proc.state.reading && !proc.state.data_work; // 1. Handle pending signals first if (!proc.pending_signals.empty() && can_interrupt) { @@ -228,9 +234,11 @@ void scheduler::execute_process_step(process &proc) { const double cpu0 = charge_cpu ? thread_cpu_seconds() : 0.0; bool done = evaluator_.step(proc.state); - // 4. Accumulate running time + // 4. Accumulate running time, and charge it to what the step was spent on auto now = std::chrono::steady_clock::now(); - proc.accumulated_time += std::chrono::duration(now - proc.last_run_start).count(); + const double step_seconds = std::chrono::duration(now - proc.last_run_start).count(); + proc.accumulated_time += step_seconds; + proc.state.stats.add_step_time(step_seconds); if (charge_cpu) proc.charge_cpu_time(thread_cpu_seconds() - cpu0); @@ -696,6 +704,7 @@ int scheduler::fork(int pid) { child->state.done = parent.state.done; child->state.stack = parent.state.stack; // Copy stack child->state.user_macros = parent.state.user_macros; + child->state.data_step_nodes = parent.state.data_step_nodes; // Create fresh env as a copy of parent's bindings auto child_env = std::make_shared(); if (parent.state.global_env) { @@ -779,6 +788,7 @@ process_info scheduler::make_info(const process &proc) const { info.uid = proc.uid; info.gid = proc.gid; info.step_count = proc.step_count(); + fill_budget_usage(info, proc.state.stats); info.elapsed_time = proc.elapsed_time(); info.current_memory = mem_tracker_.current_bytes(proc.pid); info.peak_memory = mem_tracker_.peak_bytes(proc.pid); diff --git a/src/cvc/state/state_exec/stackless_evaluator.cpp b/src/cvc/state/state_exec/stackless_evaluator.cpp index a1483a6a3..7858da2b8 100644 --- a/src/cvc/state/state_exec/stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/stackless_evaluator.cpp @@ -74,25 +74,110 @@ evaluator_state stackless_evaluator::create_program_state(std::vector f // parse_error propagates like any error from step(): the sync scheduler rethrows it to its // caller, and the async scheduler kills the process with it. bool stackless_evaluator::step_read(evaluator_state &state) { - state.stats.increment_step(); + state.stats.increment_step(step_kind::read); script_reader &reader = *state.reading; if (!reader.step(reader.options().step_tokens)) return false; evaluator_state program = create_program_state(std::move(reader.forms()), state.global_env); program.stats = std::move(state.stats); + program.data_step_nodes = state.data_step_nodes; + program.backlog = std::move(state.backlog); state = std::move(program); // also releases the reader return state.done; } +namespace { +// RAII: while alive, the native call a step applies may leave deep-data work behind +// (run_data_job), in the step's data scope. +struct native_call_scope { + explicit native_call_scope(detail::data_step_scope *data) : data(data) { + if (data) + data->allow_defer(true); + } + ~native_call_scope() { + if (data) + data->allow_defer(false); + } + native_call_scope(const native_call_scope &) = delete; + native_call_scope &operator=(const native_call_scope &) = delete; + detail::data_step_scope *data; +}; +} // namespace + +// Run a slice of the deep-data work a step left; once it is done, finish the native call or macro +// expansion that started it. +bool stackless_evaluator::step_data(evaluator_state &state) { + state.stats.increment_step(step_kind::data); + detail::data_step_scope *data = detail::data_step_scope::current(); + const std::shared_ptr work = state.data_work; + const std::size_t before = work->job->nodes(); + const bool finished = work->job->step(data ? data->remaining() : 0); + count_data_nodes(work->job->nodes() - before); + if (!finished) + return false; + state.data_work.reset(); + value_t result = work->job->result(); + if (work->what == pending_data_work::kind::macro_expansion) { + begin_expansion(state, std::move(result)); + return false; + } + if (work->then) { + native_call_scope call(data); // `then` may end in another job + result = work->then(std::move(result)); + if (auto deferred = data ? data->take_deferred() : nullptr) { + state.data_work = std::make_shared( + pending_data_work{std::move(deferred->job), std::move(deferred->then), + pending_data_work::kind::native_call}); + return false; + } + } + pop_frame(state, std::move(result)); + return state.done; +} + +void stackless_evaluator::begin_expansion(evaluator_state &state, value_t expansion) { + eval_frame &frame = state.stack.back(); + frame.expr = std::move(expansion); + frame.phase = eval_phase::init; + frame.index = 0; + frame.results.clear(); + frame.extra_vals.clear(); + frame.extra_strs.clear(); + frame.extra_lists.clear(); + frame.extra_env = nullptr; +} + bool stackless_evaluator::step(evaluator_state &state) { check_interrupted(); - if (state.reading) - return step_read(state); - if (state.done || state.stack.empty()) { + if (state.data_step_nodes > 0 && !state.backlog) + state.backlog = std::make_shared(); + // Held here so the backlog outlives this step's scope even if the step replaces the state. + const std::shared_ptr backlog = state.backlog; + detail::data_step_scope data(state.data_step_nodes, state.stats, backlog.get()); + bool done; + if (state.reading) { + done = step_read(state); + } else if (state.data_work) { + done = step_data(state); + } else if (backlog && !backlog->empty()) { + // A step that only frees what earlier steps dropped. + state.stats.increment_step(step_kind::data); + data.drain_backlog(); + return false; + } else if (state.done || state.stack.empty()) { state.done = true; return true; + } else { + state.stats.increment_step(); + step_frame(state); + done = state.done; } - state.stats.increment_step(); + data.drain_backlog(); + // Done only once what it dropped is freed too: the steps that free the rest come first. + return done && !(backlog && !backlog->empty()); +} + +void stackless_evaluator::step_frame(evaluator_state &state) { auto &frame = state.stack.back(); switch (frame.phase) { @@ -150,7 +235,6 @@ bool stackless_evaluator::step(evaluator_state &state) { step_break_unwind(state, frame); break; } - return state.done; } value_t stackless_evaluator::run(evaluator_state &state, std::optional max_steps, @@ -524,15 +608,18 @@ void stackless_evaluator::step_init(evaluator_state &state, eval_frame &frame) { std::unordered_map subst; for (size_t i = 0; i < macro_params.size(); ++i) subst[macro_params[i].name] = (i < args.size()) ? args[i] : nil_value; - // Re-init current frame with expanded form for evaluation - frame.expr = substitute(body, subst); - frame.phase = eval_phase::init; - frame.index = 0; - frame.results.clear(); - frame.extra_vals.clear(); - frame.extra_strs.clear(); - frame.extra_lists.clear(); - frame.extra_env = nullptr; + // Re-init current frame with expanded form for evaluation. The expansion is deep-data work: + // a step that slices does one slice of it here and leaves the rest to step_data(). + auto job = make_substitute_job(body, std::move(subst)); + detail::data_step_scope *data = detail::data_step_scope::current(); + const bool finished = job->step(data && data->step_nodes() > 0 ? data->remaining() : 0); + count_data_nodes(job->nodes()); + if (!finished) { + state.data_work = std::make_shared( + pending_data_work{std::move(job), {}, pending_data_work::kind::macro_expansion}); + return; + } + begin_expansion(state, job->result()); return; } @@ -581,7 +668,20 @@ void stackless_evaluator::step_apply(evaluator_state &state, eval_frame &frame) throw std::runtime_error("apply: undefined function: " + head_sym->name); if (auto *fn = std::get_if(&func_val->v)) { - pop_frame(state, (*fn)(evaled)); + detail::data_step_scope *data = detail::data_step_scope::current(); + value_t result; + { + native_call_scope call(data); + result = (*fn)(evaled); + } + if (auto deferred = data ? data->take_deferred() : nullptr) { + // The call left deep-data work: this frame waits while step_data() runs it. + state.data_work = std::make_shared( + pending_data_work{std::move(deferred->job), std::move(deferred->then), + pending_data_work::kind::native_call}); + return; + } + pop_frame(state, std::move(result)); return; } if (auto *cp = std::get_if(&func_val->v)) { @@ -1106,33 +1206,4 @@ void stackless_evaluator::handle_defclass(evaluator_state &state, eval_frame &fr pop_frame(state, *env->lookup(cls_name)); } -// --------------------------------------------------------------------------- -// macro substitution -// --------------------------------------------------------------------------- - -value_t stackless_evaluator::substitute(const value_t &tmpl, - const std::unordered_map &subst) { - // Like the evaluator, the copy keeps its work on the heap, not the native stack, so a - // template of any depth expands. (source, destination) pairs: each destination is a slot of - // a list copy allocated at its final size, so it stays put while the copy fills in. - value_t result; - std::vector> pending{{&tmpl, &result}}; - while (!pending.empty()) { - auto [src, dst] = pending.back(); - pending.pop_back(); - if (auto *sym = std::get_if(&src->v)) { - auto it = subst.find(sym->name); - *dst = it != subst.end() ? it->second : *src; - } else if (auto *lp = std::get_if(&src->v); lp && *lp) { - auto copy = std::make_shared>((*lp)->size()); - *dst = value_t{copy}; - for (std::size_t i = (*lp)->size(); i-- > 0;) - pending.emplace_back(&(**lp)[i], &(*copy)[i]); - } else { - *dst = *src; - } - } - return result; -} - } // namespace cvc::state_exec diff --git a/src/cvc/state/state_exec/state_value_codec.cpp b/src/cvc/state/state_exec/state_value_codec.cpp index 1106f07b4..cd9c0b893 100644 --- a/src/cvc/state/state_exec/state_value_codec.cpp +++ b/src/cvc/state/state_exec/state_value_codec.cpp @@ -708,6 +708,11 @@ void encode_evaluator_state(cvc::state &node, const evaluator_state &es) { r("step_tokens").value(std::to_string(es.reading->options().step_tokens)); } + // Deep-data slicing (data.step_nodes). Work in flight is not kept: after a restore the waiting + // native call or macro expansion runs again. + if (es.data_step_nodes > 0) + node("data_step_nodes").value(std::to_string(es.data_step_nodes)); + // Stats snapshot node("step_count").value(std::to_string(es.stats.get_step_count())); } @@ -756,6 +761,14 @@ evaluator_state decode_evaluator_state(cvc::state &node) { } } + if (has_child(node, "data_step_nodes")) { + const std::string v = node("data_step_nodes").value(); + std::size_t n = 0; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + if (ec == std::errc{} && end == v.data() + v.size()) + es.data_step_nodes = n; + } + // A script still being read restarts its reading from the beginning. if (has_child(node, "reading")) { auto &r = node("reading"); diff --git a/src/cvc/state/state_exec/types.cpp b/src/cvc/state/state_exec/types.cpp index 00865238c..926055145 100644 --- a/src/cvc/state/state_exec/types.cpp +++ b/src/cvc/state/state_exec/types.cpp @@ -1,10 +1,7 @@ #include -#include #include #include -#include #include -#include #include namespace cvc::state_exec { @@ -192,13 +189,24 @@ class release_queue { // inline. A raw pointer to the running release's local queue, so no thread-exit destructor is // involved and nothing outlives the release. thread_local release_queue *t_release_queue = nullptr; + +// The backlog a backlog_scope routes this thread's last owners to, if any. +thread_local detail::release_backlog *t_backlog = nullptr; } // namespace -void detail::release_nested(std::shared_ptr owner) noexcept { +void detail::release_nested(std::shared_ptr owner, std::size_t nodes) noexcept { if (release_queue *queue = t_release_queue) { queue->push(owner); // out of memory: `owner` is still here, and freed here, recursively return; } + if (release_backlog *backlog = t_backlog) { + try { + backlog->owners_.emplace_back(std::move(owner), nodes); + return; + } catch (...) { + // Out of memory: emplace_back left `owner` as it was; free it now, below. + } + } release_queue queue; t_release_queue = &queue; owner.reset(); @@ -207,6 +215,33 @@ void detail::release_nested(std::shared_ptr owner) noexcept { t_release_queue = nullptr; } +detail::release_backlog::~release_backlog() { drain(0); } + +std::size_t detail::release_backlog::drain(std::size_t budget) { + release_backlog *outer = t_backlog; + t_backlog = this; // what freeing an owner drops joins this backlog + std::size_t freed = 0; + while (!owners_.empty() && (budget == 0 || freed < budget)) { + auto [owner, nodes] = std::move(owners_.back()); + owners_.pop_back(); + owner.reset(); + freed += nodes; + } + t_backlog = outer; + return freed; +} + +detail::backlog_scope::backlog_scope(release_backlog *backlog) + : outer_(t_backlog), armed_(backlog != nullptr) { + if (armed_) + t_backlog = backlog; +} + +detail::backlog_scope::~backlog_scope() { + if (armed_) + t_backlog = outer_; +} + // -- value_tag --------------------------------------------------------------- std::string value_tag::type_name() const { @@ -287,231 +322,6 @@ environment_ptr environment::extend(environment_ptr parent) { return child; } -// -- to_string --------------------------------------------------------------- - -namespace { -// to_string caps total OUTPUT length (a shared/wide structure would otherwise expand -// exponentially, a cyclic one forever); past it, it emits an ellipsis and stops -- enough for -// debug / coercion / error messages, and impossible to weaponise into a hang or OOM. Depth -// needs no cap of its own: the walk keeps its open lists and dicts on a heap stack, not the -// native one, and each level opened costs output, so the length cap bounds that stack too. -constexpr std::size_t kToStringMaxLen = std::size_t{1} << 16; // 64 KiB -constexpr const char *kEllipsis = "\xE2\x80\xA6"; // U+2026 HORIZONTAL ELLIPSIS - -// A list or dict to_string has opened and not yet closed. -struct print_frame { - const std::vector *list = nullptr; - const ordered_map *dict = nullptr; - std::size_t next = 0; // the next element to print -}; - -// Print `val`; a non-empty list or dict is only opened (its bracket printed and a frame pushed -// for to_string's loop to print its elements and close it). -void print_value(const value_t &val, std::string &out, std::vector &open) { - if (out.size() >= kToStringMaxLen) { - out += kEllipsis; - return; - } - std::visit( - [&](auto &&arg) { - using T = std::decay_t; - if constexpr (std::is_same_v) - out += "nil"; - else if constexpr (std::is_same_v) - out += arg ? "#t" : "#f"; - else if constexpr (std::is_same_v) - out += std::to_string(arg); - else if constexpr (std::is_same_v) { - std::ostringstream oss; - oss << arg; - out += oss.str(); - } else if constexpr (std::is_same_v) { - out += '"'; - out += arg; - out += '"'; - } else if constexpr (std::is_same_v) { - // b"..." — printable ASCII verbatim, everything else (incl. all bytes >= 0x80) - // hex-escaped as \xNN, so the rendering is always valid ASCII text and never dumps raw - // binary bytes. - static const char kHex[] = "0123456789abcdef"; - out += "b\""; - for (unsigned char b : arg.data) { - if (b == '"' || b == '\\') { - out += '\\'; - out += static_cast(b); - } else if (b >= 0x20 && b < 0x7F) { - out += static_cast(b); - } else { - out += "\\x"; - out += kHex[b >> 4]; - out += kHex[b & 0x0F]; - } - } - out += '"'; - } else if constexpr (std::is_same_v) - out += arg.name; - else if constexpr (std::is_same_v) { - if (!arg || arg->empty()) { - out += "()"; - return; - } - out += '('; - open.push_back({arg.get(), nullptr, 0}); - } else if constexpr (std::is_same_v) - out += ""; - else if constexpr (std::is_same_v) { - if (!arg || arg->empty()) { - out += "{}"; - return; - } - out += '{'; - open.push_back({nullptr, arg.get(), 0}); - } else if constexpr (std::is_same_v) - out += ""; - else if constexpr (std::is_same_v) - out += arg ? "type_name + ">" : ""; - else if constexpr (std::is_same_v) - out += arg && arg->exhausted ? "" : ""; - else - out += ""; - }, - val.v); -} -} // namespace - -std::string to_string(const value_t &val) { - std::string out; - std::vector open; - print_value(val, out, open); - while (!open.empty()) { - print_frame &f = open.back(); - const std::size_t i = f.next++; - const bool is_list = f.list != nullptr; - if (i == (is_list ? f.list->size() : f.dict->size())) { - out += is_list ? ')' : '}'; - open.pop_back(); - continue; - } - if (i > 0) - out += is_list ? " " : ", "; - if (out.size() >= kToStringMaxLen) { - out += kEllipsis; - out += is_list ? ')' : '}'; - open.pop_back(); - continue; - } - if (is_list) { - print_value((*f.list)[i], out, open); // may push, invalidating f - } else { - const auto &[key, value] = (*f.dict)[i]; - out += '"'; - out += key; - out += "\": "; - print_value(value, out, open); // may push, invalidating f - } - } - return out; -} - -// -- values_equal ------------------------------------------------------------ - -namespace { -struct ptr_pair_hash { - std::size_t operator()(const std::pair &p) const noexcept { - return std::hash()(p.first) * 1000003u ^ std::hash()(p.second); - } -}; -using ptr_pair_set = std::unordered_set, ptr_pair_hash>; -using value_pairs = std::vector>; - -// Whether a and b are equal as far as one level shows; the element pairs a list or dict pair -// still needs compared are pushed onto `pending`. `seen` memoizes the container pairs already -// under comparison: re-encountering one (a shared node reached by two paths, or a cycle) -// counts as equal co-inductively. -bool shallow_equal(const value_t &a, const value_t &b, ptr_pair_set &seen, value_pairs &pending) { - if (a.v.index() != b.v.index()) - return false; - return std::visit( - [&](auto &&arg_a) -> bool { - using T = std::decay_t; - auto &arg_b = std::get(b.v); - if constexpr (std::is_same_v) - return true; - else if constexpr (std::is_same_v) - return arg_a == arg_b; - else if constexpr (std::is_same_v) - return arg_a == arg_b; - else if constexpr (std::is_same_v) - return arg_a == arg_b; - else if constexpr (std::is_same_v) - return arg_a == arg_b; - else if constexpr (std::is_same_v) - return arg_a == arg_b; - else if constexpr (std::is_same_v) { - if (arg_a == arg_b) - return true; - if (!arg_a || !arg_b) - return false; - if (arg_a->size() != arg_b->size()) - return false; - if (!seen.insert({static_cast(arg_a.get()), - static_cast(arg_b.get())}) - .second) - return true; // this pair is already being compared (shared/cyclic) -> equal - for (std::size_t i = arg_a->size(); i-- > 0;) - pending.emplace_back(&(*arg_a)[i], &(*arg_b)[i]); - return true; - } else if constexpr (std::is_same_v) - return arg_a == arg_b; // identity comparison - else if constexpr (std::is_same_v) { - if (arg_a == arg_b) - return true; - if (!arg_a || !arg_b) - return false; - if (arg_a->size() != arg_b->size()) - return false; - if (!seen.insert({static_cast(arg_a.get()), - static_cast(arg_b.get())}) - .second) - return true; - for (std::size_t i = 0; i < arg_a->size(); ++i) - if ((*arg_a)[i].first != (*arg_b)[i].first) - return false; - for (std::size_t i = arg_a->size(); i-- > 0;) - pending.emplace_back(&(*arg_a)[i].second, &(*arg_b)[i].second); - return true; - } else if constexpr (std::is_same_v) - return false; // functions are never equal - else if constexpr (std::is_same_v) - return arg_a == arg_b; // identity comparison - else if constexpr (std::is_same_v) - return arg_a == arg_b; // identity comparison - else if constexpr (std::is_same_v) - return arg_a == arg_b; // byte-wise value comparison - else - return false; - }, - a.v); -} -} // namespace - -// The memo bounds work to the number of DISTINCT node-pairs (the physical size), never the -// exponential logical unfolding of a shared DAG, and terminates on cyclic structures; for -// finite acyclic values the result is identical to a naive compare. The pairs still to compare -// wait on a heap stack, so depth costs no native stack. A deadline poll bounds a huge -// NON-shared structure (memoization alone does not: distinct nodes are never re-encountered). -bool values_equal(const value_t &a, const value_t &b) { - ptr_pair_set seen; - value_pairs pending{{&a, &b}}; - while (!pending.empty()) { - if (eval_deadline_expired()) - throw std::runtime_error("values_equal: structure too large to compare"); - const auto [x, y] = pending.back(); - pending.pop_back(); - if (!shallow_equal(*x, *y, seen, pending)) - return false; - } - return true; -} +// to_string and values_equal are walks: see data_job.cpp. } // namespace cvc::state_exec diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index 3a59b1ee7..2c65feb2b 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -77,6 +77,7 @@ add_executable(state_exec_integration_test state_exec_integration_test.cpp) add_executable(state_exec_advanced_integration_test state_exec_advanced_integration_test.cpp) add_executable(state_exec_gaps_test state_exec_gaps_test.cpp) add_executable(state_exec_deep_data_test state_exec_deep_data_test.cpp) +add_executable(state_exec_data_steps_test state_exec_data_steps_test.cpp) if(NOT WIN32) add_executable(state_exec_multiprocess_test state_exec_multiprocess_test.cpp) endif() @@ -341,6 +342,7 @@ list(APPEND TEST_TARGETS state_exec_advanced_integration_test state_exec_gaps_test state_exec_deep_data_test + state_exec_data_steps_test ) if(NOT WIN32) list(APPEND TEST_TARGETS state_exec_multiprocess_test) @@ -1215,6 +1217,13 @@ target_link_libraries(state_exec_deep_data_test GTest::gtest_main ) +target_link_libraries(state_exec_data_steps_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) + target_link_libraries(state_exec_builtins_test PRIVATE cvc @@ -1579,6 +1588,7 @@ target_compile_features(state_link_bench_test PRIVATE cxx_std_17) target_compile_features(state_exec_parser_test PRIVATE cxx_std_20) target_compile_features(state_exec_types_test PRIVATE cxx_std_20) target_compile_features(state_exec_deep_data_test PRIVATE cxx_std_20) +target_compile_features(state_exec_data_steps_test PRIVATE cxx_std_20) target_compile_features(state_exec_builtins_test PRIVATE cxx_std_20) target_compile_features(state_exec_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_stackless_evaluator_test PRIVATE cxx_std_20) @@ -1990,6 +2000,7 @@ cvc_discover_tests(state_link_bench_test) cvc_discover_tests(state_exec_parser_test) cvc_discover_tests(state_exec_types_test) cvc_discover_tests(state_exec_deep_data_test) +cvc_discover_tests(state_exec_data_steps_test) cvc_discover_tests(state_exec_builtins_test) cvc_discover_tests(state_exec_evaluator_test) cvc_discover_tests(state_exec_stackless_evaluator_test) diff --git a/src/cvc/tests/state_exec_bench_test.cpp b/src/cvc/tests/state_exec_bench_test.cpp index 52b1fef8f..f089768cd 100644 --- a/src/cvc/tests/state_exec_bench_test.cpp +++ b/src/cvc/tests/state_exec_bench_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -185,6 +186,50 @@ TEST_F(ExecBenchTest, ValueDropThroughput) { report("eval_nest_list_2000", M, std::chrono::duration_cast(t1 - t0)); } +TEST_F(ExecBenchTest, DataWalkThroughput) { + if (!bench_enabled()) { + SUCCEED(); + return; + } + + // Deep-data work per node, the unit of data.step_nodes: compare, print, copy, free. The value + // nests lists and dicts 20000 levels, each level holding a string and a number too. + constexpr int64_t kLevels = 20000; + const auto build = [] { + value_t v = value_t{int64_t{0}}; + for (int64_t i = 0; i < kLevels; ++i) + v = i % 2 ? make_dict({{"i", value_t{i}}, {"v", std::move(v)}}) + : make_list({value_t{std::string("s")}, std::move(v), value_t{1.5}}); + return v; + }; + const value_t a = build(), b = build(); + const auto per_node = [](const char *name, data_job &job) { + auto t0 = hrclock::now(); + job.step(0); + auto t1 = hrclock::now(); + report(name, static_cast(job.nodes()), std::chrono::duration_cast(t1 - t0)); + }; + auto eq = make_equal_job(a, b); + per_node("data_equal_per_node", *eq); + auto print = make_print_job(a); + per_node("data_print_per_node", *print); + auto copy = make_copy_job(a); + per_node("data_copy_per_node", *copy); + + // Freeing, through a release backlog drained whole. + value_t doomed = build(); + detail::release_backlog backlog; + { + detail::backlog_scope route(&backlog); + doomed = value_t{}; + } + auto t0 = hrclock::now(); + const std::size_t freed = backlog.drain(0); + auto t1 = hrclock::now(); + report("data_free_per_node", static_cast(freed), + std::chrono::duration_cast(t1 - t0)); +} + TEST_F(ExecBenchTest, SchedulerThroughput) { if (!bench_enabled()) { SUCCEED(); diff --git a/src/cvc/tests/state_exec_data_steps_test.cpp b/src/cvc/tests/state_exec_data_steps_test.cpp new file mode 100644 index 000000000..f1b786cce --- /dev/null +++ b/src/cvc/tests/state_exec_data_steps_test.cpp @@ -0,0 +1,432 @@ +// Deep-data work on the executor ("Deep data on the executor" in docs/STATE_EXEC_GRAMMAR.md): +// walks that run a slice at a time (data_job), the stackless evaluator slicing them +// (evaluator_state::data_step_nodes), the schedulers taking data.step_nodes from the state tree, +// and the budget split by kind (read / eval / data) that process_info, inspect and budget-usage +// report. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cvc::state_exec; + +namespace { + +value_t sym(const char *s) { return value_t{symbol{s}}; } + +value_t nested_list(std::size_t n, value_t leaf = value_t{int64_t{1}}) { + value_t v = make_list({std::move(leaf)}); + for (std::size_t i = 1; i < n; ++i) + v = make_list({std::move(v)}); + return v; +} + +// Lists and dicts alternating, each level also holding a string and a number. +value_t nested_mixed(std::size_t n) { + value_t v = value_t{int64_t{0}}; + for (std::size_t i = 0; i < n; ++i) + v = i % 2 ? make_dict({{"i", value_t{int64_t(i)}}, {"v", std::move(v)}}) + : make_list({value_t{std::string("s")}, std::move(v), value_t{1.5}}); + return v; +} + +// Run `job` in slices of `slice` nodes. Returns its result and how many step() calls it took. +std::pair run_sliced(data_job &job, std::size_t slice) { + int calls = 1; + while (!job.step(slice)) + ++calls; + return {job.result(), calls}; +} + +// The script that builds two separate n-deep lists and compares them. +std::string compare_script(int n) { + return "(begin (set a 1) (set b 1) (set i 0)" + " (while (< i " + + std::to_string(n) + + ") (begin (set a (list a)) (set b (list b)) (set i (+ i 1))))" + " (= a b))"; +} + +// Step `st` until it is done; returns the steps it took. +int step_to_done(stackless_evaluator &ev, evaluator_state &st) { + int steps = 0; + while (!ev.step(st)) + ++steps; + return steps + 1; +} + +} // namespace + +// ---- the jobs --------------------------------------------------------------- + +TEST(DataJob, SlicedWalksMatchWholeOnes) { + const value_t a = nested_mixed(300), b = nested_mixed(300), shorter = nested_mixed(299); + for (std::size_t slice : {1u, 2u, 7u, 100000u}) { + SCOPED_TRACE("slice " + std::to_string(slice)); + auto eq = make_equal_job(a, b); + auto [equal, calls] = run_sliced(*eq, slice); + EXPECT_TRUE(std::get(equal.v)); + if (slice == 1) + EXPECT_GE(calls, 600); // a node per call + auto ne = make_equal_job(a, shorter); + EXPECT_FALSE(std::get(run_sliced(*ne, slice).first.v)); + + auto print = make_print_job(a); + EXPECT_EQ(std::get(run_sliced(*print, slice).first.v), to_string(a)); + + auto copy = make_copy_job(a); + value_t c = run_sliced(*copy, slice).first; + EXPECT_TRUE(values_equal(c, a)); + EXPECT_NE(std::get(c.v).get(), std::get(a.v).get()); // 300 levels: a dict + + auto concat = make_concat_job({value_t{std::string("<")}, a, value_t{int64_t{3}}}, " "); + EXPECT_EQ(std::get(run_sliced(*concat, slice).first.v), + "< " + to_string(a) + " 3"); + + const value_t tmpl = make_list({sym("x"), nested_list(50, sym("y")), sym("z")}); + auto sub = + make_substitute_job(tmpl, {{"x", value_t{int64_t{1}}}, {"y", value_t{std::string("Y")}}}); + EXPECT_TRUE(values_equal( + run_sliced(*sub, slice).first, + make_list({value_t{int64_t{1}}, nested_list(50, value_t{std::string("Y")}), sym("z")}))); + } +} + +TEST(DataJob, NodesDoNotDependOnTheSlice) { + const value_t v = nested_mixed(200); + auto whole = make_print_job(v); + whole->step(0); + auto sliced = make_print_job(v); + run_sliced(*sliced, 3); + EXPECT_EQ(whole->nodes(), sliced->nodes()); + EXPECT_GE(whole->nodes(), 200u); +} + +TEST(DataJob, CopyKeepsSharingAndCycles) { + value_t inner = make_list({value_t{int64_t{3}}}); + auto copy = make_copy_job(make_list({inner, inner})); + value_t c = run_to_completion(*copy); + auto &l = *std::get(c.v); + EXPECT_EQ(std::get(l[0].v).get(), std::get(l[1].v).get()); + EXPECT_NE(std::get(l[0].v).get(), std::get(inner.v).get()); + + auto cyclic = std::make_shared>(); + cyclic->push_back(value_t{cyclic}); + auto cc = make_copy_job(value_t{cyclic}); + value_t cy = run_sliced(*cc, 1).first; + auto copied = std::get(cy.v); + EXPECT_EQ(std::get((*copied)[0].v).get(), copied.get()); + copied->clear(); // break both cycles + cyclic->clear(); +} + +TEST(DataJob, ValueChangedBetweenSlices) { + // A job reads what it walks by index, holding it by owning pointer: a change between two + // slices (another process's) changes the answer, never memory safety. + list_ptr inner = std::make_shared>(); + for (int i = 0; i < 10; ++i) + inner->push_back(value_t{int64_t(i)}); + value_t a = make_list({value_t{inner}}); + value_t b = make_list({make_list(std::vector(*inner))}); + auto eq = make_equal_job(a, b); + EXPECT_FALSE(eq->step(3)); + inner->clear(); + inner->shrink_to_fit(); + EXPECT_FALSE(std::get(run_sliced(*eq, 3).first.v)); + + list_ptr printed = std::make_shared>(20, value_t{int64_t{7}}); + auto print = make_print_job(value_t{printed}); + EXPECT_FALSE(print->step(4)); + printed->clear(); + printed->shrink_to_fit(); + const std::string out = std::get(run_sliced(*print, 4).first.v); + EXPECT_EQ(out.front(), '('); + EXPECT_EQ(out.back(), ')'); +} + +TEST(DataJob, RunDataJobOutsideAStepRunsToCompletion) { + value_t r = run_data_job(make_equal_job(nested_list(100), nested_list(100))); + EXPECT_TRUE(std::get(r.v)); + r = run_data_job(make_print_job(nested_list(3)), + [](value_t s) { return value_t{std::get(s.v) + "!"}; }); + EXPECT_EQ(std::get(r.v), "(((1)))!"); +} + +// ---- the evaluator ------------------------------------------------------------ + +class DataStepsEvaluator : public ::testing::Test { +protected: + environment_ptr env = builtins::make_default_environment(); + stackless_evaluator ev{env}; + + // A state evaluating `script`, with data_step_nodes = `slice`, in an environment where `a` and + // `b` are separately built n-deep lists. + evaluator_state state(const std::string &script, std::size_t slice, std::size_t n = 1000) { + auto run_env = environment::extend(env); + run_env->set("a", nested_list(n)); + run_env->set("b", nested_list(n)); + evaluator_state st = ev.create_state(parse(script), run_env); + st.data_step_nodes = slice; + st.stats.start(); + return st; + } +}; + +TEST_F(DataStepsEvaluator, NativeCallRunsASlicePerStep) { + evaluator_state whole = state("(= a b)", 0); + const int whole_steps = step_to_done(ev, whole); + EXPECT_TRUE(std::get(whole.result.v)); + EXPECT_EQ(whole.stats.steps(step_kind::data), 0u); + EXPECT_GE(whole.stats.data_nodes(), 1000u); // counted, though done in one step + + evaluator_state sliced = state("(= a b)", 64); + const int sliced_steps = step_to_done(ev, sliced); + EXPECT_TRUE(std::get(sliced.result.v)); + EXPECT_GE(sliced.stats.steps(step_kind::data), 1000u / 64 - 1); + EXPECT_EQ(sliced_steps, whole_steps + static_cast(sliced.stats.steps(step_kind::data))); + EXPECT_EQ(sliced.stats.steps(step_kind::eval), whole.stats.steps(step_kind::eval)); +} + +TEST_F(DataStepsEvaluator, PrintingAndNotEqualSlice) { + evaluator_state s = state("(str a)", 50); + step_to_done(ev, s); + EXPECT_EQ(std::get(s.result.v), to_string(nested_list(1000))); + EXPECT_GT(s.stats.steps(step_kind::data), 10u); + + evaluator_state ne = state("(!= a b)", 50); + step_to_done(ev, ne); + EXPECT_FALSE(std::get(ne.result.v)); + EXPECT_GT(ne.stats.steps(step_kind::data), 10u); + + evaluator_state cat = state("(str-concat \"a=\" a)", 50); + step_to_done(ev, cat); + EXPECT_EQ(std::get(cat.result.v), "a=" + to_string(nested_list(1000))); +} + +TEST_F(DataStepsEvaluator, SmallWorkTakesNoExtraSteps) { + evaluator_state few = state("(= (list 1 2 3) (list 1 2 3))", 64); + evaluator_state whole = state("(= (list 1 2 3) (list 1 2 3))", 0); + EXPECT_EQ(step_to_done(ev, few), step_to_done(ev, whole)); + EXPECT_EQ(few.stats.steps(step_kind::data), 0u); +} + +TEST_F(DataStepsEvaluator, MacroExpansionSlices) { + const std::size_t n = 2000; + value_t expr = make_list({sym("begin"), + make_list({sym("defmacro"), sym("m"), make_list({sym("x")}), + make_list({sym("quote"), nested_list(n, sym("x"))})}), + make_list({sym("m"), value_t{int64_t{7}}})}); + evaluator_state st = ev.create_state(expr); + st.data_step_nodes = 64; + step_to_done(ev, st); + EXPECT_TRUE(values_equal(st.result, nested_list(n, value_t{int64_t{7}}))); + EXPECT_GE(st.stats.steps(step_kind::data), n / 64 - 1); +} + +TEST_F(DataStepsEvaluator, FreeingADeepValueSpreadsOverSteps) { + const std::size_t n = 10000; + auto run_env = environment::extend(env); + run_env->set("big", nested_list(n)); // its only owner + evaluator_state st = ev.create_state(parse("(begin (set big nil) 1)"), run_env); + st.data_step_nodes = 100; + st.stats.start(); + const int steps = step_to_done(ev, st); + EXPECT_EQ(std::get(st.result.v), 1); + EXPECT_GE(st.stats.steps(step_kind::data), n / 100 - 2); + EXPECT_GE(st.stats.data_nodes(), n); + // Evaluation waits while the backlog is freed, so the last step is an evaluation step. + EXPECT_EQ(static_cast(steps), st.stats.get_step_count()); +} + +TEST_F(DataStepsEvaluator, StepBudgetStopsDeepWorkMidway) { + evaluator_state st = state("(= a b)", 16); + ev.run(st, 8); + EXPECT_FALSE(st.done); + EXPECT_NE(st.data_work, nullptr); // waiting in the comparison + ev.run(st); + EXPECT_TRUE(st.done); + EXPECT_TRUE(std::get(st.result.v)); +} + +TEST_F(DataStepsEvaluator, SnapshotMidWorkRunsTheCallAgain) { + cvc::app ctx; + evaluator_state st = state("(= a b)", 16, 300); + ev.run(st, 8); + ASSERT_NE(st.data_work, nullptr); // in the middle of the comparison + + auto &node = cvc::state::instance(ctx)("snapshot"); + encode_evaluator_state(node, st); + evaluator_state back = decode_evaluator_state(node); + EXPECT_EQ(back.data_step_nodes, 16u); + EXPECT_EQ(back.data_work, nullptr); // work in flight is not kept... + // (native functions are not serialized: put the builtins back, as the codec tests do) + auto patch = [&](environment_ptr e) { + for (; e && e->outer; e = e->outer) { + } + if (e) + for (auto &[name, val] : env->bindings) + e->set(name, val); + }; + patch(back.global_env); + for (auto &f : back.stack) + patch(f.env); + ev.run(back); // ...the waiting call runs again + ASSERT_TRUE(back.done); + EXPECT_TRUE(std::get(back.result.v)); + EXPECT_GT(back.stats.steps(step_kind::data), 0u); +} + +// ---- the schedulers ------------------------------------------------------------- + +class DataStepsScheduler : public ::testing::Test { +protected: + cvc::app app_ctx; + cvc::state &root = cvc::state::instance(app_ctx); +}; + +TEST_F(DataStepsScheduler, SettingsComeFromTheTree) { + scheduler plain; + plain.set_watch_root(&root); + plain.set_id("plain"); + plain.load_settings(); + EXPECT_EQ(plain.data_settings().step_nodes, 0u); + EXPECT_EQ(root("state_exec.schedulers.plain.data.step_nodes").value(), "0"); // published + + root("state_exec.defaults.data.step_nodes").value("32"); + scheduler sched; + sched.set_watch_root(&root); + sched.set_id("ds"); + sched.load_settings(); + EXPECT_EQ(sched.data_settings().step_nodes, 32u); + EXPECT_EQ(root("state_exec.schedulers.ds.data.step_nodes").value(), "32"); + root("state_exec.schedulers.ds.data.step_nodes").value("8"); + sched.load_settings(); + EXPECT_EQ(sched.data_settings().step_nodes, 8u); + root("state_exec.schedulers.ds.data.step_nodes").value("lots"); // skipped + sched.load_settings(); + EXPECT_EQ(sched.data_settings().step_nodes, 32u); + + async_scheduler async; + EXPECT_EQ(async.data_settings().step_nodes, 0u); + async.set_data_defaults(async_scheduler::host_data_defaults()); + EXPECT_EQ(async.data_settings().step_nodes, async_scheduler::host_data_defaults().step_nodes); + async.set_watch_root(&root); // state_exec.defaults.data.step_nodes = 32 overrides it + EXPECT_EQ(async.data_settings().step_nodes, 32u); +} + +TEST_F(DataStepsScheduler, DeepWorkCountsAgainstMaxSteps) { + root("state_exec.defaults.data.step_nodes").value("16"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + + const int done = sched.execute(compare_script(500)); + sched.run(); + auto info = sched.get_process_info(done); + ASSERT_TRUE(info); + EXPECT_EQ(info->status, process_status::terminated); + EXPECT_TRUE(std::get(sched.get_result(done)->v)); + EXPECT_GE(info->data_steps, 500u / 16 - 1); + EXPECT_GT(info->eval_steps, 0u); + EXPECT_EQ(info->read_steps, 0u); + EXPECT_GE(info->data_nodes, 500u); + EXPECT_EQ(info->eval_steps + info->data_steps + info->read_steps, info->step_count); + EXPECT_GT(info->eval_time, 0.0); + EXPECT_GT(info->data_time, 0.0); + + // The same program with room for its evaluation but not its comparison. + execute_options opts; + opts.max_steps = info->eval_steps + 2; + const int killed = sched.execute(compare_script(500), opts); + sched.run(); + EXPECT_EQ(sched.get_process_info(killed)->status, process_status::killed); +} + +TEST_F(DataStepsScheduler, OtherProcessesRunBetweenSlices) { + root("state_exec.defaults.data.step_nodes").value("4"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + const int slow = sched.execute(compare_script(400)); + // Let it reach its comparison. + while (sched.get_process_info(slow)->data_steps == 0) + sched.step(); + const int quick = sched.execute(std::string("(begin 1 2 3)")); + while (sched.get_process_info(quick)->status != process_status::terminated) + sched.step(); + EXPECT_NE(sched.get_process_info(slow)->status, process_status::terminated); + sched.run(); + EXPECT_TRUE(std::get(sched.get_result(slow)->v)); +} + +TEST_F(DataStepsScheduler, SignalsWaitForDeepWork) { + root("state_exec.defaults.data.step_nodes").value("4"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + execute_options opts; + opts.signal_handlers["SIGUSR1"] = make_list({sym("begin"), value_t{int64_t{42}}}); + const int pid = sched.execute(compare_script(200), opts); + while (sched.get_process_info(pid)->data_steps == 0) + sched.step(); + ASSERT_TRUE(sched.send_signal(pid, "SIGUSR1")); + const uint64_t before = sched.get_process_info(pid)->data_steps; + sched.step(); + EXPECT_EQ(sched.get_process_info(pid)->data_steps, before + 1); // the work went on + sched.run(); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::terminated); + EXPECT_TRUE(std::get(sched.get_result(pid)->v)); +} + +TEST_F(DataStepsScheduler, BudgetUsageAndInspectReportTheKinds) { + async_scheduler sched; + sched.set_watch_root(&root); + root("state_exec.defaults.data.step_nodes").value("16"); + memory_tracker tracker; + intrinsics_context ictx; + ictx.sched = &sched; + ictx.root = &root; + ictx.tracker = &tracker; + auto env = builtins::make_default_environment(); + register_intrinsics(env, &ictx); + execute_options opts; + opts.env = env; + const int pid = sched.execute( + std::string("(begin (set a 1) (set b 1) (set i 0)" + " (while (< i 300) (begin (set a (list a)) (set b (list b)) (set i (+ i 1))))" + " (= a b) (set used (budget-usage)) (set me (inspect (self-pid)))" + " (list used me))"), + opts); + sched.sync_run(); + auto result = sched.get_result(pid); + ASSERT_TRUE(result); + auto &pair = *std::get(result->v); + for (const value_t &d : pair) { + std::unordered_map fields; + for (auto &[k, v] : *std::get(d.v)) + fields[k] = v; + for (const char *key : {"read_steps", "eval_steps", "data_steps", "data_nodes", "read_time", + "eval_time", "data_time", "steps"}) + EXPECT_TRUE(fields.count(key)) << key; + EXPECT_GE(std::get(fields["data_steps"].v), 300 / 16 - 1); + EXPECT_GE(std::get(fields["data_nodes"].v), 300); + EXPECT_GT(std::get(fields["eval_steps"].v), 0); + } + auto info = sched.get_process_info(pid); + ASSERT_TRUE(info); + EXPECT_GT(info->data_time, 0.0); +} From ffa9a26f08c3208f2a84d5b50bf6b05aa3e35eb1 Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 15:41:03 -0500 Subject: [PATCH 12/13] pycvc: parse_options_from_state takes the State root, as in C++ Wrap the real cvc::state_exec::parse_options_from_state(state&, id, base) instead of a pycvc adapter that took the app: Python passes the tree root, pycvc.State.instance(app), exactly as C++ passes cvc::state&. This is the first decision of the pycvc/C++ parity approach (C++ cvc::state& parameters take a pycvc.State in Python). --- bindings/pycvc/README.md | 3 ++- bindings/pycvc/pycvc.i | 12 ++++++++---- bindings/pycvc/pycvc_exec.cpp | 8 -------- bindings/pycvc/pycvc_exec.h | 7 ------- bindings/pycvc/test_pycvc_exec.py | 18 ++++++++++-------- 5 files changed, 20 insertions(+), 28 deletions(-) diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index 2d3952d22..3624fa02b 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -201,7 +201,8 @@ pycvc.parse_options() # max_depth 256, step_toke pycvc.async_scheduler.host_parser_defaults() # step_tokens 1024 ex.parser_defaults() # == host_parser_defaults() pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "64") # tree overrides -pycvc.parse_options_from_state(app, "", ex.parser_defaults()) # what Exec uses +root = pycvc.State.instance(app) # the tree root, as in C++ +pycvc.parse_options_from_state(root, "", ex.parser_defaults()) # what Exec uses ``` `Exec` starts from the same defaults as the app's exec scheduler (Ariadne), and diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 404a59e87..50fc32b62 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1439,10 +1439,11 @@ class AsyncStateObserver(state_observer): // ── state_exec parser settings (docs/STATE_EXEC_GRAMMAR.md, "Parser settings") ── // DIRECT WRAP, under the C++ names: the real cvc::state_exec::parse_options -// struct and async_scheduler::host_parser_defaults(). Only these members are -// declared to SWIG; the rest of parser.h / async_scheduler.h (value_t, the -// parser, the scheduler itself) is not wrapped yet, so async_scheduler has no -// Python constructor. parse_options_from_state and Exec's parser_defaults / +// struct, parse_options_from_state (which takes a State, the tree root, as in +// C++: pycvc.State.instance(app)) and async_scheduler::host_parser_defaults(). +// Only these members are declared to SWIG; the rest of parser.h / +// async_scheduler.h (value_t, the parser, the scheduler itself) is not wrapped +// yet, so async_scheduler has no Python constructor. Exec's parser_defaults / // set_parser_defaults are in pycvc_exec.h. %{ #include @@ -1454,6 +1455,9 @@ struct parse_options { size_t max_depth; // lists and quotes that may nest (default 256) size_t step_tokens; // 0: parse at submit; N: read N tokens per process step }; +parse_options parse_options_from_state(cvc::state &root, + const std::string &scheduler_id = std::string(), + const parse_options &base = parse_options()); %nodefaultctor async_scheduler; %nodefaultdtor async_scheduler; class async_scheduler { diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 94f5c8d28..dd3893939 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -656,12 +656,4 @@ void Exec::set_parser_defaults(const cvc::state_exec::parse_options &defaults) { impl_->sched.set_parser_defaults(defaults); } -cvc::state_exec::parse_options -parse_options_from_state(const std::shared_ptr &app, const std::string &scheduler_id, - const cvc::state_exec::parse_options &base) { - if (!app) - throw std::invalid_argument("pycvc.parse_options_from_state: null app handle"); - return cvc::state_exec::parse_options_from_state(cvc::state::instance(*app), scheduler_id, base); -} - } // namespace pycvc diff --git a/bindings/pycvc/pycvc_exec.h b/bindings/pycvc/pycvc_exec.h index 55c3c9482..b774ac2c4 100644 --- a/bindings/pycvc/pycvc_exec.h +++ b/bindings/pycvc/pycvc_exec.h @@ -21,13 +21,6 @@ typedef _object PyObject; namespace pycvc { -// cvc::state_exec::parse_options_from_state on `app`'s state root (pycvc passes the app where -// C++ passes the root): per key, state_exec.schedulers..parser. (when -// scheduler_id is not empty), then state_exec.defaults.parser., then `base`. -cvc::state_exec::parse_options parse_options_from_state( - const std::shared_ptr &app, const std::string &scheduler_id = "", - const cvc::state_exec::parse_options &base = cvc::state_exec::parse_options()); - class Exec { public: // Build a DSL environment + scheduler over `app`'s state root. diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index 6d932392b..10e9f4392 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -100,20 +100,22 @@ def test_exec_starts_from_the_host_parser_defaults(): def test_parse_options_from_state_resolves_like_cpp(): + # The real C++ function: it takes the tree root, a State, as in C++. app = pycvc.make_app() + root = pycvc.State.instance(app) # Nothing set: the C++ default. - assert pycvc.parse_options_from_state(app).step_tokens == 0 + assert pycvc.parse_options_from_state(root).step_tokens == 0 # What Exec reads with: its own defaults under the tree. ex = pycvc.Exec(app) - effective = pycvc.parse_options_from_state(app, "", ex.parser_defaults()) + effective = pycvc.parse_options_from_state(root, "", ex.parser_defaults()) assert effective.step_tokens == 1024 - pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "40") - pycvc.state_set(app, "state_exec.schedulers.s1.parser.max_depth", "50") - assert pycvc.parse_options_from_state(app).max_depth == 40 - assert pycvc.parse_options_from_state(app, "s1").max_depth == 50 - pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "nope") # bad: skipped - assert pycvc.parse_options_from_state(app).max_depth == 256 + root.child("state_exec.defaults.parser.max_depth").value("40") + root.child("state_exec.schedulers.s1.parser.max_depth").value("50") + assert pycvc.parse_options_from_state(root).max_depth == 40 + assert pycvc.parse_options_from_state(root, "s1").max_depth == 50 + root.child("state_exec.defaults.parser.max_depth").value("nope") # bad: skipped + assert pycvc.parse_options_from_state(root).max_depth == 256 def test_set_parser_defaults_changes_how_exec_parses(): From a84d37e21b779e9861645639561f3804749e10df Mon Sep 17 00:00:00 2001 From: Joe Rivera Date: Sat, 17 Oct 2026 20:45:00 -0500 Subject: [PATCH 13/13] Run the master CI on #658 after retargeting it to master Retargeting a PR is an 'edited' event, which ci.yml does not run on.