state_exec: parser.max_depth 100000; the last recursive walks over parsed data

bindings/pycvc/README.md

@@ -201,7 +201,7 @@ app = pycvc.make_app()
 ex = pycvc.Exec(app)
 ex.run("(+ 1 2)")                                     # "3"
 
-pycvc.parse_options()                                 # max_depth 256, step_tokens 0
+pycvc.parse_options()                                 # max_depth 100000, step_tokens 0
 pycvc.async_scheduler.host_parser_defaults()          # step_tokens 1024
 ex.parser_defaults()                                  # == host_parser_defaults()
 pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "64")  # tree overrides

bindings/pycvc/pycvc.i

@@ -2650,7 +2650,7 @@ bool values_equal(const value_t &a, const value_t &b);
 
 // Parser (docs/STATE_EXEC_GRAMMAR.md).
 struct parse_options {
-  size_t max_depth;   // lists and quotes that may nest (default 256)
+  size_t max_depth;   // lists and quotes that may nest (default 100000)
   size_t step_tokens; // 0: parse at submit; N: read N tokens per process step
 };
 struct data_options {

bindings/pycvc/test_pycvc_exec.py

@@ -83,7 +83,7 @@ def test_parser_settings_come_from_the_app_tree():
 
 def test_parse_options_is_the_cpp_struct():
     o = pycvc.parse_options()
-    assert (o.max_depth, o.step_tokens) == (256, 0)  # C++ defaults
+    assert (o.max_depth, o.step_tokens) == (100000, 0)  # C++ defaults
     o.max_depth = 32
     o.step_tokens = 8
     assert (o.max_depth, o.step_tokens) == (32, 8)
@@ -93,7 +93,7 @@ def test_exec_starts_from_the_host_parser_defaults():
     # The same defaults as the app's exec scheduler:
     # async_scheduler::host_parser_defaults().
     host = pycvc.async_scheduler.host_parser_defaults()
-    assert (host.max_depth, host.step_tokens) == (256, 1024)
+    assert (host.max_depth, host.step_tokens) == (100000, 1024)
     ex = pycvc.Exec(pycvc.make_app())
     d = ex.parser_defaults()
     assert (d.max_depth, d.step_tokens) == (host.max_depth, host.step_tokens)
@@ -115,7 +115,7 @@ def test_parse_options_from_state_resolves_like_cpp():
     assert pycvc.parse_options_from_state(root).max_depth == 40
     assert pycvc.parse_options_from_state(root, "s1").max_depth == 50
     root.child("state_exec.defaults.parser.max_depth").value("nope")  # bad: skipped
-    assert pycvc.parse_options_from_state(root).max_depth == 256
+    assert pycvc.parse_options_from_state(root).max_depth == 100000
 
 
 def test_set_parser_defaults_changes_how_exec_parses():

bindings/pycvc/test_pycvc_state_exec.py

@@ -167,7 +167,7 @@ def test_script_reader_reads_in_slices():
     assert r.forms() == [[symbol("a"), 1], [symbol("b"), "two"]]
     assert steps > 3
     assert r.source() == '(a 1) (b "two")'
-    assert r.options().max_depth == 256
+    assert r.options().max_depth == pycvc.parse_options().max_depth  # the C++ default
 
 
 def test_script_reader_raises_parse_error():

bindings/pycvc/test_pycvc_state_exec_sched.py

@@ -377,7 +377,7 @@ def test_scheduler_settings_come_from_the_tree():
     root.child("state_exec.schedulers.py-sched.max_pending_messages").value("2048")
     root.child("state_exec.schedulers.py-sched.parser.max_depth").value("12")
     s = pycvc.scheduler()
-    assert s.parser_settings().max_depth == 256
+    assert s.parser_settings().max_depth == pycvc.parse_options().max_depth  # the default
     s.set_watch_root(root)
     s.set_id("py-sched")
     assert s.id() == "py-sched"

docs/STATE_EXEC_GRAMMAR.md

@@ -124,17 +124,17 @@ raw_bytes      ::= 'br' raw_body
 - **Positions.** `parse_error::line()` and `col()` are 1-based, and point at the character where
   the error was found, such as the non-hex character in `"\x4"`. A new line starts after LF,
   after CR, and once after CRLF. Columns count code points, not bytes.
-- **Nesting.** Lists and quotes may nest at most `max_depth` levels, 256 by default.
+- **Nesting.** Lists and quotes may nest at most `max_depth` levels, 100000 by default.
   - `(a (b (c)))` is three levels, and `'x` counts as one, since it reads as `(quote x)`.
   - The parser is stackless. Open lists and quotes live on a heap stack, not on the native
     stack, so no input depth can overflow it.
-  - The limit is for what happens to the data afterwards. Some code still walks values on the
-    native stack: destroying a nested list, the snapshot codec, `to_string`. Deeper input is a
-    `parse_error`, so a script from an untrusted source (a message, an HTTP body, the
-    multiprocess wire) can't produce data those would crash on.
-  - The default fits the 64 KiB default stack of a wasm build. Real scripts nest far less: the
-    deepest in the corpus is 13 levels, and the median is 2. Once that downstream code is
-    iterative too, the default can rise.
+  - So is the code that walks the data afterwards: freeing, printing, comparing, copying, macro
+    expansion, the snapshot codec and the Ariadne channel lint. Each is tested 100000 levels
+    deep on a 64 KiB stack (the default stack of a wasm build). Values crossing into Python go
+    through pycvc's converter, which has its own depth limit.
+  - The default was 256 while some of that code still recursed. It is now a bound on runaway
+    input, not a guard for the native stack. Real scripts nest far less: the deepest in the
+    corpus is 13 levels, and the median is 2.
   - The limit is a parser setting. Like state_exec's other settings, it lives on the state tree;
     see [Parser settings](#parser-settings).
 
@@ -153,7 +153,7 @@ next one in the order applies. `max_depth` must also be positive.
 
 | Key | Meaning | Default |
 |---|---|---|
-| `max_depth` | how many lists and quotes may nest | 256 |
+| `max_depth` | how many lists and quotes may nest | 100000 |
 | `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler and pycvc's `Exec` |
 
 Who reads them:
@@ -226,8 +226,9 @@ then `state_exec.defaults.data.step_nodes`, then the scheduler's built-in defaul
 A node is an element visited: a list element or a dict entry walked, or an element freed. With
 `step_nodes` = N:
 
-- **Native calls.** `=`, `!=`, `str`, `str-concat`, `print`, `state-data-get`, `state-json` and
-  `state-from-json` hand their walk to the evaluator (`run_data_job`). It does one slice inside the call's step; if the walk isn't
+- **Native calls.** `=`, `!=`, `str`, `str-concat`, `print`, `state-data-get`, `state-json`,
+  `state-from-json` and `state-set` of a list or dict (printed into the node) hand their walk to
+  the evaluator (`run_data_job`). It does one slice inside the call's step; if the walk isn't
   done, the call's frame waits and the following steps are **data steps** that each run N more
   nodes. The step that finishes it completes the call. A walk under N nodes finishes in the
   call's own step, so ordinary programs take no extra steps.
@@ -253,8 +254,7 @@ A node is an element visited: a list element or a dict entry walked, or an eleme
 
 The stdlib's materializers are jobs too: `string.split`/`join`/`replace`/`encode`,
 `bytes.slice`/`concat`/`decode` and all of `collections.*`. A string function counts a node per
-256 bytes it scans or copies (developer guide §11, "Sliced work"). String coercion for
-`state-set` still runs whole inside one step, but counts its nodes.
+256 bytes it scans or copies (developer guide §11, "Sliced work").
 
 Measured per node (MSVC Release, `state_exec_bench_test` `DataWalkThroughput`): ~45 ns to free,
 ~70 ns to print, ~250 ns to compare and ~380 ns to copy. So a 1024-node slice costs 0.05-0.4 ms.
@@ -396,7 +396,7 @@ strings. Every row is a case in `state_exec_grammar_test`.
 | `[a]`, `a,b`, `` `a ``, `{}`, `a\|b`, `a\b` | odd one-character or glued symbols | error (reserved) | P2 |
 | `parse("1 2")` | `1`; the rest is ignored | error | P2 |
 | `'` at end of input | `(quote nil)` | error | P2 |
-| 257 nested lists | parsed (no limit) | error at the default `max_depth` | P1b |
+| 100001 nested lists | parsed (no limit) | error at the default `max_depth` | P1b |
 | `; c` CR `1` | comment runs to LF, so no datum | `1` | P2 |
 | `0x1F` | `31.0`, a double (via `strtod`) | `31` | P2 |
 | `#x1F`, `0o17`, `0b101`, `1_000` | symbols | integers | P2 |

inc/cvc/state/state_exec/parser.h

@@ -42,11 +42,13 @@ class parse_error : public std::runtime_error {
 /// state tree under `parser.*`; parse_options_from_state() reads them from there.
 struct parse_options {
   /// How many lists and quotes may nest (`'x` reads as `(quote x)`, one level); deeper input is
-  /// a parse_error. The parser is stackless, and so are freeing, printing, comparing, copying,
-  /// macro-expanding and snapshotting a value; what still walks values recursively (the Ariadne
-  /// loader's channel lint, which stops at 256 levels, and pycvc's value conversion) is what this
-  /// keeps parsed data within. State-tree key: `parser.max_depth`.
-  std::size_t max_depth = 256;
+  /// a parse_error. The parser is stackless, and so is what walks parsed data in libcvc
+  /// (freeing, printing, comparing, copying, macro expansion, the snapshot codec, the Ariadne
+  /// channel lint), each tested 100000 levels deep on a 64 KiB stack. So the default is that
+  /// depth: a bound on runaway input, not a guard for the native stack. (Values crossing into
+  /// Python go through pycvc's converter, which has its own depth limit.)
+  /// State-tree key: `parser.max_depth`.
+  std::size_t max_depth = 100000;
 
   /// How a scheduler parses a submitted script. 0 parses all of it when it is submitted, so
   /// execute() throws a parse_error. N > 0 parses it inside the new process, N tokens per

src/cvc/ariadne/loader.cpp

@@ -1939,26 +1939,39 @@ std::vector<CustomRequirement> parse_customs(Ctx &ctx, const YAML::Node &c) {
 // references, gated on the document declaring a `channels:` block. Only literal channel names are
 // visible here; a dynamic `(msg-recv (expr))` name is the runtime policy's job (a follow-up).
 // Walk a parsed script's value_t tree, collecting the string-literal FIRST arg of every
-// (msg-send|msg-recv|msg-pending …) call — the static channel references. Depth-bounded.
-void collect_channel_refs(const cvc::state_exec::value_t &v, std::vector<std::string> &out,
-                          int depth) {
+// (msg-send|msg-recv|msg-pending …) call — the static channel references — in source order.
+// The walk keeps its open lists on a heap stack, so any depth the parser accepts is linted.
+void collect_channel_refs(const cvc::state_exec::value_t &v, std::vector<std::string> &out) {
   using namespace cvc::state_exec;
-  if (depth > 256)
-    return;
-  const auto *lp = std::get_if<list_ptr>(&v.v);
-  if (!lp || !*lp)
-    return;
-  const std::vector<value_t> &lst = **lp;
-  if (!lst.empty()) {
-    if (const auto *head = std::get_if<symbol>(&lst[0].v)) {
-      if ((head->name == "msg-send" || head->name == "msg-recv" || head->name == "msg-pending") &&
-          lst.size() >= 2)
-        if (const auto *s = std::get_if<std::string>(&lst[1].v))
-          out.push_back(*s);
+  struct open_list {
+    const std::vector<value_t> *elems;
+    std::size_t next;
+  };
+  std::vector<open_list> stack;
+  auto enter = [&](const value_t &e) {
+    const auto *lp = std::get_if<list_ptr>(&e.v);
+    if (!lp || !*lp)
+      return;
+    const std::vector<value_t> &lst = **lp;
+    if (!lst.empty()) {
+      if (const auto *head = std::get_if<symbol>(&lst[0].v)) {
+        if ((head->name == "msg-send" || head->name == "msg-recv" || head->name == "msg-pending") &&
+            lst.size() >= 2)
+          if (const auto *s = std::get_if<std::string>(&lst[1].v))
+            out.push_back(*s);
+      }
+    }
+    stack.push_back({&lst, 0});
+  };
+  enter(v);
+  while (!stack.empty()) {
+    open_list &top = stack.back();
+    if (top.next == top.elems->size()) {
+      stack.pop_back();
+      continue;
     }
+    enter((*top.elems)[top.next++]); // may push, invalidating top
   }
-  for (const value_t &e : lst)
-    collect_channel_refs(e, out, depth + 1);
 }
 // Collect static channel refs from one script string. Only an s-expr (a program, first non-space
 // '(') can hold a msg-* call; a bare event name or empty script has none. A syntactically-invalid
@@ -1968,7 +1981,7 @@ void refs_from_script(const std::string &script, std::vector<std::string> &out)
   if (i == std::string::npos || script[i] != '(')
     return;
   try {
-    collect_channel_refs(cvc::state_exec::parse(script), out, 0);
+    collect_channel_refs(cvc::state_exec::parse(script), out);
   } catch (...) {
   }
 }

src/cvc/state/state_exec/async_evaluator.cpp

@@ -10,6 +10,7 @@
 
 #include <cvc/state/state_exec/async_evaluator.h>
 #include <cvc/state/state_exec/builtins.h>
+#include <cvc/state/state_exec/data_job.h>
 #include <cvc/state/state_exec/parser.h>
 #include <thread>
 
@@ -679,22 +680,8 @@ task<value_t> async_evaluator::apply_closure(const closure_ptr &cls,
 
 value_t async_evaluator::substitute(const value_t &tmpl,
                                     const std::unordered_map<std::string, value_t> &subst) {
-  if (auto *sym = std::get_if<symbol>(&tmpl.v)) {
-    auto it = subst.find(sym->name);
-    if (it != subst.end())
-      return it->second;
-    return tmpl;
-  }
-  if (auto *lp = std::get_if<list_ptr>(&tmpl.v)) {
-    if (!*lp)
-      return tmpl;
-    auto result = std::make_shared<std::vector<value_t>>();
-    result->reserve((*lp)->size());
-    for (auto &e : **lp)
-      result->push_back(substitute(e, subst));
-    return value_t{std::move(result)};
-  }
-  return tmpl;
+  // The stackless evaluator's walk, run whole: no native recursion per template level.
+  return run_to_completion(*make_substitute_job(tmpl, subst));
 }
 
 } // namespace cvc::state_exec

src/cvc/state/state_exec/evaluator.cpp

@@ -1,5 +1,6 @@
 #include <atomic>
 #include <cvc/state/state_exec/builtins.h>
+#include <cvc/state/state_exec/data_job.h>
 #include <cvc/state/state_exec/evaluator.h>
 #include <cvc/state/state_exec/parser.h>
 #include <thread>
@@ -688,22 +689,8 @@ value_t evaluator::apply_closure(const closure_ptr &cls, const std::vector<value
 
 value_t evaluator::substitute(const value_t &tmpl,
                               const std::unordered_map<std::string, value_t> &subst) {
-  if (auto *sym = std::get_if<symbol>(&tmpl.v)) {
-    auto it = subst.find(sym->name);
-    if (it != subst.end())
-      return it->second;
-    return tmpl;
-  }
-  if (auto *lp = std::get_if<list_ptr>(&tmpl.v)) {
-    if (!*lp)
-      return tmpl;
-    auto result = std::make_shared<std::vector<value_t>>();
-    result->reserve((*lp)->size());
-    for (auto &e : **lp)
-      result->push_back(substitute(e, subst));
-    return value_t{std::move(result)};
-  }
-  return tmpl;
+  // The stackless evaluator's walk, run whole: no native recursion per template level.
+  return run_to_completion(*make_substitute_job(tmpl, subst));
 }
 
 } // namespace cvc::state_exec

src/cvc/state/state_exec/intrinsics.cpp

@@ -143,9 +143,17 @@ value_t intrinsic_state_set(intrinsics_context *ctx, std::span<const value_t> ar
   // Coerce the value to a string (state stores string-typed scalars) rather than
   // requiring the caller to pre-stringify — (state-set "n" 10) now stores "10". A
   // structured value should use state-data-set (the typed data() channel).
-  const std::string val = coerce_state_string(args[1]);
+  if (is_compound(args[1])) {
+    // A list or dict prints as a walk, a slice per step when the process slices deep data
+    // ("Deep data on the executor"); the node is written once the whole string is ready.
+    cvc::state *root = ctx->root;
+    return run_data_job(make_print_job(args[1]), [root, path = path](value_t printed) {
+      (*root)(path).value(std::get<std::string>(printed.v));
+      return nil_value;
+    });
+  }
   // operator() creates child nodes as needed
-  (*ctx->root)(path).value(val);
+  (*ctx->root)(path).value(coerce_state_string(args[1]));
   return nil_value;
 }
 

src/cvc/tests/ariadne_loader_test.cpp

@@ -3973,6 +3973,23 @@ TEST(AriadneChannelLint, StrictUndeclaredChannelFailsLoad) {
   EXPECT_NE(r.error.find("other.chan"), std::string::npos);
 }
 
+TEST(AriadneChannelLint, SeesChannelsNestedAnyDepth) {
+  // The lint walks a script on a heap stack, so a reference 1000 levels down is still checked
+  // (it used to stop at 256).
+  SKIP_WITHOUT_YAML();
+  const int n = 1000;
+  std::string on;
+  for (int i = 0; i < n; ++i)
+    on += "(begin ";
+  on += "(msg-send \"deep.chan\" \"x\")";
+  on += std::string(n, ')');
+  LoadResult r = load_string("channels:\n  - nav.done\nwindows:\n  - window: W\n    children:\n"
+                             "      - button: Go\n        on: '" +
+                             on + "'\n");
+  EXPECT_FALSE(r.ok);
+  EXPECT_NE(r.error.find("deep.chan"), std::string::npos) << r.error;
+}
+
 TEST(AriadneChannelLint, DeclaredChannelPasses) {
   SKIP_WITHOUT_YAML();
   LoadResult r = load_string(R"(

src/cvc/tests/state_exec_data_steps_test.cpp

@@ -573,6 +573,40 @@ TEST_F(DataStepsScheduler, BudgetUsageAndInspectReportTheKinds) {
   EXPECT_GT(info->data_time, 0.0);
 }
 
+TEST_F(DataStepsScheduler, StateSetOfADeepValueSlices) {
+  async_scheduler sched;
+  sched.set_watch_root(&root);
+  root("state_exec.defaults.data.step_nodes").value("16");
+  memory_tracker tracker;
+  intrinsics_context ictx;
+  ictx.sched = &sched;
+  ictx.root = &root;
+  ictx.tracker = &tracker;
+  auto env = builtins::make_default_environment();
+  register_intrinsics(env, &ictx);
+  execute_options opts;
+  opts.env = env;
+  const int pid =
+      sched.execute(std::string("(begin (set a 1) (set i 0)"
+                                " (while (< i 300) (begin (set a (list a)) (set i (+ i 1))))"
+                                " (state-set \"deep.text\" a) (state-set \"deep.n\" 7)"
+                                " (state-get \"deep.text\"))"),
+                    opts);
+  // The list prints a slice per step, and the node is written once the whole string is ready.
+  while (sched.get_process_info(pid)->data_steps == 0)
+    sched.sync_step();
+  EXPECT_EQ(root.findDescendant("deep.text"), nullptr);
+  sched.sync_run();
+  auto info = sched.get_process_info(pid);
+  ASSERT_TRUE(info);
+  EXPECT_EQ(info->status, process_status::terminated);
+  EXPECT_EQ(root("deep.text").value(), to_string(nested_list(300)));
+  EXPECT_EQ(root("deep.n").value(), "7");
+  EXPECT_EQ(std::get<std::string>(sched.get_result(pid)->v), root("deep.text").value());
+  EXPECT_GE(info->data_steps, 300u / 16 - 1);
+  EXPECT_GE(info->data_nodes, 300u);
+}
+
 // ---- state-json / state-from-json ------------------------------------------------------
 //
 // cvc::state::json() and json(j) as data jobs: the same writer and reader (state_json.h), a