state_exec: read submitted scripts inside the process, a slice per step

bindings/pycvc/README.md

@@ -184,3 +184,28 @@ Every widget reads/writes `cvc::state`, so the same value is drivable from a
 script, a config file or a replicated peer. Raw custom-widget ImGui (free-form
 windows) is a separate, deferred design (a curated subset compiled inside cvcGL).
 Contract test: `test_pycvc_gl_imgui.py`.
+
+## state_exec DSL (`Exec`) and parser settings
+
+`pycvc.Exec(app)` runs state_exec programs over the app's state tree. The parser
+settings are the C++ surface, under the C++ names (see
+`docs/STATE_EXEC_GRAMMAR.md`, "Parser settings"):
+
+```python
+import pycvc
+app = pycvc.make_app()
+ex = pycvc.Exec(app)
+ex.run("(+ 1 2)")                                     # "3"
+
+pycvc.parse_options()                                 # max_depth 256, step_tokens 0
+pycvc.async_scheduler.host_parser_defaults()          # step_tokens 1024
+ex.parser_defaults()                                  # == host_parser_defaults()
+pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "64")  # tree overrides
+root = pycvc.State.instance(app)                      # the tree root, as in C++
+pycvc.parse_options_from_state(root, "", ex.parser_defaults())      # what Exec uses
+```
+
+`Exec` starts from the same defaults as the app's exec scheduler (Ariadne), and
+the same `state_exec.defaults.parser.*` keys override both. A syntax error
+raises a `RuntimeError` with the parse error's message. Contract test:
+`test_pycvc_exec.py`.

bindings/pycvc/pycvc.i

@@ -1437,6 +1437,36 @@ class AsyncStateObserver(state_observer):
         self._workers = []
 %}
 
+// ── state_exec parser settings (docs/STATE_EXEC_GRAMMAR.md, "Parser settings") ──
+// DIRECT WRAP, under the C++ names: the real cvc::state_exec::parse_options
+// struct, parse_options_from_state (which takes a State, the tree root, as in
+// C++: pycvc.State.instance(app)) and async_scheduler::host_parser_defaults().
+// Only these members are declared to SWIG; the rest of parser.h /
+// async_scheduler.h (value_t, the parser, the scheduler itself) is not wrapped
+// yet, so async_scheduler has no Python constructor. Exec's parser_defaults /
+// set_parser_defaults are in pycvc_exec.h.
+%{
+#include <cvc/state/state_exec/async_scheduler.h>
+#include <cvc/state/state_exec/parser.h>
+%}
+namespace cvc {
+namespace state_exec {
+struct parse_options {
+  size_t max_depth;   // lists and quotes that may nest (default 256)
+  size_t step_tokens; // 0: parse at submit; N: read N tokens per process step
+};
+parse_options parse_options_from_state(cvc::state &root,
+                                       const std::string &scheduler_id = std::string(),
+                                       const parse_options &base = parse_options());
+%nodefaultctor async_scheduler;
+%nodefaultdtor async_scheduler;
+class async_scheduler {
+public:
+  static parse_options host_parser_defaults();
+};
+}
+}
+
 // ── Phase 4: DSL execution + Python-authored DSL functions ──────────────
 // pycvc.Exec(app) runs state_exec DSL programs over app's state root and lets a
 // Python callable be registered as a DSL function (register_fn). PyObject* is

bindings/pycvc/pycvc_exec.cpp

@@ -378,6 +378,10 @@ Exec::Exec(const std::shared_ptr<cvc::app> &app) : impl_(std::make_shared<ExecIm
   // only the scheduler is private.
   app->computePool();
   impl_->sched.set_watch_root(impl_->root);
+  // The same state_exec defaults as the app's exec scheduler, so a program behaves the same from
+  // Python as from Ariadne: scripts are read inside the process (host_parser_defaults), and the
+  // app tree's state_exec.defaults.parser.* keys override them for both.
+  impl_->sched.set_parser_defaults(async_scheduler::host_parser_defaults());
 
   impl_->host_proc = make_process();
   impl_->host_proc->pid = 0;
@@ -554,7 +558,10 @@ std::string Exec::run(const std::string &src) {
   execute_options opts;
   opts.name = "pycvc";
   opts.env = impl_->env;
-  const int pid = sched.execute(src, opts); // may throw on parse error (nothing to clean up yet)
+  // A syntax error is read inside the process (parser.step_tokens) and surfaces below as the
+  // killed process's exit error; with step_tokens = 0 on the tree, execute() throws it here
+  // instead (nothing to clean up yet). Either way run() raises with the parse error's message.
+  const int pid = sched.execute(src, opts);
 
   // Drive the private scheduler to completion. A purely synchronous program finishes in the first
   // slice; a program that parks (await / an async fn) yields, and the pump waits for a compute-pool
@@ -641,4 +648,12 @@ std::string Exec::run(const std::string &src) {
   return render_result(*result);
 }
 
+cvc::state_exec::parse_options Exec::parser_defaults() const {
+  return impl_->sched.parser_defaults();
+}
+
+void Exec::set_parser_defaults(const cvc::state_exec::parse_options &defaults) {
+  impl_->sched.set_parser_defaults(defaults);
+}
+
 } // namespace pycvc

bindings/pycvc/pycvc_exec.h

@@ -7,6 +7,7 @@
 // registered here becomes callable from DSL source like any builtin.
 #pragma once
 
+#include <cvc/state/state_exec/parser.h> // parse_options
 #include <memory>
 #include <string>
 
@@ -86,6 +87,13 @@ class Exec {
   // (each owns its own scheduler, so they do not interfere).
   std::string run(const std::string &src);
 
+  // The parser settings this Exec's scheduler parses programs with when the app tree's
+  // state_exec.defaults.parser.* keys don't say otherwise; the same calls as
+  // async_scheduler::parser_defaults() / set_parser_defaults(). They start as
+  // async_scheduler::host_parser_defaults(), as the app's exec scheduler's do.
+  cvc::state_exec::parse_options parser_defaults() const;
+  void set_parser_defaults(const cvc::state_exec::parse_options &defaults);
+
 private:
   struct ExecImpl;
   std::shared_ptr<ExecImpl> impl_;

bindings/pycvc/test_pycvc_exec.py

@@ -36,6 +36,99 @@ def test_parse_error_raises():
         raise AssertionError("a parse error must raise")
 
 
+def _run_error(ex, src):
+    """run(src) must raise; return the message."""
+    try:
+        ex.run(src)
+    except Exception as e:  # noqa: BLE001 - pycvc surfaces C++ errors as RuntimeError
+        return str(e)
+    raise AssertionError(f"{src!r} must raise")
+
+
+def test_parse_error_names_the_problem():
+    # Exec uses the same state_exec defaults as the app's exec scheduler: the script
+    # is read inside its process, so a syntax error comes back as the process's exit
+    # error. run() raises with the parse error's own message, as it did when the
+    # script was parsed at submit.
+    app = pycvc.make_app()
+    ex = pycvc.Exec(app)
+    assert "unclosed parenthesis" in _run_error(ex, "(begin (set x")
+
+
+def test_parser_settings_come_from_the_app_tree():
+    # Exec's scheduler watches the app's state tree, so state_exec.defaults.parser.*
+    # configures it exactly as it configures the app's exec scheduler
+    # (docs/STATE_EXEC_GRAMMAR.md).
+    app = pycvc.make_app()
+    ex = pycvc.Exec(app)
+    deep = "(+ 1 (+ 2 (+ 3 (+ 4 5))))"  # four levels of nesting
+    assert ex.run(deep) == "15"
+
+    pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "3")
+    assert "nesting deeper than 3" in _run_error(ex, deep)
+    assert ex.run("(+ 1 (+ 2 3))") == "6"  # three levels still parse
+
+    # Parsing at submit (step_tokens 0) gives the same results and the same errors.
+    pycvc.state_set(app, "state_exec.defaults.parser.step_tokens", "0")
+    assert "nesting deeper than 3" in _run_error(ex, deep)
+    assert "unclosed parenthesis" in _run_error(ex, "(begin (set x")
+    assert ex.run("(+ 1 (+ 2 3))") == "6"
+
+    pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "256")
+    assert ex.run(deep) == "15"
+
+
+# ── state_exec parser settings: the C++ surface, under the C++ names ────
+
+
+def test_parse_options_is_the_cpp_struct():
+    o = pycvc.parse_options()
+    assert (o.max_depth, o.step_tokens) == (256, 0)  # C++ defaults
+    o.max_depth = 32
+    o.step_tokens = 8
+    assert (o.max_depth, o.step_tokens) == (32, 8)
+
+
+def test_exec_starts_from_the_host_parser_defaults():
+    # The same defaults as the app's exec scheduler:
+    # async_scheduler::host_parser_defaults().
+    host = pycvc.async_scheduler.host_parser_defaults()
+    assert (host.max_depth, host.step_tokens) == (256, 1024)
+    ex = pycvc.Exec(pycvc.make_app())
+    d = ex.parser_defaults()
+    assert (d.max_depth, d.step_tokens) == (host.max_depth, host.step_tokens)
+
+
+def test_parse_options_from_state_resolves_like_cpp():
+    # The real C++ function: it takes the tree root, a State, as in C++.
+    app = pycvc.make_app()
+    root = pycvc.State.instance(app)
+    # Nothing set: the C++ default.
+    assert pycvc.parse_options_from_state(root).step_tokens == 0
+    # What Exec reads with: its own defaults under the tree.
+    ex = pycvc.Exec(app)
+    effective = pycvc.parse_options_from_state(root, "", ex.parser_defaults())
+    assert effective.step_tokens == 1024
+
+    root.child("state_exec.defaults.parser.max_depth").value("40")
+    root.child("state_exec.schedulers.s1.parser.max_depth").value("50")
+    assert pycvc.parse_options_from_state(root).max_depth == 40
+    assert pycvc.parse_options_from_state(root, "s1").max_depth == 50
+    root.child("state_exec.defaults.parser.max_depth").value("nope")  # bad: skipped
+    assert pycvc.parse_options_from_state(root).max_depth == 256
+
+
+def test_set_parser_defaults_changes_how_exec_parses():
+    ex = pycvc.Exec(pycvc.make_app())
+    o = pycvc.parse_options()
+    o.max_depth = 2
+    ex.set_parser_defaults(o)
+    assert ex.parser_defaults().max_depth == 2
+    assert ex.parser_defaults().step_tokens == 0
+    assert "nesting deeper than 2" in _run_error(ex, "(+ 1 (+ 2 (+ 3 4)))")
+    assert ex.run("(+ 1 (+ 2 3))") == "6"
+
+
 # ── DSL <-> state <-> Python round trips (one shared app) ───────────────
 
 

docs/STATE_EXEC_GRAMMAR.md

@@ -148,26 +148,66 @@ this order:
 2. `state_exec.defaults.parser.<key>`, for everything on that tree;
 3. the built-in default.
 
-A value that is missing, empty, not a positive integer or out of range is skipped, and the next
-one in the order applies.
+A value that is missing, empty, not a non-negative integer or out of range is skipped, and the
+next one in the order applies. `max_depth` must also be positive.
 
 | Key | Meaning | Default |
 |---|---|---|
 | `max_depth` | how many lists and quotes may nest | 256 |
+| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler and pycvc's `Exec` |
 
 Who reads them:
 
 - **`scheduler`** resolves them in `load_settings()`, like `max_pending_messages`. It publishes
   the effective values under `state_exec.schedulers.<id>.parser.*` and uses them for every
   script submitted to it afterwards.
 - **`async_scheduler`** has no settings loader. It reads `state_exec.defaults.parser.*` from its
-  watch root each time a script is submitted.
+  watch root each time a script is submitted. Keys that aren't set fall back to the scheduler's
+  own defaults (`set_parser_defaults()`), which are the built-in ones unless the host changes
+  them.
+  - **libcvc's own hosts all use `async_scheduler::host_parser_defaults()`** (`step_tokens` =
+    1024), so a script behaves the same in each:
+    - the app's exec scheduler (`app::exec_scheduler()`, which runs Ariadne's actions and
+      residents);
+    - pycvc's `Exec`, on its private scheduler. That scheduler watches the same app tree, so the
+      same keys override both.
+  - **Why 1024:** every script in the corpus fits in one slice (the largest is 159 tokens, and
+    `.ari` scripts are at most 24). So a normal action pays one extra step out of its 5000. A
+    slice costs about 80 µs at the parser's measured ~80 ns per token, so a pathologically large
+    script is spread over steps and frames instead of stalling a drain.
+  - pycvc's `Exec.run()` raises a `RuntimeError` carrying the parse error either way: from the
+    killed process's exit error, or straight from `execute()` when the tree sets
+    `step_tokens` = 0.
 - **Ariadne** reads `state_exec.defaults.parser.*` from the app's tree when it compiles a
   predicate.
 - **Code without a state tree** (a standalone evaluator, the `.ari` channel lint) uses the
   built-in defaults, or passes `parse_options` to `parse()` and `parse_all()` explicitly.
   `parse_options_from_state()` does the resolution above for any host that has a tree.
 
+### Reading inside the scheduler
+
+The parser is stackless and resumable. Its whole state is an input position plus a heap stack
+of open lists and quotes, and `parser::read(out, budget)` stops after `budget` tokens. So a
+scheduler can parse a script the same way it evaluates one: a slice per step. With
+`parser.step_tokens` = N:
+
+- **Submitting.** `execute(script)` returns at once. The new process starts in a reading phase.
+  Each of its scheduler steps reads N more tokens (parens, quotes, atoms, strings). The step
+  after the last token sets up evaluation, exactly as parsing at submit would have, and
+  evaluation steps follow.
+- **Budgets.** Reading steps are process steps. They count against `max_steps` and `max_time`,
+  and they interleave with other processes. A huge script costs its own process's budget instead
+  of stalling the scheduler before the process exists.
+- **Errors.** A syntax error surfaces where a runtime error would. The async scheduler kills the
+  process, with the parse error as its exit error. The sync scheduler throws it from `step()` or
+  `run()`. With step_tokens = 0, `execute()` throws it, as before. Ariadne reports a killed
+  action or resident with its exit error ("ari: on: action program did not complete: …",
+  "ari: resident [...] stopped: …").
+- **Signals and watch events** wait until the script is read, because their handlers run on the
+  evaluation stack.
+- **Snapshots.** A process snapshotted while reading stores its source and parser settings. After
+  a restore it reads the script again from the start.
+
 ## Data
 
 | Production | Value (`value_t` alternative) |

inc/cvc/state/state_exec/async_scheduler.h

@@ -163,6 +163,25 @@ class async_scheduler : public scheduler_base {
 
   void set_watch_root(cvc::state *root) override { watch_root_ = root; }
 
+  /// This scheduler's built-in parser settings: what a submitted script is parsed with when the
+  /// watch root's `state_exec.defaults.parser.*` keys don't say otherwise. parse_options{} (parse
+  /// at submit) unless the host sets it.
+  void set_parser_defaults(const parse_options &defaults) { parser_defaults_ = defaults; }
+  const parse_options &parser_defaults() const { return parser_defaults_; }
+
+  /// The parser defaults libcvc's own hosts give their async schedulers, so a script behaves the
+  /// same in each: app::exec_scheduler() (Ariadne) and pycvc's Exec. Scripts are read inside
+  /// their process, 1024 tokens per step (docs/STATE_EXEC_GRAMMAR.md, "Reading inside the
+  /// scheduler"). Every script the repo carries fits in one slice (the largest is 159 tokens;
+  /// .ari scripts are at most 24), so a normal action pays one extra step of its 5000. A slice
+  /// costs about 80 us at the parser's measured ~80 ns per token, so a pathologically large
+  /// script is spread over steps instead of stalling a drain.
+  static parse_options host_parser_defaults() {
+    parse_options defaults;
+    defaults.step_tokens = 1024;
+    return defaults;
+  }
+
 private:
   scheduling_policy policy_;
   int next_pid_ = 1;
@@ -180,6 +199,7 @@ class async_scheduler : public scheduler_base {
   async_stackless_evaluator evaluator_;
   memory_tracker mem_tracker_;
   cvc::state *watch_root_ = nullptr;
+  parse_options parser_defaults_;
 
   std::unordered_map<int, process_ptr> processes_;
 

inc/cvc/state/state_exec/parser.h

@@ -46,13 +46,21 @@ struct parse_options {
   /// recursively (destroying a nested list, the snapshot codec, to_string), so this keeps parsed
   /// data within what they can handle. State-tree key: `parser.max_depth`.
   std::size_t max_depth = 256;
+
+  /// How a scheduler parses a submitted script. 0 parses all of it when it is submitted, so
+  /// execute() throws a parse_error. N > 0 parses it inside the new process, N tokens per
+  /// scheduler step, before evaluation starts. Those steps count against the process's max_steps
+  /// and max_time, and a syntax error then surfaces like a runtime error. State-tree key:
+  /// `parser.step_tokens`.
+  std::size_t step_tokens = 0;
 };
 
 /// Resolve parser settings from a state tree, per key: `state_exec.schedulers.<scheduler_id>.
 /// parser.<key>` (when scheduler_id is not empty), then `state_exec.defaults.parser.<key>`, then
-/// the parse_options default. A value that is missing, empty, not a positive integer or out of
-/// range is skipped.
-parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {});
+/// `base` (a host's built-in defaults; parse_options{} unless it has its own). A value that is
+/// missing, empty, malformed or out of range is skipped; max_depth must also be positive.
+parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {},
+                                       const parse_options &base = {});
 
 /// Parse a single S-expression from the input string.
 /// Returns the parsed value_t.
@@ -122,6 +130,31 @@ class parser {
   value_t parse_number_or_symbol(char first);
 };
 
+/// Reads a whole script a slice at a time. A scheduler drives it from a process's steps when
+/// parse_options::step_tokens is set. It owns its copy of the source, which its parser views, so
+/// it can be neither copied nor moved; hold it by pointer.
+class script_reader {
+public:
+  script_reader(std::string source, const parse_options &opts);
+  script_reader(const script_reader &) = delete;
+  script_reader &operator=(const script_reader &) = delete;
+
+  /// Read up to `budget` more tokens. Returns true once the whole script has been read; forms()
+  /// then holds every top-level datum, as parse_all() would return them.
+  /// @throws parse_error on syntax errors.
+  bool step(std::size_t budget);
+
+  std::vector<value_t> &forms() { return _forms; }
+  const std::string &source() const { return _source; }
+  const parse_options &options() const { return _opts; }
+
+private:
+  std::string _source; // declared before _parser, which views it
+  parse_options _opts;
+  parser _parser;
+  std::vector<value_t> _forms;
+};
+
 } // namespace cvc::state_exec
 
 #endif // CVC_STATE_EXEC_PARSER_H