diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index c77d513de..3624fa02b 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -184,3 +184,28 @@ Every widget reads/writes `cvc::state`, so the same value is drivable from a script, a config file or a replicated peer. Raw custom-widget ImGui (free-form windows) is a separate, deferred design (a curated subset compiled inside cvcGL). Contract test: `test_pycvc_gl_imgui.py`. + +## state_exec DSL (`Exec`) and parser settings + +`pycvc.Exec(app)` runs state_exec programs over the app's state tree. The parser +settings are the C++ surface, under the C++ names (see +`docs/STATE_EXEC_GRAMMAR.md`, "Parser settings"): + +```python +import pycvc +app = pycvc.make_app() +ex = pycvc.Exec(app) +ex.run("(+ 1 2)") # "3" + +pycvc.parse_options() # max_depth 256, step_tokens 0 +pycvc.async_scheduler.host_parser_defaults() # step_tokens 1024 +ex.parser_defaults() # == host_parser_defaults() +pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "64") # tree overrides +root = pycvc.State.instance(app) # the tree root, as in C++ +pycvc.parse_options_from_state(root, "", ex.parser_defaults()) # what Exec uses +``` + +`Exec` starts from the same defaults as the app's exec scheduler (Ariadne), and +the same `state_exec.defaults.parser.*` keys override both. A syntax error +raises a `RuntimeError` with the parse error's message. Contract test: +`test_pycvc_exec.py`. diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 535391b20..50fc32b62 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1437,6 +1437,36 @@ class AsyncStateObserver(state_observer): self._workers = [] %} +// ── state_exec parser settings (docs/STATE_EXEC_GRAMMAR.md, "Parser settings") ── +// DIRECT WRAP, under the C++ names: the real cvc::state_exec::parse_options +// struct, parse_options_from_state (which takes a State, the tree root, as in +// C++: pycvc.State.instance(app)) and async_scheduler::host_parser_defaults(). +// Only these members are declared to SWIG; the rest of parser.h / +// async_scheduler.h (value_t, the parser, the scheduler itself) is not wrapped +// yet, so async_scheduler has no Python constructor. Exec's parser_defaults / +// set_parser_defaults are in pycvc_exec.h. +%{ +#include +#include +%} +namespace cvc { +namespace state_exec { +struct parse_options { + size_t max_depth; // lists and quotes that may nest (default 256) + size_t step_tokens; // 0: parse at submit; N: read N tokens per process step +}; +parse_options parse_options_from_state(cvc::state &root, + const std::string &scheduler_id = std::string(), + const parse_options &base = parse_options()); +%nodefaultctor async_scheduler; +%nodefaultdtor async_scheduler; +class async_scheduler { +public: + static parse_options host_parser_defaults(); +}; +} +} + // ── Phase 4: DSL execution + Python-authored DSL functions ────────────── // pycvc.Exec(app) runs state_exec DSL programs over app's state root and lets a // Python callable be registered as a DSL function (register_fn). PyObject* is diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 27ea4d652..dd3893939 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -378,6 +378,10 @@ Exec::Exec(const std::shared_ptr &app) : impl_(std::make_sharedcomputePool(); impl_->sched.set_watch_root(impl_->root); + // The same state_exec defaults as the app's exec scheduler, so a program behaves the same from + // Python as from Ariadne: scripts are read inside the process (host_parser_defaults), and the + // app tree's state_exec.defaults.parser.* keys override them for both. + impl_->sched.set_parser_defaults(async_scheduler::host_parser_defaults()); impl_->host_proc = make_process(); impl_->host_proc->pid = 0; @@ -554,7 +558,10 @@ std::string Exec::run(const std::string &src) { execute_options opts; opts.name = "pycvc"; opts.env = impl_->env; - const int pid = sched.execute(src, opts); // may throw on parse error (nothing to clean up yet) + // A syntax error is read inside the process (parser.step_tokens) and surfaces below as the + // killed process's exit error; with step_tokens = 0 on the tree, execute() throws it here + // instead (nothing to clean up yet). Either way run() raises with the parse error's message. + const int pid = sched.execute(src, opts); // Drive the private scheduler to completion. A purely synchronous program finishes in the first // slice; a program that parks (await / an async fn) yields, and the pump waits for a compute-pool @@ -641,4 +648,12 @@ std::string Exec::run(const std::string &src) { return render_result(*result); } +cvc::state_exec::parse_options Exec::parser_defaults() const { + return impl_->sched.parser_defaults(); +} + +void Exec::set_parser_defaults(const cvc::state_exec::parse_options &defaults) { + impl_->sched.set_parser_defaults(defaults); +} + } // namespace pycvc diff --git a/bindings/pycvc/pycvc_exec.h b/bindings/pycvc/pycvc_exec.h index 192de81d5..b774ac2c4 100644 --- a/bindings/pycvc/pycvc_exec.h +++ b/bindings/pycvc/pycvc_exec.h @@ -7,6 +7,7 @@ // registered here becomes callable from DSL source like any builtin. #pragma once +#include // parse_options #include #include @@ -86,6 +87,13 @@ class Exec { // (each owns its own scheduler, so they do not interfere). std::string run(const std::string &src); + // The parser settings this Exec's scheduler parses programs with when the app tree's + // state_exec.defaults.parser.* keys don't say otherwise; the same calls as + // async_scheduler::parser_defaults() / set_parser_defaults(). They start as + // async_scheduler::host_parser_defaults(), as the app's exec scheduler's do. + cvc::state_exec::parse_options parser_defaults() const; + void set_parser_defaults(const cvc::state_exec::parse_options &defaults); + private: struct ExecImpl; std::shared_ptr impl_; diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index 33f610b49..3bf8f1eee 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -36,6 +36,99 @@ def test_parse_error_raises(): raise AssertionError("a parse error must raise") +def _run_error(ex, src): + """run(src) must raise; return the message.""" + try: + ex.run(src) + except Exception as e: # noqa: BLE001 - pycvc surfaces C++ errors as RuntimeError + return str(e) + raise AssertionError(f"{src!r} must raise") + + +def test_parse_error_names_the_problem(): + # Exec uses the same state_exec defaults as the app's exec scheduler: the script + # is read inside its process, so a syntax error comes back as the process's exit + # error. run() raises with the parse error's own message, as it did when the + # script was parsed at submit. + app = pycvc.make_app() + ex = pycvc.Exec(app) + assert "unclosed parenthesis" in _run_error(ex, "(begin (set x") + + +def test_parser_settings_come_from_the_app_tree(): + # Exec's scheduler watches the app's state tree, so state_exec.defaults.parser.* + # configures it exactly as it configures the app's exec scheduler + # (docs/STATE_EXEC_GRAMMAR.md). + app = pycvc.make_app() + ex = pycvc.Exec(app) + deep = "(+ 1 (+ 2 (+ 3 (+ 4 5))))" # four levels of nesting + assert ex.run(deep) == "15" + + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "3") + assert "nesting deeper than 3" in _run_error(ex, deep) + assert ex.run("(+ 1 (+ 2 3))") == "6" # three levels still parse + + # Parsing at submit (step_tokens 0) gives the same results and the same errors. + pycvc.state_set(app, "state_exec.defaults.parser.step_tokens", "0") + assert "nesting deeper than 3" in _run_error(ex, deep) + assert "unclosed parenthesis" in _run_error(ex, "(begin (set x") + assert ex.run("(+ 1 (+ 2 3))") == "6" + + pycvc.state_set(app, "state_exec.defaults.parser.max_depth", "256") + assert ex.run(deep) == "15" + + +# ── state_exec parser settings: the C++ surface, under the C++ names ──── + + +def test_parse_options_is_the_cpp_struct(): + o = pycvc.parse_options() + assert (o.max_depth, o.step_tokens) == (256, 0) # C++ defaults + o.max_depth = 32 + o.step_tokens = 8 + assert (o.max_depth, o.step_tokens) == (32, 8) + + +def test_exec_starts_from_the_host_parser_defaults(): + # The same defaults as the app's exec scheduler: + # async_scheduler::host_parser_defaults(). + host = pycvc.async_scheduler.host_parser_defaults() + assert (host.max_depth, host.step_tokens) == (256, 1024) + ex = pycvc.Exec(pycvc.make_app()) + d = ex.parser_defaults() + assert (d.max_depth, d.step_tokens) == (host.max_depth, host.step_tokens) + + +def test_parse_options_from_state_resolves_like_cpp(): + # The real C++ function: it takes the tree root, a State, as in C++. + app = pycvc.make_app() + root = pycvc.State.instance(app) + # Nothing set: the C++ default. + assert pycvc.parse_options_from_state(root).step_tokens == 0 + # What Exec reads with: its own defaults under the tree. + ex = pycvc.Exec(app) + effective = pycvc.parse_options_from_state(root, "", ex.parser_defaults()) + assert effective.step_tokens == 1024 + + root.child("state_exec.defaults.parser.max_depth").value("40") + root.child("state_exec.schedulers.s1.parser.max_depth").value("50") + assert pycvc.parse_options_from_state(root).max_depth == 40 + assert pycvc.parse_options_from_state(root, "s1").max_depth == 50 + root.child("state_exec.defaults.parser.max_depth").value("nope") # bad: skipped + assert pycvc.parse_options_from_state(root).max_depth == 256 + + +def test_set_parser_defaults_changes_how_exec_parses(): + ex = pycvc.Exec(pycvc.make_app()) + o = pycvc.parse_options() + o.max_depth = 2 + ex.set_parser_defaults(o) + assert ex.parser_defaults().max_depth == 2 + assert ex.parser_defaults().step_tokens == 0 + assert "nesting deeper than 2" in _run_error(ex, "(+ 1 (+ 2 (+ 3 4)))") + assert ex.run("(+ 1 (+ 2 3))") == "6" + + # ── DSL <-> state <-> Python round trips (one shared app) ─────────────── diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index c9e2ce74b..c6045b560 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -148,12 +148,13 @@ this order: 2. `state_exec.defaults.parser.`, for everything on that tree; 3. the built-in default. -A value that is missing, empty, not a positive integer or out of range is skipped, and the next -one in the order applies. +A value that is missing, empty, not a non-negative integer or out of range is skipped, and the +next one in the order applies. `max_depth` must also be positive. | Key | Meaning | Default | |---|---|---| | `max_depth` | how many lists and quotes may nest | 256 | +| `step_tokens` | 0: a scheduler parses a submitted script all at once, when it is submitted. N: it parses the script inside the new process, N tokens per step (see [Reading inside the scheduler](#reading-inside-the-scheduler)) | 0; **1024** for the app's exec scheduler and pycvc's `Exec` | Who reads them: @@ -161,13 +162,52 @@ Who reads them: the effective values under `state_exec.schedulers..parser.*` and uses them for every script submitted to it afterwards. - **`async_scheduler`** has no settings loader. It reads `state_exec.defaults.parser.*` from its - watch root each time a script is submitted. + watch root each time a script is submitted. Keys that aren't set fall back to the scheduler's + own defaults (`set_parser_defaults()`), which are the built-in ones unless the host changes + them. + - **libcvc's own hosts all use `async_scheduler::host_parser_defaults()`** (`step_tokens` = + 1024), so a script behaves the same in each: + - the app's exec scheduler (`app::exec_scheduler()`, which runs Ariadne's actions and + residents); + - pycvc's `Exec`, on its private scheduler. That scheduler watches the same app tree, so the + same keys override both. + - **Why 1024:** every script in the corpus fits in one slice (the largest is 159 tokens, and + `.ari` scripts are at most 24). So a normal action pays one extra step out of its 5000. A + slice costs about 80 µs at the parser's measured ~80 ns per token, so a pathologically large + script is spread over steps and frames instead of stalling a drain. + - pycvc's `Exec.run()` raises a `RuntimeError` carrying the parse error either way: from the + killed process's exit error, or straight from `execute()` when the tree sets + `step_tokens` = 0. - **Ariadne** reads `state_exec.defaults.parser.*` from the app's tree when it compiles a predicate. - **Code without a state tree** (a standalone evaluator, the `.ari` channel lint) uses the built-in defaults, or passes `parse_options` to `parse()` and `parse_all()` explicitly. `parse_options_from_state()` does the resolution above for any host that has a tree. +### Reading inside the scheduler + +The parser is stackless and resumable. Its whole state is an input position plus a heap stack +of open lists and quotes, and `parser::read(out, budget)` stops after `budget` tokens. So a +scheduler can parse a script the same way it evaluates one: a slice per step. With +`parser.step_tokens` = N: + +- **Submitting.** `execute(script)` returns at once. The new process starts in a reading phase. + Each of its scheduler steps reads N more tokens (parens, quotes, atoms, strings). The step + after the last token sets up evaluation, exactly as parsing at submit would have, and + evaluation steps follow. +- **Budgets.** Reading steps are process steps. They count against `max_steps` and `max_time`, + and they interleave with other processes. A huge script costs its own process's budget instead + of stalling the scheduler before the process exists. +- **Errors.** A syntax error surfaces where a runtime error would. The async scheduler kills the + process, with the parse error as its exit error. The sync scheduler throws it from `step()` or + `run()`. With step_tokens = 0, `execute()` throws it, as before. Ariadne reports a killed + action or resident with its exit error ("ari: on: action program did not complete: …", + "ari: resident [...] stopped: …"). +- **Signals and watch events** wait until the script is read, because their handlers run on the + evaluation stack. +- **Snapshots.** A process snapshotted while reading stores its source and parser settings. After + a restore it reads the script again from the start. + ## Data | Production | Value (`value_t` alternative) | diff --git a/inc/cvc/state/state_exec/async_scheduler.h b/inc/cvc/state/state_exec/async_scheduler.h index 750c61d68..8af85f86f 100644 --- a/inc/cvc/state/state_exec/async_scheduler.h +++ b/inc/cvc/state/state_exec/async_scheduler.h @@ -163,6 +163,25 @@ class async_scheduler : public scheduler_base { void set_watch_root(cvc::state *root) override { watch_root_ = root; } + /// This scheduler's built-in parser settings: what a submitted script is parsed with when the + /// watch root's `state_exec.defaults.parser.*` keys don't say otherwise. parse_options{} (parse + /// at submit) unless the host sets it. + void set_parser_defaults(const parse_options &defaults) { parser_defaults_ = defaults; } + const parse_options &parser_defaults() const { return parser_defaults_; } + + /// The parser defaults libcvc's own hosts give their async schedulers, so a script behaves the + /// same in each: app::exec_scheduler() (Ariadne) and pycvc's Exec. Scripts are read inside + /// their process, 1024 tokens per step (docs/STATE_EXEC_GRAMMAR.md, "Reading inside the + /// scheduler"). Every script the repo carries fits in one slice (the largest is 159 tokens; + /// .ari scripts are at most 24), so a normal action pays one extra step of its 5000. A slice + /// costs about 80 us at the parser's measured ~80 ns per token, so a pathologically large + /// script is spread over steps instead of stalling a drain. + static parse_options host_parser_defaults() { + parse_options defaults; + defaults.step_tokens = 1024; + return defaults; + } + private: scheduling_policy policy_; int next_pid_ = 1; @@ -180,6 +199,7 @@ class async_scheduler : public scheduler_base { async_stackless_evaluator evaluator_; memory_tracker mem_tracker_; cvc::state *watch_root_ = nullptr; + parse_options parser_defaults_; std::unordered_map processes_; diff --git a/inc/cvc/state/state_exec/parser.h b/inc/cvc/state/state_exec/parser.h index 102d447c1..5267c0c3d 100644 --- a/inc/cvc/state/state_exec/parser.h +++ b/inc/cvc/state/state_exec/parser.h @@ -46,13 +46,21 @@ struct parse_options { /// recursively (destroying a nested list, the snapshot codec, to_string), so this keeps parsed /// data within what they can handle. State-tree key: `parser.max_depth`. std::size_t max_depth = 256; + + /// How a scheduler parses a submitted script. 0 parses all of it when it is submitted, so + /// execute() throws a parse_error. N > 0 parses it inside the new process, N tokens per + /// scheduler step, before evaluation starts. Those steps count against the process's max_steps + /// and max_time, and a syntax error then surfaces like a runtime error. State-tree key: + /// `parser.step_tokens`. + std::size_t step_tokens = 0; }; /// Resolve parser settings from a state tree, per key: `state_exec.schedulers.. /// parser.` (when scheduler_id is not empty), then `state_exec.defaults.parser.`, then -/// the parse_options default. A value that is missing, empty, not a positive integer or out of -/// range is skipped. -parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {}); +/// `base` (a host's built-in defaults; parse_options{} unless it has its own). A value that is +/// missing, empty, malformed or out of range is skipped; max_depth must also be positive. +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id = {}, + const parse_options &base = {}); /// Parse a single S-expression from the input string. /// Returns the parsed value_t. @@ -122,6 +130,31 @@ class parser { value_t parse_number_or_symbol(char first); }; +/// Reads a whole script a slice at a time. A scheduler drives it from a process's steps when +/// parse_options::step_tokens is set. It owns its copy of the source, which its parser views, so +/// it can be neither copied nor moved; hold it by pointer. +class script_reader { +public: + script_reader(std::string source, const parse_options &opts); + script_reader(const script_reader &) = delete; + script_reader &operator=(const script_reader &) = delete; + + /// Read up to `budget` more tokens. Returns true once the whole script has been read; forms() + /// then holds every top-level datum, as parse_all() would return them. + /// @throws parse_error on syntax errors. + bool step(std::size_t budget); + + std::vector &forms() { return _forms; } + const std::string &source() const { return _source; } + const parse_options &options() const { return _opts; } + +private: + std::string _source; // declared before _parser, which views it + parse_options _opts; + parser _parser; + std::vector _forms; +}; + } // namespace cvc::state_exec #endif // CVC_STATE_EXEC_PARSER_H diff --git a/inc/cvc/state/state_exec/stackless_evaluator.h b/inc/cvc/state/state_exec/stackless_evaluator.h index 17a6b1b08..10902797c 100644 --- a/inc/cvc/state/state_exec/stackless_evaluator.h +++ b/inc/cvc/state/state_exec/stackless_evaluator.h @@ -74,6 +74,9 @@ struct evaluator_state { evaluator::macro_map user_macros; evaluation_stats stats; bool yielded = false; // Set by (yield val) — signals generator_next to stop + /// Set while the script is still being read, a slice per step (parse_options::step_tokens); + /// step() reads instead of evaluating until the whole script is read. + std::shared_ptr reading; }; /// Synchronous stackless evaluator (Tier 2 — local contexts only). @@ -94,7 +97,9 @@ class stackless_evaluator { evaluator_state create_state(const value_t &expr, environment_ptr env = nullptr); /// Create a fresh evaluator_state from a script string, parsed with `popts` (a scheduler passes - /// the parser settings it resolved from the state tree). + /// the parser settings it resolved from the state tree). With popts.step_tokens > 0 nothing is + /// parsed yet: the state reads the script popts.step_tokens tokens per step() first, and a + /// syntax error is thrown from step() rather than from here. evaluator_state create_state(const std::string &script, environment_ptr env = nullptr, const parse_options &popts = {}); @@ -137,6 +142,12 @@ class stackless_evaluator { private: void check_interrupted(); + // A state that evaluates `forms` as a script: done when empty, the form itself when one, a + // (begin ...) of them otherwise. + evaluator_state create_program_state(std::vector forms, environment_ptr env); + // One step of a state that is still reading its script (evaluator_state::reading). + bool step_read(evaluator_state &state); + // Frame management void pop_frame(evaluator_state &state, value_t result); void push_frame(evaluator_state &state, value_t expr, environment_ptr env); diff --git a/src/cvc/ariadne/ariadne.cpp b/src/cvc/ariadne/ariadne.cpp index 7463d5e8e..dc725bcba 100644 --- a/src/cvc/ariadne/ariadne.cpp +++ b/src/cvc/ariadne/ariadne.cpp @@ -1494,12 +1494,28 @@ void Runtime::Impl::pump_and_sweep_actions() { if (!info || info->status == se::process_status::terminated) { it = live_actions_.erase(it); // normal completion (or already reaped from the process map) } else if (info->status == se::process_status::killed) { - warn_once("ari: on: action program did not complete (a runtime error or resource limit)"); + // The exit error says why: a syntax error (scripts are read inside the process), a runtime + // error or a resource limit. + warn_once("ari: on: action program did not complete: " + + sched.get_exit_error(it->first).value_or("killed")); it = live_actions_.erase(it); } else { ++it; // ready/running/waiting — still in flight (e.g. suspended on await/msg-recv/sleep) } } + // A resident never finishes on its own, so one that was killed (a syntax error in its body, + // read inside the process; a runtime error; its per-step cap) is reported, once per reason. + const auto report_killed_resident = [&](int pid, const std::string &channel) { + if (pid < 0) + return; + auto info = sched.get_process_info(pid); + if (info && info->status == se::process_status::killed) + warn_once("ari: resident [" + channel + + "] stopped: " + sched.get_exit_error(pid).value_or("killed")); + }; + report_killed_resident(tick_resident_pid_, tick_channel()); + report_killed_resident(key_resident_pid_, key_channel()); + report_killed_resident(pointer_resident_pid_, pointer_channel()); } void Runtime::Impl::ensure_resident(const std::string &channel, const std::string &body, diff --git a/src/cvc/core/app.cpp b/src/cvc/core/app.cpp index d776c16bf..5af747911 100644 --- a/src/cvc/core/app.cpp +++ b/src/cvc/core/app.cpp @@ -1022,6 +1022,9 @@ cvc::state_exec::async_scheduler &app::exec_scheduler() { // Wire it to this app's state root so state-watches (state-watch/on_change:) // resolve against the app tree. Processes are chrooted per-owner on submit. _execScheduler->set_watch_root(&root()); + // Read submitted scripts inside their processes, like pycvc's Exec (see host_parser_defaults + // for why 1024 tokens a step). state_exec.defaults.parser.* on this tree overrides it. + _execScheduler->set_parser_defaults(cvc::state_exec::async_scheduler::host_parser_defaults()); } return *_execScheduler; } diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index 625f82c2e..46b210f24 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -41,9 +41,10 @@ int async_scheduler::execute(const std::string &script, const execute_options &o proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); - // No load_settings() here: read the parser settings from the tree on every submission. + // No load_settings() here: read the parser settings from the tree on every submission, over + // this scheduler's own defaults (set_parser_defaults). const parse_options popts = - watch_root_ ? parse_options_from_state(*watch_root_) : parse_options{}; + watch_root_ ? parse_options_from_state(*watch_root_, {}, parser_defaults_) : parser_defaults_; proc.state = evaluator_.create_state(script, opts.env, popts); proc.state.stats.start(); processes_.emplace(pid, std::make_shared(std::move(proc))); @@ -146,12 +147,17 @@ void async_scheduler::execute_process_step(process &proc) { } } reset_guard{current_pid_, current_proc_}; - if (!proc.pending_signals.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + // A process still reading its script (parser.step_tokens) takes signals and watch events once + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. + const bool can_interrupt = + !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + + if (!proc.pending_signals.empty() && can_interrupt) { handle_signal(proc); return; } - if (!proc.pending_watch_events.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_watch_events.empty() && can_interrupt) { handle_watch_event(proc); return; } diff --git a/src/cvc/state/state_exec/parser.cpp b/src/cvc/state/state_exec/parser.cpp index 340003b07..dd024c405 100644 --- a/src/cvc/state/state_exec/parser.cpp +++ b/src/cvc/state/state_exec/parser.cpp @@ -27,29 +27,37 @@ std::vector parse_all(const std::string &input, const parse_options &op return results; } -// A positive integer setting at `path`, or nullopt when it is missing, empty, malformed, zero or -// out of range (so the caller falls through to the next place in the resolution order). -static std::optional read_positive_setting(cvc::state &root, const std::string &path) { +// A non-negative integer setting at `path` (positive unless `allow_zero`), or nullopt when it is +// missing, empty, malformed or out of range, so the caller falls through to the next place in +// the resolution order. +static std::optional read_size_setting(cvc::state &root, const std::string &path, + bool allow_zero) { cvc::state *node = root.findDescendant(path); if (!node) return std::nullopt; const std::string v = node->value(); std::size_t n = 0; const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); - if (ec != std::errc{} || end != v.data() + v.size() || n == 0) + if (ec != std::errc{} || end != v.data() + v.size() || v.empty() || (n == 0 && !allow_zero)) return std::nullopt; return n; } -parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id) { - parse_options opts; - const std::string key = ".parser.max_depth"; - if (auto v = read_positive_setting(root, std::string(scheduler::defaults_prefix) + key)) - opts.max_depth = *v; - if (!scheduler_id.empty()) - if (auto v = read_positive_setting(root, std::string(scheduler::sched_prefix) + "." + - scheduler_id + key)) - opts.max_depth = *v; +parse_options parse_options_from_state(cvc::state &root, const std::string &scheduler_id, + const parse_options &base) { + parse_options opts = base; + const auto resolve = [&](const char *key, bool allow_zero, std::size_t &out) { + const std::string suffix = std::string(".parser.") + key; + if (auto v = + read_size_setting(root, std::string(scheduler::defaults_prefix) + suffix, allow_zero)) + out = *v; + if (!scheduler_id.empty()) + if (auto v = read_size_setting( + root, std::string(scheduler::sched_prefix) + "." + scheduler_id + suffix, allow_zero)) + out = *v; + }; + resolve("max_depth", /*allow_zero=*/false, opts.max_depth); + resolve("step_tokens", /*allow_zero=*/true, opts.step_tokens); // 0 = parse at submit return opts; } @@ -184,6 +192,26 @@ parser::read_status parser::read(value_t &out, std::size_t &budget) { return read_status::more; } +// -- script_reader ------------------------------------------------------------- + +script_reader::script_reader(std::string source, const parse_options &opts) + : _source(std::move(source)), _opts(opts), _parser(_source, opts) {} + +bool script_reader::step(std::size_t budget) { + for (;;) { + value_t form; + switch (_parser.read(form, budget)) { + case parser::read_status::datum: + _forms.push_back(std::move(form)); + break; + case parser::read_status::end: + return true; + case parser::read_status::more: + return false; + } + } +} + value_t parser::parse_string() { advance(); // consume opening '"' std::string result; diff --git a/src/cvc/state/state_exec/scheduler.cpp b/src/cvc/state/state_exec/scheduler.cpp index c2f55ece1..b3ae0f1aa 100644 --- a/src/cvc/state/state_exec/scheduler.cpp +++ b/src/cvc/state/state_exec/scheduler.cpp @@ -65,6 +65,7 @@ void scheduler::load_settings() { std::string base = std::string(sched_prefix) + "." + id_; (*watch_root_)(base + ".max_pending_messages").value(std::to_string(max_pending_messages)); (*watch_root_)(base + ".parser.max_depth").value(std::to_string(parse_options_.max_depth)); + (*watch_root_)(base + ".parser.step_tokens").value(std::to_string(parse_options_.step_tokens)); (*watch_root_)(base + ".limits.max_alloc_bytes").value(std::to_string(max_alloc_bytes_)); (*watch_root_)(base + ".policy") .value(policy_ == scheduling_policy::round_robin ? "round_robin" @@ -213,14 +214,19 @@ void scheduler::execute_process_step(process &proc) { } } reset_guard{current_pid_, current_proc_}; + // A process still reading its script (parser.step_tokens) takes signals and watch events once + // the script is read: their handlers run on the evaluation stack, which doesn't exist yet. + const bool can_interrupt = + !proc.in_signal_handler && !proc.in_watch_handler && !proc.state.reading; + // 1. Handle pending signals first - if (!proc.pending_signals.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_signals.empty() && can_interrupt) { handle_signal(proc); return; // Signal setup counts as a step } // 1b. Handle pending watch events - if (!proc.pending_watch_events.empty() && !proc.in_signal_handler && !proc.in_watch_handler) { + if (!proc.pending_watch_events.empty() && can_interrupt) { handle_watch_event(proc); return; } diff --git a/src/cvc/state/state_exec/stackless_evaluator.cpp b/src/cvc/state/state_exec/stackless_evaluator.cpp index 74b318cf1..a1483a6a3 100644 --- a/src/cvc/state/state_exec/stackless_evaluator.cpp +++ b/src/cvc/state/state_exec/stackless_evaluator.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace cvc::state_exec { @@ -41,23 +42,52 @@ evaluator_state stackless_evaluator::create_state(const value_t &expr, environme evaluator_state stackless_evaluator::create_state(const std::string &script, environment_ptr env, const parse_options &popts) { - auto exprs = parse_all(script, popts); - if (exprs.empty()) { + if (popts.step_tokens > 0) { + // Read inside the process: step_read() parses a slice per step, then sets up evaluation. + evaluator_state st; + st.global_env = env ? env : environment::extend(global_env_); + st.reading = std::make_shared(script, popts); + return st; + } + return create_program_state(parse_all(script, popts), env); +} + +evaluator_state stackless_evaluator::create_program_state(std::vector forms, + environment_ptr env) { + if (forms.empty()) { evaluator_state st; st.done = true; return st; } - if (exprs.size() == 1) - return create_state(exprs[0], env); + if (forms.size() == 1) + return create_state(forms[0], env); std::vector begin_exprs; - begin_exprs.reserve(1 + exprs.size()); + begin_exprs.reserve(1 + forms.size()); begin_exprs.push_back(value_t{symbol{"begin"}}); - begin_exprs.insert(begin_exprs.end(), exprs.begin(), exprs.end()); + begin_exprs.insert(begin_exprs.end(), std::make_move_iterator(forms.begin()), + std::make_move_iterator(forms.end())); return create_state(make_list(std::move(begin_exprs)), env); } +// Read the next slice of the script. Once all of it is read, the state becomes exactly what +// create_state(script) would have built, keeping the step count and timing of the reading. A +// parse_error propagates like any error from step(): the sync scheduler rethrows it to its +// caller, and the async scheduler kills the process with it. +bool stackless_evaluator::step_read(evaluator_state &state) { + state.stats.increment_step(); + script_reader &reader = *state.reading; + if (!reader.step(reader.options().step_tokens)) + return false; + evaluator_state program = create_program_state(std::move(reader.forms()), state.global_env); + program.stats = std::move(state.stats); + state = std::move(program); // also releases the reader + return state.done; +} + bool stackless_evaluator::step(evaluator_state &state) { check_interrupted(); + if (state.reading) + return step_read(state); if (state.done || state.stack.empty()) { state.done = true; return true; diff --git a/src/cvc/state/state_exec/state_value_codec.cpp b/src/cvc/state/state_exec/state_value_codec.cpp index 527525a12..1106f07b4 100644 --- a/src/cvc/state/state_exec/state_value_codec.cpp +++ b/src/cvc/state/state_exec/state_value_codec.cpp @@ -699,6 +699,15 @@ void encode_evaluator_state(cvc::state &node, const evaluator_state &es) { } } + // A script still being read (parser.step_tokens): keep its source and parser settings; the + // restored state reads it again from the start. + if (es.reading) { + auto &r = node("reading"); + r("source").value(es.reading->source()); + r("max_depth").value(std::to_string(es.reading->options().max_depth)); + r("step_tokens").value(std::to_string(es.reading->options().step_tokens)); + } + // Stats snapshot node("step_count").value(std::to_string(es.stats.get_step_count())); } @@ -747,6 +756,23 @@ evaluator_state decode_evaluator_state(cvc::state &node) { } } + // A script still being read restarts its reading from the beginning. + if (has_child(node, "reading")) { + auto &r = node("reading"); + parse_options opts; + const auto size_of = [&](const char *key, std::size_t fallback) { + const std::string v = r(key).value(); + std::size_t n = fallback; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), n); + return ec == std::errc{} && end == v.data() + v.size() ? n : fallback; + }; + opts.max_depth = size_of("max_depth", opts.max_depth); + opts.step_tokens = size_of("step_tokens", 1); // a reading state always has a positive slice + if (opts.step_tokens == 0) + opts.step_tokens = 1; + es.reading = std::make_shared(r("source").value(), opts); + } + return es; } diff --git a/src/cvc/tests/ariadne_runtime_test.cpp b/src/cvc/tests/ariadne_runtime_test.cpp index 00388e839..dbca01741 100644 --- a/src/cvc/tests/ariadne_runtime_test.cpp +++ b/src/cvc/tests/ariadne_runtime_test.cpp @@ -520,6 +520,27 @@ TEST(AriadneResident, TickProgramRunsOncePerDrain) { EXPECT_TRUE(rt.take_reactive_warnings().empty()); } +// The app's exec scheduler reads a submitted script inside its process (parser.step_tokens), so a +// resident whose body doesn't parse is killed by the syntax error once it is read, and the sweep +// reports it with the parse error rather than letting it vanish. +TEST(AriadneResident, ResidentWithASyntaxErrorIsReported) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + rt.set_tick_program("(state-set \"tick.flag\" \"1\""); // unclosed + rt.render(); + rt.drain(); + rt.drain(); + const std::vector warns = rt.take_reactive_warnings(); + bool reported = false; + for (const std::string &w : warns) + reported = reported || (w.find("stopped") != std::string::npos && + w.find("unclosed parenthesis") != std::string::npos); + EXPECT_TRUE(reported) << "warnings: " << ::testing::PrintToString(warns); + EXPECT_NE(cvc::state::instance(app)("tick.flag").value(), "1"); // the body never ran +} + // set_tick_program("") clears the resident: after clearing, a drain no longer re-fires the body. TEST(AriadneResident, ClearingTickProgramStopsIt) { cvc::app app; diff --git a/src/cvc/tests/state_exec_codec_test.cpp b/src/cvc/tests/state_exec_codec_test.cpp index 277c27d05..20fea8d41 100644 --- a/src/cvc/tests/state_exec_codec_test.cpp +++ b/src/cvc/tests/state_exec_codec_test.cpp @@ -291,6 +291,37 @@ TEST_F(EvaluatorStateCodecTest, CompletedState) { EXPECT_TRUE(restored.stack.empty()); } +TEST_F(EvaluatorStateCodecTest, StateStillReadingItsScript) { + // parser.step_tokens: a process snapshotted mid-read keeps its source and settings, and the + // restored state reads the script again and evaluates it. + parse_options popts; + popts.step_tokens = 2; + popts.max_depth = 64; + auto state = ev->create_state(std::string("(set x 20) (+ x 22)"), nullptr, popts); + ev->step(state); // part way through reading + ASSERT_TRUE(state.reading); + + auto &node = ctx.root()("__es_reading__"); + encode_evaluator_state(node, state); + auto restored = decode_evaluator_state(node); + + ASSERT_TRUE(restored.reading); + EXPECT_EQ(restored.reading->source(), "(set x 20) (+ x 22)"); + EXPECT_EQ(restored.reading->options().step_tokens, 2u); + EXPECT_EQ(restored.reading->options().max_depth, 64u); + + // Native builtins aren't serialized; put them back on the restored environment's root, as + // PauseResumeProducesCorrectResult does. Nothing is on the frame stack yet. + ASSERT_TRUE(restored.global_env); + environment_ptr root_env = restored.global_env; + while (root_env->outer) + root_env = root_env->outer; + const auto builtins_env = builtins::make_default_environment(); // must outlive the loop + for (auto &[name, val] : builtins_env->bindings) + root_env->set(name, val); + EXPECT_EQ(std::get(ev->run(restored).v), 42); +} + TEST_F(EvaluatorStateCodecTest, PausedState) { // Run a program partially (10 steps of a loop) auto state = diff --git a/src/cvc/tests/state_exec_integration_test.cpp b/src/cvc/tests/state_exec_integration_test.cpp index 0fc33685e..561f38ca7 100644 --- a/src/cvc/tests/state_exec_integration_test.cpp +++ b/src/cvc/tests/state_exec_integration_test.cpp @@ -2403,6 +2403,138 @@ TEST_F(SchedulerSettingsTest, AsyncSchedulerReadsParserSettingsOnEachSubmit) { EXPECT_NO_THROW(sched.execute(three_deep)); } +// --- parser.step_tokens: reading a script inside the process, a slice per step --- + +TEST_F(SchedulerSettingsTest, StepTokensResolveFromTreeAndAllowZero) { + auto &root = cvc::state::instance(app_ctx); + EXPECT_EQ(parse_options_from_state(root).step_tokens, 0u); // built-in: parse at submit + root("state_exec.defaults.parser.step_tokens").value("64"); + EXPECT_EQ(parse_options_from_state(root).step_tokens, 64u); + root("state_exec.schedulers.eager.parser.step_tokens").value("0"); // 0 is a real setting here + EXPECT_EQ(parse_options_from_state(root, "eager").step_tokens, 0u); + root("state_exec.schedulers.eager.parser.step_tokens").value("-1"); // bad: falls through + EXPECT_EQ(parse_options_from_state(root, "eager").step_tokens, 64u); +} + +TEST_F(SchedulerSettingsTest, SyncSchedulerEvaluatesAScriptReadInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("2"); + scheduler sched; + sched.set_watch_root(&root); + sched.set_id("reader"); + sched.load_settings(); + EXPECT_EQ(sched.parser_settings().step_tokens, 2u); + EXPECT_EQ(root("state_exec.schedulers.reader.parser.step_tokens").value(), "2"); + + const int pid = sched.execute(std::string("(set a 40) (+ a 2)")); + auto results = sched.run(); + EXPECT_EQ(std::get(results[pid].v), 42); +} + +TEST_F(SchedulerSettingsTest, SyncSchedulerThrowsSyntaxErrorsFromRunWhenReadingInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("2"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + EXPECT_NO_THROW(sched.execute(std::string("(+ 1 2) (+ 3"))); // nothing is parsed yet + EXPECT_THROW(sched.run(), parse_error); // the process reads it +} + +TEST_F(SchedulerSettingsTest, ReadingStepsCountAgainstMaxSteps) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("1"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + std::string big = "(list"; + for (int i = 0; i < 100; ++i) + big += " " + std::to_string(i); + big += ")"; + execute_options opts; + opts.max_steps = 5; // far fewer than the 103 tokens to read + const int pid = sched.execute(big, opts); + sched.run(); + auto info = sched.get_process_info(pid); + ASSERT_TRUE(info); + EXPECT_EQ(info->status, process_status::killed); + EXPECT_EQ(info->step_count, 5u); +} + +TEST_F(SchedulerSettingsTest, SignalsWaitUntilTheScriptIsRead) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("1"); + scheduler sched; + sched.set_watch_root(&root); + sched.load_settings(); + execute_options opts; + opts.signal_handlers["SIGUSR1"] = make_list({value_t{symbol{"begin"}}, value_t{int64_t(7)}}); + const int pid = sched.execute(std::string("(begin 1 2 (+ 40 2))"), opts); + sched.step(); // reading has begun + EXPECT_TRUE(sched.send_signal(pid, "SIGUSR1")); + auto results = sched.run(); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::terminated); + EXPECT_EQ(std::get(results[pid].v), 42); +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerKillsAProcessWhoseScriptDoesNotParse) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("4"); + async_scheduler sched; + sched.set_watch_root(&root); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1 2) (+ 3"))); + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + const auto err = sched.get_exit_error(pid); + ASSERT_TRUE(err); + EXPECT_NE(err->find("unclosed parenthesis"), std::string::npos) << *err; +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerParserDefaultsSitUnderTheTree) { + auto &root = cvc::state::instance(app_ctx); + async_scheduler sched; + sched.set_watch_root(&root); + EXPECT_EQ(sched.parser_defaults().step_tokens, 0u); // a plain async_scheduler parses at submit + EXPECT_THROW(sched.execute(std::string("(+ 1")), parse_error); + + parse_options defaults; + defaults.step_tokens = 1024; + sched.set_parser_defaults(defaults); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1"))); // now read inside the process + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + + root("state_exec.defaults.parser.step_tokens").value("0"); // the tree overrides the defaults + EXPECT_THROW(sched.execute(std::string("(+ 1")), parse_error); +} + +TEST_F(SchedulerSettingsTest, AppExecSchedulerReadsScriptsInSteps) { + auto &sched = app_ctx.exec_scheduler(); + EXPECT_EQ(async_scheduler::host_parser_defaults().step_tokens, 1024u); + EXPECT_EQ(sched.parser_defaults().step_tokens, + async_scheduler::host_parser_defaults().step_tokens); // shared with pycvc's Exec + EXPECT_EQ(sched.parser_defaults().max_depth, async_scheduler::host_parser_defaults().max_depth); + int pid = -1; + ASSERT_NO_THROW(pid = sched.execute(std::string("(+ 1 (+ 2"))); + sched.sync_run(1000, 1.0); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + const auto err = sched.get_exit_error(pid); + ASSERT_TRUE(err); + EXPECT_NE(err->find("unclosed parenthesis"), std::string::npos) << *err; +} + +TEST_F(SchedulerSettingsTest, AsyncSchedulerEvaluatesAScriptReadInSlices) { + auto &root = cvc::state::instance(app_ctx); + root("state_exec.defaults.parser.step_tokens").value("3"); + async_scheduler sched; + sched.set_watch_root(&root); + const int pid = sched.execute(std::string("(set a 40) (+ a 2)")); + auto results = sched.sync_run(1000, 1.0); + EXPECT_EQ(std::get(results[pid].v), 42); +} + // =========================================================================== // §12 message-channel scoping — end-to-end through the scheduler + chroot // =========================================================================== diff --git a/src/cvc/tests/state_exec_parser_test.cpp b/src/cvc/tests/state_exec_parser_test.cpp index ce00c4fee..888bc8b86 100644 --- a/src/cvc/tests/state_exec_parser_test.cpp +++ b/src/cvc/tests/state_exec_parser_test.cpp @@ -328,6 +328,35 @@ TEST(StateExecParserTest, ReadResumesAcrossTokenBudgets) { EXPECT_GT(pauses, 5); // the lists really were read across several calls } +TEST(StateExecParserTest, ScriptReaderReadsAWholeScriptInSlices) { + const std::string src = "(define x 1) ; a comment\n(+ x '(2 3)) \"s\""; + std::vector whole; + for (const value_t &v : parse_all(src)) + whole.push_back(to_string(v)); + + script_reader reader(src, parse_options{}); + int slices = 1; + while (!reader.step(2)) // two tokens per slice + ++slices; + std::vector sliced; + for (const value_t &v : reader.forms()) + sliced.push_back(to_string(v)); + EXPECT_EQ(sliced, whole); + EXPECT_GT(slices, 5); + EXPECT_EQ(reader.source(), src); // owns its copy +} + +TEST(StateExecParserTest, ScriptReaderReportsSyntaxErrorsWhenItReachesThem) { + script_reader reader("(a) (b", parse_options{}); + EXPECT_FALSE(reader.step(2)); // '(' 'a' + EXPECT_THROW( + { + while (!reader.step(2)) { + } + }, + parse_error); +} + TEST(StateExecParserTest, ReadCountsTokensAgainstTheBudget) { parser p("(a b) c"); std::size_t budget = 3; // '(' 'a' 'b' diff --git a/src/cvc/tests/state_exec_stackless_evaluator_test.cpp b/src/cvc/tests/state_exec_stackless_evaluator_test.cpp index 906f59c63..3728ce8ed 100644 --- a/src/cvc/tests/state_exec_stackless_evaluator_test.cpp +++ b/src/cvc/tests/state_exec_stackless_evaluator_test.cpp @@ -416,3 +416,43 @@ TEST_F(StacklessEvaluatorTest, DeadlineIsNotSpentWhileTheThreadDoesNotRun) { auto st = napper.create_state(parse("(nap)")); EXPECT_EQ(std::get(napper.run(st, std::nullopt, /*timeout_sec=*/0.02).v), 7); } + +// ─── Reading inside the steps (parse_options::step_tokens) ───────────────── + +TEST_F(StacklessEvaluatorTest, SlicedReadingEvaluatesLikeParsingUpFront) { + const std::string script = "(defun sq (x) (* x x)) (set a (sq 7)) (+ a 1)"; + auto eager = ev->create_state(script); + ASSERT_FALSE(eager.reading); + const value_t expected = ev->run(eager); + + parse_options popts; + popts.step_tokens = 3; + auto sliced = ev->create_state(script, nullptr, popts); + ASSERT_TRUE(sliced.reading); // nothing parsed yet + EXPECT_TRUE(sliced.stack.empty()); + const value_t got = ev->run(sliced); + EXPECT_TRUE(sliced.done); + EXPECT_FALSE(sliced.reading); + EXPECT_EQ(to_string(got), to_string(expected)); + EXPECT_EQ(std::get(got.v), 50); + // The reading steps are steps: 25 tokens at 3 per step take 9 (the ninth reads the last token, + // finds the end of the script and sets up evaluation). + EXPECT_EQ(sliced.stats.get_step_count(), eager.stats.get_step_count() + 9); +} + +TEST_F(StacklessEvaluatorTest, SlicedReadingThrowsSyntaxErrorsFromStep) { + parse_options popts; + popts.step_tokens = 2; + evaluator_state st; + ASSERT_NO_THROW(st = ev->create_state(std::string("(+ 1 2) (+ 3"), nullptr, popts)); + EXPECT_THROW(ev->run(st), parse_error); +} + +TEST_F(StacklessEvaluatorTest, SlicedReadingOfAnEmptyScriptFinishesWithNil) { + parse_options popts; + popts.step_tokens = 4; + auto st = ev->create_state(std::string(" ; nothing here\n"), nullptr, popts); + EXPECT_TRUE(ev->step(st)); + EXPECT_TRUE(st.done); + EXPECT_TRUE(st.result.is_nil()); +}