diff --git a/bindings/pycvc/CMakeLists.txt b/bindings/pycvc/CMakeLists.txt index 4cac1bcd6..e2cfc52bd 100644 --- a/bindings/pycvc/CMakeLists.txt +++ b/bindings/pycvc/CMakeLists.txt @@ -273,6 +273,7 @@ if(CVC_BUILD_PYCVC_CORE) pycvc_exec_coro:test_pycvc_exec_coro.py pycvc_state_exec:test_pycvc_state_exec.py pycvc_state_exec_sched:test_pycvc_state_exec_sched.py + pycvc_state_exec_eval:test_pycvc_state_exec_eval.py pycvc_world_clock:test_pycvc_world_clock.py pycvc_world_units:test_pycvc_world_units.py pycvc_dataprep:test_pycvc_dataprep.py diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index 8ebbd3995..e6bb80260 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -259,5 +259,22 @@ The enums are `enum.IntEnum` classes, and an enum argument must be one of their members. Where C++ returns an empty `std::optional` or a null pointer, Python gets `None`, except for a value: `get_result`, `pop_pending_message` and `environment.lookup` raise `KeyError` instead, because `None` is the nil value. + +The evaluators, their state, and the intrinsics are wrapped the same way: + +```python +ev = pycvc.stackless_evaluator(pycvc.builtins.make_default_environment()) +st = ev.create_state("(begin (set x 1) (+ x 41))") +while not ev.step(st): # releases the GIL; ev.interrupt() from + pass # another thread raises evaluation_interrupted +st.result, st.stats.get_step_count() # 42, the step count +pycvc.encode_evaluator_state(root.child("snap"), st) # snapshot, decode_... to restore + +ctx = pycvc.intrinsics_context() # state-get, msg-send, spawn, ... over a tree +ctx.sched, ctx.root = sched, root +env = pycvc.builtins.make_default_environment() +pycvc.register_intrinsics(env, ctx) # each function keeps ctx alive +``` + Contract tests: `test_pycvc_exec.py`, `test_pycvc_state_exec.py`, -`test_pycvc_state_exec_sched.py`. +`test_pycvc_state_exec_sched.py`, `test_pycvc_state_exec_eval.py`. diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 3888257c8..5de4c547d 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1455,17 +1455,24 @@ class AsyncStateObserver(state_observer): #include #include #include +#include +#include #include #include +#include #include #include +#include #include #include #include "pycvc_value.h" #include +#include #include #include #include +#include +#include #include #include #include @@ -1473,6 +1480,11 @@ class AsyncStateObserver(state_observer): #include #include #include + +namespace pycvc { +// The evaluators' on_complete parameter type, under a name SWIG can match a typemap on. +using value_callback = std::function; +} // namespace pycvc %} // ── values: the one value_t <-> Python conversion (pycvc_value.h) ────── @@ -1888,6 +1900,368 @@ bool py_to_value_map(PyObject *o, value_map &out) { } %} +// ── members and results of the evaluator and process types ── +// A value_t member reads as a Python value. +%typemap(out) const cvc::state_exec::value_tag & %{ + $result = pycvc::value_to_py(*$1); + if (!$result) + SWIG_fail; +%} + +%{ +namespace pycvc { +namespace { +using cvc::state_exec::value_t; + +PyObject *value_list_to_py(const std::vector &vals) { + PyObject *list = PyList_New(static_cast(vals.size())); + if (list == nullptr) + return nullptr; + for (size_t i = 0; i < vals.size(); ++i) { + PyObject *item = value_to_py(vals[i]); + if (item == nullptr) { + Py_DECREF(list); + return nullptr; + } + PyList_SET_ITEM(list, static_cast(i), item); + } + return list; +} + +// Each item of the sequence `o` converted with `convert`, or false with a Python error set. +template bool py_to_list(PyObject *o, std::vector &out, F convert) { + PyObject *seq = PySequence_Fast(o, "pycvc: expected a list"); + if (seq == nullptr) + return false; + const Py_ssize_t n = PySequence_Fast_GET_SIZE(seq); + std::vector items; + items.reserve(static_cast(n)); + for (Py_ssize_t i = 0; i < n; ++i) { + T item; + if (!convert(PySequence_Fast_GET_ITEM(seq, i), item)) { + Py_DECREF(seq); + return false; + } + items.push_back(std::move(item)); + } + Py_DECREF(seq); + out = std::move(items); + return true; +} + +bool py_to_value_list(PyObject *o, std::vector &out) { + return py_to_list(o, out, [](PyObject *item, value_t &v) { return py_to_value(item, v); }); +} + +// A struct proxy of `type` copied out (a list item), or false with a TypeError set. +template bool py_to_struct(PyObject *o, swig_type_info *type, T &out) { + void *p = nullptr; + if (!SWIG_IsOK(SWIG_ConvertPtr(o, &p, type, 0)) || p == nullptr) { + PyErr_Format(PyExc_TypeError, "pycvc: expected a pycvc.%s, got %.200s", + type ? type->str : "?", Py_TYPE(o)->tp_name); + return false; + } + out = *static_cast(p); + return true; +} + +// A new owned proxy of a copy of `v`, or nullptr with a Python error set. +template PyObject *struct_to_py(const T &v, swig_type_info *type) { + return SWIG_NewPointerObj(new T(v), type, SWIG_POINTER_OWN); +} +} // namespace +} // namespace pycvc +%} + +// A list of values, and a list of lists of values (eval_frame members). +%typemap(in) const std::vector & + (std::vector temp) %{ + if (!pycvc::py_to_value_list($input, temp)) + SWIG_fail; + $1 = &temp; +%} +%typemap(out) const std::vector & %{ + $result = pycvc::value_list_to_py(*$1); + if (!$result) + SWIG_fail; +%} +%typemap(in) const std::vector > & + (std::vector > temp) %{ + if (!pycvc::py_to_list($input, temp, [](PyObject *item, std::vector &v) { + return pycvc::py_to_value_list(item, v); + })) + SWIG_fail; + $1 = &temp; +%} +%typemap(out) const std::vector > & %{ + $result = PyList_New(static_cast($1->size())); + if (!$result) + SWIG_fail; + for (size_t i = 0; i < $1->size(); ++i) { + PyObject *item = pycvc::value_list_to_py((*$1)[i]); + if (!item) { + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + PyList_SET_ITEM($result, static_cast(i), item); + } +%} + +// A list of structs (eval_frame, process::watch_event) reads as a list of copies and is set from a +// list of them. +%define PYCVC_STATE_EXEC_STRUCT_LIST(TYPE) +%typemap(in) const std::vector & (std::vector temp) { + if (!pycvc::py_to_list($input, temp, [](PyObject *item, TYPE &v) { + return pycvc::py_to_struct(item, $descriptor(TYPE *), v); + })) + SWIG_fail; + $1 = &temp; +} +%typemap(out) const std::vector & { + $result = PyList_New(static_cast($1->size())); + if (!$result) + SWIG_fail; + for (size_t i = 0; i < $1->size(); ++i) { + PyObject *item = pycvc::struct_to_py((*$1)[i], $descriptor(TYPE *)); + if (!item) { + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + PyList_SET_ITEM($result, static_cast(i), item); + } +} +%enddef +PYCVC_STATE_EXEC_STRUCT_LIST(cvc::state_exec::eval_frame) +PYCVC_STATE_EXEC_STRUCT_LIST(cvc::state_exec::process::watch_event) + +// process::watch_handlers: a dict of watch id -> watch_handler_entry (copies). +%typemap(in) const std::unordered_map & + (std::unordered_map temp) { + if (!PyDict_Check($input)) { + PyErr_SetString(PyExc_TypeError, "pycvc: expected a dict of int to watch_handler_entry"); + SWIG_fail; + } + PyObject *k = nullptr; + PyObject *v = nullptr; + Py_ssize_t pos = 0; + while (PyDict_Next($input, &pos, &k, &v)) { + const long id = PyLong_AsLong(k); + if (id == -1 && PyErr_Occurred()) + SWIG_fail; + cvc::state_exec::process::watch_handler_entry entry; + if (!pycvc::py_to_struct(v, $descriptor(cvc::state_exec::process::watch_handler_entry *), + entry)) + SWIG_fail; + temp[static_cast(id)] = std::move(entry); + } + $1 = &temp; +} +%typemap(out) const std::unordered_map & { + $result = PyDict_New(); + if (!$result) + SWIG_fail; + for (const auto &kv : *$1) { + PyObject *k = PyLong_FromLong(kv.first); + PyObject *v = k ? pycvc::struct_to_py( + kv.second, $descriptor(cvc::state_exec::process::watch_handler_entry *)) + : nullptr; + const int rc = v ? PyDict_SetItem($result, k, v) : -1; + Py_XDECREF(k); + Py_XDECREF(v); + if (rc != 0) { + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + } +} + +// evaluator::macro_map: a dict of macro name -> (params, body), params a list of pycvc.symbol. +%{ +namespace pycvc { +namespace { +using macro_map = cvc::state_exec::evaluator::macro_map; + +PyObject *macro_map_to_py(const macro_map &m) { + PyObject *dict = PyDict_New(); + if (dict == nullptr) + return nullptr; + for (const auto &kv : m) { + std::vector params(kv.second.first.begin(), kv.second.first.end()); + PyObject *key = text_to_py(kv.first); + PyObject *ps = key ? value_list_to_py(params) : nullptr; + PyObject *body = ps ? value_to_py(kv.second.second) : nullptr; + PyObject *pair = body ? PyTuple_Pack(2, ps, body) : nullptr; + const int rc = pair ? PyDict_SetItem(dict, key, pair) : -1; + Py_XDECREF(key); + Py_XDECREF(ps); + Py_XDECREF(body); + Py_XDECREF(pair); + if (rc != 0) { + Py_DECREF(dict); + return nullptr; + } + } + return dict; +} + +bool py_to_macro_map(PyObject *o, macro_map &out) { + if (!PyDict_Check(o)) { + PyErr_SetString(PyExc_TypeError, "pycvc: expected a dict of name to (params, body)"); + return false; + } + macro_map m; + PyObject *key = nullptr; + PyObject *val = nullptr; + Py_ssize_t pos = 0; + while (PyDict_Next(o, &pos, &key, &val)) { + if (!PyUnicode_Check(key) || !PyTuple_Check(val) || PyTuple_GET_SIZE(val) != 2) { + PyErr_SetString(PyExc_TypeError, + "pycvc: a macro is name (str) -> (params, body), params a list of symbols"); + return false; + } + std::vector params; + value_t body; + if (!py_to_value_list(PyTuple_GET_ITEM(val, 0), params) || + !py_to_value(PyTuple_GET_ITEM(val, 1), body)) + return false; + std::vector syms; + for (const value_t &p : params) { + const auto *s = std::get_if(&p.v); + if (s == nullptr) { + PyErr_SetString(PyExc_TypeError, "pycvc: macro params must be pycvc.symbol"); + return false; + } + syms.push_back(*s); + } + const char *name = PyUnicode_AsUTF8(key); + if (name == nullptr) + return false; + m[name] = {std::move(syms), std::move(body)}; + } + out = std::move(m); + return true; +} +} // namespace +} // namespace pycvc +%} +%typemap(in) const cvc::state_exec::evaluator::macro_map & (cvc::state_exec::evaluator::macro_map temp) %{ + if (!pycvc::py_to_macro_map($input, temp)) + SWIG_fail; + $1 = &temp; +%} +%typemap(out) const cvc::state_exec::evaluator::macro_map & %{ + $result = pycvc::macro_map_to_py(*$1); + if (!$result) + SWIG_fail; +%} +%typemap(out) cvc::state_exec::evaluator::macro_map %{ + $result = pycvc::macro_map_to_py(static_cast($1)); + if (!$result) + SWIG_fail; +%} + +// Optional strings as members (process::recv_path, exit_error): None or str. +%typemap(in) const std::optional & (std::optional temp) %{ + if ($input != Py_None) { + Py_ssize_t pycvc_n = 0; + const char *pycvc_s = + PyUnicode_Check($input) ? PyUnicode_AsUTF8AndSize($input, &pycvc_n) : nullptr; + if (pycvc_s == nullptr) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "pycvc: expected str or None"); + SWIG_fail; + } + temp = std::string(pycvc_s, static_cast(pycvc_n)); + } + $1 = &temp; +%} +%typemap(out) const std::optional & %{ + if (*$1) { + $result = PyUnicode_DecodeUTF8((*$1)->data(), static_cast((*$1)->size()), + "surrogateescape"); + if (!$result) + SWIG_fail; + } else { + $result = Py_None; + Py_INCREF(Py_None); + } +%} +// process::time_left() and alloc_limit(): None when unlimited. +%typemap(out) std::optional %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over this initialization + const std::optional &pycvc_opt = $1; + $result = pycvc_opt ? PyFloat_FromDouble(*pycvc_opt) : (Py_INCREF(Py_None), Py_None); + } +%} +%typemap(out) std::optional %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over this initialization + const std::optional &pycvc_opt = $1; + $result = pycvc_opt ? PyLong_FromSize_t(*pycvc_opt) : (Py_INCREF(Py_None), Py_None); + } +%} + +// restrict_special_forms takes any iterable of str, or None for no restriction; +// all_special_forms() returns a set. +// $1 is always assigned: SWIG holds this argument in a SwigValueWrapper, which is empty until then. +%typemap(in) std::shared_ptr > %{ + $1 = std::shared_ptr>(); + if ($input != Py_None) { + auto pycvc_forms = std::make_shared>(); + PyObject *pycvc_it = PyObject_GetIter($input); + if (pycvc_it == nullptr) + SWIG_fail; + while (PyObject *pycvc_item = PyIter_Next(pycvc_it)) { + const char *pycvc_s = PyUnicode_Check(pycvc_item) ? PyUnicode_AsUTF8(pycvc_item) : nullptr; + if (pycvc_s != nullptr) + pycvc_forms->insert(pycvc_s); + Py_DECREF(pycvc_item); + if (pycvc_s == nullptr) { + Py_DECREF(pycvc_it); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "pycvc: special forms must be str"); + SWIG_fail; + } + } + Py_DECREF(pycvc_it); + if (PyErr_Occurred()) + SWIG_fail; + $1 = std::move(pycvc_forms); + } +%} +%typemap(out) const std::set & %{ + $result = PySet_New(nullptr); + if (!$result) + SWIG_fail; + for (const std::string &pycvc_form : *$1) { + PyObject *pycvc_s = PyUnicode_FromStringAndSize(pycvc_form.data(), + static_cast(pycvc_form.size())); + if (!pycvc_s || PySet_Add($result, pycvc_s) != 0) { + Py_XDECREF(pycvc_s); + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + Py_DECREF(pycvc_s); + } +%} + +// The evaluators' on_complete: a Python callable or None, called with the result. +%typemap(in) pycvc::value_callback %{ + $1 = pycvc::value_callback(); // always assigned, should SWIG hold it in a SwigValueWrapper + if ($input != Py_None) { + if (!PyCallable_Check($input)) { // checked here: a typemap runs outside the try block + PyErr_SetString(PyExc_TypeError, "pycvc: on_complete must be callable or None"); + SWIG_fail; + } + $1 = pycvc::make_python_on_complete($input); + } +%} +%typemap(typecheck, precedence=SWIG_TYPECHECK_POINTER) pycvc::value_callback + "$1 = ($input == Py_None || PyCallable_Check($input)) ? 1 : 0;" + // ── enums: pycvc.process_status and pycvc.scheduling_policy are enum.IntEnum classes ── // pycvc.process_status.killed is process_status::killed. An enum argument must be a member of its // IntEnum (strict, like values), which is also what lets to_string(process_status) and @@ -1963,6 +2337,19 @@ static bool pycvc_enum_from_py(PyObject *o, PyObject *type, const char *name, in %typemap(typecheck, precedence=SWIG_TYPECHECK_INT32) TYPE { $1 = pycvc_is_enum($input, PYTYPE) ? 1 : 0; } +// By const reference too, for a %naturalvar member of an enum type SWIG wasn't shown +// (resource_policy::enforce). +%typemap(in) const TYPE & (TYPE pycvc_ev_val, int pycvc_ev) { + if (!pycvc_enum_from_py($input, PYTYPE, NAME, pycvc_ev)) + SWIG_fail; + pycvc_ev_val = static_cast(pycvc_ev); + $1 = &pycvc_ev_val; +} +%typemap(out) const TYPE & { + $result = PyObject_CallFunction(PYTYPE, "i", static_cast(*$1)); + if (!$result) + SWIG_fail; +} %enddef PYCVC_STATE_EXEC_ENUM(cvc::state_exec::process_status, pycvc_enum_process_status, "process_status") PYCVC_STATE_EXEC_ENUM(cvc::state_exec::scheduling_policy, pycvc_enum_scheduling_policy, @@ -1992,6 +2379,63 @@ process_status = _pycvc.process_status scheduling_policy = _pycvc.scheduling_policy %} +// eval_phase, step_kind, and resource_policy::mode (pycvc.resource_policy.mode, set on the class +// below). +%{ +static PyObject *pycvc_enum_eval_phase = nullptr; +static PyObject *pycvc_enum_step_kind = nullptr; +static PyObject *pycvc_enum_resource_policy_mode = nullptr; +%} +PYCVC_STATE_EXEC_ENUM(cvc::state_exec::eval_phase, pycvc_enum_eval_phase, "eval_phase") +PYCVC_STATE_EXEC_ENUM(cvc::state_exec::step_kind, pycvc_enum_step_kind, "step_kind") +PYCVC_STATE_EXEC_ENUM(cvc::state_exec::resource_policy::mode, pycvc_enum_resource_policy_mode, + "resource_policy.mode") +%init %{ + { + using cvc::state_exec::eval_phase; + pycvc_enum_eval_phase = pycvc_make_int_enum( + "eval_phase", + {{"init", static_cast(eval_phase::init)}, + {"eval_args", static_cast(eval_phase::eval_args)}, + {"apply", static_cast(eval_phase::apply)}, + {"if_cond", static_cast(eval_phase::if_cond)}, + {"if_branch", static_cast(eval_phase::if_branch)}, + {"begin_next", static_cast(eval_phase::begin_next)}, + {"while_cond", static_cast(eval_phase::while_cond)}, + {"while_body", static_cast(eval_phase::while_body)}, + {"for_body", static_cast(eval_phase::for_body)}, + {"set_value", static_cast(eval_phase::set_value)}, + {"let_bindings", static_cast(eval_phase::let_bindings)}, + {"let_body", static_cast(eval_phase::let_body)}, + {"return_value", static_cast(eval_phase::return_value)}, + {"eval_inner", static_cast(eval_phase::eval_inner)}, + {"super_args", static_cast(eval_phase::super_args)}, + {"defclass_methods", static_cast(eval_phase::defclass_methods)}, + {"yield_value", static_cast(eval_phase::yield_value)}, + {"break_unwind", static_cast(eval_phase::break_unwind)}}); + if (pycvc_enum_eval_phase != NULL) + PyDict_SetItemString(d, "eval_phase", pycvc_enum_eval_phase); + using cvc::state_exec::step_kind; + pycvc_enum_step_kind = + pycvc_make_int_enum("step_kind", {{"eval", static_cast(step_kind::eval)}, + {"read", static_cast(step_kind::read)}, + {"data", static_cast(step_kind::data)}}); + if (pycvc_enum_step_kind != NULL) + PyDict_SetItemString(d, "step_kind", pycvc_enum_step_kind); + using rp_mode = cvc::state_exec::resource_policy::mode; + pycvc_enum_resource_policy_mode = + pycvc_make_int_enum("mode", {{"strict", static_cast(rp_mode::strict)}, + {"clamp", static_cast(rp_mode::clamp)}, + {"warn", static_cast(rp_mode::warn)}}); + if (pycvc_enum_resource_policy_mode != NULL) + PyDict_SetItemString(d, "_resource_policy_mode", pycvc_enum_resource_policy_mode); + } +%} +%pythoncode %{ +eval_phase = _pycvc.eval_phase +step_kind = _pycvc.step_kind +%} + // ── parse_error: pycvc.parse_error, a RuntimeError with .line and .col ── %{ static PyObject *pycvc_parse_error_type = nullptr; @@ -2021,6 +2465,26 @@ static void pycvc_raise_parse_error(const cvc::state_exec::parse_error &e) { parse_error = _pycvc.parse_error %} +// ── evaluation_interrupted / evaluation_timeout: RuntimeError subclasses, as in C++ ── +%{ +static PyObject *pycvc_evaluation_interrupted_type = nullptr; +static PyObject *pycvc_evaluation_timeout_type = nullptr; +%} +%init %{ + pycvc_evaluation_interrupted_type = + PyErr_NewException("pycvc.evaluation_interrupted", PyExc_RuntimeError, NULL); + if (pycvc_evaluation_interrupted_type != NULL) + PyDict_SetItemString(d, "evaluation_interrupted", pycvc_evaluation_interrupted_type); + pycvc_evaluation_timeout_type = + PyErr_NewException("pycvc.evaluation_timeout", PyExc_RuntimeError, NULL); + if (pycvc_evaluation_timeout_type != NULL) + PyDict_SetItemString(d, "evaluation_timeout", pycvc_evaluation_timeout_type); +%} +%pythoncode %{ +evaluation_interrupted = _pycvc.evaluation_interrupted +evaluation_timeout = _pycvc.evaluation_timeout +%} + // ── no value: KeyError ── // Where C++ returns an empty std::optional or a null value_t* (an unbound name, a pid with // no result, an empty message queue), the wrapper throws pycvc::no_value and Python gets @@ -2057,6 +2521,12 @@ static void pycvc_raise_no_value(const pycvc::no_value &e) { } catch (const cvc::state_exec::parse_error &e) { pycvc_raise_parse_error(e); SWIG_fail; + } catch (const cvc::state_exec::evaluation_interrupted &e) { + PyErr_SetString(pycvc_evaluation_interrupted_type, e.what()); + SWIG_fail; + } catch (const cvc::state_exec::evaluation_timeout &e) { + PyErr_SetString(pycvc_evaluation_timeout_type, e.what()); + SWIG_fail; } catch (const cvc::exception &e) { SWIG_exception(SWIG_RuntimeError, e.what()); } catch (const std::exception &e) { @@ -2088,11 +2558,44 @@ PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::scheduler::step) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::scheduler::run) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::async_scheduler::sync_step) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::async_scheduler::sync_run) - -// environment is shared (environment_ptr), as in C++. The map members read and write as dicts. +// The evaluators, likewise. +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::step) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::run) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::evaluate) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::evaluate_script) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::evaluator::evaluate) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::evaluator::evaluate_script) + +// Shared as in C++ (environment_ptr, process_ptr, ...). Struct members that are maps, lists or +// values read and write as Python values (copies). %shared_ptr(cvc::state_exec::environment) +%shared_ptr(cvc::state_exec::script_reader) +%shared_ptr(cvc::state_exec::process) +%shared_ptr(cvc::state_exec::intrinsics_context) +%naturalvar cvc::state_exec::value_tag; %naturalvar cvc::state_exec::environment::bindings; %naturalvar cvc::state_exec::execute_options::signal_handlers; +%naturalvar cvc::state_exec::eval_frame::results; +%naturalvar cvc::state_exec::eval_frame::extra_vals; +%naturalvar cvc::state_exec::eval_frame::extra_strs; +%naturalvar cvc::state_exec::eval_frame::extra_lists; +%naturalvar cvc::state_exec::evaluator_state::stack; +%naturalvar cvc::state_exec::evaluator_state::user_macros; +%naturalvar cvc::state_exec::process::recv_path; +%naturalvar cvc::state_exec::process::exit_error; +%naturalvar cvc::state_exec::process::signal_handlers; +%naturalvar cvc::state_exec::process::pending_signals; +%naturalvar cvc::state_exec::process::saved_stack; +%naturalvar cvc::state_exec::process::pending_watch_events; +%naturalvar cvc::state_exec::process::watch_handlers; +%naturalvar cvc::state_exec::resource_policy::enforce; +%naturalvar cvc::state_exec::channel_policy::declared; +%naturalvar cvc::state_exec::channel_policy::global; +%rename(global_) cvc::state_exec::channel_policy::global; // `global` is a Python keyword +// process::watch_event and process::watch_handler_entry are pycvc.watch_event and +// pycvc.watch_handler_entry (Python has no nested classes here); process.watch_event names them too. +%feature("flatnested") cvc::state_exec::process::watch_event; +%feature("flatnested") cvc::state_exec::process::watch_handler_entry; namespace cvc { namespace state_exec { @@ -2229,6 +2732,132 @@ public: std::string percent_encode(const std::string &s); std::string percent_decode(const std::string &s); +// ── evaluators ── +// evaluation_stats is the C++ class; it can't be copied (it holds a mutex), so the evaluators and +// states hand out references, which keep their owner alive in Python. evaluation_interrupted and +// evaluation_timeout are pycvc exceptions (RuntimeError subclasses). The evaluators release the +// GIL while they evaluate, so another thread can interrupt(), pause() or resume() them. +// step_kind (#658) is an IntEnum: the steps and time are counted by kind. +enum class step_kind; +class evaluation_stats { +public: + evaluation_stats(); + void start(); + void mark_complete(); + void increment_step(); + void increment_step(step_kind kind); + unsigned long long get_step_count() const; + unsigned long long steps(step_kind kind) const; + step_kind last_step_kind() const; + void add_data_nodes(unsigned long long nodes); + unsigned long long data_nodes() const; + void add_step_time(double seconds); + double step_time(step_kind kind) const; + bool is_complete() const; + double elapsed_seconds() const; +}; + +// eval_phase is the IntEnum above. A frame reads and writes as copies of its parts. +enum class eval_phase; +struct eval_frame { + value_t expr; + environment_ptr env; + eval_phase phase; + long long index; + std::vector results; + std::vector extra_vals; + std::vector extra_strs; + std::vector > extra_lists; + environment_ptr extra_env; +}; + +// The recursive evaluator. user_macros() returns a copy of the macro map: a dict of name -> +// (params, body). +%pythonappend cvc::state_exec::evaluator::stats %{ + val._pycvc_owner = self +%} +class evaluator { +public: + typedef std::unordered_map, value_t> > macro_map; + evaluator(environment_ptr global_env); + value_t evaluate(const value_t &expr, environment_ptr env = environment_ptr(), + std::optional timeout_sec = std::nullopt, + pycvc::value_callback on_complete = pycvc::value_callback()); + value_t evaluate_script(const std::string &script, environment_ptr env = environment_ptr(), + std::optional timeout_sec = std::nullopt, + pycvc::value_callback on_complete = pycvc::value_callback()); + void interrupt(); + void reset_interrupt(); + void pause(); + void resume(); + bool is_paused() const; + evaluation_stats &stats(); + macro_map user_macros(); +}; + +// A stackless evaluator's whole state. The members read and write as copies; stats is a reference +// into the state. +%pythonappend cvc::state_exec::evaluator_state::_stats %{ + val._pycvc_owner = self +%} +struct evaluator_state { + std::vector stack; + value_t result; + bool done; + value_t root_expr; + environment_ptr global_env; + evaluator::macro_map user_macros; + bool yielded; + std::shared_ptr reading; + size_t data_step_nodes; +}; +%extend evaluator_state { + evaluation_stats *_stats() { return &$self->stats; } + %pythoncode %{ + stats = property(_stats) + %} +} + +// evaluator_state can't be copied (its stats hold a mutex), and SWIG returns objects by copy: +// move a returned state into its Python object instead. +%typemap(out) cvc::state_exec::evaluator_state %{ + $result = SWIG_NewPointerObj(new cvc::state_exec::evaluator_state(std::move($1)), + $descriptor(cvc::state_exec::evaluator_state *), SWIG_POINTER_OWN); +%} + +// The stackless evaluator. on_complete is a Python callable or None. A str passed to +// create_state is a script (the std::string overload), as in C++. +class stackless_evaluator { +public: + stackless_evaluator(environment_ptr global_env); + evaluator_state create_state(const value_t &expr, environment_ptr env = environment_ptr()); + evaluator_state create_state(const std::string &script, environment_ptr env = environment_ptr(), + const parse_options &popts = parse_options()); + bool step(evaluator_state &state); + value_t run(evaluator_state &state, std::optional max_steps = std::nullopt, + std::optional timeout_sec = std::nullopt, + pycvc::value_callback on_complete = pycvc::value_callback()); + value_t evaluate(const value_t &expr, environment_ptr env = environment_ptr(), + std::optional timeout_sec = std::nullopt, + pycvc::value_callback on_complete = pycvc::value_callback()); + value_t evaluate_script(const std::string &script, environment_ptr env = environment_ptr(), + std::optional timeout_sec = std::nullopt, + pycvc::value_callback on_complete = pycvc::value_callback()); + void interrupt(); + void reset_interrupt(); + void pause(); + void resume(); + bool is_paused() const; + void restrict_special_forms(std::shared_ptr > allowed); + static const std::set &all_special_forms(); +}; + +// Snapshot codec for an environment chain and for a whole evaluator state. +void encode_environment(cvc::state &node, const environment_ptr &env); +environment_ptr decode_environment(cvc::state &node); +void encode_evaluator_state(cvc::state &node, const evaluator_state &es); +evaluator_state decode_evaluator_state(cvc::state &node); + // ── processes and schedulers ── // process_status and scheduling_policy are the IntEnums above; declared opaque here so SWIG adds // no flat constants of its own. @@ -2256,7 +2885,8 @@ struct execute_options { std::unordered_map signal_handlers; environment_ptr env; }; -%extend execute_options { +%define PYCVC_STATE_EXEC_ON_COMPLETE(CLASS) +%extend CLASS { PyObject *_on_complete_get() const { if (!$self->on_complete) { Py_INCREF(Py_None); @@ -2266,7 +2896,7 @@ struct execute_options { Py_INCREF(f->callable.get()); return f->callable.get(); } - throw std::runtime_error("pycvc.execute_options.on_complete: holds a C++ callback"); + throw std::runtime_error("pycvc: on_complete holds a C++ callback"); } void _on_complete_set(PyObject *fn) { if (fn == Py_None) @@ -2278,6 +2908,8 @@ struct execute_options { on_complete = property(_on_complete_get, _on_complete_set) %} } +%enddef +PYCVC_STATE_EXEC_ON_COMPLETE(execute_options) struct process_info { int pid; @@ -2333,10 +2965,105 @@ struct scheduler_stats { %} } +// ── process: a scheduler's record of one program (shared, as process_ptr) ── +// Its members read and write as copies, except state, a reference into the process. The +// steady_clock time points (create_time, last_run_start, wake_time) aren't wrapped; elapsed_time() +// and accumulated_time give the same information. inbox reads as a list. +const size_t kDefaultMaxAllocBytes = 16777216; +// The per-call allocation limit (#660). A Python function a program calls runs on the thread +// stepping the scheduler, where the step's limit is armed, so it can honour the limit the way a +// builtin does: eval_alloc_check() before building a large result. eval_alloc_guard (RAII, which +// must nest last-in first-out) isn't wrapped. +std::optional eval_alloc_limit(); +void eval_alloc_check(size_t bytes, const char *who); +size_t alloc_bytes(size_t count, size_t each); +%pythonappend cvc::state_exec::process::_state %{ + val._pycvc_owner = self +%} +struct process { + struct watch_event { + int watch_id; + }; + struct watch_handler_entry { + value_t handler; + std::string path; + std::string last_value; + }; + int pid; + std::string name; + process_status status; + int priority; + std::string uid; + std::string gid; + std::string root_path; + std::string owner; + bool awaiting_frame; + unsigned long long max_steps; + double max_time; + unsigned long long max_memory; + unsigned long long max_messages; + unsigned long long max_message_bytes; + double max_step_time; + unsigned long long max_alloc_bytes; + double accumulated_time; + double cpu_time_charged; + std::optional recv_path; + value_t exit_code; + std::optional exit_error; + std::unordered_map signal_handlers; + std::vector pending_signals; + bool in_signal_handler; + bool in_watch_handler; + std::vector saved_stack; + value_t saved_result; + std::vector pending_watch_events; + int next_watch_id; + std::unordered_map watch_handlers; + unsigned long long message_count; + unsigned long long message_bytes; + int parent_pid; + double elapsed_time() const; + bool charges_cpu_time() const; + void charge_cpu_time(double seconds); + bool over_time_limit() const; + std::optional time_left() const; + std::optional alloc_limit(size_t used_bytes) const; + unsigned long long step_count() const; +}; +%extend process { + evaluator_state *_state() { return &$self->state; } + PyObject *_inbox_get() const { + std::queue q = $self->inbox; + std::vector items; + for (; !q.empty(); q.pop()) + items.push_back(q.front()); + return pycvc::value_list_to_py(items); + } + %pythoncode %{ + state = property(_state) + inbox = property(_inbox_get) + %} +} +PYCVC_STATE_EXEC_ON_COMPLETE(process) +%pythoncode %{ +process.watch_event = watch_event +process.watch_handler_entry = watch_handler_entry +%} +typedef std::shared_ptr process_ptr; +process_ptr make_process(); + +// The common base of the two schedulers, which intrinsics_context.sched points to. Its members are +// on the two concrete classes. +%nodefaultctor scheduler_base; +%nodefaultdtor scheduler_base; +class scheduler_base {}; + // The members scheduler and async_scheduler share (scheduler_base plus their common extras). -// Returns of a C++ reference come back as copies. Not wrapped: mem_tracker(), and -// current_process() / queue_watch_event(), which take the process struct (PR 3). +// Returns of a C++ reference come back as copies (current_process() shares the process, as +// process_ptr does). Not wrapped: mem_tracker(). %define PYCVC_STATE_EXEC_SCHEDULER_MEMBERS + process_ptr current_process() const; + void queue_watch_event(int pid, process::watch_event evt); int execute(const std::string &script, const execute_options &opts = execute_options()); int execute(const value_t &expr, const execute_options &opts = execute_options()); void stop(); @@ -2389,7 +3116,7 @@ struct scheduler_stats { %enddef // The synchronous scheduler. run() and step() release the GIL while programs run. -class scheduler { +class scheduler : public scheduler_base { public: scheduler(); scheduler(scheduling_policy policy); @@ -2402,16 +3129,19 @@ public: void load_settings(); parse_options parser_settings() const; data_options data_settings() const; + unsigned long long max_alloc_bytes() const; static constexpr const char *settings_root = "state_exec"; static constexpr const char *defaults_prefix = "state_exec.defaults"; static constexpr const char *sched_prefix = "state_exec.schedulers"; }; PYCVC_STATE_EXEC_SCHEDULER_EXTEND(scheduler) +unsigned long long max_alloc_bytes_from_state(cvc::state &root, const std::string &scheduler_id, + unsigned long long base = kDefaultMaxAllocBytes); // The scheduler libcvc's hosts use (app::exec_scheduler(), Exec). The coroutine step()/run() aren't // wrapped; Python drives it with sync_step()/sync_run(), which release the GIL while programs run. // post_message() is the one thread-safe call. -class async_scheduler { +class async_scheduler : public scheduler_base { public: async_scheduler(); async_scheduler(scheduling_policy policy); @@ -2435,9 +3165,133 @@ public: static data_options host_data_defaults(); }; PYCVC_STATE_EXEC_SCHEDULER_EXTEND(async_scheduler) + +// ── resource_policy ── +// mode is pycvc.resource_policy.mode, an IntEnum. validate_limits raises RuntimeError for +// out-of-range limits in strict mode, as C++ throws. +struct resource_policy { + double max_time_min; + double max_time_max; + double max_time_default; + unsigned long long max_memory_min; + unsigned long long max_memory_max; + unsigned long long max_memory_default; + unsigned long long max_messages_min; + unsigned long long max_messages_max; + unsigned long long max_messages_default; + unsigned long long max_message_bytes_min; + unsigned long long max_message_bytes_max; + unsigned long long max_message_bytes_default; + unsigned long long max_steps_min; + unsigned long long max_steps_max; + unsigned long long max_steps_default; + int max_processes; + unsigned long long max_total_memory; + cvc::state_exec::resource_policy::mode enforce; +}; +%pythoncode %{ +resource_policy.mode = _pycvc._resource_policy_mode +%} +struct process_limits { + unsigned long long max_steps; + double max_time; + unsigned long long max_memory; + unsigned long long max_messages; + unsigned long long max_message_bytes; +}; +process_limits validate_limits(const resource_policy &policy, const process_limits &requested); + +// ── intrinsics ── +// channel_policy.global is global_ (a Python keyword). intrinsics_context is shared; its sched and +// channels are the Python objects assigned to them, which the context keeps alive. Its root is a +// tree node, which lives as long as its app (keep the app alive, as in C++). Not wrapped: tracker +// (memory_tracker is internal), document (document_scope's typed slots are C++ templates), and +// watches (the intrinsics' own registry). +struct channel_policy { + bool enforce; + bool quiet; + std::vector declared; + std::vector global; +}; +%rename(_sched) intrinsics_context::sched; +%rename(_channels) intrinsics_context::channels; +struct intrinsics_context { + scheduler_base *sched; + cvc::state *root; + std::shared_ptr proc; + int pid; + std::string uid; + std::string cluster_id; + std::string node_id; + std::string root_path; + const channel_policy *channels; +}; +%extend intrinsics_context { + %pythoncode %{ + def _get_sched(self): + return self.__dict__.get("_pycvc_sched") if self._sched is not None else None + def _set_sched(self, sched): + self._sched = sched + self.__dict__["_pycvc_sched"] = sched + sched = property(_get_sched, _set_sched) + def _get_channels(self): + return self.__dict__.get("_pycvc_channels") if self._channels is not None else None + def _set_channels(self, channels): + self._channels = channels + self.__dict__["_pycvc_channels"] = channels + channels = property(_get_channels, _set_channels) + %} +} +void apply_chroot(intrinsics_context &ctx, cvc::state &tree_root, const std::string &root_path); +std::string resolve_channel_key(const std::string &root_path, const std::string &channel); +value_t make_future(const std::string &channel); +std::optional future_channel_of(const value_t &v); } } +// register_intrinsics(env, ctx): the C++ call, except that every function it adds keeps ctx (and +// so the scheduler and channel policy ctx holds) alive. C++ requires the caller to keep the context +// valid instead; in Python an environment may outlive any object. +%{ +namespace pycvc { +void register_intrinsics_keeping(cvc::state_exec::environment_ptr env, + std::shared_ptr ctx, + PyObject *owner) { + using namespace cvc::state_exec; + if (!env || !ctx) + throw std::invalid_argument("pycvc.register_intrinsics: env and ctx must not be None"); + // register_intrinsics only adds functions, so add them to a scratch scope and copy each one in, + // wrapped to hold the context. + auto scratch = std::make_shared(); + register_intrinsics(scratch, ctx.get()); + std::shared_ptr keep = python_ref(owner); + for (auto &[name, v] : scratch->bindings) { + const auto *fn = std::get_if(&v.v); + if (fn == nullptr) { + env->set(name, v); + continue; + } + env->set(name, value_t(native_fn([inner = *fn, ctx, keep](std::span args) { + return inner(args); + }))); + } +} +} // namespace pycvc +%} +%rename(_register_intrinsics) pycvc::register_intrinsics_keeping; +namespace pycvc { +void register_intrinsics_keeping(cvc::state_exec::environment_ptr env, + std::shared_ptr ctx, + PyObject *owner); +} +%pythoncode %{ +def register_intrinsics(env, ctx): + """cvc::state_exec::register_intrinsics: add the DSL intrinsics over ctx to env. + + Each function added keeps ctx alive, and with it the scheduler and channel policy it holds.""" + _register_intrinsics(env, ctx, ctx) +%} + // Exec.scheduler() returns a reference into the Exec, so the scheduler keeps the Exec alive. %pythonappend pycvc::Exec::scheduler %{ val._pycvc_exec = self diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index 534875a87..cbe62299f 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -274,10 +274,12 @@ void python_on_complete::operator()(value_t result) const { python_on_complete make_python_on_complete(PyObject *callable) { if (callable == nullptr || !PyCallable_Check(callable)) - throw std::invalid_argument("pycvc.execute_options.on_complete: not callable"); + throw std::invalid_argument("pycvc: on_complete is not callable"); return python_on_complete{own_callable(callable)}; } +std::shared_ptr python_ref(PyObject *o) { return own_callable(o); } + // Owns everything an evaluation needs. The intrinsics capture &ictx (a raw // pointer), so ictx must outlive any run — holding it here (Exec-lifetime) // satisfies that. Mirrors the C++ tests' make_exec_env(). diff --git a/bindings/pycvc/pycvc_value.h b/bindings/pycvc/pycvc_value.h index 8cc17b305..5fb0fa844 100644 --- a/bindings/pycvc/pycvc_value.h +++ b/bindings/pycvc/pycvc_value.h @@ -57,4 +57,8 @@ struct python_on_complete { /// The python_on_complete for `callable`. Throws std::invalid_argument when it isn't callable. python_on_complete make_python_on_complete(PyObject *callable); +/// A shared owner of a new reference to `o`. The last owner to let go drops the reference under +/// the GIL, so C++ can keep a Python object alive from any thread. +std::shared_ptr python_ref(PyObject *o); + } // namespace pycvc diff --git a/bindings/pycvc/test_pycvc_state_exec_eval.py b/bindings/pycvc/test_pycvc_state_exec_eval.py new file mode 100644 index 000000000..d9d2960e4 --- /dev/null +++ b/bindings/pycvc/test_pycvc_state_exec_eval.py @@ -0,0 +1,418 @@ +"""state_exec parity, part 3: evaluators, evaluator state, processes, policy, intrinsics. + +pycvc.stackless_evaluator and pycvc.evaluator are the real C++ evaluators, with +evaluator_state, eval_frame, evaluation_stats, the process struct, the environment +and evaluator-state codec, resource_policy, and register_intrinsics over an +intrinsics_context. Most tests repeat a C++ test from src/cvc/tests +(state_exec_stackless_evaluator_test.cpp, state_exec_evaluator_test.cpp, +state_exec_codec_test.cpp, state_exec_intrinsics_test.cpp). +docs/PYCVC_STATE_EXEC_PARITY.md has the inventory. +""" + +import gc +import threading +import time + +import pycvc +from pycvc import eval_phase, process_status, symbol + + +def _raises(exc, fn, *args): + try: + fn(*args) + except exc as e: + return e + raise AssertionError(f"{fn.__name__}{args!r} did not raise {exc.__name__}") + + +def _builtins(): + return pycvc.builtins.make_default_environment() + + +LOOP = ( + "(begin (set sum 0) (set i 0)" + " (while (< i 100) (begin (set sum (+ sum i)) (set i (+ i 1)))) sum)" +) + + +# ── evaluators ────────────────────────────────────────────────────────── + + +def test_both_evaluators_evaluate(): + for ev in (pycvc.stackless_evaluator(_builtins()), pycvc.evaluator(_builtins())): + assert ev.evaluate_script("(+ 1 2)") == 3 + assert ev.evaluate([symbol("*"), 6, 7]) == 42 # an expression, as evaluate(value_t) + assert ev.evaluate("text") == "text" # a str value evaluates to itself + done = [] + assert ev.evaluate_script(LOOP, None, None, done.append) == 4950 + assert done == [4950] + env = pycvc.environment.extend(_builtins()) + env.set("x", 40) + assert ev.evaluate_script("(+ x 2)", env) == 42 + + +def test_state_steps_to_a_result(): + ev = pycvc.stackless_evaluator(_builtins()) + st = ev.create_state("(begin (set x 1) (+ x 41))") # a str is a script, as in C++ + assert isinstance(st, pycvc.evaluator_state) and not st.done + steps = 0 + while not ev.step(st): + steps += 1 + assert st.done and st.result == 42 and st.stack == [] + assert st.stats.get_step_count() > steps > 0 + # A value is an expression. + assert ev.run(ev.create_state([symbol("+"), 1, 2])) == 3 + + +def test_run_stops_at_max_steps_and_exposes_the_frames(): + ev = pycvc.stackless_evaluator(_builtins()) + st = ev.create_state(LOOP) + assert ev.run(st, 10) is None and not st.done # nil: not done + frames = st.stack + assert frames and all(isinstance(f, pycvc.eval_frame) for f in frames) + assert all(isinstance(f.phase, eval_phase) for f in frames) + assert frames[0].expr[0] == symbol("begin") + assert ev.run(st) == 4950 and st.done + _raises(OverflowError, ev.run, ev.create_state("1"), -1) + + +def test_state_reading_its_script_in_slices(): + ev = pycvc.stackless_evaluator(_builtins()) + popts = pycvc.parse_options() + popts.step_tokens = 2 + st = ev.create_state("(set x 20) (+ x 22)", None, popts) + assert isinstance(st.reading, pycvc.script_reader) + assert st.reading.source() == "(set x 20) (+ x 22)" + ev.step(st) + assert ev.run(st) == 42 and st.reading is None + + +def test_stats_is_a_reference_that_keeps_its_owner(): + st = pycvc.stackless_evaluator(_builtins()).create_state("(+ 1 2)") + stats = st.stats + del st + gc.collect() + assert stats.get_step_count() == 0 and not stats.is_complete() + rec = pycvc.evaluator(_builtins()) + rec.evaluate_script("(+ 1 2)") + s = rec.stats() + del rec + gc.collect() + assert s.get_step_count() > 0 and s.elapsed_seconds() >= 0.0 + own = pycvc.evaluation_stats() + own.start() + own.increment_step() + own.mark_complete() + assert own.get_step_count() == 1 and own.is_complete() + + +def test_stats_by_kind_of_step(): + # #658 counts steps and time by kind: reading the script, evaluating, deep-data work. + kind = pycvc.step_kind + assert [k.name for k in kind] == ["eval", "read", "data"] + s = pycvc.evaluation_stats() + s.increment_step(kind.read) + s.increment_step(kind.read) + s.increment_step() # eval, as in C++ + assert (s.steps(kind.read), s.steps(kind.eval), s.get_step_count()) == (2, 1, 3) + assert s.last_step_kind() is kind.eval + s.add_step_time(0.25) # charged to the last step's kind + s.add_data_nodes(5) + assert s.step_time(kind.eval) == 0.25 and s.step_time(kind.read) == 0.0 + assert s.data_nodes() == 5 + ev = pycvc.stackless_evaluator(_builtins()) + st = ev.create_state("(+ 1 2)") + ev.run(st) + assert st.stats.steps(kind.eval) > 0 and st.data_step_nodes == 0 + + +def test_user_macros(): + ev = pycvc.evaluator(_builtins()) + assert ev.evaluate_script("(begin (defmacro double-it (x) (* x 2)) (double-it 21))") == 42 + assert ev.user_macros() == {"double-it": ([symbol("x")], [symbol("*"), symbol("x"), 2])} + st = pycvc.evaluator_state() + st.user_macros = {"m": ([symbol("a")], symbol("a"))} + assert st.user_macros == {"m": ([symbol("a")], symbol("a"))} + _raises(TypeError, setattr, st, "user_macros", {"m": (["a"], 1)}) # params are symbols + + +def test_interrupt_from_another_thread(): + # The evaluators release the GIL while they evaluate, so this thread can interrupt them. + for ev in (pycvc.stackless_evaluator(_builtins()), pycvc.evaluator(_builtins())): + t = threading.Timer(0.1, ev.interrupt) + t.start() + # The 10 s timeout only bounds a failure; the interrupt comes first. + _raises(pycvc.evaluation_interrupted, ev.evaluate_script, "(while 1 1)", None, 10.0) + t.join() + ev.reset_interrupt() + assert ev.evaluate_script("(+ 1 2)") == 3 + + +def test_timeout_and_pause(): + for ev in (pycvc.stackless_evaluator(_builtins()), pycvc.evaluator(_builtins())): + e = _raises(pycvc.evaluation_timeout, ev.evaluate_script, "(while 1 1)", None, 0.05) + assert isinstance(e, RuntimeError) + ev.pause() + assert ev.is_paused() + t = threading.Timer(0.1, ev.resume) + t.start() + t0 = time.perf_counter() + assert ev.evaluate_script("(+ 1 2)") == 3 # waits for resume() + t.join() + assert time.perf_counter() - t0 >= 0.05 and not ev.is_paused() + + +def test_restrict_special_forms(): + assert {"if", "begin", "quote", "defclass"} <= pycvc.stackless_evaluator.all_special_forms() + ev = pycvc.stackless_evaluator(_builtins()) + ev.restrict_special_forms(["if", "begin"]) + assert ev.evaluate_script("(if #t 1 2)") == 1 + _raises(RuntimeError, ev.evaluate_script, "(quote 1)") + assert ev.evaluate_script("(+ 1 2)") == 3 # not a special form + ev.restrict_special_forms(None) + assert ev.evaluate_script("(quote 1)") == 1 + _raises(TypeError, ev.restrict_special_forms, [1]) + + +# ── codec ─────────────────────────────────────────────────────────────── + + +def test_environment_and_evaluator_state_codec(): + app = pycvc.make_app() + root = pycvc.State.instance(app) + env = pycvc.environment.extend(pycvc.environment()) + env.set("a", [1, "two"]) + env.outer.set("b", symbol("c")) + pycvc.encode_environment(root.child("snap.env"), env) + back = pycvc.decode_environment(root.child("snap.env")) + assert back.bindings == {"a": [1, "two"]} and back.lookup("b") == symbol("c") + + ev = pycvc.stackless_evaluator(_builtins()) + paused = ev.create_state(LOOP) + ev.run(paused, 10) + pycvc.encode_evaluator_state(root.child("snap.loop"), paused) + restored = pycvc.decode_evaluator_state(root.child("snap.loop")) + assert not restored.done and len(restored.stack) == len(paused.stack) + + # Resuming, as the C++ PauseResumeProducesCorrectResult does: the codec keeps each frame's + # scope by value (not shared identity), so the expression mutates nothing; and native + # builtins aren't serialized, so they go back on the root of every restored scope chain. + st = ev.create_state("(+ 1 2 3 4 5 6 7 8 9 10)") + ev.run(st, 3) + assert not st.done + pycvc.encode_evaluator_state(root.child("snap.es"), st) + restored = pycvc.decode_evaluator_state(root.child("snap.es")) + builtins = _builtins().bindings + for scope in [restored.global_env] + [f.env for f in restored.stack]: + while scope is not None and scope.outer is not None: + scope = scope.outer + if scope is not None: + for name, fn in builtins.items(): + scope.set(name, fn) + assert ev.run(restored) == 55 + + +# ── process ───────────────────────────────────────────────────────────── + + +def test_process_struct(): + p = pycvc.make_process() + assert isinstance(p, pycvc.process) + assert (p.pid, p.status, p.priority, p.parent_pid) == (-1, process_status.ready, 0, -1) + assert p.max_alloc_bytes == pycvc.kDefaultMaxAllocBytes == 16 << 20 + assert p.recv_path is None and p.exit_error is None and p.inbox == [] + assert p.time_left() is None # no max_time + assert p.alloc_limit(0) == 16 << 20 + p.max_memory = 1000 + assert p.alloc_limit(400) == 600 # the tighter of the two + p.recv_path = "a.b" + p.exit_error = "boom" + p.signal_handlers = {"SIGUSR1": [symbol("begin"), 1]} + p.pending_signals = ["SIGUSR1"] + p.status = process_status.paused + assert (p.recv_path, p.exit_error, p.status) == ("a.b", "boom", process_status.paused) + assert p.signal_handlers == {"SIGUSR1": [symbol("begin"), 1]} + assert list(p.pending_signals) == ["SIGUSR1"] + ev = pycvc.watch_event() + ev.watch_id = 3 + p.pending_watch_events = [ev] + assert [e.watch_id for e in p.pending_watch_events] == [3] + assert pycvc.process.watch_event is pycvc.watch_event + entry = pycvc.watch_handler_entry() + entry.handler = [symbol("begin"), 1] + entry.path = "a.b" + p.watch_handlers = {3: entry} + got = p.watch_handlers[3] + assert (got.handler, got.path) == ([symbol("begin"), 1], "a.b") + seen = [] + p.on_complete = seen.append + assert p.on_complete == seen.append + state = p.state # a reference into the process, which it keeps alive + del p + gc.collect() + assert not state.done and state.stack == [] + + +def test_current_process_during_a_run(): + for sched in (pycvc.scheduler(), pycvc.async_scheduler()): + run = sched.run if isinstance(sched, pycvc.scheduler) else sched.sync_run + assert sched.current_process() is None # nothing is being stepped + seen = [] + env = _builtins() + pycvc.builtins.register_fn(env, "whoami", + lambda: seen.append(sched.current_process()) or 0) + o = pycvc.execute_options() + o.env = env + o.name = "me" + pid = sched.execute("(whoami)", o) + run() + assert len(seen) == 1 and seen[0].pid == pid and seen[0].name == "me" + # Shared, as process_ptr is: it shows the process as it is now, finished. + assert seen[0].status is process_status.terminated and seen[0].exit_code == 0 + sched.queue_watch_event(pid, pycvc.watch_event()) # accepted for a finished pid + assert isinstance(sched, pycvc.scheduler_base) + + +def test_max_alloc_bytes_from_the_tree(): + app = pycvc.make_app() + root = pycvc.State.instance(app) + assert pycvc.scheduler().max_alloc_bytes() == pycvc.kDefaultMaxAllocBytes + assert pycvc.max_alloc_bytes_from_state(root, "") == pycvc.kDefaultMaxAllocBytes + root.child("state_exec.defaults.limits.max_alloc_bytes").value("1024") + root.child("state_exec.schedulers.s1.limits.max_alloc_bytes").value("2048") + assert pycvc.max_alloc_bytes_from_state(root, "") == 1024 + assert pycvc.max_alloc_bytes_from_state(root, "s1") == 2048 + assert pycvc.max_alloc_bytes_from_state(root, "s2", 7) == 1024 + s = pycvc.scheduler() + s.set_watch_root(root) + s.set_id("s1") + s.load_settings() + assert s.max_alloc_bytes() == 2048 + + +def test_allocation_limit_seen_by_a_python_function(): + assert pycvc.eval_alloc_limit() == pycvc.kDefaultMaxAllocBytes # no guard armed + e = _raises(RuntimeError, pycvc.eval_alloc_check, 1 << 40, "py-fn") + assert "py-fn" in str(e) + pycvc.eval_alloc_check(10, "py-fn") # under the limit + assert pycvc.alloc_bytes(3, 4) == 12 + assert pycvc.alloc_bytes(1 << 40, 1 << 40) == 2**64 - 1 # saturates at SIZE_MAX + # During a step, a Python function sees the limit the scheduler armed: the tighter of + # limits.max_alloc_bytes and what is left of the process's max_memory. + seen = [] + env = _builtins() + pycvc.builtins.register_fn(env, "limit", lambda: seen.append(pycvc.eval_alloc_limit()) or 0) + for sched in (pycvc.scheduler(), pycvc.async_scheduler()): + run = sched.run if isinstance(sched, pycvc.scheduler) else sched.sync_run + o = pycvc.execute_options() + o.env = env + o.max_memory = 4096 + sched.execute("(limit)", o) + run() + assert len(seen) == 2 and all(lim is not None and lim <= 4096 for lim in seen) + + +# ── resource_policy ───────────────────────────────────────────────────── + + +def _limits(steps=0, time_=0.0, memory=0, messages=0): + lim = pycvc.process_limits() + lim.max_steps, lim.max_time, lim.max_memory, lim.max_messages = steps, time_, memory, messages + return lim + + +def test_validate_limits(): + mode = pycvc.resource_policy.mode + assert [m.name for m in mode] == ["strict", "clamp", "warn"] + policy = pycvc.resource_policy() + assert policy.enforce is mode.clamp + r = pycvc.validate_limits(policy, _limits(1000, 60.0, 1024, 50)) + assert (r.max_steps, r.max_time, r.max_memory, r.max_messages) == (1000, 60.0, 1024, 50) + policy.max_steps_default = 500 + policy.max_time_default = 30.0 + assert (pycvc.validate_limits(policy, _limits()).max_steps, + pycvc.validate_limits(policy, _limits()).max_time) == (500, 30.0) + policy.max_steps_max = 100 + assert pycvc.validate_limits(policy, _limits(200)).max_steps == 100 # clamp + policy.enforce = mode.strict + _raises(RuntimeError, pycvc.validate_limits, policy, _limits(200)) + _raises(TypeError, setattr, policy, "enforce", 0) # an enum argument must be the enum + + +# ── intrinsics ────────────────────────────────────────────────────────── + + +def _intrinsic_env(app, sched): + ctx = pycvc.intrinsics_context() + ctx.sched = sched + ctx.root = pycvc.State.instance(app) + ctx.pid = 1 + env = _builtins() + pycvc.register_intrinsics(env, ctx) + return env, ctx + + +def test_register_intrinsics_over_a_context(): + app = pycvc.make_app() + sched = pycvc.scheduler() + env, ctx = _intrinsic_env(app, sched) + assert ctx.sched is sched and ctx.channels is None + for name in ("state-get", "state-set", "msg-send", "msg-recv", "self-pid", "spawn"): + assert isinstance(env.lookup(name), pycvc.value), name + o = pycvc.execute_options() + o.env = env + pid = sched.execute('(begin (state-set "py.x" "hello") (state-get "py.x"))', o) + assert sched.run()[pid] == "hello" + assert pycvc.state_get(app, "py.x") == "hello" + + +def test_registered_intrinsics_keep_their_context_alive(): + app = pycvc.make_app() + env, _ = _intrinsic_env(app, pycvc.scheduler()) # the context and scheduler go out of scope + gc.collect() + other = pycvc.scheduler() + o = pycvc.execute_options() + o.env = env + pid = other.execute('(begin (state-set "kept.x" "yes") (state-get "kept.x"))', o) + assert other.run()[pid] == "yes" + + +def test_apply_chroot_and_channels(): + app = pycvc.make_app() + sched = pycvc.scheduler() + ctx = pycvc.intrinsics_context() + ctx.sched = sched + pycvc.apply_chroot(ctx, pycvc.State.instance(app), "sandbox.user1") + assert ctx.root_path == "sandbox.user1" + env = _builtins() + pycvc.register_intrinsics(env, ctx) + o = pycvc.execute_options() + o.env = env + sched.execute('(state-set "x" "inside")', o) + sched.run() + assert pycvc.state_get(app, "sandbox.user1.x") == "inside" + + cp = pycvc.channel_policy() + cp.enforce = True + cp.declared = ["inbox"] + cp.global_ = ["g"] + ctx.channels = cp + assert ctx.channels is cp and list(ctx.channels.declared) == ["inbox"] + assert list(cp.global_) == ["g"] + + assert pycvc.resolve_channel_key("doc", "nav.done") == "doc.channels.nav.done" + assert pycvc.resolve_channel_key("doc", "#tick") == "#tick" + assert pycvc.resolve_channel_key("doc", "/g") == "g" + assert pycvc.resolve_channel_key("", "x") == "x" + fut = pycvc.make_future("ch.reply") + assert fut == {"__future__": "ch.reply"} + assert pycvc.future_channel_of(fut) == "ch.reply" + assert pycvc.future_channel_of(5) is None + + +if __name__ == "__main__": + tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] + for t in tests: + t() + print(" ok:", t.__name__) + print("pycvc state_exec evaluator parity tests: OK") diff --git a/docs/PYCVC_STATE_EXEC_PARITY.md b/docs/PYCVC_STATE_EXEC_PARITY.md index c64e37db8..ca3ea3ef5 100644 --- a/docs/PYCVC_STATE_EXEC_PARITY.md +++ b/docs/PYCVC_STATE_EXEC_PARITY.md @@ -49,19 +49,31 @@ approach agreed for closing the gap, and the plan. `None`, except for a value: `None` is nil, so `get_result`, `pop_pending_message` and `environment::lookup` raise `KeyError(pid / path / name)` instead. 8. **Callables cross as Python callables.** Where C++ takes a `native_fn` or a - `std::function` (`builtins::register_fn`, `execute_options::on_complete`), Python passes a - callable. It runs on the thread stepping the scheduler and takes the GIL for its call; a - Python exception becomes the C++ error a throwing callback would be. `on_complete` reads back - as the callable that was set. -9. **The GIL is released while programs run:** `scheduler.run`/`step` and - `async_scheduler.sync_run`/`sync_step`, so other Python threads keep going. The schedulers' + `std::function` (`builtins::register_fn`, the `on_complete` of `execute_options`, `process` + and the evaluators), Python passes a callable. It runs on the thread stepping the scheduler + or evaluator and takes the GIL for its call. A Python exception becomes the C++ error a + throwing callback would be. An `on_complete` member reads back as the callable that was set. +9. **The GIL is released while programs run:** `scheduler.run`/`step`, + `async_scheduler.sync_run`/`sync_step`, `stackless_evaluator.step`/`run`/`evaluate`/ + `evaluate_script` and `evaluator.evaluate`/`evaluate_script`. Other Python threads keep going, + and can call an evaluator's thread-safe `interrupt`, `pause` and `resume`. The schedulers' thread safety is the C++ one: while a scheduler runs, another thread may call only `async_scheduler.post_message`. +10. **References keep their owner alive.** A member or call that hands out a reference into its + object (`evaluator_state.stats`, `evaluator.stats()`, `process.state`, `Exec.scheduler()`) + returns a Python object that holds the owner. Every other reference return, such as a + struct's list or map member, comes back as a copy. +11. **Intrinsics keep their context alive.** C++'s `register_intrinsics(env, ctx)` requires the + caller to keep `ctx` valid for as long as `env` is used. In Python, every function it adds + holds the context, and the context holds the scheduler and channel policy assigned to it. + The context's `root` is a tree node, which lives as long as its app, so keep the app alive + as in C++. ## Inventory The audit (2026-10-17) found 97 public items in the 20 headers. Counted individually, with the 38 -`builtin_*` functions and the 7 `MIME_*` constants, it is 140. Status after PR 2: +`builtin_*` functions and the 7 `MIME_*` constants, it is 140. Items added to master since then +are listed too. Status after PR 3: | Header | Item | Python | Status | |---|---|---|---| @@ -80,25 +92,32 @@ The audit (2026-10-17) found 97 public items in the 20 headers. Counted individu | | `environment` (`bindings`, `outer`, `frozen`, `lookup`, `set`, `set_existing`, `extend`) | `pycvc.environment` | wrapped (PR 2), shared like `environment_ptr`. `bindings` reads and writes as a dict (a copy); `lookup` raises `KeyError` where C++ returns nullptr; `set` on a frozen scope (#660) raises `RuntimeError` | | | `closure`, `data_object`, `generator` | `pycvc.value` (opaque) | wrapped as opaque handles; no member access | | | `time_budget`, `eval_deadline_guard`, `eval_deadline_expired`, `thread_cpu_seconds` | none | exception: thread-local evaluator internals | +| | `kDefaultMaxAllocBytes`, `eval_alloc_limit`, `eval_alloc_check`, `alloc_bytes` (#660) | the same names under `pycvc.` | wrapped (PR 3). A Python function a program calls runs on the stepping thread, where the step's limit is armed, so it can check the limit as a builtin does | +| | `eval_alloc_guard` (#660) | none | exception: RAII that must nest last-in first-out, which a Python object's lifetime doesn't promise | +| | `detail::release_nested`, `detail::drop_owner` (#657) | none | exception: internals of `value_t`'s destructor | | state_value_codec.h | `encode_value`, `decode_value` | `pycvc.encode_value(State, v)`, `pycvc.decode_value(State)` | wrapped (PR 1) | -| | `encode_environment`/`decode_environment`, `encode_evaluator_state`/`decode_evaluator_state` | | PR 3, with the environment and evaluator types | +| | `encode_environment`/`decode_environment`, `encode_evaluator_state`/`decode_evaluator_state` | `pycvc.encode_environment(State, env)`, ... | wrapped (PR 3) | | scheduler_base.h | `execute_options`, `process_info`, `scheduler_stats` | `pycvc.execute_options`, `pycvc.process_info`, `pycvc.scheduler_stats` | wrapped (PR 2). `on_complete` is a Python callable, `signal_handlers` a dict, the `uint64_t` fields Python ints. `process_info` includes #658's steps and times by kind (`read_steps`, `eval_steps`, `data_steps`, `data_nodes`, `read_time`, `eval_time`, `data_time`) | | | `fill_budget_usage` (#658) | none | exception: how the schedulers fill `process_info`, from the evaluator's stats | | | `scheduling_policy` | `pycvc.scheduling_policy` | wrapped (PR 2), an `IntEnum` | -| | `scheduler_base` | none | exception: the abstract interface the intrinsics use; Python has the two concrete classes | -| scheduler.h | `scheduler` (about 45 members) | `pycvc.scheduler` | wrapped (PR 2). `run`/`step` release the GIL. `execute(str)` is the script overload, so a string can't be passed as a `value_t` expression (write it as the script `"\"text\""`). `id()`, `parser_settings()` and `data_settings()` (#658) return copies. Not wrapped: `mem_tracker()` (see memory_tracker.h), and `current_process()` / `queue_watch_event()`, which take the `process` struct (PR 3) | +| | `scheduler_base` | `pycvc.scheduler_base` | wrapped (PR 3) as the common base of the two classes (what `intrinsics_context.sched` holds). Its members are on the concrete classes | +| scheduler.h | `scheduler` (about 45 members) | `pycvc.scheduler` | wrapped (PR 2). `run`/`step` release the GIL. `execute(str)` is the script overload, so a string can't be passed as a `value_t` expression (write it as the script `"\"text\""`). `id()`, `parser_settings()` and `data_settings()` (#658) return copies. `current_process()` (shares the process), `queue_watch_event()` and `max_alloc_bytes()` (#660) are wrapped in PR 3. Not wrapped: `mem_tracker()` (see memory_tracker.h) | +| | `max_alloc_bytes_from_state(state&, id, base)` (#660) | `pycvc.max_alloc_bytes_from_state(State, …)` | wrapped (PR 3) | | async_scheduler.h | `async_scheduler` (about 55 members) | `pycvc.async_scheduler` | wrapped (PR 2), with the same notes as `scheduler`; `sync_run`/`sync_step` release the GIL, and `parser_defaults()` returns a copy. #658's `set_data_defaults`/`data_defaults`/`data_settings`/`host_data_defaults` sit beside the parser ones. Coroutine `step()`/`run()` are an exception: Python gets `sync_step()`/`sync_run()` | | process.h | `process_status`, `to_string(process_status)`, `parse_process_status` | `pycvc.process_status` (an `IntEnum`), `pycvc.to_string`, `pycvc.parse_process_status` | wrapped (PR 2) | -| | `process` (about 35 fields), `make_process` | | PR 3, with `evaluator_state`, which a process holds | +| | `process`, `process::watch_event`, `process::watch_handler_entry`, `make_process` | `pycvc.process` (shared, as `process_ptr`), `pycvc.watch_event`, `pycvc.watch_handler_entry` (also `process.watch_event`, ...), `pycvc.make_process()` | wrapped (PR 3). `state` is a reference into the process; `inbox` reads as a list. The `steady_clock` time points (`create_time`, `last_run_start`, `wake_time`) are an exception: `elapsed_time()` and `accumulated_time` carry the same information | | builtins.h | `builtins::make_default_environment`, `builtins::register_fn` | `pycvc.builtins.make_default_environment()`, `pycvc.builtins.register_fn(env, name, fn)` | wrapped (PR 2). `fn` is a Python callable, or a `pycvc.value` holding a native fn | | | 38 `builtin_*` functions | none | exception: DSL builtins, called from DSL source | | | `percent_encode`, `percent_decode` | `pycvc.percent_encode`, `pycvc.percent_decode` | wrapped (PR 2) | -| stackless_evaluator.h, evaluator.h | `stackless_evaluator`, `evaluator`, `evaluator_state`, `evaluation_stats`, the evaluation exceptions | | PR 3 | +| stackless_evaluator.h, evaluator.h | `stackless_evaluator`, `evaluator`, `evaluator_state`, `eval_frame`, `eval_phase`, `evaluation_stats`, `step_kind` (#658), `evaluation_interrupted`, `evaluation_timeout` | the same names under `pycvc.` | wrapped (PR 3). `eval_phase` and `step_kind` are `IntEnum`s, and `evaluation_stats` has #658's counts by kind (`steps`, `step_time`, `data_nodes`, ...); the two exceptions are `RuntimeError` subclasses. `on_complete` is a Python callable. `evaluator.user_macros()` returns a copy, a dict of name -> (params, body). `restrict_special_forms` takes any iterable of str, or None | +| | `evaluator_state.data_work`, `evaluator_state.backlog`, `pending_data_work` (#658) | none | exception: deep-data work in flight between steps, internal to the evaluator; `data_step_nodes` is wrapped | +| | `return_exception` | none | exception: the internal unwinding of `(return ...)`, never seen outside an evaluation | | async_evaluator.h, async_stackless_evaluator.h | the async evaluators | none | exception: coroutine API | -| resource_policy.h | `resource_policy`, `process_limits`, `validate_limits` | | PR 3 | -| intrinsics.h | `register_intrinsics`, `intrinsics_context`, `apply_chroot`, `channel_policy` | `Exec` uses them internally | PR 3, with the intrinsics context's raw-pointer fields as `State`/scheduler references | -| | `document_scope`, `park_on_channel`, `make_future`, `future_channel_of` | none | exception: Ariadne/runtime internals (templates, raw process pointers) | -| stdlib.h | `stdlib_registry` | | after the stdlib is wired into the DSL (separate work: today only tests import it) | +| resource_policy.h | `resource_policy`, `process_limits`, `validate_limits` | `pycvc.resource_policy` (its `mode` is `pycvc.resource_policy.mode`, an `IntEnum`), `pycvc.process_limits`, `pycvc.validate_limits` | wrapped (PR 3) | +| intrinsics.h | `register_intrinsics`, `intrinsics_context`, `apply_chroot`, `channel_policy`, `resolve_channel_key`, `make_future`, `future_channel_of` | the same names under `pycvc.` | wrapped (PR 3). See approach item 11 for the context's lifetime. `channel_policy.global` is `global_` (a Python keyword). The context's `tracker`, `document` and `watches` aren't wrapped: `memory_tracker` is internal, `document_scope`'s slots are C++ templates, and `watches` is the intrinsics' own registry | +| | `document_scope`, `park_on_channel` | none | exception: Ariadne/runtime internals. `document_scope` is typed C++ template slots; `park_on_channel` takes raw process pointers and is valid only inside a running native function | +| stdlib.h | `stdlib_registry` | | next (PR 4). #660 loads the stdlib modules into every environment, so DSL programs already reach them | +| heap.h (#667) | the heap measurement, and the fields it adds to `process_info` and `process` | | PR 4, after #667 merges | | exec_coordinator.h | `exec_coordinator` and its result types | none | deferred: it needs the cluster shard, membership and message-bus objects, which pycvc doesn't wrap | | memory_tracker.h | `memory_tracker` | none | exception: per-node byte accounting, internal to the schedulers | | generator.h, task.h | `generator`, `generator_next`, `task<>` | none | exception: coroutine and evaluator internals; a generator crosses as a `pycvc.value` | @@ -115,18 +134,21 @@ Each PR is stacked on the previous one and includes pytest checks against the C+ `values_equal`, `encode_value`, `decode_value`. - `bindings/pycvc/test_pycvc_state_exec.py`. - The pycvc README's outdated Status and Design sections are rewritten. -2. **PR 2 (this):** `environment` and `builtins`; `execute_options`, `process_info`, +2. **PR 2 (#671):** `environment` and `builtins`; `execute_options`, `process_info`, `scheduler_stats` and the enums; `scheduler` and `async_scheduler` as real classes, with their non-coroutine methods. `Exec.scheduler()` returns its scheduler. `bindings/pycvc/test_pycvc_state_exec_sched.py` repeats the C++ scheduler tests on both classes. -3. **PR 3:** the evaluators and `evaluator_state`, the `process` struct and the scheduler calls - that take it, the environment and evaluator-state codec, `resource_policy`, and - `register_intrinsics` / `intrinsics_context`. With `register_intrinsics`, a program submitted - to `Exec.scheduler()` can get the intrinsics `Exec.run` programs have; until then an - environment built in Python has only the builtins and what `register_fn` adds. +3. **PR 3 (this):** the evaluators, `evaluator_state`, `eval_frame` and `evaluation_stats`; the + `process` struct and the scheduler calls that take it; the environment and evaluator-state + codec; `resource_policy`; `register_intrinsics` and `intrinsics_context`, so an environment + built in Python can have the intrinsics `Exec.run` programs have; and master's + `max_alloc_bytes` additions (#660). `bindings/pycvc/test_pycvc_state_exec_eval.py` repeats the + C++ tests. +4. **PR 4:** `stdlib_registry`, and the API the deep-data (#658) and heap (#667) work adds, once + those land. -Known gaps after PR 1: +Known gaps: - Invalid UTF-8 in a DSL string raises when it crosses to Python, but no Python test produces such a string. Python can't create one, and the stdlib's `bytes.decode` isn't reachable from DSL code yet.