diff --git a/bindings/pycvc/CMakeLists.txt b/bindings/pycvc/CMakeLists.txt index 396b2f413..4cac1bcd6 100644 --- a/bindings/pycvc/CMakeLists.txt +++ b/bindings/pycvc/CMakeLists.txt @@ -272,6 +272,7 @@ if(CVC_BUILD_PYCVC_CORE) pycvc_exec_async:test_pycvc_exec_async.py pycvc_exec_coro:test_pycvc_exec_coro.py pycvc_state_exec:test_pycvc_state_exec.py + pycvc_state_exec_sched:test_pycvc_state_exec_sched.py pycvc_world_clock:test_pycvc_world_clock.py pycvc_world_units:test_pycvc_world_units.py pycvc_dataprep:test_pycvc_dataprep.py diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index 35c138f1b..8ebbd3995 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -236,4 +236,28 @@ Values convert to native Python types, with `pycvc.symbol` for symbols and `pycvc.value` as an opaque handle for closures and other non-data values. Such a handle converts back to the same value when passed into the DSL. Conversion is strict: non-`str` dict keys, ints outside 64 bits and unsupported types -raise. Contract tests: `test_pycvc_exec.py`, `test_pycvc_state_exec.py`. +raise. + +The schedulers are the real classes too, with environments, builtins and the +process types: + +```python +from pycvc import process_status, scheduling_policy +env = pycvc.builtins.make_default_environment() +pycvc.builtins.register_fn(env, "greet", lambda n: "hi " + n) # a Python callable +opts = pycvc.execute_options() +opts.env = env +opts.on_complete = print # called with the result +sched = pycvc.scheduler(scheduling_policy.round_robin) # or pycvc.async_scheduler() +pid = sched.execute('(greet "ann")', opts) +sched.run() # {pid: 'hi ann'}; releases the GIL +sched.get_process_info(pid).status # process_status.terminated +ex.scheduler().list_processes() # Exec's own async_scheduler +``` + +The enums are `enum.IntEnum` classes, and an enum argument must be one of their +members. Where C++ returns an empty `std::optional` or a null pointer, Python gets +`None`, except for a value: `get_result`, `pop_pending_message` and +`environment.lookup` raise `KeyError` instead, because `None` is the nil value. +Contract tests: `test_pycvc_exec.py`, `test_pycvc_state_exec.py`, +`test_pycvc_state_exec_sched.py`. diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index bdd4098d5..3888257c8 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1453,15 +1453,25 @@ class AsyncStateObserver(state_observer): // executor") are wrapped the same way as the parser settings. %{ #include +#include #include #include +#include +#include +#include #include #include #include "pycvc_value.h" +#include +#include #include +#include #include +#include #include +#include #include +#include #include %} @@ -1684,7 +1694,10 @@ bool py_to_value(PyObject *o, cvc::state_exec::value_t &out) { return from_py(o, SWIG_fail; $1 = &temp; %} -%typemap(typecheck, precedence=SWIG_TYPECHECK_POINTER) cvc::state_exec::value_tag, +// Any Python object may be a value, so the value overload is tried last (SWIG tries the lowest +// precedence first): a str goes to execute(script) and a process_status to +// to_string(process_status), not to their value_t overloads. +%typemap(typecheck, precedence=10000) cvc::state_exec::value_tag, const cvc::state_exec::value_tag & "$1 = 1;" %typemap(out) cvc::state_exec::value_tag %{ $result = pycvc::value_to_py(static_cast($1)); @@ -1709,6 +1722,276 @@ bool py_to_value(PyObject *o, cvc::state_exec::value_t &out) { return from_py(o, } %} +// ── maps and optionals (the scheduler surface) ────────────────────────── +// A name -> value map (environment::bindings, execute_options::signal_handlers) is a dict with str +// keys, in key order. A pid -> value map (run()'s results) is a dict with int keys. +%{ +namespace pycvc { +namespace { +using value_map = std::unordered_map; + +PyObject *value_map_to_py(const value_map &m) { + std::vector entries; + entries.reserve(m.size()); + for (const auto &kv : m) + entries.push_back(&kv); + std::sort(entries.begin(), entries.end(), + [](const auto *a, const auto *b) { return a->first < b->first; }); + PyObject *dict = PyDict_New(); + if (dict == nullptr) + return nullptr; + for (const auto *kv : entries) { + PyObject *key = text_to_py(kv->first); + PyObject *val = key != nullptr ? value_to_py(kv->second) : nullptr; + const int rc = val != nullptr ? PyDict_SetItem(dict, key, val) : -1; + Py_XDECREF(key); + Py_XDECREF(val); + if (rc != 0) { + Py_DECREF(dict); + return nullptr; + } + } + return dict; +} + +bool py_to_value_map(PyObject *o, value_map &out) { + if (!PyDict_Check(o)) { + PyErr_Format(PyExc_TypeError, "pycvc: expected a dict of str to values, got %.200s", + Py_TYPE(o)->tp_name); + return false; + } + value_map m; + PyObject *key = nullptr; + PyObject *val = nullptr; + Py_ssize_t pos = 0; + while (PyDict_Next(o, &pos, &key, &val)) { + if (!PyUnicode_Check(key)) { + PyErr_Format(PyExc_TypeError, "pycvc: dict keys must be str, got %.200s", + Py_TYPE(key)->tp_name); + return false; + } + Py_ssize_t n = 0; + const char *s = PyUnicode_AsUTF8AndSize(key, &n); + cvc::state_exec::value_t v; + if (s == nullptr || !py_to_value(val, v)) + return false; + m[std::string(s, static_cast(n))] = std::move(v); + } + out = std::move(m); + return true; +} +} // namespace +} // namespace pycvc +%} + +%typemap(in) const std::unordered_map & + (std::unordered_map temp) %{ + if (!pycvc::py_to_value_map($input, temp)) + SWIG_fail; + $1 = &temp; +%} +%typemap(out) const std::unordered_map & %{ + $result = pycvc::value_map_to_py(*$1); + if (!$result) + SWIG_fail; +%} +%typemap(out) std::unordered_map %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over these initializations + const std::unordered_map &pycvc_map = $1; + $result = PyDict_New(); + if (!$result) + SWIG_fail; + for (const auto &kv : pycvc_map) { + PyObject *k = PyLong_FromLong(kv.first); + PyObject *v = k ? pycvc::value_to_py(kv.second) : nullptr; + const int rc = v ? PyDict_SetItem($result, k, v) : -1; + Py_XDECREF(k); + Py_XDECREF(v); + if (rc != 0) { + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + } + } +%} + +// Limits passed as std::optional: None is std::nullopt. +%typemap(in) std::optional %{ + if ($input != Py_None) { + const unsigned long long pycvc_v = PyLong_AsUnsignedLongLong($input); + if (pycvc_v == static_cast(-1) && PyErr_Occurred()) + SWIG_fail; + $1 = pycvc_v; + } +%} +%typemap(typecheck, precedence=SWIG_TYPECHECK_UINT64) std::optional + "$1 = ($input == Py_None || PyLong_Check($input)) ? 1 : 0;" +%typemap(in) std::optional %{ + if ($input != Py_None) { + const double pycvc_v = PyFloat_AsDouble($input); + if (pycvc_v == -1.0 && PyErr_Occurred()) + SWIG_fail; + $1 = pycvc_v; + } +%} +%typemap(typecheck, precedence=SWIG_TYPECHECK_DOUBLE) std::optional + "$1 = ($input == Py_None || PyFloat_Check($input) || PyLong_Check($input)) ? 1 : 0;" + +// Optional results other than values are None when empty (a value can be nil, so get_result and +// pop_pending_message raise KeyError instead; see their %extend below). +%typemap(out) std::optional %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over this initialization + const std::optional &pycvc_opt = $1; + if (pycvc_opt) { + $result = PyUnicode_DecodeUTF8(pycvc_opt->data(), static_cast(pycvc_opt->size()), + "surrogateescape"); + if (!$result) + SWIG_fail; + } else { + $result = Py_None; + Py_INCREF(Py_None); + } + } +%} +%typemap(out) std::optional %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over this initialization + const std::optional &pycvc_opt = $1; + if (pycvc_opt) { + $result = SWIG_NewPointerObj(new cvc::state_exec::process_info(*pycvc_opt), + $descriptor(cvc::state_exec::process_info *), SWIG_POINTER_OWN); + if (!$result) + SWIG_fail; + } else { + $result = Py_None; + Py_INCREF(Py_None); + } + } +%} +%typemap(out) std::vector %{ + { // a block, so the wrapper's later `goto fail` doesn't jump over this initialization + const std::vector &pycvc_infos = $1; + $result = PyList_New(static_cast(pycvc_infos.size())); + if (!$result) + SWIG_fail; + for (size_t i = 0; i < pycvc_infos.size(); ++i) { + PyObject *item = SWIG_NewPointerObj(new cvc::state_exec::process_info(pycvc_infos[i]), + $descriptor(cvc::state_exec::process_info *), + SWIG_POINTER_OWN); + if (!item) { + Py_DECREF($result); + $result = nullptr; + SWIG_fail; + } + PyList_SET_ITEM($result, static_cast(i), item); + } + } +%} + +// ── enums: pycvc.process_status and pycvc.scheduling_policy are enum.IntEnum classes ── +// pycvc.process_status.killed is process_status::killed. An enum argument must be a member of its +// IntEnum (strict, like values), which is also what lets to_string(process_status) and +// to_string(value) share their C++ name. +%{ +static PyObject *pycvc_enum_process_status = nullptr; +static PyObject *pycvc_enum_scheduling_policy = nullptr; + +// enum.IntEnum(name, members) in module pycvc, or nullptr with a Python error set. +static PyObject *pycvc_make_int_enum(const char *name, + std::initializer_list> members) { + PyObject *enum_mod = PyImport_ImportModule("enum"); + PyObject *int_enum = enum_mod ? PyObject_GetAttrString(enum_mod, "IntEnum") : nullptr; + Py_XDECREF(enum_mod); + PyObject *pairs = int_enum ? PyList_New(0) : nullptr; + if (pairs == nullptr) { + Py_XDECREF(int_enum); + return nullptr; + } + for (const auto &m : members) { + PyObject *pair = Py_BuildValue("(si)", m.first, m.second); + if (pair == nullptr || PyList_Append(pairs, pair) != 0) { + Py_XDECREF(pair); + Py_DECREF(pairs); + Py_DECREF(int_enum); + return nullptr; + } + Py_DECREF(pair); + } + PyObject *args = Py_BuildValue("(sO)", name, pairs); + PyObject *kwargs = Py_BuildValue("{s:s}", "module", "pycvc"); + PyObject *cls = args && kwargs ? PyObject_Call(int_enum, args, kwargs) : nullptr; + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_DECREF(pairs); + Py_DECREF(int_enum); + return cls; +} + +static bool pycvc_is_enum(PyObject *o, PyObject *type) { + const int is = type != nullptr ? PyObject_IsInstance(o, type) : 0; + if (is < 0) + PyErr_Clear(); + return is == 1; +} + +// An enum argument's value: `o` must be a member of `type`. False, with a TypeError set, otherwise. +static bool pycvc_enum_from_py(PyObject *o, PyObject *type, const char *name, int &out) { + if (!pycvc_is_enum(o, type)) { + PyErr_Format(PyExc_TypeError, "pycvc: expected a pycvc.%s member, got %.200s", name, + Py_TYPE(o)->tp_name); + return false; + } + const long v = PyLong_AsLong(o); + if (v == -1 && PyErr_Occurred()) + return false; + out = static_cast(v); + return true; +} +%} + +%define PYCVC_STATE_EXEC_ENUM(TYPE, PYTYPE, NAME) +%typemap(in) TYPE (int pycvc_ev) { + if (!pycvc_enum_from_py($input, PYTYPE, NAME, pycvc_ev)) + SWIG_fail; + $1 = static_cast(pycvc_ev); +} +%typemap(out) TYPE { + $result = PyObject_CallFunction(PYTYPE, "i", static_cast($1)); + if (!$result) + SWIG_fail; +} +%typemap(typecheck, precedence=SWIG_TYPECHECK_INT32) TYPE { + $1 = pycvc_is_enum($input, PYTYPE) ? 1 : 0; +} +%enddef +PYCVC_STATE_EXEC_ENUM(cvc::state_exec::process_status, pycvc_enum_process_status, "process_status") +PYCVC_STATE_EXEC_ENUM(cvc::state_exec::scheduling_policy, pycvc_enum_scheduling_policy, + "scheduling_policy") + +%init %{ + pycvc_enum_process_status = pycvc_make_int_enum( + "process_status", + {{"ready", static_cast(cvc::state_exec::process_status::ready)}, + {"running", static_cast(cvc::state_exec::process_status::running)}, + {"paused", static_cast(cvc::state_exec::process_status::paused)}, + {"waiting", static_cast(cvc::state_exec::process_status::waiting)}, + {"terminated", static_cast(cvc::state_exec::process_status::terminated)}, + {"killed", static_cast(cvc::state_exec::process_status::killed)}}); + if (pycvc_enum_process_status != NULL) + PyDict_SetItemString(d, "process_status", pycvc_enum_process_status); + pycvc_enum_scheduling_policy = pycvc_make_int_enum( + "scheduling_policy", + {{"round_robin", static_cast(cvc::state_exec::scheduling_policy::round_robin)}, + {"priority", static_cast(cvc::state_exec::scheduling_policy::priority)}, + {"priority_rr", static_cast(cvc::state_exec::scheduling_policy::priority_rr)}}); + if (pycvc_enum_scheduling_policy != NULL) + PyDict_SetItemString(d, "scheduling_policy", pycvc_enum_scheduling_policy); +%} +%pythoncode %{ +process_status = _pycvc.process_status +scheduling_policy = _pycvc.scheduling_policy +%} + // ── parse_error: pycvc.parse_error, a RuntimeError with .line and .col ── %{ static PyObject *pycvc_parse_error_type = nullptr; @@ -1738,12 +2021,39 @@ static void pycvc_raise_parse_error(const cvc::state_exec::parse_error &e) { parse_error = _pycvc.parse_error %} +// ── no value: KeyError ── +// Where C++ returns an empty std::optional or a null value_t* (an unbound name, a pid with +// no result, an empty message queue), the wrapper throws pycvc::no_value and Python gets +// KeyError(key). It can't return None: None is the nil value. +%{ +namespace pycvc { +struct no_value { + std::variant key; +}; +} // namespace pycvc + +static void pycvc_raise_no_value(const pycvc::no_value &e) { + PyObject *key = nullptr; + if (const int *pid = std::get_if(&e.key)) { + key = PyLong_FromLong(*pid); + } else { + const std::string &s = std::get(e.key); + key = PyUnicode_DecodeUTF8(s.data(), static_cast(s.size()), "surrogateescape"); + } + if (key != nullptr) { + PyErr_SetObject(PyExc_KeyError, key); + Py_DECREF(key); + } +} +%} + // For the state_exec declarations (and Exec, below): the default mapping, with // parse_error first so it raises pycvc.parse_error. Scoped to this section, so // the modules that %import pycvc.i keep PYCVC_DEFAULT_EXCEPTION. -%exception { - try { - $action +%define PYCVC_STATE_EXEC_CATCH + catch (const pycvc::no_value &e) { + pycvc_raise_no_value(e); + SWIG_fail; } catch (const cvc::state_exec::parse_error &e) { pycvc_raise_parse_error(e); SWIG_fail; @@ -1754,8 +2064,36 @@ parse_error = _pycvc.parse_error } catch (...) { SWIG_exception(SWIG_RuntimeError, "pycvc: C++ exception (see libcvc)"); } +%enddef +%exception { + try { + $action + } + PYCVC_STATE_EXEC_CATCH } +// Stepping and running a scheduler release the GIL around the C++ call (pycvc::gil_release, from +// pycvc_lod.i; a throw restores it before the catch arms run), so other Python threads keep going +// while DSL programs run. A Python function a program calls takes the GIL back for its call. +%define PYCVC_STATE_EXEC_NOGIL(NAME) +%exception NAME { + try { + pycvc::gil_release pycvc_nogil; + $action + } + PYCVC_STATE_EXEC_CATCH +} +%enddef +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::scheduler::step) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::scheduler::run) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::async_scheduler::sync_step) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::async_scheduler::sync_run) + +// environment is shared (environment_ptr), as in C++. The map members read and write as dicts. +%shared_ptr(cvc::state_exec::environment) +%naturalvar cvc::state_exec::environment::bindings; +%naturalvar cvc::state_exec::execute_options::signal_handlers; + namespace cvc { namespace state_exec { @@ -1841,16 +2179,270 @@ public: void encode_value(cvc::state &node, const value_t &val); value_t decode_value(cvc::state &node); -%nodefaultctor async_scheduler; -%nodefaultdtor async_scheduler; +// ── environment and builtins ── +// lookup() returns the bound value, or raises KeyError (C++ returns nullptr). bindings is the +// scope's own map, read and written as a dict (a copy: change bindings with set()). set() on a +// frozen scope raises RuntimeError, as C++ throws std::logic_error. +struct environment { + std::unordered_map bindings; + std::shared_ptr outer; + bool frozen; + void set(const std::string &name, value_t val); + void set_existing(const std::string &name, value_t val); + static std::shared_ptr extend(std::shared_ptr parent); +}; +%extend environment { + value_t lookup(const std::string &name) { + if (const cvc::state_exec::value_t *v = $self->lookup(name)) + return *v; + throw pycvc::no_value{name}; + } +} +typedef std::shared_ptr environment_ptr; + +// register_fn takes a Python callable (or a pycvc.value holding a native fn) where C++ takes a +// native_fn. The callable runs when a program calls it, on the thread stepping the scheduler; it +// gets the arguments as Python values and its result converts back, as for Exec.register_fn. +%nodefaultctor builtins; +%nodefaultdtor builtins; +class builtins { +public: + static environment_ptr make_default_environment(); +}; +%extend builtins { + static void register_fn(environment_ptr env, const std::string &name, PyObject *fn) { + if (!env) + throw std::invalid_argument("pycvc.builtins.register_fn: env is None"); + cvc::state_exec::value_t held; + void *p = nullptr; + if (SWIG_IsOK(SWIG_ConvertPtr(fn, &p, pycvc::value_swig_type(), 0)) && p != nullptr) + held = *static_cast(p); + if (const auto *native = std::get_if(&held.v)) { + cvc::state_exec::builtins::register_fn(env, name, *native); + return; + } + cvc::state_exec::builtins::register_fn( + env, name, pycvc::python_native_fn("pycvc.builtins.register_fn", name, fn)); + } +} + +std::string percent_encode(const std::string &s); +std::string percent_decode(const std::string &s); + +// ── processes and schedulers ── +// process_status and scheduling_policy are the IntEnums above; declared opaque here so SWIG adds +// no flat constants of its own. +enum class process_status; +enum class scheduling_policy; +const char *to_string(process_status s); +process_status parse_process_status(const std::string &s); + +// on_complete is a Python callable (or None) called with the process's result when it terminates, +// on the thread stepping the scheduler. Its return value is ignored; an exception it raises is +// raised from that step()/run() call. The uint64_t limits are Python ints. +struct execute_options { + std::string name; + int priority; + std::string uid; + std::string gid; + std::string root_path; + std::string owner; + unsigned long long max_steps; + double max_time; + double max_step_time; + unsigned long long max_memory; + unsigned long long max_messages; + unsigned long long max_message_bytes; + std::unordered_map signal_handlers; + environment_ptr env; +}; +%extend execute_options { + PyObject *_on_complete_get() const { + if (!$self->on_complete) { + Py_INCREF(Py_None); + return Py_None; + } + if (const auto *f = $self->on_complete.target()) { + Py_INCREF(f->callable.get()); + return f->callable.get(); + } + throw std::runtime_error("pycvc.execute_options.on_complete: holds a C++ callback"); + } + void _on_complete_set(PyObject *fn) { + if (fn == Py_None) + $self->on_complete = nullptr; + else + $self->on_complete = pycvc::make_python_on_complete(fn); + } + %pythoncode %{ + on_complete = property(_on_complete_get, _on_complete_set) + %} +} + +struct process_info { + int pid; + std::string name; + process_status status; + int priority; + std::string uid; + std::string gid; + unsigned long long step_count; + double elapsed_time; + unsigned long long current_memory; + unsigned long long peak_memory; + unsigned long long max_memory; + double max_time; + unsigned long long message_count; + unsigned long long max_messages; + unsigned long long message_bytes; + unsigned long long max_message_bytes; + int parent_pid; + // The budget by kind of step (#658): reading the script, evaluating, deep-data work. + unsigned long long read_steps; + unsigned long long eval_steps; + unsigned long long data_steps; + unsigned long long data_nodes; + double read_time; + double eval_time; + double data_time; +}; +%extend process_info { + %pythoncode %{ + def __repr__(self): + return "" % ( + self.pid, self.name, self.status.name) + %} +} + +struct scheduler_stats { + int total_processes; + int running; + int ready; + int paused; + int terminated; + int killed; + unsigned long long total_steps; +}; +%extend scheduler_stats { + %pythoncode %{ + def __repr__(self): + return ("" % ( + self.total_processes, self.running, self.ready, self.paused, + self.terminated, self.killed, self.total_steps)) + %} +} + +// The members scheduler and async_scheduler share (scheduler_base plus their common extras). +// Returns of a C++ reference come back as copies. Not wrapped: mem_tracker(), and +// current_process() / queue_watch_event(), which take the process struct (PR 3). +%define PYCVC_STATE_EXEC_SCHEDULER_MEMBERS + int execute(const std::string &script, const execute_options &opts = execute_options()); + int execute(const value_t &expr, const execute_options &opts = execute_options()); + void stop(); + bool is_running() const; + bool pause(int pid); + bool resume(int pid); + bool kill(int pid); + int fork(int pid); + bool sleep(int pid, double seconds); + bool yield_frame(int pid); + bool receive_message(int pid, const std::string &path); + int deliver_to_receivers(const std::string &path, const value_t &msg); + size_t pending_message_count(const std::string &path) const; + size_t total_pending_messages() const; + size_t max_pending_messages; + bool set_priority(int pid, int priority); + bool set_max_steps(int pid, unsigned long long max_steps); + bool set_max_time(int pid, double seconds); + bool set_max_memory(int pid, unsigned long long bytes); + bool set_max_messages(int pid, unsigned long long count); + bool set_max_message_bytes(int pid, unsigned long long bytes); + bool send_signal(int pid, const std::string &signal); + std::vector list_processes() const; + std::optional get_process_info(int pid) const; + std::unordered_map get_results() const; + scheduler_stats get_stats() const; + scheduling_policy policy() const; + int process_count() const; + bool has_runnable() const; + int current_pid() const; + void register_watch_handler(int pid, int watch_id, value_t handler, const std::string &path); + void unregister_watch_handler(int pid, int watch_id); + void set_watch_root(cvc::state *root); +%enddef + +// get_result and pop_pending_message raise KeyError(pid / path) where C++ returns std::nullopt. +%define PYCVC_STATE_EXEC_SCHEDULER_EXTEND(CLASS) +%extend CLASS { + value_t get_result(int pid) const { + if (std::optional r = $self->get_result(pid)) + return *r; + throw pycvc::no_value{pid}; + } + value_t pop_pending_message(const std::string &path) { + if (std::optional m = $self->pop_pending_message(path)) + return *m; + throw pycvc::no_value{path}; + } +} +%enddef + +// The synchronous scheduler. run() and step() release the GIL while programs run. +class scheduler { +public: + scheduler(); + scheduler(scheduling_policy policy); + PYCVC_STATE_EXEC_SCHEDULER_MEMBERS + int step(); + std::unordered_map run(std::optional max_steps = std::nullopt, + std::optional max_time = std::nullopt); + void set_id(const std::string &id); + std::string id() const; + void load_settings(); + parse_options parser_settings() const; + data_options data_settings() const; + static constexpr const char *settings_root = "state_exec"; + static constexpr const char *defaults_prefix = "state_exec.defaults"; + static constexpr const char *sched_prefix = "state_exec.schedulers"; +}; +PYCVC_STATE_EXEC_SCHEDULER_EXTEND(scheduler) + +// The scheduler libcvc's hosts use (app::exec_scheduler(), Exec). The coroutine step()/run() aren't +// wrapped; Python drives it with sync_step()/sync_run(), which release the GIL while programs run. +// post_message() is the one thread-safe call. class async_scheduler { public: + async_scheduler(); + async_scheduler(scheduling_policy policy); + PYCVC_STATE_EXEC_SCHEDULER_MEMBERS + int sync_step(); + std::unordered_map sync_run( + std::optional max_steps = std::nullopt, + std::optional max_time = std::nullopt); + int kill_owner(const std::string &owner); + int wake_awaiting(); + void post_message(const std::string &path, const value_t &msg); + void drain_ingress(); + std::optional get_exit_error(int pid) const; + bool reap(int pid); + void set_parser_defaults(const parse_options &defaults); + parse_options parser_defaults() const; static parse_options host_parser_defaults(); + void set_data_defaults(const data_options &defaults); + data_options data_defaults() const; + data_options data_settings() const; static data_options host_data_defaults(); }; +PYCVC_STATE_EXEC_SCHEDULER_EXTEND(async_scheduler) } } +// Exec.scheduler() returns a reference into the Exec, so the scheduler keeps the Exec alive. +%pythonappend pycvc::Exec::scheduler %{ + val._pycvc_exec = self +%} + // ── Phase 4: DSL execution + Python-authored DSL functions ────────────── // pycvc.Exec(app) runs state_exec DSL programs over app's state root and lets a // Python callable be registered as a DSL function (register_fn). PyObject* is diff --git a/bindings/pycvc/pycvc_exec.cpp b/bindings/pycvc/pycvc_exec.cpp index e0d4bf0fc..534875a87 100644 --- a/bindings/pycvc/pycvc_exec.cpp +++ b/bindings/pycvc/pycvc_exec.cpp @@ -92,9 +92,10 @@ std::shared_ptr own_callable(PyObject *callable) { // async fn): PyGILState_Ensure creates/binds a thread state and blocks until the // GIL is free, so the only rule is that whoever is idle-waiting for this worker // must have released the GIL (run()'s pump does). Contains a Python exception as -// a C++ error carrying the message. +// a C++ error carrying the message, prefixed with `what`. With `want_result` +// false the return value is ignored (an on_complete callback) and nil returned. value_t call_python(const std::shared_ptr &holder, std::span args, - const std::string &name) { + const std::string &what, bool want_result = true) { PyGILState_STATE gil = PyGILState_Ensure(); PyObject *pyargs = values_to_py_tuple(args); // nullptr (error set) if an arg can't convert PyObject *res = pyargs ? PyObject_CallObject(holder.get(), pyargs) : nullptr; @@ -102,30 +103,21 @@ value_t call_python(const std::shared_ptr &holder, std::span holder = own_callable(callable); - return [name, holder](std::span args) -> value_t { - return call_python(holder, args, name); - }; -} - std::string render_result(const value_t &v) { if (const auto *s = std::get_if(&v.v)) return *s; @@ -260,6 +252,32 @@ struct EndGuard { }; } // namespace +// A SYNCHRONOUS DSL native_fn: it runs inline on the thread stepping the scheduler (Exec::run's, +// or the caller of a wrapped scheduler's run/step, which release the GIL; call_python retakes it). +native_fn python_native_fn(const std::string &who, const std::string &name, PyObject *callable) { + if (callable == nullptr || !PyCallable_Check(callable)) + throw std::invalid_argument(who + ": '" + name + "' is not callable"); + if (is_coroutine_function(callable)) + throw std::invalid_argument(who + ": '" + name + + "' is an async def — use Exec.register_async_coro"); + std::shared_ptr holder = own_callable(callable); + const std::string what = "python DSL fn '" + name + "'"; + return [what, holder](std::span args) -> value_t { + return call_python(holder, args, what); + }; +} + +void python_on_complete::operator()(value_t result) const { + call_python(callable, std::span(&result, 1), "python on_complete", + /*want_result=*/false); +} + +python_on_complete make_python_on_complete(PyObject *callable) { + if (callable == nullptr || !PyCallable_Check(callable)) + throw std::invalid_argument("pycvc.execute_options.on_complete: not callable"); + return python_on_complete{own_callable(callable)}; +} + // Owns everything an evaluation needs. The intrinsics capture &ictx (a raw // pointer), so ictx must outlive any run — holding it here (Exec-lifetime) // satisfies that. Mirrors the C++ tests' make_exec_env(). @@ -341,12 +359,8 @@ Exec::~Exec() { } void Exec::register_fn(const std::string &name, PyObject *callable) { - if (callable == nullptr || !PyCallable_Check(callable)) - throw std::invalid_argument("pycvc.Exec.register_fn: '" + name + "' is not callable"); - if (is_coroutine_function(callable)) - throw std::invalid_argument("pycvc.Exec.register_fn: '" + name + - "' is an async def — use register_async_coro"); - builtins::register_fn(impl_->env, name, make_python_native_fn(name, callable)); + builtins::register_fn(impl_->env, name, + python_native_fn("pycvc.Exec.register_fn", name, callable)); } void Exec::register_async_fn(const std::string &name, PyObject *callable) { @@ -379,7 +393,7 @@ void Exec::register_async_fn(const std::string &name, PyObject *callable) { [holder, argv, name, inflight]() -> value_t { EndGuard guard{ inflight}; // --in-flight when this body returns/throws, before the post - return call_python(holder, *argv, name); + return call_python(holder, *argv, "python DSL fn '" + name + "'"); }, [name](const std::string &e) -> value_t { return async_error_dict("python async fn '" + name + "': " + e); @@ -585,4 +599,6 @@ void Exec::set_data_defaults(const cvc::state_exec::data_options &defaults) { impl_->sched.set_data_defaults(defaults); } +cvc::state_exec::async_scheduler &Exec::scheduler() { return impl_->sched; } + } // namespace pycvc diff --git a/bindings/pycvc/pycvc_exec.h b/bindings/pycvc/pycvc_exec.h index 27da26732..58d879a60 100644 --- a/bindings/pycvc/pycvc_exec.h +++ b/bindings/pycvc/pycvc_exec.h @@ -14,7 +14,10 @@ namespace cvc { class app; +namespace state_exec { +class async_scheduler; } +} // namespace cvc // Forward-declare PyObject so this SWIG-visible header needs no . struct _object; @@ -100,6 +103,11 @@ class Exec { cvc::state_exec::data_options data_defaults() const; void set_data_defaults(const cvc::state_exec::data_options &defaults); + // This Exec's private scheduler, the one run() drives. A program submitted to it directly + // (scheduler().execute(...)) runs alongside run()'s programs, and run() steps it too. In Python + // the returned scheduler keeps this Exec alive. + cvc::state_exec::async_scheduler &scheduler(); + private: struct ExecImpl; std::shared_ptr impl_; diff --git a/bindings/pycvc/pycvc_value.h b/bindings/pycvc/pycvc_value.h index c9f4bf28a..8cc17b305 100644 --- a/bindings/pycvc/pycvc_value.h +++ b/bindings/pycvc/pycvc_value.h @@ -18,7 +18,9 @@ #pragma once #include +#include #include +#include struct _object; typedef _object PyObject; @@ -35,4 +37,24 @@ PyObject *values_to_py_tuple(std::span values); /// Convert `o` into `out`. Returns false, with a Python exception set, when it can't. bool py_to_value(PyObject *o, cvc::state_exec::value_t &out); +// Python callables as state_exec callbacks. These are defined in pycvc_exec.cpp. Each one owns a +// reference to its callable and takes the GIL for the call, so it can run on any thread. + +/// A DSL native function that calls `callable`: its arguments convert to Python and its return +/// value converts back, and a Python exception becomes a C++ error naming `name`. Throws +/// std::invalid_argument, prefixed with `who`, when `callable` isn't callable or is an async def. +cvc::state_exec::native_fn python_native_fn(const std::string &who, const std::string &name, + PyObject *callable); + +/// An execute_options::on_complete that calls `callable` with the process's result and ignores +/// what it returns. A Python exception becomes a C++ error, thrown from the scheduler call that +/// finished the process. It keeps the callable, so pycvc can hand it back. +struct python_on_complete { + std::shared_ptr callable; + void operator()(cvc::state_exec::value_t result) const; +}; + +/// The python_on_complete for `callable`. Throws std::invalid_argument when it isn't callable. +python_on_complete make_python_on_complete(PyObject *callable); + } // namespace pycvc diff --git a/bindings/pycvc/test_pycvc_state_exec_sched.py b/bindings/pycvc/test_pycvc_state_exec_sched.py new file mode 100644 index 000000000..c885101b4 --- /dev/null +++ b/bindings/pycvc/test_pycvc_state_exec_sched.py @@ -0,0 +1,511 @@ +"""state_exec parity, part 2: environments, builtins and the schedulers. + +pycvc.scheduler and pycvc.async_scheduler are the real C++ classes, with +execute_options, process_info, scheduler_stats, and the process_status and +scheduling_policy enums (enum.IntEnum classes). Most tests here repeat a C++ test +from src/cvc/tests/state_exec_scheduler_test.cpp or state_exec_async_test.cpp, +on both schedulers where both have the call. docs/PYCVC_STATE_EXEC_PARITY.md has +the inventory. +""" + +import threading +import time + +import pycvc +from pycvc import process_status, scheduling_policy, symbol + + +def _raises(exc, fn, *args): + try: + fn(*args) + except exc as e: + return e + raise AssertionError(f"{fn.__name__}{args!r} did not raise {exc.__name__}") + + +def _schedulers(): + """A fresh sync and async scheduler, each with its run and step calls.""" + s = pycvc.scheduler() + a = pycvc.async_scheduler() + return [(s, s.run, s.step), (a, a.sync_run, a.sync_step)] + + +def _opts(**fields): + o = pycvc.execute_options() + for k, v in fields.items(): + setattr(o, k, v) + return o + + +# ── enums ─────────────────────────────────────────────────────────────── + + +def test_enums_are_int_enums_under_the_cpp_names(): + assert [m.name for m in process_status] == [ + "ready", "running", "paused", "waiting", "terminated", "killed"] + assert [m.name for m in scheduling_policy] == ["round_robin", "priority", "priority_rr"] + assert process_status.killed == 5 # an IntEnum, with the C++ values + assert pycvc.to_string(process_status.waiting) == "waiting" + assert pycvc.parse_process_status("paused") is process_status.paused + assert pycvc.parse_process_status("bogus") is process_status.ready # as in C++ + # A plain int is a value, not an enum, so to_string keeps both C++ overloads. + assert pycvc.to_string(2) == "2" + # An enum argument must be the enum. + _raises(TypeError, pycvc.scheduler, 1) + assert pycvc.scheduler(scheduling_policy.priority).policy() is scheduling_policy.priority + assert pycvc.async_scheduler().policy() is scheduling_policy.round_robin + + +# ── environment and builtins ──────────────────────────────────────────── + + +def test_environment_is_the_scope_chain(): + env = pycvc.environment() + env.set("x", 1) + assert env.lookup("x") == 1 + child = pycvc.environment.extend(env) + assert child.outer is not None and child.outer.lookup("x") == 1 + assert child.lookup("x") == 1 # lookup walks the chain + child.set("y", [1, "two"]) + assert child.bindings == {"y": [1, "two"]} # this scope's own map + child.set_existing("x", 5) # the nearest scope that has it + assert env.lookup("x") == 5 and "x" not in child.bindings + child.set_existing("z", None) # bound nowhere: the outermost unfrozen scope + assert "z" not in child.bindings and env.bindings == {"x": 5, "z": None} + # C++ returns nullptr for an unbound name; Python raises KeyError (None is nil). + assert _raises(KeyError, child.lookup, "nope").args == ("nope",) + env.bindings = {"a": symbol("b")} # replaces the map, as assigning it does in C++ + assert env.bindings == {"a": symbol("b")} + _raises(KeyError, child.lookup, "x") + _raises(TypeError, setattr, env, "bindings", {1: 2}) + assert pycvc.environment.extend(None).outer is None + + +def test_a_frozen_scope_is_read_only(): + shared = pycvc.environment() + shared.set("k", 1) + assert not shared.frozen + shared.frozen = True + _raises(RuntimeError, shared.set, "k", 2) # C++ throws std::logic_error + child = pycvc.environment.extend(shared) + assert child.lookup("k") == 1 + child.set_existing("k", 3) # never written into the frozen scope: shadowed here instead + assert child.bindings == {"k": 3} and shared.bindings == {"k": 1} + + +def test_builtins_and_register_fn(): + env = pycvc.builtins.make_default_environment() + plus = env.lookup("+") + assert isinstance(plus, pycvc.value) and plus.type_name() == "native_fn" + calls = [] + + def greet(name): + calls.append(name) + return "hi " + name + + pycvc.builtins.register_fn(env, "greet", greet) + pycvc.builtins.register_fn(env, "plus", plus) # a native fn moves as a pycvc.value + for sched, run, _ in _schedulers(): + pid = sched.execute('(list (greet "ann") (plus 2 3))', _opts(env=env)) + assert run()[pid] == ["hi ann", 5] + assert calls == ["ann", "ann"] + # Not callable, no environment, or an async def: refused at registration. + _raises(RuntimeError, pycvc.builtins.register_fn, env, "bad", 42) + _raises(RuntimeError, pycvc.builtins.register_fn, None, "f", greet) + + async def coro(): + return 1 + + _raises(RuntimeError, pycvc.builtins.register_fn, env, "c", coro) + + +def test_a_python_fn_error_surfaces_as_each_scheduler_reports_it(): + env = pycvc.builtins.make_default_environment() + pycvc.builtins.register_fn(env, "boom", lambda: 1 / 0) + # The sync scheduler throws from run(), as a throwing native fn does in C++. + s = pycvc.scheduler() + s.execute("(boom)", _opts(env=env)) + assert "division by zero" in str(_raises(RuntimeError, s.run)) + # The async scheduler kills the process and keeps the reason as its exit error. + a = pycvc.async_scheduler() + pid = a.execute("(boom)", _opts(env=env)) + assert a.sync_run() == {} + assert a.get_process_info(pid).status is process_status.killed + assert "division by zero" in a.get_exit_error(pid) + assert a.reap(pid) and a.get_process_info(pid) is None + assert a.get_exit_error(999) is None + + +def test_percent_encode_and_decode(): + assert pycvc.percent_encode("a b/é~") == "a%20b%2F%C3%A9~" + assert pycvc.percent_decode("a%20b%2F%C3%A9~") == "a b/é~" + assert pycvc.percent_decode("x+y%zz") == "x y%zz" # '+' is a space; a bad escape stays + + +# ── execute_options, process_info, scheduler_stats ────────────────────── + + +def test_execute_options_has_the_cpp_fields_and_defaults(): + o = pycvc.execute_options() + assert (o.name, o.priority, o.uid, o.gid, o.root_path, o.owner) == ("", 0, "", "", "", "") + assert (o.max_steps, o.max_time, o.max_step_time, o.max_memory) == (0, 0.0, 0.0, 0) + assert (o.max_messages, o.max_message_bytes) == (0, 0) + assert o.signal_handlers == {} and o.env is None and o.on_complete is None + o.max_memory = 2**63 # the uint64_t limits take the full range + assert o.max_memory == 2**63 + _raises(OverflowError, setattr, o, "max_steps", -1) + + +def test_process_info_and_stats_report_each_process(): + for sched, run, _ in _schedulers(): + p1 = sched.execute("42", _opts(name="my-process", uid="user1", gid="group1")) + p2 = sched.execute("(+ 1 2)") + info = sched.get_process_info(p1) + assert (info.pid, info.name, info.uid, info.gid) == (p1, "my-process", "user1", "group1") + assert info.status is process_status.ready and info.parent_pid == -1 + assert sched.get_process_info(p2).name == "proc-%d" % p2 + assert sched.get_process_info(999) is None + stats = sched.get_stats() + assert (stats.total_processes, stats.ready, stats.terminated) == (2, 2, 0) + run() + stats = sched.get_stats() + assert stats.terminated == 2 and stats.total_steps > 0 + procs = sched.list_processes() + assert [p.pid for p in procs] == [p1, p2] # sorted by pid + assert all(p.status is process_status.terminated for p in procs) + assert procs[1].step_count > 0 + + +# ── running programs ──────────────────────────────────────────────────── + + +def test_both_schedulers_run_programs(): + for sched, run, step in _schedulers(): + assert not sched.has_runnable() and sched.process_count() == 0 + p1 = sched.execute("(+ 1 2)") + p2 = sched.execute("(* 3 4)") + fib = sched.execute( + "(begin (defun fib (n) (if (<= n 1) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10))") + nil = sched.execute("nil") + assert sched.has_runnable() and sched.process_count() == 4 + # No result yet: C++ returns std::nullopt, Python raises KeyError(pid). + assert _raises(KeyError, sched.get_result, p1).args == (p1,) + assert run() == {p1: 3, p2: 12, fib: 55, nil: None} + assert sched.get_result(p1) == 3 + assert sched.get_result(nil) is None # nil is a result, not a missing one + assert sched.get_results() == {p1: 3, p2: 12, fib: 55, nil: None} + assert not sched.has_runnable() and not sched.is_running() + + +def test_execute_takes_a_script_or_a_value(): + for sched, run, _ in _schedulers(): + script = sched.execute("(* 6 7)") + expr = sched.execute(pycvc.parse("(* 6 7)")) # an expression, as execute(value_t) + quoted = sched.execute([symbol("quote"), [1, symbol("a")]]) + assert run() == {script: 42, expr: 42, quoted: [1, symbol("a")]} + + +def test_step_by_step_and_run_limits(): + for sched, run, step in _schedulers(): + pid = sched.execute("(+ 1 2)") + stepped = 0 + while sched.has_runnable(): + step() + stepped += 1 + assert stepped > 0 and sched.get_result(pid) == 3 + sched.execute("(begin 1 2 3 4 5 6 7 8 9 10)") + run(3) # at most three steps in all + assert sched.has_runnable() + _raises(OverflowError, run, -1) + run(None, 30.0) + assert not sched.has_runnable() + + +def test_priority_policies(): + for policy in (scheduling_policy.priority, scheduling_policy.priority_rr): + for sched in (pycvc.scheduler(policy), pycvc.async_scheduler(policy)): + run = sched.run if isinstance(sched, pycvc.scheduler) else sched.sync_run + low = sched.execute("(begin 1 2 3)", _opts(priority=10)) + high = sched.execute("(begin 4 5 6)", _opts(priority=-10)) + assert run() == {low: 3, high: 6} + + +def test_process_control(): + for sched, run, step in _schedulers(): + pid = sched.execute("(begin 1 2 3 4 5)") + step() + assert sched.pause(pid) + assert sched.get_process_info(pid).status is process_status.paused + assert step() == 0 # nothing runnable while paused + assert sched.resume(pid) and not sched.resume(pid) + assert sched.get_process_info(pid).status is process_status.ready + assert sched.set_priority(pid, -5) and sched.get_process_info(pid).priority == -5 + assert sched.sleep(pid, 0.01) + assert sched.get_process_info(pid).status is process_status.waiting + # A sleeper counts as runnable for the sync scheduler only; the async one's driver + # steps it after the deadline (as the C++ tests do). + assert sched.has_runnable() is isinstance(sched, pycvc.scheduler) + time.sleep(0.03) + for _ in range(64): + if sched.get_process_info(pid).status is process_status.terminated: + break + step() + assert sched.get_result(pid) == 5 + assert not sched.kill(pid) # already terminated + for call in (sched.pause, sched.kill, sched.yield_frame): + assert not call(999) + assert sched.fork(999) == -1 + + doomed = sched.execute("(begin 1 2 3 4 5)") + step() + assert sched.kill(doomed) + assert sched.get_process_info(doomed).status is process_status.killed + + +def test_fork_copies_the_process(): + for sched, run, step in _schedulers(): + parent = sched.execute("(begin 1 2 3)", _opts(priority=-5, uid="user1", gid="group1")) + step() + child = sched.fork(parent) + assert child > 0 and child != parent + info = sched.get_process_info(child) + assert (info.parent_pid, info.priority) == (parent, -5) + assert (info.uid, info.gid) == ("user1", "group1") + assert info.status is process_status.ready + run() + assert sched.get_result(parent) == 3 and sched.get_result(child) == 3 + + +def test_limits_kill_the_process(): + for sched, run, step in _schedulers(): + capped = sched.execute("(begin 1 2 3 4 5 6 7 8 9 10)", _opts(max_steps=5)) + later = sched.execute("(begin 1 2 3 4 5 6 7 8 9 10)") + assert sched.set_max_steps(later, 3) + assert sched.set_max_time(later, 10.0) and sched.set_max_memory(later, 1 << 20) + assert sched.set_max_messages(later, 100) and sched.set_max_message_bytes(later, 4096) + assert not sched.set_max_steps(999, 1) + info = sched.get_process_info(later) + assert (info.max_time, info.max_memory, info.max_messages, info.max_message_bytes) == ( + 10.0, 1 << 20, 100, 4096) + run() + assert sched.get_process_info(capped).status is process_status.killed + assert sched.get_process_info(later).status is process_status.killed + + +def test_signals(): + handler = [symbol("begin"), 42] + for sched, run, step in _schedulers(): + o = _opts(signal_handlers={"SIGUSR1": handler}) + assert o.signal_handlers == {"SIGUSR1": handler} + pid = sched.execute("(begin 1 2 3 4 5)", o) + step() + assert sched.send_signal(pid, "SIGUSR1") + run() + assert sched.get_process_info(pid).status is process_status.terminated + assert not sched.send_signal(pid, "SIGUSR1") # terminated + doomed = sched.execute("(begin 1 2 3 4 5 6 7 8 9 10)") + step() + assert sched.send_signal(doomed, "SIGKILL") + assert sched.get_process_info(doomed).status is process_status.killed + assert not sched.send_signal(999, "SIGUSR1") + + +def test_on_complete_is_a_python_callable(): + results = [] + + def done(value): + results.append(value) + return {1, 2} # ignored, so not converted + + o = _opts(on_complete=done) + assert o.on_complete is done + for sched, run, _ in _schedulers(): + sched.execute("(list 1 2)", o) + run() + assert results == [[1, 2], [1, 2]] + o.on_complete = None + assert o.on_complete is None + _raises(RuntimeError, setattr, o, "on_complete", 42) + # A raising on_complete is raised from the call that finished the process. + s = pycvc.scheduler() + s.execute("7", _opts(on_complete=lambda v: 1 / 0)) + assert "division by zero" in str(_raises(RuntimeError, s.run)) + + +def test_messages_queue_until_received(): + for sched, run, step in _schedulers(): + # No receiver: the message queues under its path (FIFO). + assert sched.deliver_to_receivers("test.q", {"n": 1}) == 0 + sched.deliver_to_receivers("test.q", {"n": 2}) + assert sched.pending_message_count("test.q") == 2 and sched.total_pending_messages() == 2 + assert sched.pop_pending_message("test.q") == {"n": 1} + assert sched.pop_pending_message("test.q") == {"n": 2} + # Empty: C++ returns std::nullopt, Python raises KeyError(path). + assert _raises(KeyError, sched.pop_pending_message, "test.q").args == ("test.q",) + # A waiting receiver gets the message instead, and wakes. + pid = sched.execute("(begin 1 2 3 4 5)") + step() + assert sched.receive_message(pid, "test.inbox") + assert sched.get_process_info(pid).status is process_status.waiting + assert sched.deliver_to_receivers("test.inbox", "hello") == 1 + assert sched.get_process_info(pid).status is process_status.ready + assert not sched.receive_message(999, "test.inbox") + # The queue is bounded by max_pending_messages. + assert sched.max_pending_messages == 1024 + sched.max_pending_messages = 2 + for n in range(4): + sched.deliver_to_receivers("test.b", n) + assert sched.pending_message_count("test.b") == 2 + + +def test_watch_handlers_register_and_unregister(): + app = pycvc.make_app() + for sched, run, step in _schedulers(): + sched.set_watch_root(pycvc.State.instance(app)) + pid = sched.execute("(begin 1 2 3)") + sched.register_watch_handler(pid, 1, [symbol("begin"), 1], "watched.path") + sched.unregister_watch_handler(pid, 1) + run() + assert sched.get_result(pid) == 3 + sched.set_watch_root(None) + + +def test_scheduler_settings_come_from_the_tree(): + app = pycvc.make_app() + root = pycvc.State.instance(app) + root.child("state_exec.defaults.max_pending_messages").value("512") + root.child("state_exec.schedulers.py-sched.max_pending_messages").value("2048") + root.child("state_exec.schedulers.py-sched.parser.max_depth").value("12") + s = pycvc.scheduler() + assert s.parser_settings().max_depth == 256 + s.set_watch_root(root) + s.set_id("py-sched") + assert s.id() == "py-sched" + s.load_settings() + assert s.max_pending_messages == 2048 + assert s.parser_settings().max_depth == 12 + assert pycvc.scheduler.settings_root == "state_exec" + assert pycvc.scheduler.defaults_prefix == "state_exec.defaults" + assert pycvc.scheduler.sched_prefix == "state_exec.schedulers" + + +def test_data_settings_and_the_budget_by_kind(): + # #658's deep-data settings sit beside the parser settings, and process_info splits a + # process's steps and time by kind. + a = pycvc.async_scheduler() + assert a.data_defaults().step_nodes == 0 # a bare async_scheduler does deep data in place + assert pycvc.async_scheduler.host_data_defaults().step_nodes == 1024 + d = pycvc.data_options() + d.step_nodes = 7 + a.set_data_defaults(d) + assert a.data_defaults().step_nodes == 7 + assert a.data_settings().step_nodes == 7 # no watch root: the defaults + assert pycvc.scheduler().data_settings().step_nodes == 0 + for sched, run, _ in _schedulers(): + pid = sched.execute("(+ 1 2)") + run() + info = sched.get_process_info(pid) + assert info.eval_steps > 0 and info.read_steps == 0 and info.data_steps == 0 + assert info.eval_time >= 0.0 and info.data_nodes == 0 + + +# ── async_scheduler's own calls ───────────────────────────────────────── + + +def test_async_scheduler_extras(): + a = pycvc.async_scheduler() + assert a.parser_defaults().step_tokens == 0 # a bare async_scheduler parses at submit + assert pycvc.async_scheduler.host_parser_defaults().step_tokens == 1024 + a.set_parser_defaults(pycvc.async_scheduler.host_parser_defaults()) + assert a.parser_defaults().step_tokens == 1024 + + a1 = a.execute("(begin 1 2 3 4 5)", _opts(owner="docA")) + a2 = a.execute("(begin 1 2 3 4 5)", _opts(owner="docA")) + b1 = a.execute("(begin 1 2 3 4 5)", _opts(owner="docB")) + a.sync_step() + assert a.kill_owner("docA") == 2 and a.kill_owner("docA") == 0 + assert a.get_process_info(a1).status is process_status.killed + assert a.get_process_info(a2).status is process_status.killed + assert a.get_process_info(b1).status is not process_status.killed + + # (await ...) parking: only wake_awaiting, once per frame, readies it. + assert a.yield_frame(b1) + a.sync_step() + assert a.get_process_info(b1).status is process_status.waiting + assert a.wake_awaiting() == 1 + assert a.get_process_info(b1).status is not process_status.waiting + + +def test_post_message_from_another_thread(): + a = pycvc.async_scheduler() + pid = a.execute("(begin 1 2 3 4 5)") + a.sync_step() + assert a.receive_message(pid, "chan.x") + worker = threading.Thread(target=a.post_message, args=("chan.x", "done")) + worker.start() + worker.join() + a.drain_ingress() # applied on this (the scheduler's) thread + assert a.get_process_info(pid).status is not process_status.waiting + assert a.sync_run()[pid] == 5 + + +def test_run_releases_the_gil(): + # A Python thread keeps running while a scheduler runs a long program: run() and + # sync_run() release the GIL around the C++ call. + for sched, run, _ in _schedulers(): + sched.execute("(while 1 1)") + ticks = [] + stop = threading.Event() + + def spin(): + while not stop.is_set(): + ticks.append(time.perf_counter()) + + t = threading.Thread(target=spin) + t.start() + while not ticks: + time.sleep(0.001) + t0 = time.perf_counter() + run(None, 0.3) + t1 = time.perf_counter() + stop.set() + t.join() + assert t1 - t0 >= 0.1 + # A call that held the GIL would stop the thread for its whole duration; it could + # only run at the call's edges. So look in the middle third. + third = (t1 - t0) / 3 + assert any(t0 + third < x < t1 - third for x in ticks) + + +# ── Exec ──────────────────────────────────────────────────────────────── + + +def test_exec_exposes_its_scheduler(): + ex = pycvc.Exec(pycvc.make_app()) + sched = ex.scheduler() + assert isinstance(sched, pycvc.async_scheduler) + assert sched.parser_defaults().step_tokens == 1024 # host_parser_defaults, as set by Exec + o = pycvc.parse_options() + o.max_depth = 40 + ex.set_parser_defaults(o) + assert sched.parser_defaults().max_depth == 40 # the same scheduler + o.max_depth = 256 + sched.set_parser_defaults(o) + assert ex.parser_defaults().max_depth == 256 + # A program submitted to it directly runs while run() drives the scheduler. + pid = sched.execute("(+ 20 22)") + assert ex.run("1") == "1" + assert sched.get_result(pid) == 42 + # run() reaps its own processes, so only ours is left. + assert [p.pid for p in sched.list_processes()] == [pid] + # The scheduler keeps its Exec alive. + del ex + assert sched.process_count() == 1 and sched.reap(pid) and sched.process_count() == 0 + + +if __name__ == "__main__": + tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] + for t in tests: + t() + print(" ok:", t.__name__) + print("pycvc state_exec scheduler parity tests: OK") diff --git a/docs/PYCVC_STATE_EXEC_PARITY.md b/docs/PYCVC_STATE_EXEC_PARITY.md index 6e9ce0813..c64e37db8 100644 --- a/docs/PYCVC_STATE_EXEC_PARITY.md +++ b/docs/PYCVC_STATE_EXEC_PARITY.md @@ -40,11 +40,28 @@ approach agreed for closing the gap, and the plan. GIL around Python callbacks, an asyncio loop for coroutines, and compute-pool offload. It uses the same value conversion as everything else. `Exec.run` raises `pycvc.parse_error` for a syntax error whether the script was parsed at submit or read inside the process. + `Exec.scheduler()` returns its `async_scheduler`, which keeps the `Exec` alive. +6. **Enums are `enum.IntEnum` classes** with the C++ names and values: + `pycvc.process_status.killed` is `process_status::killed`. An enum argument must be a member + of its enum, so a plain int stays a value: `to_string(process_status)` and `to_string(value_t)` + keep their shared C++ name. +7. **Empty results.** Where C++ returns an empty `std::optional` or a null pointer, Python gets + `None`, except for a value: `None` is nil, so `get_result`, `pop_pending_message` and + `environment::lookup` raise `KeyError(pid / path / name)` instead. +8. **Callables cross as Python callables.** Where C++ takes a `native_fn` or a + `std::function` (`builtins::register_fn`, `execute_options::on_complete`), Python passes a + callable. It runs on the thread stepping the scheduler and takes the GIL for its call; a + Python exception becomes the C++ error a throwing callback would be. `on_complete` reads back + as the callable that was set. +9. **The GIL is released while programs run:** `scheduler.run`/`step` and + `async_scheduler.sync_run`/`sync_step`, so other Python threads keep going. The schedulers' + thread safety is the C++ one: while a scheduler runs, another thread may call only + `async_scheduler.post_message`. ## Inventory The audit (2026-10-17) found 97 public items in the 20 headers. Counted individually, with the 38 -`builtin_*` functions and the 7 `MIME_*` constants, it is 140. Status after PR 1: +`builtin_*` functions and the 7 `MIME_*` constants, it is 140. Status after PR 2: | Header | Item | Python | Status | |---|---|---|---| @@ -54,23 +71,28 @@ The audit (2026-10-17) found 97 public items in the 20 headers. Counted individu | | `parse_error` (line, col) | `pycvc.parse_error` | wrapped (PR 1) | | | `script_reader` | `pycvc.script_reader` | wrapped (PR 1). `forms()`, `source()` and `options()` return copies | | | `parser` (`read`, `parse_expr`) | none | exception: it holds a `string_view` of its input, which Python can't keep alive. `script_reader` owns its source and gives the same slice-by-slice reading | +| data_job.h (#658) | `data_options`, `data_options_from_state(state&, id, base)` | `pycvc.data_options`, `pycvc.data_options_from_state(State, …)` | wrapped (#658) | +| | the data jobs (`data_job`, `make_*_job`, `run_data_job`, ...) | | PR 4 | | types.h | `symbol` | `pycvc.symbol` | wrapped (PR 1) | | | `value_t` | native types, or `pycvc.value` | wrapped (PR 1), see the table above | | | `to_string`, `values_equal` | `pycvc.to_string`, `pycvc.values_equal` | wrapped (PR 1) | | | `make_list`, `make_dict`, `make_bytes`, `nil_value`, `true_value`, `false_value` | `list`, `dict`, `bytes`, `None`, `True`, `False` | exception: the native Python values are the equivalents | -| | `environment` | | PR 2 | +| | `environment` (`bindings`, `outer`, `frozen`, `lookup`, `set`, `set_existing`, `extend`) | `pycvc.environment` | wrapped (PR 2), shared like `environment_ptr`. `bindings` reads and writes as a dict (a copy); `lookup` raises `KeyError` where C++ returns nullptr; `set` on a frozen scope (#660) raises `RuntimeError` | | | `closure`, `data_object`, `generator` | `pycvc.value` (opaque) | wrapped as opaque handles; no member access | | | `time_budget`, `eval_deadline_guard`, `eval_deadline_expired`, `thread_cpu_seconds` | none | exception: thread-local evaluator internals | | state_value_codec.h | `encode_value`, `decode_value` | `pycvc.encode_value(State, v)`, `pycvc.decode_value(State)` | wrapped (PR 1) | | | `encode_environment`/`decode_environment`, `encode_evaluator_state`/`decode_evaluator_state` | | PR 3, with the environment and evaluator types | -| scheduler_base.h | `execute_options`, `process_info`, `scheduler_stats`, `scheduling_policy` | | PR 2 | -| scheduler.h | `scheduler` (about 45 members) | | PR 2 | -| async_scheduler.h | `async_scheduler` (about 55 members) | `pycvc.async_scheduler.host_parser_defaults()` only | PR 2. Coroutine `step()`/`run()` are an exception: Python gets the existing `sync_step()`/`sync_run()` | -| process.h | `process_status`, `to_string(process_status)`, `parse_process_status` | | PR 2 | -| | `process` (about 35 fields), `make_process` | | PR 2, read-only where safe | -| builtins.h | `builtins::make_default_environment`, `builtins::register_fn` | `Exec.register_fn` | PR 2. `register_fn` takes a Python callable | +| scheduler_base.h | `execute_options`, `process_info`, `scheduler_stats` | `pycvc.execute_options`, `pycvc.process_info`, `pycvc.scheduler_stats` | wrapped (PR 2). `on_complete` is a Python callable, `signal_handlers` a dict, the `uint64_t` fields Python ints. `process_info` includes #658's steps and times by kind (`read_steps`, `eval_steps`, `data_steps`, `data_nodes`, `read_time`, `eval_time`, `data_time`) | +| | `fill_budget_usage` (#658) | none | exception: how the schedulers fill `process_info`, from the evaluator's stats | +| | `scheduling_policy` | `pycvc.scheduling_policy` | wrapped (PR 2), an `IntEnum` | +| | `scheduler_base` | none | exception: the abstract interface the intrinsics use; Python has the two concrete classes | +| scheduler.h | `scheduler` (about 45 members) | `pycvc.scheduler` | wrapped (PR 2). `run`/`step` release the GIL. `execute(str)` is the script overload, so a string can't be passed as a `value_t` expression (write it as the script `"\"text\""`). `id()`, `parser_settings()` and `data_settings()` (#658) return copies. Not wrapped: `mem_tracker()` (see memory_tracker.h), and `current_process()` / `queue_watch_event()`, which take the `process` struct (PR 3) | +| async_scheduler.h | `async_scheduler` (about 55 members) | `pycvc.async_scheduler` | wrapped (PR 2), with the same notes as `scheduler`; `sync_run`/`sync_step` release the GIL, and `parser_defaults()` returns a copy. #658's `set_data_defaults`/`data_defaults`/`data_settings`/`host_data_defaults` sit beside the parser ones. Coroutine `step()`/`run()` are an exception: Python gets `sync_step()`/`sync_run()` | +| process.h | `process_status`, `to_string(process_status)`, `parse_process_status` | `pycvc.process_status` (an `IntEnum`), `pycvc.to_string`, `pycvc.parse_process_status` | wrapped (PR 2) | +| | `process` (about 35 fields), `make_process` | | PR 3, with `evaluator_state`, which a process holds | +| builtins.h | `builtins::make_default_environment`, `builtins::register_fn` | `pycvc.builtins.make_default_environment()`, `pycvc.builtins.register_fn(env, name, fn)` | wrapped (PR 2). `fn` is a Python callable, or a `pycvc.value` holding a native fn | | | 38 `builtin_*` functions | none | exception: DSL builtins, called from DSL source | -| | `percent_encode`, `percent_decode` | | PR 2 | +| | `percent_encode`, `percent_decode` | `pycvc.percent_encode`, `pycvc.percent_decode` | wrapped (PR 2) | | stackless_evaluator.h, evaluator.h | `stackless_evaluator`, `evaluator`, `evaluator_state`, `evaluation_stats`, the evaluation exceptions | | PR 3 | | async_evaluator.h, async_stackless_evaluator.h | the async evaluators | none | exception: coroutine API | | resource_policy.h | `resource_policy`, `process_limits`, `validate_limits` | | PR 3 | @@ -86,18 +108,23 @@ The audit (2026-10-17) found 97 public items in the 20 headers. Counted individu Each PR is stacked on the previous one and includes pytest checks against the C++ behaviour. -1. **PR 1 (this):** values, parsing and the value codec. +1. **PR 1 (#661):** values, parsing and the value codec. - `pycvc_value.h`: the strict shared converter, used by `Exec` too. - `pycvc.symbol` and `pycvc.value`. - `parse`, `parse_all`, `parse_error` with position, `script_reader`, `to_string`, `values_equal`, `encode_value`, `decode_value`. - `bindings/pycvc/test_pycvc_state_exec.py`. - The pycvc README's outdated Status and Design sections are rewritten. -2. **PR 2:** `environment` and `builtins`; `execute_options`, `process_info`, `scheduler_stats` and - the enums; `scheduler` and `async_scheduler` as real classes, with their non-coroutine - methods. `Exec` exposes its scheduler. -3. **PR 3:** the evaluators and `evaluator_state`, the environment and evaluator-state codec, - `resource_policy`, and `register_intrinsics` / `intrinsics_context`. +2. **PR 2 (this):** `environment` and `builtins`; `execute_options`, `process_info`, + `scheduler_stats` and the enums; `scheduler` and `async_scheduler` as real classes, with their + non-coroutine methods. `Exec.scheduler()` returns its scheduler. + `bindings/pycvc/test_pycvc_state_exec_sched.py` repeats the C++ scheduler tests on both + classes. +3. **PR 3:** the evaluators and `evaluator_state`, the `process` struct and the scheduler calls + that take it, the environment and evaluator-state codec, `resource_policy`, and + `register_intrinsics` / `intrinsics_context`. With `register_intrinsics`, a program submitted + to `Exec.scheduler()` can get the intrinsics `Exec.run` programs have; until then an + environment built in Python has only the builtins and what `register_fn` adds. Known gaps after PR 1: - Invalid UTF-8 in a DSL string raises when it crosses to Python, but no Python test produces such