diff --git a/bindings/pycvc/CMakeLists.txt b/bindings/pycvc/CMakeLists.txt index c0f87b6a2..f92b20dd3 100644 --- a/bindings/pycvc/CMakeLists.txt +++ b/bindings/pycvc/CMakeLists.txt @@ -274,6 +274,7 @@ if(CVC_BUILD_PYCVC_CORE) pycvc_state_exec:test_pycvc_state_exec.py pycvc_state_exec_sched:test_pycvc_state_exec_sched.py pycvc_state_exec_eval:test_pycvc_state_exec_eval.py + pycvc_state_exec_data:test_pycvc_state_exec_data.py pycvc_world_clock:test_pycvc_world_clock.py pycvc_world_units:test_pycvc_world_units.py pycvc_dataprep:test_pycvc_dataprep.py diff --git a/bindings/pycvc/README.md b/bindings/pycvc/README.md index 43a21cb86..f7639d8f6 100644 --- a/bindings/pycvc/README.md +++ b/bindings/pycvc/README.md @@ -292,4 +292,5 @@ pycvc.register_intrinsics(env, ctx) # each function keeps ctx alive ``` Contract tests: `test_pycvc_exec.py`, `test_pycvc_state_exec.py`, -`test_pycvc_state_exec_sched.py`, `test_pycvc_state_exec_eval.py`. +`test_pycvc_state_exec_sched.py`, `test_pycvc_state_exec_eval.py`, +`test_pycvc_state_exec_data.py`. diff --git a/bindings/pycvc/pycvc.i b/bindings/pycvc/pycvc.i index 845acbbd5..1b32e66fb 100644 --- a/bindings/pycvc/pycvc.i +++ b/bindings/pycvc/pycvc.i @@ -1464,6 +1464,7 @@ class AsyncStateObserver(state_observer): #include #include #include +#include #include #include #include @@ -1472,6 +1473,7 @@ class AsyncStateObserver(state_observer): #include #include #include +#include #include #include "pycvc_value.h" #include @@ -1485,10 +1487,18 @@ class AsyncStateObserver(state_observer): #include #include #include +#include #include #include #include +// SWIG passes by-value arguments with SWIG_STD_MOVE, which is std::move only when __cplusplus says +// C++11; MSVC reports 199711L without /Zc:__cplusplus, so there it copies, and a std::unique_ptr +// (a data job given to run_data_job) can't be copied. Move, as on GCC and Clang, for the wrapper +// functions, which come after this. +#undef SWIG_STD_MOVE +#define SWIG_STD_MOVE(OBJ) std::move(OBJ) + namespace pycvc { // The evaluators' on_complete parameter type, under a name SWIG can match a typemap on. using value_callback = std::function; @@ -1503,7 +1513,6 @@ using value_callback = std::function; namespace pycvc { namespace { using cvc::state_exec::value_t; -constexpr int kMaxValueDepth = 1000; swig_type_info *symbol_swig_type() { static swig_type_info *t = SWIG_TypeQuery("cvc::state_exec::symbol *"); @@ -1514,22 +1523,20 @@ swig_type_info *value_swig_type() { return t; } -bool too_deep(int depth) { - if (depth <= kMaxValueDepth) - return false; - PyErr_SetString(PyExc_ValueError, - "pycvc: value nests deeper than 1000 levels (or contains itself)"); - return true; -} - PyObject *text_to_py(const std::string &s) { return PyUnicode_DecodeUTF8(s.data(), static_cast(s.size()), "strict"); } -PyObject *to_py(const value_t &v, int depth) { +// The two directions walk a value with an explicit stack rather than recursing, so a value nested +// any depth converts (a parsed script may nest 100000 deep). A list or dict shared within a value +// converts once, to one object, so sharing survives the trip and a shared structure converts in +// time proportional to its size: a DSL list that holds itself becomes a Python list that holds +// itself. The other way, a Python value that holds itself raises ValueError, because only the +// evaluator's own containers are tracked for cycle collection (heap.h). + +// Everything but a list or dict: a new reference, or nullptr with an error set. +PyObject *leaf_to_py(const value_t &v) { using namespace cvc::state_exec; - if (too_deep(depth)) - return nullptr; if (v.is_nil()) Py_RETURN_NONE; if (const auto *b = std::get_if(&v.v)) @@ -1544,112 +1551,216 @@ PyObject *to_py(const value_t &v, int depth) { return PyBytes_FromStringAndSize(by->data.data(), static_cast(by->data.size())); if (const auto *sym = std::get_if(&v.v)) return SWIG_NewPointerObj(new symbol(*sym), symbol_swig_type(), SWIG_POINTER_OWN); - if (const auto *l = std::get_if(&v.v)) { - const Py_ssize_t n = *l ? static_cast((*l)->size()) : 0; - PyObject *out = PyList_New(n); + // closure / native_fn / data_object / generator: an opaque handle that converts back to the + // same value, so Python can hold it and pass it into the DSL again. + return SWIG_NewPointerObj(new value_t(v), value_swig_type(), SWIG_POINTER_OWN); +} + +PyObject *to_py(const value_t &root) { + using namespace cvc::state_exec; + struct frame { + const value_t *v; // a list or dict + PyObject *out; // its Python object, being filled + size_t next; // the next element to convert + size_t size; // the Python list's length + }; + std::vector stack; + std::unordered_map made; // container storage -> its Python object + // A new reference to `v` in Python. A list or dict not made yet is created empty and pushed to + // be filled; one already made (shared, or an enclosing one) is the same object. + auto convert = [&](const value_t &v) -> PyObject * { + const auto *l = std::get_if(&v.v); + const auto *d = std::get_if(&v.v); + if (l == nullptr && d == nullptr) + return leaf_to_py(v); + const void *storage = l ? static_cast(l->get()) : d->get(); + if (storage == nullptr) + return l ? PyList_New(0) : PyDict_New(); + if (const auto it = made.find(storage); it != made.end()) { + Py_INCREF(it->second); + return it->second; + } + const size_t size = l ? (*l)->size() : 0; + PyObject *out = l ? PyList_New(static_cast(size)) : PyDict_New(); if (out == nullptr) return nullptr; - for (Py_ssize_t k = 0; k < n; ++k) { - PyObject *item = to_py((**l)[static_cast(k)], depth + 1); + made.emplace(storage, out); + stack.push_back({&v, out, 0, size}); + return out; + }; + PyObject *result = convert(root); + if (result == nullptr) + return nullptr; + while (!stack.empty()) { + const size_t top = stack.size() - 1; // an index: convert() may grow the stack + const value_t &v = *stack[top].v; + PyObject *out = stack[top].out; + const size_t i = stack[top].next++; + if (const auto *l = std::get_if(&v.v)) { + if (i == stack[top].size) { + stack.pop_back(); + continue; + } + // The list can't change while the GIL is held, but a slot is filled with None rather than + // left empty should it be shorter than the Python list made for it. + PyObject *item = i < (*l)->size() ? convert((**l)[i]) : (Py_INCREF(Py_None), Py_None); if (item == nullptr) { - Py_DECREF(out); + Py_DECREF(result); return nullptr; } - PyList_SET_ITEM(out, k, item); // steals the new reference - } - return out; - } - if (const auto *dd = std::get_if(&v.v)) { - PyObject *out = PyDict_New(); - if (out == nullptr) - return nullptr; - if (*dd) { - for (const auto &kv : **dd) { - PyObject *key = text_to_py(kv.first); - PyObject *val = key ? to_py(kv.second, depth + 1) : nullptr; - const int rc = val ? PyDict_SetItem(out, key, val) : -1; - Py_XDECREF(key); - Py_XDECREF(val); - if (rc < 0) { - Py_DECREF(out); - return nullptr; - } + PyList_SET_ITEM(out, static_cast(i), item); // steals the reference + } else { + const auto &entries = **std::get_if(&v.v); + if (i >= entries.size()) { + stack.pop_back(); + continue; + } + PyObject *key = text_to_py(entries[i].first); + PyObject *val = key ? convert(entries[i].second) : nullptr; + const int rc = val ? PyDict_SetItem(out, key, val) : -1; + Py_XDECREF(key); + Py_XDECREF(val); + if (rc < 0) { + Py_DECREF(result); + return nullptr; } } - return out; } - // closure / native_fn / data_object / generator: an opaque handle that converts back to the - // same value, so Python can hold it and pass it into the DSL again. - return SWIG_NewPointerObj(new value_t(v), value_swig_type(), SWIG_POINTER_OWN); + return result; } -bool from_py(PyObject *o, value_t &out, int depth) { +// 1: `o` converted into `out`; 0: `o` is a list, tuple or dict, which from_py walks; -1: an error +// is set. +int leaf_from_py(PyObject *o, value_t &out) { using namespace cvc::state_exec; - if (too_deep(depth)) - return false; if (o == nullptr || o == Py_None) { out = value_t(); - return true; + return 1; } if (PyBool_Check(o)) { // before int: bool is an int subclass out = value_t(o == Py_True); - return true; + return 1; } if (PyLong_Check(o)) { int overflow = 0; const long long x = PyLong_AsLongLongAndOverflow(o, &overflow); if (overflow != 0) { PyErr_SetString(PyExc_OverflowError, "pycvc: int does not fit in 64 bits"); - return false; + return -1; } if (x == -1 && PyErr_Occurred()) - return false; + return -1; out = value_t(static_cast(x)); - return true; + return 1; } if (PyFloat_Check(o)) { const double d = PyFloat_AsDouble(o); if (d == -1.0 && PyErr_Occurred()) - return false; + return -1; out = value_t(d); - return true; + return 1; } if (PyUnicode_Check(o)) { Py_ssize_t n = 0; const char *s = PyUnicode_AsUTF8AndSize(o, &n); // fails (and raises) on lone surrogates if (s == nullptr) - return false; + return -1; out = value_t(std::string(s, static_cast(n))); - return true; + return 1; } if (PyBytes_Check(o)) { out = make_bytes(std::string(PyBytes_AS_STRING(o), static_cast(PyBytes_GET_SIZE(o)))); - return true; + return 1; } if (PyByteArray_Check(o)) { out = make_bytes( std::string(PyByteArray_AS_STRING(o), static_cast(PyByteArray_GET_SIZE(o)))); - return true; + return 1; } - if (PyList_Check(o) || PyTuple_Check(o)) { - const bool tup = PyTuple_Check(o); - const Py_ssize_t n = tup ? PyTuple_GET_SIZE(o) : PyList_GET_SIZE(o); - std::vector elems; - elems.reserve(static_cast(n)); - for (Py_ssize_t k = 0; k < n; ++k) { - value_t e; - if (!from_py(tup ? PyTuple_GET_ITEM(o, k) : PyList_GET_ITEM(o, k), e, depth + 1)) - return false; - elems.push_back(std::move(e)); + if (PyList_Check(o) || PyTuple_Check(o) || PyDict_Check(o)) + return 0; + void *p = nullptr; + if (SWIG_IsOK(SWIG_ConvertPtr(o, &p, symbol_swig_type(), 0)) && p != nullptr) { + out = value_t(*static_cast(p)); + return 1; + } + if (SWIG_IsOK(SWIG_ConvertPtr(o, &p, value_swig_type(), 0)) && p != nullptr) { + out = *static_cast(p); + return 1; + } + PyErr_Clear(); // a failed proxy lookup above may leave an attribute error behind + PyErr_Format(PyExc_TypeError, "pycvc: cannot convert %.200s to a state_exec value", + Py_TYPE(o)->tp_name); + return -1; +} + +bool from_py(PyObject *root, value_t &out) { + using namespace cvc::state_exec; + const int leaf = leaf_from_py(root, out); + if (leaf != 0) + return leaf > 0; + struct frame { + PyObject *o; // a list, tuple or dict + value_t v; // its value, being filled (shares its storage with the value handed out) + Py_ssize_t pos; + }; + std::vector stack; + std::unordered_map made; // container -> its value + std::unordered_set filling; // containers on the stack: met again, a cycle + // References on every container made, so none goes while its value is in `made` (a lookup of a + // proxy can run Python code). Dropped when this returns. + struct holds { + std::vector objects; + ~holds() { + for (PyObject *o : objects) + Py_DECREF(o); + } + } held; + // `o`'s value: one already made is shared; otherwise an empty list or dict, pushed to be filled. + auto container = [&](PyObject *o, value_t &v) -> bool { + if (filling.count(o) != 0) { + PyErr_SetString(PyExc_ValueError, "pycvc: value contains itself"); + return false; + } + if (const auto it = made.find(o); it != made.end()) { + v = it->second; + return true; } - out = make_list(std::move(elems)); + if (PyDict_Check(o)) { + v = make_dict(); + } else { + v = make_list(); + std::get(v.v)->reserve( + static_cast(PyTuple_Check(o) ? PyTuple_GET_SIZE(o) : PyList_GET_SIZE(o))); + } + Py_INCREF(o); + held.objects.push_back(o); + made.emplace(o, v); + filling.insert(o); + stack.push_back({o, v, 0}); return true; - } - if (PyDict_Check(o)) { - std::vector> entries; - PyObject *k = nullptr, *v = nullptr; - Py_ssize_t pos = 0; - while (PyDict_Next(o, &pos, &k, &v)) { + }; + // An element's value, into `v`. + auto element = [&](PyObject *item, value_t &v) -> bool { + Py_INCREF(item); // the container could drop it while a proxy lookup runs Python code + const int r = leaf_from_py(item, v); + const bool ok = r > 0 || (r == 0 && container(item, v)); + Py_DECREF(item); + return ok; + }; + if (!container(root, out)) + return false; + while (!stack.empty()) { + const size_t top = stack.size() - 1; // an index: container() may grow the stack + PyObject *o = stack[top].o; + if (PyDict_Check(o)) { + PyObject *k = nullptr; + PyObject *item = nullptr; + if (!PyDict_Next(o, &stack[top].pos, &k, &item)) { + filling.erase(o); + stack.pop_back(); + continue; + } if (!PyUnicode_Check(k)) { PyErr_Format(PyExc_TypeError, "pycvc: dict keys must be str, not %.200s", Py_TYPE(k)->tp_name); @@ -1659,38 +1770,38 @@ bool from_py(PyObject *o, value_t &out, int depth) { const char *ks = PyUnicode_AsUTF8AndSize(k, &n); if (ks == nullptr) return false; + std::string key(ks, static_cast(n)); + value_t e; + if (!element(item, e)) + return false; + std::get(stack[top].v.v)->emplace_back(std::move(key), std::move(e)); + } else { + const bool tup = PyTuple_Check(o); + const Py_ssize_t i = stack[top].pos; + if (i >= (tup ? PyTuple_GET_SIZE(o) : PyList_GET_SIZE(o))) { + filling.erase(o); + stack.pop_back(); + continue; + } + stack[top].pos = i + 1; value_t e; - if (!from_py(v, e, depth + 1)) + if (!element(tup ? PyTuple_GET_ITEM(o, i) : PyList_GET_ITEM(o, i), e)) return false; - entries.emplace_back(std::string(ks, static_cast(n)), std::move(e)); + std::get(stack[top].v.v)->push_back(std::move(e)); } - out = make_dict(std::move(entries)); - return true; - } - void *p = nullptr; - if (SWIG_IsOK(SWIG_ConvertPtr(o, &p, symbol_swig_type(), 0)) && p != nullptr) { - out = value_t(*static_cast(p)); - return true; - } - if (SWIG_IsOK(SWIG_ConvertPtr(o, &p, value_swig_type(), 0)) && p != nullptr) { - out = *static_cast(p); - return true; } - PyErr_Clear(); // a failed proxy lookup above may leave an attribute error behind - PyErr_Format(PyExc_TypeError, "pycvc: cannot convert %.200s to a state_exec value", - Py_TYPE(o)->tp_name); - return false; + return true; } } // namespace -PyObject *value_to_py(const cvc::state_exec::value_t &v) { return to_py(v, 0); } +PyObject *value_to_py(const cvc::state_exec::value_t &v) { return to_py(v); } PyObject *values_to_py_tuple(std::span values) { PyObject *t = PyTuple_New(static_cast(values.size())); if (t == nullptr) return nullptr; for (size_t i = 0; i < values.size(); ++i) { - PyObject *item = to_py(values[i], 0); + PyObject *item = to_py(values[i]); if (item == nullptr) { Py_DECREF(t); return nullptr; @@ -1700,7 +1811,7 @@ PyObject *values_to_py_tuple(std::span values) { return t; } -bool py_to_value(PyObject *o, cvc::state_exec::value_t &out) { return from_py(o, out, 0); } +bool py_to_value(PyObject *o, cvc::state_exec::value_t &out) { return from_py(o, out); } } // namespace pycvc %} @@ -2270,6 +2381,37 @@ bool py_to_macro_map(PyObject *o, macro_map &out) { %typemap(typecheck, precedence=SWIG_TYPECHECK_POINTER) pycvc::value_callback "$1 = ($input == Py_None || PyCallable_Check($input)) ? 1 : 0;" +// ── native functions from Python, and the names C++ keeps ── +%{ +#include +namespace pycvc { +namespace { +// A native_fn from `fn`: the native function a pycvc.value holds, or a Python callable wrapped by +// python_native_fn (which throws std::invalid_argument, prefixed with `who`, for anything else). +cvc::state_exec::native_fn native_from_py(const std::string &who, const std::string &name, + PyObject *fn) { + void *p = nullptr; + if (SWIG_IsOK(SWIG_ConvertPtr(fn, &p, value_swig_type(), 0)) && p != nullptr) { + const auto &held = *static_cast(p); + if (const auto *native = std::get_if(&held.v)) + return *native; + } + return python_native_fn(who, name, fn); +} + +// A `who` that lives for good. C++ keeps the const char* a job or call is given (for its errors, +// which can come after the call returns), where C++ passes a string literal; Python's string +// would go. Bounded by the distinct names used. +const char *stable_who(const std::string &who) { + static std::mutex mu; + static std::set names; // node-based: c_str() stays put + std::lock_guard lock(mu); + return names.insert(who).first->c_str(); +} +} // namespace +} // namespace pycvc +%} + // ── enums: pycvc.process_status and pycvc.scheduling_policy are enum.IntEnum classes ── // pycvc.process_status.killed is process_status::killed. An enum argument must be a member of its // IntEnum (strict, like values), which is also what lets to_string(process_status) and @@ -2573,6 +2715,10 @@ PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::evaluate) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::stackless_evaluator::evaluate_script) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::evaluator::evaluate) PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::evaluator::evaluate_script) +// And the long walks: a job run to completion, a heap measurement, a cycle collection. +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::run_to_completion) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::measure_heap_share) +PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::collect_cycles) // Shared as in C++ (environment_ptr, process_ptr, ...). Struct members that are maps, lists or // values read and write as Python values (copies). @@ -2580,6 +2726,10 @@ PYCVC_STATE_EXEC_NOGIL(cvc::state_exec::evaluator::evaluate_script) %shared_ptr(cvc::state_exec::script_reader) %shared_ptr(cvc::state_exec::process) %shared_ptr(cvc::state_exec::intrinsics_context) +// A data job is owned (std::unique_ptr): Python owns one it is handed, and gives it up to +// run_data_job. +%include +%unique_ptr(cvc::state_exec::data_job) %naturalvar cvc::state_exec::value_tag; %naturalvar cvc::state_exec::environment::bindings; %naturalvar cvc::state_exec::execute_options::signal_handlers; @@ -2724,16 +2874,8 @@ public: static void register_fn(environment_ptr env, const std::string &name, PyObject *fn) { if (!env) throw std::invalid_argument("pycvc.builtins.register_fn: env is None"); - cvc::state_exec::value_t held; - void *p = nullptr; - if (SWIG_IsOK(SWIG_ConvertPtr(fn, &p, pycvc::value_swig_type(), 0)) && p != nullptr) - held = *static_cast(p); - if (const auto *native = std::get_if(&held.v)) { - cvc::state_exec::builtins::register_fn(env, name, *native); - return; - } cvc::state_exec::builtins::register_fn( - env, name, pycvc::python_native_fn("pycvc.builtins.register_fn", name, fn)); + env, name, pycvc::native_from_py("pycvc.builtins.register_fn", name, fn)); } } @@ -2821,8 +2963,18 @@ struct evaluator_state { }; %extend evaluator_state { evaluation_stats *_stats() { return &$self->stats; } + // A heap measurement's result while evaluation waits to resume (#667), or None. Read-only: it + // belongs to the measurement in flight. + PyObject *_heap_measured() const { + if (!$self->heap_measured) { + Py_INCREF(Py_None); + return Py_None; + } + return PyLong_FromUnsignedLongLong(static_cast(*$self->heap_measured)); + } %pythoncode %{ stats = property(_stats) + heap_measured = property(_heap_measured) %} } @@ -2858,6 +3010,7 @@ public: bool is_paused() const; void restrict_special_forms(std::shared_ptr > allowed); static const std::set &all_special_forms(); + environment_ptr global_env() const; // (#667) }; // Snapshot codec for an environment chain and for a whole evaluator state. @@ -2945,6 +3098,9 @@ struct process_info { double read_time; double eval_time; double data_time; + // The measured heap share (#667, heap.h), which max_memory covers too. + unsigned long long heap_memory; + unsigned long long heap_measurements; }; %extend process_info { %pythoncode %{ @@ -2985,6 +3141,7 @@ const size_t kDefaultMaxAllocBytes = 16777216; std::optional eval_alloc_limit(); void eval_alloc_check(size_t bytes, const char *who); size_t alloc_bytes(size_t count, size_t each); +void eval_alloc_note(size_t bytes); // (#667) report bytes allocated, for the heap's re-measures %pythonappend cvc::state_exec::process::_state %{ val._pycvc_owner = self %} @@ -3030,6 +3187,15 @@ struct process { unsigned long long message_count; unsigned long long message_bytes; int parent_pid; + // The heap share (#667, heap.h). + unsigned long long heap_bytes; + unsigned long long heap_alloc_bytes; + unsigned long long heap_steps; + unsigned long long heap_measurements; + size_t heap_check_steps; + bool heap_measuring; + bool heap_measure_due(unsigned long long state_bytes) const; + void release_state(); double elapsed_time() const; bool charges_cpu_time() const; void charge_cpu_time(double seconds); @@ -3254,9 +3420,168 @@ void apply_chroot(intrinsics_context &ctx, cvc::state &tree_root, const std::str std::string resolve_channel_key(const std::string &root_path, const std::string &channel); value_t make_future(const std::string &channel); std::optional future_channel_of(const value_t &v); + +// ── the heap (#667, heap.h) ── +// A process's heap share, and the per-thread reference-cycle collector, which a Python driver can +// run between scheduler runs on the thread that ran them. Not wrapped: heap_after_step (the +// schedulers' own per-step accounting) and track_captured_env / track_mutated (a value from +// Python is a fresh copy, so tracking it would mean nothing; the evaluator tracks its own). +struct heap_options { + size_t check_steps; +}; +heap_options heap_options_from_state(cvc::state &root, + const std::string &scheduler_id = std::string(), + const heap_options &base = heap_options()); +size_t measure_heap_share(const process &proc); +size_t heap_remeasure_bytes(size_t last_measured); +struct cycle_stats { + size_t candidates; + size_t examined; + size_t freed; +}; +cycle_stats collect_cycles(); +bool cycle_collection_due(); +void note_process_exit(); +size_t tracked_objects(); + +// ── deep-data jobs (#658, #665, data_job.h) ── +// A job runs a slice at a time: step(nodes) until it says it's done, then result(). The factories +// hand Python an owned job. From a Python function a program calls (it runs on the stepping +// thread), run_data_job and call_function work as for a C++ native: return what they return at +// once, and the evaluator finishes the call. Not wrapped: the call protocol for C++ job +// implementations (call_request, take_call, pending_call, call_returned, own_inputs) and +// detail::. make_concat_job, run_data_job and call_function are below, in pycvc's own wrappers. +%nodefaultctor data_job; +class data_job { +public: + virtual ~data_job(); + virtual bool step(size_t nodes) = 0; + virtual value_t result() = 0; + size_t nodes() const; +}; +const size_t kDataNodeBytes = 256; +// The native-call nesting limits (#682): a level is refused past kMaxNativeNesting, or when less +// than kNativeStackReserve bytes of the thread's stack are left (128 KiB on Windows, 32 KiB +// elsewhere). detail::native_nesting, the RAII level, isn't wrapped. +const size_t kMaxNativeNesting = 256; +const size_t kNativeStackReserve = 131072; +bool is_compound(const value_t &v); +std::unique_ptr make_equal_job(value_t a, value_t b); +std::unique_ptr make_print_job(value_t v); +std::unique_ptr make_copy_job(value_t v); +std::unique_ptr make_substitute_job(value_t tmpl, + const std::unordered_map &subst); +std::unique_ptr make_heap_share_job(const process &proc); +value_t run_to_completion(data_job &job); +void count_data_nodes(size_t nodes); + +// ── stdlib_registry (stdlib.h) ── +// standard() returns a copy (C++ returns a const reference to the shared registry), so Python +// can't add to the one every environment shares. register_function takes a Python callable or a +// pycvc.value holding a native function, as builtins.register_fn does. lookup_qualified returns +// the function as a pycvc.value, or raises KeyError where C++ returns nullptr. +class stdlib_registry { +public: + stdlib_registry(); + static environment_ptr shared_scope(); + std::vector list_functions(const std::string &module) const; + std::vector list_modules() const; +}; +%extend stdlib_registry { + static cvc::state_exec::stdlib_registry standard() { + return cvc::state_exec::stdlib_registry::standard(); + } + void register_function(const std::string &module, const std::string &func_name, PyObject *fn) { + $self->register_function( + module, func_name, + pycvc::native_from_py("pycvc.stdlib_registry.register_function", func_name, fn)); + } + void import_module(const std::string &module, environment_ptr env, + const std::vector &specific_fns = std::vector()) { + if (!env) + throw std::invalid_argument("pycvc.stdlib_registry.import_module: env is None"); + $self->import_module(module, env, specific_fns); + } + value_t lookup_qualified(const std::string &qualified_name) const { + if (const cvc::state_exec::native_fn *fn = $self->lookup_qualified(qualified_name)) + return cvc::state_exec::value_t(*fn); + throw pycvc::no_value{qualified_name}; + } +} } } +// make_concat_job, run_data_job and call_function: the C++ calls, except that `who` is kept for +// good (C++ holds the pointer for its errors), and run_data_job's `then` is a Python callable or +// None. +%{ +namespace pycvc { +std::unique_ptr +make_concat_job_kept(const std::vector &parts, const std::string &sep, + const std::string &who) { + return cvc::state_exec::make_concat_job(parts, sep, stable_who(who)); +} + +cvc::state_exec::value_t run_data_job_then(std::unique_ptr job, + PyObject *then = nullptr) { + if (!job) + throw std::invalid_argument("pycvc.run_data_job: job is None"); + std::function after; + if (then != nullptr && then != Py_None) { + cvc::state_exec::native_fn fn = python_native_fn("pycvc.run_data_job", "then", then); + after = [fn](cvc::state_exec::value_t v) { + return fn(std::span(&v, 1)); + }; + } + return cvc::state_exec::run_data_job(std::move(job), std::move(after)); +} + +cvc::state_exec::value_t call_function_kept(const cvc::state_exec::value_t &fn, + const std::vector &args, + const std::string &who, PyObject *then = nullptr) { + const std::span argv(args); + if (then == nullptr || then == Py_None) + return cvc::state_exec::call_function(fn, argv, stable_who(who)); + // The overload with a continuation (#683): `then` runs on the call's value once the function + // has finished, however the evaluator ran it. + cvc::state_exec::native_fn after = python_native_fn("pycvc.call_function", "then", then); + return cvc::state_exec::call_function( + fn, argv, stable_who(who), [after](cvc::state_exec::value_t v) { + return after(std::span(&v, 1)); + }); +} + +// make_generator_job takes the generator as a pycvc.value (C++ takes its generator_ptr). +std::unique_ptr make_generator_job_from(const cvc::state_exec::value_t &gen, + bool collect = false) { + const auto *g = std::get_if(&gen.v); + if (g == nullptr || !*g) + throw std::invalid_argument("pycvc.make_generator_job: not a generator"); + return cvc::state_exec::make_generator_job(*g, collect); +} +} // namespace pycvc +%} +%rename(make_concat_job) pycvc::make_concat_job_kept; +%rename(run_data_job) pycvc::run_data_job_then; +%rename(call_function) pycvc::call_function_kept; +%rename(make_generator_job) pycvc::make_generator_job_from; +// One wrapper each, defaults filled in, rather than an overload per default (a value list has no +// overload typecheck). +%feature("compactdefaultargs") pycvc::call_function_kept; +%feature("compactdefaultargs") pycvc::make_generator_job_from; +namespace pycvc { +std::unique_ptr +make_concat_job_kept(const std::vector &parts, const std::string &sep, + const std::string &who); +cvc::state_exec::value_t run_data_job_then(std::unique_ptr job, + PyObject *then = nullptr); +cvc::state_exec::value_t call_function_kept(const cvc::state_exec::value_t &fn, + const std::vector &args, + const std::string &who, PyObject *then = nullptr); +std::unique_ptr make_generator_job_from(const cvc::state_exec::value_t &gen, + bool collect = false); +} + // register_intrinsics(env, ctx): the C++ call, except that every function it adds keeps ctx (and // so the scheduler and channel policy ctx holds) alive. C++ requires the caller to keep the context // valid instead; in Python an environment may outlive any object. diff --git a/bindings/pycvc/pycvc_value.h b/bindings/pycvc/pycvc_value.h index 5fb0fa844..f346fea8c 100644 --- a/bindings/pycvc/pycvc_value.h +++ b/bindings/pycvc/pycvc_value.h @@ -12,8 +12,10 @@ // - closure, native_fn, data_object, generator -> pycvc.value, an opaque handle that converts // back to the same value, so it can be passed into the DSL again. // -// Strict: an int outside 64 bits raises OverflowError, any other Python type raises TypeError, and -// a value nested deeper than 1000 levels (or a cyclic one) raises ValueError. Defined in the SWIG +// Strict: an int outside 64 bits raises OverflowError, and any other Python type raises TypeError. +// Any depth converts (the conversion doesn't recurse). A list or dict shared within a value +// converts to one shared object, both ways, so a DSL value that holds itself becomes a Python value +// that holds itself; a Python value that holds itself raises ValueError. Defined in the SWIG // wrapper (pycvc.i), which owns the pycvc.symbol / pycvc.value types. Call with the GIL held. #pragma once diff --git a/bindings/pycvc/test_pycvc_state_exec.py b/bindings/pycvc/test_pycvc_state_exec.py index 27c11b294..4aacb479b 100644 --- a/bindings/pycvc/test_pycvc_state_exec.py +++ b/bindings/pycvc/test_pycvc_state_exec.py @@ -104,13 +104,31 @@ def test_conversion_is_strict(): _raises(TypeError, pycvc.to_string, {1: 2}) # dict keys must be str _raises(OverflowError, pycvc.to_string, 2**70) _raises(TypeError, pycvc.to_string, {1, 2}) # a set has no state_exec equivalent - deep = [] - for _ in range(1100): - deep = [deep] - _raises(ValueError, pycvc.to_string, deep) loop = [] loop.append(loop) - _raises(ValueError, pycvc.to_string, loop) + _raises(ValueError, pycvc.to_string, loop) # only the evaluator's own cycles are collected + + +def test_conversion_takes_any_depth_and_keeps_sharing(): + # The conversion walks with an explicit stack, so a value nests any depth, both ways: here + # Python -> C++ (the quoted argument) -> Python (the result), 200000 deep. + deep = [] + for _ in range(200000): + deep = [deep] + ev = pycvc.stackless_evaluator(pycvc.builtins.make_default_environment()) + back = ev.evaluate([symbol("quote"), deep]) + depth = 0 + while back: + (back,) = back + depth += 1 + assert depth == 200000 + # A list shared within a value stays one object, both ways. + shared = [1, 2] + pair = ev.evaluate([symbol("quote"), [shared, shared]]) + assert pair[0] is pair[1] and pair[0] == [1, 2] + # A DSL list that holds itself comes out as a Python list that holds itself. + looped = ev.evaluate_script("(begin (set l (list 1)) (set-nth l 0 l) l)") + assert looped[0] is looped def test_registered_fns_get_symbols_and_strict_returns(): diff --git a/bindings/pycvc/test_pycvc_state_exec_data.py b/bindings/pycvc/test_pycvc_state_exec_data.py new file mode 100644 index 000000000..a94c89f40 --- /dev/null +++ b/bindings/pycvc/test_pycvc_state_exec_data.py @@ -0,0 +1,198 @@ +"""state_exec parity, part 4: deep-data jobs, the heap, and the stdlib registry. + +The deep-data jobs (data_job.h, #658/#665), the heap share and reference-cycle collector +(heap.h, #667) and stdlib_registry, under the C++ names. The value conversion itself has no +depth limit and keeps sharing (test_pycvc_state_exec.py). docs/PYCVC_STATE_EXEC_PARITY.md has +the inventory. +""" + +import pycvc +from pycvc import symbol + + +def _raises(exc, fn, *args): + try: + fn(*args) + except exc as e: + return e + raise AssertionError(f"{fn.__name__}{args!r} did not raise {exc.__name__}") + + +def _builtins(): + return pycvc.builtins.make_default_environment() + + +# ── deep-data jobs ────────────────────────────────────────────────────── + + +def test_a_job_runs_a_slice_at_a_time(): + job = pycvc.make_copy_job([1, [2, [3, [4]]]]) + assert isinstance(job, pycvc.data_job) + slices = 1 + while not job.step(1): + slices += 1 + assert slices > 1 and job.nodes() > 0 + assert job.result() == [1, [2, [3, [4]]]] + + +def test_the_job_factories(): + run = pycvc.run_to_completion + assert run(pycvc.make_equal_job([1, [2]], [1, [2]])) is True + assert run(pycvc.make_equal_job([1], [2])) is False + assert run(pycvc.make_print_job([symbol("a"), 1])) == pycvc.to_string([symbol("a"), 1]) + assert run(pycvc.make_concat_job(["a", 1, "b"], "-", "py-join")) == "a-1-b" + tmpl = [symbol("+"), symbol("x"), [symbol("x")]] + assert run(pycvc.make_substitute_job(tmpl, {"x": 41})) == [symbol("+"), 41, [41]] + assert pycvc.is_compound([1]) and not pycvc.is_compound([]) and not pycvc.is_compound(5) + assert pycvc.kDataNodeBytes == 256 + pycvc.count_data_nodes(3) # outside an evaluator step: nothing to count against + + +def test_run_data_job_outside_an_evaluator_runs_it_whole(): + assert pycvc.run_data_job(pycvc.make_copy_job([1, 2])) == [1, 2] + assert pycvc.run_data_job(pycvc.make_copy_job([1, 2]), len) == 2 # `then` maps the result + job = pycvc.make_copy_job([1]) + pycvc.run_data_job(job) # the job is given up to C++ + assert not job.thisown + + +def test_a_python_function_calls_dsl_functions(): + # call_function from a Python function a program calls: a native function is called + # there; a lambda runs as the process's own frames, and the call returns its result. + def apply_one(fn, arg): + return pycvc.call_function(fn, [arg], "py-apply") + + env = _builtins() + pycvc.builtins.register_fn(env, "apply-one", apply_one) + for sched in (pycvc.scheduler(), pycvc.async_scheduler()): + run = sched.run if isinstance(sched, pycvc.scheduler) else sched.sync_run + o = pycvc.execute_options() + o.env = env + native = sched.execute("(apply-one string.length \"four\")", o) + closure = sched.execute("(apply-one (lambda (x) (* x 2)) 21)", o) + results = run() + assert results[native] == 4 and results[closure] == 42 + # Outside a running program a lambda can't be called. + lam = pycvc.stackless_evaluator(_builtins()).evaluate_script("(lambda (x) x)") + e = _raises(RuntimeError, pycvc.call_function, lam, [1], "py-outside") + assert "py-outside" in str(e) + + +def test_call_function_with_a_continuation(): + # The #683 overload: `then` gets the call's value once the function has run, however the + # evaluator ran it (a lambda runs as frames, so `then` comes after them). + def apply_then(fn, arg): + return pycvc.call_function(fn, [arg], "py-then", lambda v: v + 1) + + env = _builtins() + pycvc.builtins.register_fn(env, "apply-then", apply_then) + s = pycvc.scheduler() + o = pycvc.execute_options() + o.env = env + pid = s.execute("(apply-then (lambda (x) (* x 2)) 20)", o) + assert s.run()[pid] == 41 + + +def test_generator_jobs_and_nesting_limits(): + ev = pycvc.stackless_evaluator(_builtins()) + gen = ev.evaluate_script( + "(generator (lambda () (begin (yield 1) (yield 2) (yield 3))))") + assert isinstance(gen, pycvc.value) and gen.type_name() == "generator" + assert pycvc.run_to_completion(pycvc.make_generator_job(gen)) == [1] # the next value + assert pycvc.run_to_completion(pycvc.make_generator_job(gen, True)) == [2, 3] # the rest + assert pycvc.run_to_completion(pycvc.make_generator_job(gen)) == [] # exhausted + _raises(RuntimeError, pycvc.make_generator_job, 5) + assert pycvc.kMaxNativeNesting == 256 + assert pycvc.kNativeStackReserve in (128 << 10, 32 << 10) # Windows, elsewhere + + +# ── the heap ──────────────────────────────────────────────────────────── + + +def test_heap_settings_and_fields(): + app = pycvc.make_app() + root = pycvc.State.instance(app) + assert pycvc.heap_options().check_steps == 4096 + root.child("state_exec.defaults.limits.heap_check_steps").value("100") + root.child("state_exec.schedulers.s1.limits.heap_check_steps").value("50") + assert pycvc.heap_options_from_state(root).check_steps == 100 + assert pycvc.heap_options_from_state(root, "s1").check_steps == 50 + assert pycvc.heap_remeasure_bytes(0) == 64 << 10 + assert pycvc.heap_remeasure_bytes(1 << 20) == 1 << 19 # half the last measurement + p = pycvc.make_process() + assert (p.heap_bytes, p.heap_measurements, p.heap_check_steps) == (0, 0, 4096) + assert not p.heap_measuring and not p.heap_measure_due(0) # no memory budget + assert isinstance(pycvc.measure_heap_share(p), int) + assert pycvc.run_to_completion(pycvc.make_heap_share_job(p)) == pycvc.measure_heap_share(p) + p.release_state() + assert p.state.heap_measured is None + pycvc.eval_alloc_note(10) # outside a step: nothing armed to note it + + +def test_a_budgeted_process_is_measured(): + for sched in (pycvc.scheduler(), pycvc.async_scheduler()): + run = sched.run if isinstance(sched, pycvc.scheduler) else sched.sync_run + o = pycvc.execute_options() + o.max_memory = 64 << 20 + pid = sched.execute("(begin (set big (collections.range 50000)) (length big))", o) + assert run()[pid] == 50000 + info = sched.get_process_info(pid) + assert info.heap_measurements >= 1 and info.heap_memory > 0 + + +def test_cycles_are_collected_on_this_thread(): + ev = pycvc.stackless_evaluator(_builtins()) + ev.evaluate_script( + "(begin (set i 0) (while (< i 300) (begin (let ((h nil)) (set h (lambda () h)))" + " (set i (+ i 1)))) 0)") + assert pycvc.tracked_objects() > 0 + stats = pycvc.collect_cycles() + assert isinstance(stats, pycvc.cycle_stats) and stats.freed > 0 + assert isinstance(pycvc.cycle_collection_due(), bool) + pycvc.note_process_exit() + + +def test_evaluator_global_env(): + env = _builtins() + ev = pycvc.stackless_evaluator(env) + assert ev.global_env().lookup("+") is not None + + +# ── stdlib_registry ───────────────────────────────────────────────────── + + +def test_the_standard_registry(): + reg = pycvc.stdlib_registry.standard() # a copy of the shared one + assert {"string", "bytes", "math", "collections"} <= set(reg.list_modules()) + assert "string.length" in reg.list_functions("string") + assert isinstance(reg.lookup_qualified("string.length"), pycvc.value) + assert _raises(KeyError, reg.lookup_qualified, "string.nope").args == ("string.nope",) + shared = pycvc.stdlib_registry.shared_scope() + assert shared.frozen and isinstance(shared.lookup("string.length"), pycvc.value) + # Adding to the copy leaves the shared registry alone. + reg.register_function("py", "py.twice", lambda x: 2 * x) + assert "py" in reg.list_modules() + assert "py" not in pycvc.stdlib_registry.standard().list_modules() + + +def test_a_registry_of_ones_own(): + reg = pycvc.stdlib_registry() + reg.register_function("py", "py.twice", lambda x: 2 * x) + reg.register_function("py", "py.len", reg.lookup_qualified("string.length")) # a native + env = pycvc.environment.extend(_builtins()) + reg.import_module("py", env) + ev = pycvc.stackless_evaluator(_builtins()) + assert ev.evaluate_script('(list (py.twice 21) (py.len "abc"))', env) == [42, 3] + only = pycvc.environment() + reg.import_module("math", only, ["math.sqrt"]) + assert list(only.bindings) == ["math.sqrt"] + _raises(RuntimeError, reg.import_module, "nope", only) + _raises(RuntimeError, reg.import_module, "py", None) + + +if __name__ == "__main__": + tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] + for t in tests: + t() + print(" ok:", t.__name__) + print("pycvc state_exec data/heap/stdlib parity tests: OK") diff --git a/docs/PYCVC_STATE_EXEC_PARITY.md b/docs/PYCVC_STATE_EXEC_PARITY.md index ca3ea3ef5..2b051ea3e 100644 --- a/docs/PYCVC_STATE_EXEC_PARITY.md +++ b/docs/PYCVC_STATE_EXEC_PARITY.md @@ -29,7 +29,13 @@ approach agreed for closing the gap, and the plan. Conversion is strict: - an int outside 64 bits raises `OverflowError`; - a non-`str` dict key or any other Python type raises `TypeError`; - - a value nested deeper than 1000 levels, or one that contains itself, raises `ValueError`. + - a Python value that contains itself raises `ValueError`, since only the evaluator's own + containers are tracked for cycle collection. + + It takes any depth: it walks with an explicit stack, not recursion (PR 4), as the parser and + the evaluator do. A list or dict shared within a value converts to one object, both ways, so + sharing survives the trip, and a DSL list that holds itself comes out as a Python list that + holds itself. (PR 1's converter recursed and stopped at 1000 levels.) Before this, symbols became plain strings, closures became the text `""`, and any object became its `str()`. The old converter could also pass a null object into a list when a @@ -73,7 +79,7 @@ approach agreed for closing the gap, and the plan. The audit (2026-10-17) found 97 public items in the 20 headers. Counted individually, with the 38 `builtin_*` functions and the 7 `MIME_*` constants, it is 140. Items added to master since then -are listed too. Status after PR 3: +are listed too. Status after PR 4: | Header | Item | Python | Status | |---|---|---|---| @@ -84,7 +90,8 @@ are listed too. Status after PR 3: | | `script_reader` | `pycvc.script_reader` | wrapped (PR 1). `forms()`, `source()` and `options()` return copies | | | `parser` (`read`, `parse_expr`) | none | exception: it holds a `string_view` of its input, which Python can't keep alive. `script_reader` owns its source and gives the same slice-by-slice reading | | data_job.h (#658) | `data_options`, `data_options_from_state(state&, id, base)` | `pycvc.data_options`, `pycvc.data_options_from_state(State, …)` | wrapped (#658) | -| | the data jobs (`data_job`, `make_*_job`, `run_data_job`, ...) | | PR 4 | +| | `data_job` (`step`, `result`, `nodes`), `make_equal_job`, `make_print_job`, `make_concat_job`, `make_copy_job`, `make_substitute_job`, `run_to_completion`, `run_data_job`, `call_function` (with or without #683's `then`), `make_generator_job` (#683; the generator is a `pycvc.value`), `is_compound`, `count_data_nodes`, `kDataNodeBytes`, `kMaxNativeNesting`, `kNativeStackReserve` (#682) (#658, #665) | the same names under `pycvc.` | wrapped (PR 4). A factory hands Python an owned job, which `run_data_job` takes over. `who` (C++ keeps the pointer for its errors) is kept for good. `run_data_job`'s `then` is a Python callable. From a Python function a program calls, `run_data_job` and `call_function` work as for a C++ native: return what they return at once | +| | the call protocol (`data_job::call_request`, `take_call`, `pending_call`, `call_returned`, `own_inputs`), `detail::deferred_data_job`, `detail::data_step_scope`, `detail::native_nesting` (#682) | none | exception: for C++ job implementations and the evaluator's own bookkeeping; Python can't subclass a data job | | types.h | `symbol` | `pycvc.symbol` | wrapped (PR 1) | | | `value_t` | native types, or `pycvc.value` | wrapped (PR 1), see the table above | | | `to_string`, `values_equal` | `pycvc.to_string`, `pycvc.values_equal` | wrapped (PR 1) | @@ -116,8 +123,11 @@ are listed too. Status after PR 3: | resource_policy.h | `resource_policy`, `process_limits`, `validate_limits` | `pycvc.resource_policy` (its `mode` is `pycvc.resource_policy.mode`, an `IntEnum`), `pycvc.process_limits`, `pycvc.validate_limits` | wrapped (PR 3) | | intrinsics.h | `register_intrinsics`, `intrinsics_context`, `apply_chroot`, `channel_policy`, `resolve_channel_key`, `make_future`, `future_channel_of` | the same names under `pycvc.` | wrapped (PR 3). See approach item 11 for the context's lifetime. `channel_policy.global` is `global_` (a Python keyword). The context's `tracker`, `document` and `watches` aren't wrapped: `memory_tracker` is internal, `document_scope`'s slots are C++ templates, and `watches` is the intrinsics' own registry | | | `document_scope`, `park_on_channel` | none | exception: Ariadne/runtime internals. `document_scope` is typed C++ template slots; `park_on_channel` takes raw process pointers and is valid only inside a running native function | -| stdlib.h | `stdlib_registry` | | next (PR 4). #660 loads the stdlib modules into every environment, so DSL programs already reach them | -| heap.h (#667) | the heap measurement, and the fields it adds to `process_info` and `process` | | PR 4, after #667 merges | +| stdlib.h | `stdlib_registry` (`standard`, `shared_scope`, `register_function`, `list_functions`, `list_modules`, `import_module`, `lookup_qualified`) | `pycvc.stdlib_registry` | wrapped (PR 4). `standard()` returns a copy, so Python can't add to the shared registry; `register_function` takes a Python callable or a `pycvc.value` native; `lookup_qualified` raises `KeyError` where C++ returns nullptr | +| heap.h (#667) | `heap_options`, `heap_options_from_state`, `measure_heap_share`, `make_heap_share_job`, `heap_remeasure_bytes`, `cycle_stats`, `collect_cycles`, `cycle_collection_due`, `note_process_exit`, `tracked_objects` | the same names under `pycvc.` | wrapped (PR 4). The collector is per thread: run it on the thread that ran the programs. `measure_heap_share` and `collect_cycles` release the GIL | +| | `heap_after_step`, `track_captured_env`, `track_mutated` | none | exception: the schedulers' per-step accounting, and tracking that only means something for the evaluator's own containers (a value from Python is a fresh copy) | +| | #667's fields: `process.heap_*`, `heap_measure_due`, `release_state`; `process_info.heap_memory`/`heap_measurements`; `evaluator_state.heap_measured`; `stackless_evaluator::global_env()`; `eval_alloc_note` | the same names | wrapped (PR 4). `heap_measured` is read-only (the result of a measurement in flight) | +| | `eval_alloc_guard::allocated()` (#667) | none | exception: the guard isn't wrapped (see types.h) | | exec_coordinator.h | `exec_coordinator` and its result types | none | deferred: it needs the cluster shard, membership and message-bus objects, which pycvc doesn't wrap | | memory_tracker.h | `memory_tracker` | none | exception: per-node byte accounting, internal to the schedulers | | generator.h, task.h | `generator`, `generator_next`, `task<>` | none | exception: coroutine and evaluator internals; a generator crosses as a `pycvc.value` | @@ -145,8 +155,7 @@ Each PR is stacked on the previous one and includes pytest checks against the C+ built in Python can have the intrinsics `Exec.run` programs have; and master's `max_alloc_bytes` additions (#660). `bindings/pycvc/test_pycvc_state_exec_eval.py` repeats the C++ tests. -4. **PR 4:** `stdlib_registry`, and the API the deep-data (#658) and heap (#667) work adds, once - those land. +4. **PR 4 (this):** `stdlib_registry`, the deep-data jobs (#658, #665) and the heap (#667); and the value conversion without a depth limit (it walks with an explicit stack, as the parser and evaluator do, and keeps shared structure). `bindings/pycvc/test_pycvc_state_exec_data.py` covers them. Known gaps: - Invalid UTF-8 in a DSL string raises when it crosses to Python, but no Python test produces such