diff --git a/docs/STATE_EXEC_DEVELOPER_GUIDE.md b/docs/STATE_EXEC_DEVELOPER_GUIDE.md index 7bc6e359f..ac57fd3da 100644 --- a/docs/STATE_EXEC_DEVELOPER_GUIDE.md +++ b/docs/STATE_EXEC_DEVELOPER_GUIDE.md @@ -562,8 +562,19 @@ register_intrinsics(env, &ictx); ;; Store/retrieve binary data objects (state-data-set "myapp.blob" some-value) (state-data-get "myapp.blob") + +;; A subtree's values as JSON, and back: the document cvc::state::json() writes, keyed by paths +;; relative to the node ("" for the node itself). Both run as sliced data steps. +(state-json "myapp.config") ;; => "{\n \"timeout\": \"30\"\n}\n" +(state-from-json "myapp.copy" (state-json "myapp.config")) ``` +`state-json` returns nil for a missing node and reads through a transparent link, as +`state-get` does. `state-from-json` writes through a writable transparent link, as `state-set` +does. It sets every non-empty string, number, `true`, `false` and `null` at its key, and returns +nil. It checks the whole document before it sets anything, so a malformed document changes +nothing. Its errors name the line, as in `state-from-json: line 3: expected ':'`. + ### State Expiry ```lisp @@ -1447,8 +1458,16 @@ process with it as the exit error; the sync scheduler's `run()` throws it: These functions do their work as a data job (`data_job.h`; "Deep data on the executor" in `docs/STATE_EXEC_GRAMMAR.md`): `string.split`, `string.join`, `string.replace`, -`string.encode`, `bytes.slice`, `bytes.concat`, `bytes.decode` and every `collections.*` -function. +`string.encode`, `bytes.slice`, `bytes.concat`, `bytes.decode`, every `collections.*` +function, and the intrinsics `state-json` and `state-from-json`. + +`state-json` and `state-from-json` run `cvc::state`'s own writer and reader +(`cvc/state/state_json.h`), the code behind `json()`, `json(j)`, `save()` and `restore()`, so +they produce and accept the same documents. The writer's unit is a node visited. The reader's +is a value scanned, and it scans each value twice: once to check the document, once to set it. +Both also count one unit per 256 bytes. The writer holds the nodes it has yet to visit, so a +node another process removes between slices is still written, as it was when its parent was +read. The document is held to the allocation limit as it grows. - **Slices.** A slice is `data.step_nodes` nodes: one per element visited, and one per 256 bytes (`kDataNodeBytes`) scanned or copied. With `data.step_nodes` = N the call does one diff --git a/docs/STATE_EXEC_GRAMMAR.md b/docs/STATE_EXEC_GRAMMAR.md index 2485bcc02..58bb72fc1 100644 --- a/docs/STATE_EXEC_GRAMMAR.md +++ b/docs/STATE_EXEC_GRAMMAR.md @@ -226,8 +226,8 @@ then `state_exec.defaults.data.step_nodes`, then the scheduler's built-in defaul A node is an element visited: a list element or a dict entry walked, or an element freed. With `step_nodes` = N: -- **Native calls.** `=`, `!=`, `str`, `str-concat`, `print` and `state-data-get` hand their walk - to the evaluator (`run_data_job`). It does one slice inside the call's step; if the walk isn't +- **Native calls.** `=`, `!=`, `str`, `str-concat`, `print`, `state-data-get`, `state-json` and + `state-from-json` hand their walk to the evaluator (`run_data_job`). It does one slice inside the call's step; if the walk isn't done, the call's frame waits and the following steps are **data steps** that each run N more nodes. The step that finishes it completes the call. A walk under N nodes finishes in the call's own step, so ordinary programs take no extra steps. diff --git a/inc/cvc/state/state.h b/inc/cvc/state/state.h index b64112beb..233412f07 100644 --- a/inc/cvc/state/state.h +++ b/inc/cvc/state/state.h @@ -485,6 +485,12 @@ class state : public boost::enable_shared_from_this { void ptree(const boost::property_tree::ptree &); // returns a json version of the property map + // + // The document is written and read by state_json_writer / state_json_reader + // (cvc/state/state_json.h), whose walks state_exec's state-json and state-from-json run a + // slice per scheduler step. The text is what write_json(ptree()) writes, and the reader keeps + // read_json()'s grammar, but its container stack is on the heap, so a deeply nested document + // costs no native stack. A malformed document sets nothing. std::string json(); // sets this property tree based on a json @@ -859,6 +865,8 @@ class state : public boost::enable_shared_from_this { static state_ptr instancePtr(app &ctx); private: + friend class state_subtree_cursor; // snapshots a node's value and children under its lock + state(const state &); }; diff --git a/inc/cvc/state/state_json.h b/inc/cvc/state/state_json.h new file mode 100644 index 000000000..01ac6255c --- /dev/null +++ b/inc/cvc/state/state_json.h @@ -0,0 +1,151 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +// A state subtree's JSON document (state::json(), state::save()) and loading one +// (state::json(j), state::restore()), as walks that can stop after any number of nodes and go +// on later. cvc::state runs them to completion; state_exec's state-json and state-from-json run +// them a slice per scheduler step. One implementation, so the two cannot drift. +// +// The document is one level: an entry per node, keyed by its path relative to the node it +// describes ("" for that node), in pre-order with children in name order, holding the node's +// value, or "" for a node below it with neither a value nor children. Values only, not data(). +// It is written exactly as boost::property_tree::write_json() writes that one-level tree (the +// format json() had), and read as read_json() followed by ptree(pt) would read it. + +#ifndef CVC_STATE_STATE_JSON_H +#define CVC_STATE_STATE_JSON_H + +#include +#include +#include +#include +#include +#include + +namespace cvc { + +// Pre-order over a subtree, a node per next(). A heap stack of levels, not a call per level. +// Each node's value and children are snapshotted under its own lock, released before going +// down, and the children are held by owning pointer, so a node removed from the tree between two +// calls is still visited (as it was when its parent was snapshotted) and none dangles. Changes +// made between calls show in the nodes visited after them. +class state_subtree_cursor { +public: + // Walk `start`'s subtree. The cursor keeps `start` alive while it walks. + explicit state_subtree_cursor(state &start); + + enum class visit { + listed, // the node belongs in the document: it has a value, or it is childless below start + unlisted, // it does not: an empty node with children, or an empty start + done // no nodes are left + }; + + // Visit the next node: its path relative to start ("" for start) and its value. + visit next(std::string &path, std::string &value); + + // Nodes visited so far. + std::size_t visited() const { return visited_; } + +private: + struct level { + std::vector> kids; // with their relative paths + std::size_t next = 0; + }; + // Snapshot `n`'s value and children, whose paths are `rel`'s plus their names. + level open(state &n, const std::string &rel, std::string &value); + + state::state_ptr keep_; + state *start_; + bool started_ = false; + std::vector stack_; + std::size_t visited_ = 0; +}; + +// The document of a subtree, written a slice at a time. +class state_json_writer { +public: + explicit state_json_writer(state &start) : cursor_(start) {} + + // Visit at most `nodes` more nodes (0: all that remain). True once the document is complete. + bool step(std::size_t nodes); + + std::size_t nodes() const { return cursor_.visited(); } + // The text so far (all of it once step() has returned true). + const std::string &text() const { return out_; } + std::string take() { return std::move(out_); } + +private: + state_subtree_cursor cursor_; + std::string out_ = "{\n"; + std::string path_, value_; + bool any_ = false, done_ = false; +}; + +// Load a document into a node, a slice at a time, as state::json(j) does. Every string, number, +// true, false and null at any depth of the document is a value; one that is not empty is set +// at its key relative to the target ("" inside an array: the target itself), in document order. +// Numbers keep their text; true, false and null are those words. Objects and arrays only nest. +// The whole document is checked first, so a malformed one sets nothing; errors are +// boost::property_tree::json_parser_error, as read_json() throws them. +class state_json_reader { +public: + // `doc` is borrowed: it must outlive the reader, or be passed again to rebind(). + state_json_reader(state &target, std::string_view doc, std::string filename = {}); + + // The same text at a new address (the reader keeps only offsets into it). + void rebind(std::string_view doc) { doc_ = doc; } + std::string_view doc() const { return doc_; } + + // Scan at most `values` more values (0: all that remain); each is scanned twice, once to + // check and once to set. True once the document is loaded. + bool step(std::size_t values); + + // Values scanned so far (both passes), and bytes. + std::size_t values() const { return values_; } + std::size_t bytes() const { return bytes_; } + // Values set so far. + std::size_t applied() const { return applied_; } + +private: + enum class want { start, value, key_or_end, value_or_end, key, comma_or_end, end, done }; + struct frame { + bool array; + std::string key; // an object's current key + }; + + [[noreturn]] void fail(const char *msg) const; + bool at_end() const { return pos_ >= doc_.size(); } + char peek() const { return doc_[pos_]; } + void advance(); + bool have(char c); + void expect(char c, const char *msg); + char need(const char *msg) const; + void skip_ws(); + void scan_string(std::string &out); + void scan_escape(std::string &out); + unsigned scan_hex_quad(); + void scan_scalar(std::string &out); + void emit(const std::string &text); + want after_value() const { return stack_.empty() ? want::end : want::comma_or_end; } + + state::state_ptr keep_; + state *target_; + std::string_view doc_; + std::string filename_; + bool applying_ = false; // second pass + want want_ = want::start; + std::vector stack_; + std::size_t pos_ = 0, line_ = 1, scanned_ = 0; + std::size_t values_ = 0, bytes_ = 0, applied_ = 0; +}; + +} // namespace cvc + +#endif // CVC_STATE_STATE_JSON_H diff --git a/src/cvc/CMakeLists.txt b/src/cvc/CMakeLists.txt index 32a091091..4cf535c75 100644 --- a/src/cvc/CMakeLists.txt +++ b/src/cvc/CMakeLists.txt @@ -76,6 +76,7 @@ set(INCLUDE_FILES ../../inc/cvc/utility/utility.h ../../inc/cvc/state/state_object.h ../../inc/cvc/state/state_list.h + ../../inc/cvc/state/state_json.h ../../inc/cvc/state/state_eviction_store.h ../../inc/cvc/state/state_memory_manager.h ../../inc/cvc/state/state_exec/types.h @@ -200,6 +201,7 @@ set(SOURCE_FILES processing/smoothing.cpp volume/spider_io.cpp state/state.cpp + state/state_json.cpp state/state_list.cpp state/state_eviction_store.cpp state/state_memory_manager.cpp diff --git a/src/cvc/ariadne/ariadne.cpp b/src/cvc/ariadne/ariadne.cpp index af6ff5c67..8423e19be 100644 --- a/src/cvc/ariadne/ariadne.cpp +++ b/src/cvc/ariadne/ariadne.cpp @@ -421,9 +421,10 @@ class ReactiveEngine { "is-float", "is-string", "is-null", "is-list", "type-of", "not", "and", "or", // bounded compound construction (by-reference; not the doubling materializers) + access "list", "cons", "car", "cdr", "nth", "length", "dict", "get-attr", - // side-effect-free state readers (state-data-get deep-copies; no aliasing, no callables) - "state-get", "state-exists", "state-children", "state-data-get", "state-root-path", - "state-has-expiry", "state-is-expired", + // side-effect-free state readers (state-data-get deep-copies; no aliasing, no callables; + // state-json writes its document as sliced data steps, under the allocation limit) + "state-get", "state-exists", "state-children", "state-data-get", "state-json", + "state-root-path", "state-has-expiry", "state-is-expired", // the stdlib: the pure, bounded functions (a scalar, or a result no bigger than its // argument), and the materializers, which do their work as sliced data steps (each // eval slices, see data_step_nodes_) under the per-call allocation limit. A function diff --git a/src/cvc/state/state.cpp b/src/cvc/state/state.cpp index da09f33f8..780a11145 100644 --- a/src/cvc/state/state.cpp +++ b/src/cvc/state/state.cpp @@ -36,8 +36,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -680,44 +683,16 @@ boost::property_tree::ptree state::ptree() { // holding the node's value -- or "" for a node other than this one with neither a value nor // children. For the root, relative paths are full names, as they always were. boost's ptree and // its JSON writer and parser recurse once per nesting level, so with one level, building this, - // writing it as JSON, reading it back, copying and destroying it don't recurse once per state - // level. A heap stack of levels; each node's value and children are snapshotted under its own - // lock, released before going down, and a child's path is its parent's, a separator and its - // name, carried down. - struct level { - std::vector> kids; // with their relative paths - std::size_t next = 0; - }; - const char *const fn = BOOST_CURRENT_FUNCTION; - const auto open = [&pt, fn, this](state &n, const std::string &n_rel, bool top) { - level l; - std::string v; - { - boost::mutex::scoped_lock lock(n._mutex); - v = n._value; - l.kids.reserve(n._children.size()); - for (const child_map::value_type &kv : n._children) - if (kv.second) - l.kids.emplace_back(kv.second, n_rel.empty() ? kv.first : n_rel + SEPARATOR + kv.first); - } - _ctx.log(6, boost::str(boost::format("%s :: %s = %s\n") % fn % n_rel % v)); - if (!v.empty()) - pt.push_back(property_tree::ptree::value_type(n_rel, property_tree::ptree(v))); - else if (!top && l.kids.empty()) - pt.push_back(property_tree::ptree::value_type(n_rel, property_tree::ptree())); - return l; - }; - std::vector stack; - stack.push_back(open(*this, std::string(), true)); - while (!stack.empty()) { - level &l = stack.back(); - if (l.next == l.kids.size()) { - stack.pop_back(); - continue; - } - std::pair kid = std::move(l.kids[l.next++]); - boost::this_thread::interruption_point(); - stack.push_back(open(*kid.first, kid.second, false)); // invalidates `l` + // copying and destroying it don't recurse once per state level. The walk is + // state_subtree_cursor's, which json() and state_exec's state-json share. + state_subtree_cursor cursor(*this); + std::string path, value; + for (;;) { + const state_subtree_cursor::visit v = cursor.next(path, value); + if (v == state_subtree_cursor::visit::done) + break; + if (v == state_subtree_cursor::visit::listed) + pt.push_back(property_tree::ptree::value_type(path, property_tree::ptree(value))); } return pt; } @@ -772,7 +747,13 @@ void state::ptree(const boost::property_tree::ptree &pt) { apply_state_ptree(*th // Returns a json string version of the property map. // ---- Change History ---- // 01/12/2014 -- Joe R. -- Creation. -std::string state::json() { return cvc::json(ptree()); } +// The text write_json(ptree()) writes, from state_json_writer (state_exec's state-json runs the +// same writer a slice at a time). +std::string state::json() { + state_json_writer w(*this); + w.step(0); + return w.take(); +} // ----------- // state::json @@ -781,7 +762,13 @@ std::string state::json() { return cvc::json(ptree()); } // Sets this state object and its children based on an incoming json. // ---- Change History ---- // 01/13/2014 -- Joe R. -- Creation. -void state::json(const std::string &j) { ptree(cvc::json(j)); } +// What read_json() and ptree(pt) did, from state_json_reader (state_exec's state-from-json runs +// the same reader a slice at a time): the document is checked whole first, so a malformed one +// sets nothing, and its nesting costs no native stack. +void state::json(const std::string &j) { + state_json_reader r(*this, j); + r.step(0); +} // ----------- // state::save @@ -791,7 +778,19 @@ void state::json(const std::string &j) { ptree(cvc::json(j)); } // ---- Change History ---- // 03/16/2012 -- Joe R. -- Creation. // 01/12/2014 -- Joe R. -- Forcing json. -void state::save(const std::string &filename) { write_json(filename, ptree()); } +// The file write_json(filename, ptree()) wrote: the same text, through a text-mode stream, and +// the same errors. +void state::save(const std::string &filename) { + using boost::property_tree::json_parser::json_parser_error; + state_json_writer w(*this); + w.step(0); + std::ofstream out(filename.c_str()); + if (!out) + BOOST_PROPERTY_TREE_THROW(json_parser_error("cannot open file", filename, 0)); + out << w.text() << std::flush; + if (!out.good()) + BOOST_PROPERTY_TREE_THROW(json_parser_error("write error", filename, 0)); +} // -------------- // state::restore @@ -801,11 +800,16 @@ void state::save(const std::string &filename) { write_json(filename, ptree()); } // ---- Change History ---- // 03/16/2012 -- Joe R. -- Creation. // 01/12/2014 -- Joe R. -- Forcing json. +// read_json(filename, pt) then ptree(pt): a text-mode stream, the same errors, and the reader +// json(j) uses. void state::restore(const std::string &filename) { - using namespace boost; - property_tree::ptree pt; - read_json(filename, pt); - ptree(pt); + using boost::property_tree::json_parser::json_parser_error; + std::ifstream in(filename.c_str()); + if (!in) + BOOST_PROPERTY_TREE_THROW(json_parser_error("cannot open file", filename, 0)); + const std::string doc((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + state_json_reader r(*this, doc, filename); + r.step(0); } // --------------- diff --git a/src/cvc/state/state_exec/intrinsics.cpp b/src/cvc/state/state_exec/intrinsics.cpp index 88c37bd59..a453fc49b 100644 --- a/src/cvc/state/state_exec/intrinsics.cpp +++ b/src/cvc/state/state_exec/intrinsics.cpp @@ -1,6 +1,7 @@ #include // ยง12 enforce_channel_policy: std::find over the declared/global channel lists #include #include +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -239,6 +241,116 @@ value_t intrinsic_state_data_set(intrinsics_context *ctx, std::spanisLink() && node->linkMode() == cvc::state::link_mode::transparent && + node->linkWritable()) { + const cvc::state::link_resolution lr = node->resolveLink(); + if (lr.kind == cvc::state::link_resolution_kind::resolved && lr.target) + return lr.target; + } + return node; +} + +// The writer holds the nodes it has yet to visit by owning pointer: a node another process +// removes between slices is still written, as it was when its parent was snapshotted. A slice +// ends when its units are spent, a long value counting for its bytes. The text is held to the +// allocation limit as it grows. +class state_json_job final : public data_job { +public: + explicit state_json_job(cvc::state &node) : writer_(node) {} + + bool step(std::size_t budget) override { + const std::size_t start = nodes_; + const std::optional limit = eval_alloc_limit(); + bool done = false; + while (!done && (budget == 0 || nodes_ - start < budget)) { + done = writer_.step(budget == 0 ? 0 : 1); + const std::size_t size = writer_.text().size(); + nodes_ = writer_.nodes() + size / kDataNodeBytes; + if (limit && size > *limit) + eval_alloc_check(size, "state-json"); // throws + } + eval_alloc_note(writer_.text().size() - noted_); + noted_ = writer_.text().size(); + return done; + } + + value_t result() override { return value_t(writer_.take()); } + +private: + cvc::state_json_writer writer_; + std::size_t noted_ = 0; // text bytes reported to heap accounting so far +}; + +// The reader checks the whole document before it sets anything, so a malformed one changes +// nothing even when the check spans many steps. A slice ends when its units are spent, a long +// string counting for its bytes. +class state_from_json_job final : public data_job { +public: + state_from_json_job(cvc::state &node, std::string_view doc) : reader_(node, doc) {} + + bool step(std::size_t budget) override { + const std::size_t start = nodes_; + bool done = false; + try { + while (!done && (budget == 0 || nodes_ - start < budget)) { + done = reader_.step(budget == 0 ? 0 : 1); + nodes_ = reader_.values() + reader_.bytes() / kDataNodeBytes; + } + } catch (const boost::property_tree::json_parser::json_parser_error &e) { + throw std::runtime_error("state-from-json: line " + std::to_string(e.line()) + ": " + + e.message()); + } + return done; + } + + value_t result() override { return nil_value; } + + // The document was the call's argument: keep a copy once the job outlives the call. + void own_inputs() override { + doc_ = std::string(reader_.doc()); + reader_.rebind(doc_); + } + +private: + std::string doc_; + cvc::state_json_reader reader_; +}; + +} // namespace + +value_t intrinsic_state_json(intrinsics_context *ctx, std::span args) { + expect_exact(args, 1, "state-json"); + require_root(ctx, "state-json"); + auto &path = as_string(args[0], "state-json"); + cvc::state *node = ctx->root->findDescendant(path); + if (!node) + return nil_value; + return run_data_job(std::make_unique(*read_through_link(node))); +} + +value_t intrinsic_state_from_json(intrinsics_context *ctx, std::span args) { + expect_exact(args, 2, "state-from-json"); + require_root(ctx, "state-from-json"); + auto &path = as_string(args[0], "state-from-json"); + auto &doc = as_string(args[1], "state-from-json"); + cvc::state *node = write_through_link(&(*ctx->root)(path)); + return run_data_job(std::make_unique(*node, doc)); +} + value_t intrinsic_state_root_path(intrinsics_context *ctx, std::span args) { expect_exact(args, 0, "state-root-path"); return value_t(ctx->root_path); @@ -828,6 +940,8 @@ void register_intrinsics(environment_ptr env, intrinsics_context *ctx) { reg("state-delete", intrinsic_state_delete); reg("state-data-get", intrinsic_state_data_get); reg("state-data-set", intrinsic_state_data_set); + reg("state-json", intrinsic_state_json); + reg("state-from-json", intrinsic_state_from_json); reg("state-root-path", intrinsic_state_root_path); reg("state-watch", intrinsic_state_watch); reg("state-unwatch", intrinsic_state_unwatch); diff --git a/src/cvc/state/state_json.cpp b/src/cvc/state/state_json.cpp new file mode 100644 index 000000000..6cf2e7cab --- /dev/null +++ b/src/cvc/state/state_json.cpp @@ -0,0 +1,473 @@ +/* + Copyright 2026 The University of Texas at Austin + + This file is part of libcvc. + + libcvc is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License version 2.1 as published by the Free Software Foundation. +*/ + +#include +#include +#include +#include +#include +#include + +namespace cvc { + +// --------------------------------------------------------------------------- +// state_subtree_cursor +// --------------------------------------------------------------------------- + +state_subtree_cursor::state_subtree_cursor(state &start) + : keep_(start.weak_from_this().lock()), start_(&start) {} + +state_subtree_cursor::level state_subtree_cursor::open(state &n, const std::string &rel, + std::string &value) { + level l; + { + boost::mutex::scoped_lock lock(n._mutex); + value = n._value; + l.kids.reserve(n._children.size()); + for (const state::child_map::value_type &kv : n._children) + if (kv.second) + l.kids.emplace_back(kv.second, rel.empty() ? kv.first : rel + state::SEPARATOR + kv.first); + } + n._ctx.log(6, boost::str(boost::format("cvc::state_subtree_cursor :: %s = %s\n") % rel % value)); + return l; +} + +state_subtree_cursor::visit state_subtree_cursor::next(std::string &path, std::string &value) { + boost::this_thread::interruption_point(); + if (!started_) { + started_ = true; + ++visited_; + path.clear(); + stack_.push_back(open(*start_, path, value)); + return value.empty() ? visit::unlisted : visit::listed; + } + while (!stack_.empty()) { + level &l = stack_.back(); + if (l.next == l.kids.size()) { + stack_.pop_back(); + continue; + } + std::pair kid = std::move(l.kids[l.next++]); + ++visited_; + path = std::move(kid.second); + level opened = open(*kid.first, path, value); + const bool childless = opened.kids.empty(); + stack_.push_back(std::move(opened)); // invalidates `l` + return !value.empty() || childless ? visit::listed : visit::unlisted; + } + return visit::done; +} + +// --------------------------------------------------------------------------- +// state_json_writer +// --------------------------------------------------------------------------- + +namespace { + +// boost::property_tree::json_parser::create_escapes(), byte for byte: printable ASCII and every +// byte from 0x5D up (so UTF-8 passes through) stay; " \ / and the named controls get their +// escapes; any other control byte becomes \u00XX. +void append_escaped(std::string &out, const std::string &s) { + static const char kHex[] = "0123456789ABCDEF"; + for (const char ch : s) { + const unsigned char c = static_cast(ch); + if (c == 0x20 || c == 0x21 || (c >= 0x23 && c <= 0x2E) || (c >= 0x30 && c <= 0x5B) || c >= 0x5D) + out += ch; + else if (ch == '\b') + out += "\\b"; + else if (ch == '\f') + out += "\\f"; + else if (ch == '\n') + out += "\\n"; + else if (ch == '\r') + out += "\\r"; + else if (ch == '\t') + out += "\\t"; + else if (ch == '/') + out += "\\/"; + else if (ch == '"') + out += "\\\""; + else if (ch == '\\') + out += "\\\\"; + else { + out += "\\u00"; + out += kHex[c >> 4]; + out += kHex[c & 0x0F]; + } + } +} + +} // namespace + +// write_json()'s pretty layout for a one-level tree: "{\n", then each entry as +// ` "key": "value"` with ",\n" between entries and "\n" after the last, then "}\n". +bool state_json_writer::step(std::size_t nodes) { + if (done_) + return true; + for (std::size_t n = 0; nodes == 0 || n < nodes; ++n) { + const state_subtree_cursor::visit v = cursor_.next(path_, value_); + if (v == state_subtree_cursor::visit::done) { + if (any_) + out_ += '\n'; + out_ += "}\n"; + done_ = true; + return true; + } + if (v == state_subtree_cursor::visit::unlisted) + continue; + out_ += any_ ? ",\n \"" : " \""; + append_escaped(out_, path_); + out_ += "\": \""; + append_escaped(out_, value_); + out_ += '"'; + any_ = true; + } + return false; +} + +// --------------------------------------------------------------------------- +// state_json_reader +// --------------------------------------------------------------------------- + +namespace { + +bool is_digit(char c) { return c >= '0' && c <= '9'; } + +int hex_value(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + return -1; +} + +// How many continuation bytes a UTF-8 lead byte takes, or -1 when it cannot lead (read_json's +// utf8_utf8_encoding::trail_table). +int trailing_bytes(unsigned char c) { + if (c >= 0xC0 && c <= 0xDF) + return 1; + if (c >= 0xE0 && c <= 0xEF) + return 2; + if (c >= 0xF0 && c <= 0xF7) + return 3; + return -1; +} + +void append_utf8(std::string &out, unsigned cp) { + if (cp <= 0x7F) { + out += static_cast(cp); + } else if (cp <= 0x7FF) { + out += static_cast(0xC0 | (cp >> 6)); + out += static_cast(0x80 | (cp & 0x3F)); + } else if (cp <= 0xFFFF) { + out += static_cast(0xE0 | (cp >> 12)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } else { + out += static_cast(0xF0 | (cp >> 18)); + out += static_cast(0x80 | ((cp >> 12) & 0x3F)); + out += static_cast(0x80 | ((cp >> 6) & 0x3F)); + out += static_cast(0x80 | (cp & 0x3F)); + } +} + +} // namespace + +state_json_reader::state_json_reader(state &target, std::string_view doc, std::string filename) + : keep_(target.weak_from_this().lock()), target_(&target), doc_(doc), + filename_(std::move(filename)) {} + +void state_json_reader::fail(const char *msg) const { + BOOST_PROPERTY_TREE_THROW( + boost::property_tree::json_parser::json_parser_error(msg, filename_, line_)); +} + +void state_json_reader::advance() { + if (doc_[pos_] == '\n') + ++line_; + ++pos_; +} + +bool state_json_reader::have(char c) { + if (at_end() || peek() != c) + return false; + advance(); + return true; +} + +void state_json_reader::expect(char c, const char *msg) { + if (!have(c)) + fail(msg); +} + +char state_json_reader::need(const char *msg) const { + if (at_end()) + fail(msg); + return peek(); +} + +void state_json_reader::skip_ws() { + while (!at_end()) { + const char c = peek(); + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') + return; + advance(); + } +} + +// The rest of a string whose opening quote was read. +void state_json_reader::scan_string(std::string &out) { + for (;;) { + const unsigned char c = static_cast(need("unterminated string")); + if (c == '"') { + ++pos_; + return; + } + if (c == '\\') { + ++pos_; + scan_escape(out); + continue; + } + if (c <= 0x7F) { + if (c < 0x20) + fail("invalid code sequence"); + out += static_cast(c); + ++pos_; + continue; + } + const int trailing = trailing_bytes(c); + if (trailing < 0) + fail("invalid code sequence"); + out += static_cast(c); + ++pos_; + for (int i = 0; i < trailing; ++i) { + if (at_end() || (static_cast(peek()) & 0xC0) != 0x80) + fail("invalid code sequence"); + out += peek(); + ++pos_; + } + } +} + +unsigned state_json_reader::scan_hex_quad() { + unsigned cp = 0; + for (int i = 0; i < 4; ++i) { + const int v = hex_value(need("invalid escape sequence")); + if (v < 0) + fail("invalid escape sequence"); + cp = cp * 16 + static_cast(v); + ++pos_; + } + return cp; +} + +void state_json_reader::scan_escape(std::string &out) { + if (have('"')) + out += '"'; + else if (have('\\')) + out += '\\'; + else if (have('/')) + out += '/'; + else if (have('b')) + out += '\b'; + else if (have('f')) + out += '\f'; + else if (have('n')) + out += '\n'; + else if (have('r')) + out += '\r'; + else if (have('t')) + out += '\t'; + else if (have('u')) { + unsigned cp = scan_hex_quad(); + if ((cp & 0xFC00) == 0xDC00) + fail("invalid codepoint, stray low surrogate"); + if ((cp & 0xFC00) == 0xD800) { + expect('\\', "invalid codepoint, stray high surrogate"); + expect('u', "expected codepoint reference after high surrogate"); + const unsigned low = scan_hex_quad(); + if ((low & 0xFC00) != 0xDC00) + fail("expected low surrogate after high surrogate"); + cp = 0x10000 + (((cp & 0x3FF) << 10) | (low & 0x3FF)); + } + append_utf8(out, cp); + } else + fail("invalid escape sequence"); +} + +// A string, true, false, null or number (kept as its text). +void state_json_reader::scan_scalar(std::string &out) { + const char c = need("expected value"); + if (c == '"') { + ++pos_; + scan_string(out); + return; + } + if (c == 't') { + ++pos_; + expect('r', "expected 'true'"); + expect('u', "expected 'true'"); + expect('e', "expected 'true'"); + out = "true"; + return; + } + if (c == 'f') { + ++pos_; + expect('a', "expected 'false'"); + expect('l', "expected 'false'"); + expect('s', "expected 'false'"); + expect('e', "expected 'false'"); + out = "false"; + return; + } + if (c == 'n') { + ++pos_; + expect('u', "expected 'null'"); + expect('l', "expected 'null'"); + expect('l', "expected 'null'"); + out = "null"; + return; + } + const std::size_t start = pos_; + const bool minus = have('-'); + if (!have('0')) { + if (at_end() || peek() < '1' || peek() > '9') + fail(minus ? "expected digits after -" : "expected value"); + while (!at_end() && is_digit(peek())) + ++pos_; + } + if (have('.')) { + if (at_end() || !is_digit(peek())) + fail("need at least one digit after '.'"); + while (!at_end() && is_digit(peek())) + ++pos_; + } + if (!at_end() && (peek() == 'e' || peek() == 'E')) { + ++pos_; + if (!at_end() && (peek() == '+' || peek() == '-')) + ++pos_; + if (at_end() || !is_digit(peek())) + fail("need at least one digit in exponent"); + while (!at_end() && is_digit(peek())) + ++pos_; + } + out.assign(doc_.substr(start, pos_ - start)); +} + +void state_json_reader::emit(const std::string &text) { + if (!applying_ || stack_.empty() || text.empty()) + return; + const frame &f = stack_.back(); + (*target_)(f.array ? std::string() : f.key).value(text); + ++applied_; +} + +// The grammar is read_json()'s: a value, optionally preceded by a byte-order mark (a leading +// 0xEF skips three bytes, as read_json's does) and followed by nothing but whitespace (space, +// tab, CR, LF). The first pass checks the document; the second, identical one sets its values. +// The container stack is on the heap, so nesting costs no native stack. +bool state_json_reader::step(std::size_t values) { + std::size_t n = 0; + for (;;) { + switch (want_) { + case want::start: + pos_ = 0; + line_ = 1; + stack_.clear(); + if (!doc_.empty() && static_cast(doc_[0]) == 0xEF) + pos_ = std::min(3, doc_.size()); + want_ = want::value; + break; + case want::value: { + if (values != 0 && n >= values) { + bytes_ = scanned_ + pos_; + return false; + } + ++n; + ++values_; + skip_ws(); + if (have('{')) { + stack_.push_back(frame{false, {}}); + want_ = want::key_or_end; + } else if (have('[')) { + stack_.push_back(frame{true, {}}); + want_ = want::value_or_end; + } else { + std::string text; + scan_scalar(text); + emit(text); + want_ = after_value(); + } + break; + } + case want::key_or_end: + skip_ws(); + if (have('}')) { + stack_.pop_back(); + want_ = after_value(); + } else { + want_ = want::key; + } + break; + case want::value_or_end: + skip_ws(); + if (have(']')) { + stack_.pop_back(); + want_ = after_value(); + } else { + want_ = want::value; + } + break; + case want::key: { + skip_ws(); + if (!have('"')) + fail("expected key string"); + std::string &key = stack_.back().key; + key.clear(); + scan_string(key); + skip_ws(); + expect(':', "expected ':'"); + want_ = want::value; + break; + } + case want::comma_or_end: { + skip_ws(); + const bool array = stack_.back().array; + if (have(',')) { + want_ = array ? want::value : want::key; + break; + } + expect(array ? ']' : '}', array ? "expected ']' or ','" : "expected '}' or ','"); + stack_.pop_back(); + want_ = after_value(); + break; + } + case want::end: + skip_ws(); + if (!at_end()) + fail("garbage after data"); + scanned_ += pos_; + bytes_ = scanned_; + if (!applying_) { + applying_ = true; + want_ = want::start; + break; + } + want_ = want::done; + return true; + case want::done: + return true; + } + } +} + +} // namespace cvc diff --git a/src/cvc/tests/state_deep_tree_test.cpp b/src/cvc/tests/state_deep_tree_test.cpp index 06cb4d42d..62f4b4f0c 100644 --- a/src/cvc/tests/state_deep_tree_test.cpp +++ b/src/cvc/tests/state_deep_tree_test.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -486,6 +487,176 @@ TEST_F(StateDeepTree, ResetSignalsWithoutTheAncestorsLocks) { c2.disconnect(); } +// ---- the JSON writer and reader (state_json.h) ----------------------------------- + +namespace { +// What json() used to return: boost's writer over ptree(). +std::string boost_json(cvc::state &node) { + std::stringstream ss; + boost::property_tree::write_json(ss, node.ptree()); + return ss.str(); +} + +// What json(j) used to do: boost's parser, then ptree(pt). +void boost_load(cvc::state &node, const std::string &doc) { + std::stringstream ss(doc); + boost::property_tree::ptree pt; + boost::property_tree::read_json(ss, pt); + node.ptree(pt); +} + +// Every value under `node`, as (path, value) pairs in pre-order: what a load left behind. +std::vector> tree_values(cvc::state &node) { + std::vector> out{{"", node.value()}}; + for (const std::string &name : node.children()) { + const std::string rel = name.substr(node.fullName().size() + 1); + out.emplace_back(rel, node.findDescendant(rel)->value()); + } + return out; +} +} // namespace + +TEST_F(StateDeepTree, JsonIsWhatWriteJsonWrites) { + // Every byte but '.' in a key and a value, a valued node, an empty childless one and one with + // only children: the writer escapes and lays out the document exactly as write_json() does. + cvc::state &top = root("bytes"); + top.value("top \"/\\ value"); + for (int c = 1; c < 256; ++c) { + if (c == '.') + continue; + const std::string ch(1, static_cast(c)); + top("k" + ch + "x").value("v" + ch + "w"); + } + top("empty"); + top("only.children").value("x"); + top("utf8").value("\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80"); + EXPECT_EQ(top.json(), boost_json(top)); + EXPECT_EQ(root("bytes.only").json(), boost_json(root("bytes.only"))); + EXPECT_EQ(root.json(), boost_json(root)); + cvc::app empty_app; + EXPECT_EQ(cvc::state::instance(empty_app)("none").json(), "{\n}\n"); + + // A slice at a time, the same text. + cvc::state_json_writer w(top); + int calls = 1; + while (!w.step(1)) + ++calls; + EXPECT_EQ(w.text(), top.json()); + EXPECT_GT(calls, 250); +} + +TEST_F(StateDeepTree, JsonLoadsWhatReadJsonLoads) { + const std::vector docs{ + R"({"a": "1", "b.c": "2", "": "self"})", + R"({"j": "1", "j.b": {"j.b.c": {"j.b.c": "3"}, "j.b.e": {"j.b.e": "5"}}, "j.d": ""})", + R"({"n": 1.50, "m": -0, "e": 1e5, "E": 2E-3, "x": -12.5e+3, "t": true, "f": false, "z": null})", + R"({"arr": ["x", "y", {"in": "z"}], "deep": [[["w"]]], "empty": [], "o": {}})", + " {\"esc\": \"\\u00e9\\ud83d\\ude00\\/\\\"\\\\\\b\\f\\n\\r\\t\", \"k\\u0041\": \"v\"}\r\n ", + "\xEF\xBB\xBF{\"bom\": \"1\"}", + "{}", + "\"top\"", + "[]", + "[\"own\"]", + R"({"empty": "", "dup": "1", "dup": "2"})", + "{\n\t\"a\" :\n\"b\" ,\"c\":{\"d\":\"e\"}}", + "{\"utf8\": \"\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\"}", + }; + for (const std::string &doc : docs) { + SCOPED_TRACE(doc); + cvc::app a, b; + cvc::state &na = cvc::state::instance(a)("t"); + cvc::state &nb = cvc::state::instance(b)("t"); + na.json(doc); + boost_load(nb, doc); + EXPECT_EQ(tree_values(na), tree_values(nb)); + + // A value at a time, the same tree. + cvc::app c; + cvc::state &nc = cvc::state::instance(c)("t"); + cvc::state_json_reader r(nc, doc); + while (!r.step(1)) { + } + EXPECT_EQ(tree_values(nc), tree_values(nb)); + } +} + +TEST_F(StateDeepTree, AMalformedDocumentFailsAsReadJsonDoesAndSetsNothing) { + // Each starts with a good entry, which must not be set: the document is checked first. + const std::vector bad{ + R"({"ok": "1", "a": })", + R"({"ok": "1", "a" "b"})", + R"({"ok": "1", "a": "b",})", + R"({"ok": "1", "a": [1,]})", + "{\"ok\": \"1\", \"a\": \"\x01\"}", + R"({"ok": "1", "a": "\uD800"})", + R"({"ok": "1", "a": "\uDC00"})", + R"({"ok": "1", "a": "\uD800A"})", + R"({"ok": "1", "a": tru})", + R"({"ok": "1", "a": -})", + R"({"ok": "1", "a": 01})", + R"({"ok": "1", "a": 1.})", + R"({"ok": "1", "a": 1e})", + R"({"ok": "1", "a": "unterminated)", + R"({"ok": "1"} x)", + R"({"ok": "1", "a": "\q"})", + "{\"ok\": \"1\", \"a\": \"\xC3\"}", + "{\"ok\": \"1\", \"a\": \"\x80\"}", + "{\"ok\": \"1\",\n\"a\":\n}", + "", + "{\"ok\": \"1\"", + }; + using boost::property_tree::json_parser::json_parser_error; + for (const std::string &doc : bad) { + SCOPED_TRACE(doc); + std::string want, got; + unsigned long want_line = 0, got_line = 0; + try { + std::stringstream ss(doc); + boost::property_tree::ptree pt; + boost::property_tree::read_json(ss, pt); + ADD_FAILURE() << "read_json accepted it"; + } catch (const json_parser_error &e) { + want = e.message(); + want_line = e.line(); + } + cvc::state &node = root("bad"); + try { + node.json(doc); + ADD_FAILURE() << "json(j) accepted it"; + } catch (const json_parser_error &e) { + got = e.message(); + got_line = e.line(); + } + EXPECT_EQ(got, want); + EXPECT_EQ(got_line, want_line); + EXPECT_EQ(node.findDescendant("ok"), nullptr); + } +} + +TEST_F(StateDeepTree, ADeeplyNestedDocumentLoads) { + // read_json() recursed once per nesting level, and so did destroying the ptree it built. The + // reader keeps its containers on the heap. + const std::size_t n = depth(); + std::string objects, arrays; + objects.reserve(8 * n + 8); + arrays.reserve(2 * n + 8); + for (std::size_t i = 0; i < n; ++i) { + objects += "{\"k\": "; + arrays += '['; + } + objects += "\"v\""; + arrays += "\"w\""; + objects.append(n, '}'); + arrays.append(n, ']'); + cvc::state &node = root("nested"); + on_small_stack([&] { + node.json(objects); + node.json(arrays); + }); + EXPECT_EQ(root("nested.k").value(), "v"); + EXPECT_EQ(node.value(), "w"); // an array's values have no key: the node itself +} + TEST_F(StateDeepTree, WriteBench) { if (env_size("CVC_STATE_BENCH", 0) != 1) { SUCCEED(); diff --git a/src/cvc/tests/state_exec_data_steps_test.cpp b/src/cvc/tests/state_exec_data_steps_test.cpp index db5dfc6d2..2b02c40d1 100644 --- a/src/cvc/tests/state_exec_data_steps_test.cpp +++ b/src/cvc/tests/state_exec_data_steps_test.cpp @@ -572,3 +572,146 @@ TEST_F(DataStepsScheduler, BudgetUsageAndInspectReportTheKinds) { ASSERT_TRUE(info); EXPECT_GT(info->data_time, 0.0); } + +// ---- state-json / state-from-json ------------------------------------------------------ +// +// cvc::state::json() and json(j) as data jobs: the same writer and reader (state_json.h), a +// slice of data.step_nodes units per step. + +class DataStepsStateJson : public ::testing::Test { +protected: + cvc::app app_ctx; + cvc::state &root = cvc::state::instance(app_ctx); + async_scheduler sched; + memory_tracker tracker; + intrinsics_context ictx; + environment_ptr env = builtins::make_default_environment(); + + void SetUp() override { + root("state_exec.defaults.data.step_nodes").value("64"); + sched.set_watch_root(&root); + ictx.sched = &sched; + ictx.root = &root; + ictx.tracker = &tracker; + register_intrinsics(env, &ictx); + // deep: a 1000-level chain with a value on every node; wide: 20000 children. + cvc::state *node = &root("deep"); + node->value("0"); + for (int i = 1; i <= 1000; ++i) { + node = &(*node)("n"); + node->value(std::to_string(i)); + } + cvc::state &wide = root("wide"); + for (int i = 0; i < 20000; ++i) + wide("k" + std::to_string(i)).value("v" + std::to_string(i)); + } + + int submit(const std::string &script, execute_options opts = {}) { + opts.env = environment::extend(env); + return sched.execute(script, opts); + } + + std::string error(int pid) const { return sched.get_exit_error(pid).value_or(""); } +}; + +TEST_F(DataStepsStateJson, StateJsonWritesWhatJsonWritesASliceAStep) { + for (const char *path : {"deep", "wide"}) { + SCOPED_TRACE(path); + const int pid = submit(std::string("(state-json \"") + path + "\")"); + sched.sync_run(); + const auto info = sched.get_process_info(pid); + ASSERT_TRUE(info); + ASSERT_EQ(info->status, process_status::terminated) << error(pid); + const std::string doc = root(path).json(); + EXPECT_EQ(std::get(sched.get_result(pid)->v), doc); + // A unit per node and per 256 bytes of text, 64 units a step (a slice overruns by at most its + // last node's bytes: deep's longest key is ~2 KB, 8 units). + const std::size_t units = root(path).children().size() + 1 + doc.size() / kDataNodeBytes; + EXPECT_GE(info->data_nodes, units); + EXPECT_GE(info->data_steps, units / (64 + 8) - 1); + EXPECT_GT(info->data_steps, 50u); + } + // A missing node is nil, like state-get. + const int missing = submit("(state-json \"no.such.node\")"); + sched.sync_run(); + EXPECT_TRUE(sched.get_result(missing)->is_nil()); +} + +TEST_F(DataStepsStateJson, StateFromJsonLoadsWhatJsonLoadsASliceAStep) { + const int pid = submit("(state-from-json \"copy\" (state-json \"wide\"))"); + sched.sync_run(); + const auto info = sched.get_process_info(pid); + ASSERT_EQ(info->status, process_status::terminated) << error(pid); + EXPECT_EQ(root("copy").json(), root("wide").json()); + EXPECT_GT(info->data_steps, 200u); + + // Keys are relative to the node, so a document goes back into the node it came from. + const std::string doc = root("deep").json(); + root("saved").value(doc); + root("deep").reset(false); // its value and the whole chain + const int restore = submit("(state-from-json \"deep\" (state-get \"saved\"))"); + sched.sync_run(); + ASSERT_EQ(sched.get_process_info(restore)->status, process_status::terminated) << error(restore); + EXPECT_EQ(root("deep").json(), doc); +} + +TEST_F(DataStepsStateJson, OtherProcessesRunBetweenSlices) { + root("state_exec.defaults.data.step_nodes").value("4"); + const int slow = submit("(state-json \"wide\")"); + while (sched.get_process_info(slow)->data_steps == 0) + sched.sync_step(); + const int quick = submit("(begin 1 2 3)"); + while (sched.get_process_info(quick)->status != process_status::terminated) + sched.sync_step(); + EXPECT_NE(sched.get_process_info(slow)->status, process_status::terminated); + sched.sync_run(); + EXPECT_EQ(std::get(sched.get_result(slow)->v), root("wide").json()); +} + +TEST_F(DataStepsStateJson, TheWorkCountsAgainstMaxSteps) { + execute_options opts; + opts.max_steps = 20; + const int pid = submit("(state-json \"wide\")", opts); + sched.sync_run(); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); +} + +TEST_F(DataStepsStateJson, AMalformedDocumentSetsNothingThoughItsCheckSpansSteps) { + // Thousands of good entries, then an error: the first pass finds it before any is set. + std::string doc = "{"; + for (int i = 0; i < 5000; ++i) + doc += "\"k" + std::to_string(i) + "\": \"v\", "; + doc += "\"bad\": }"; + root("doc").value(doc); + const int pid = submit("(state-from-json \"target\" (state-get \"doc\"))"); + sched.sync_run(); + const auto info = sched.get_process_info(pid); + EXPECT_EQ(info->status, process_status::killed); + EXPECT_NE(error(pid).find("state-from-json: line 1: expected value"), std::string::npos) + << error(pid); + EXPECT_GT(info->data_steps, 10u); + EXPECT_EQ(root.findDescendant("target.k0"), nullptr); +} + +TEST_F(DataStepsStateJson, TheDocumentIsHeldToTheAllocationLimit) { + root("state_exec.defaults.limits.max_alloc_bytes").value("4096"); + const int pid = submit("(state-json \"wide\")"); + sched.sync_run(); + EXPECT_EQ(sched.get_process_info(pid)->status, process_status::killed); + EXPECT_NE(error(pid).find("state-json"), std::string::npos) << error(pid); +} + +TEST_F(DataStepsStateJson, NodesRemovedBetweenSlicesAreStillWritten) { + // The walk holds the children it has yet to visit, so dropping them from the tree mid-walk + // neither crashes it nor loses them: they are written as they were. + root("state_exec.defaults.data.step_nodes").value("16"); + const std::string before = root("wide").json(); + const int pid = submit("(state-json \"wide\")"); + while (sched.get_process_info(pid)->data_steps < 3) + sched.sync_step(); + root("wide").reset(false); // destroys the children the tree owned + sched.sync_run(); + ASSERT_EQ(sched.get_process_info(pid)->status, process_status::terminated) << error(pid); + EXPECT_EQ(std::get(sched.get_result(pid)->v), before); + EXPECT_EQ(root("wide").numChildren(), 0u); +}