diff --git a/bindings/pycvc/test_pycvc_exec.py b/bindings/pycvc/test_pycvc_exec.py index dff90e373..33f610b49 100644 --- a/bindings/pycvc/test_pycvc_exec.py +++ b/bindings/pycvc/test_pycvc_exec.py @@ -149,6 +149,42 @@ def capture(x): assert seen["val"] == blob, seen["val"] +# ── the stdlib modules (string, bytes, math, collections) ─────────────── + + +def test_stdlib_modules_are_always_loaded(): + app = pycvc.make_app() + ex = pycvc.Exec(app) + assert ex.run('(string.length "café")') == "4" # codepoints, not bytes + assert ex.run('(string.upper "go")') == "GO" + assert ex.run('(bytes.length (string.encode "café"))') == "5" + assert ex.run("(math.floor 3.7)") == "3" + assert ex.run("(collections.reduce (list 1 2 3) + 0)") == "6" + # A program's own binding shadows a stdlib name without touching other programs. + assert ex.run("(begin (set string.length 7) string.length)") == "7" + assert pycvc.Exec(app).run('(string.length "ab")') == "2" + + +def test_stdlib_allocation_limit_comes_from_the_app_tree(): + app = pycvc.make_app() + ex = pycvc.Exec(app) + + def run_error(src): + try: + ex.run(src) + except Exception as e: # noqa: BLE001 - any surfaced DSL error + return str(e) + return "" + + # The 16 MiB default stops a huge one-call allocation. + assert "allocation limit" in run_error("(collections.range 1000000000)") + # limits.max_alloc_bytes on the app tree is read on every run. + pycvc.state_set(app, "state_exec.defaults.limits.max_alloc_bytes", "1024") + assert "collections.range" in run_error("(collections.range 1000)") + pycvc.state_set(app, "state_exec.defaults.limits.max_alloc_bytes", "1048576") + assert ex.run("(length (collections.range 1000))") == "1000" + + if __name__ == "__main__": tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] for t in tests: diff --git a/docs/ARIADNE.md b/docs/ARIADNE.md index 33df701db..90da9dc59 100644 --- a/docs/ARIADNE.md +++ b/docs/ARIADNE.md @@ -122,11 +122,15 @@ A fire-once handler is *submitted* (`submit_action`, plus host intrinsics, its writes chrooted to the action's mount prefix (§7), under per-activation caps ([`ariadne.cpp:1382`](../src/cvc/ariadne/ariadne.cpp)): -| bound | constant | value | -|-------|----------|-------| +| bound | source | value | +|-------|--------|-------| | step cap | `kActionMaxSteps` | `5000` | | wall-time cap | `kActionMaxSeconds` | `0.1 s` | -| write budget | `kActionMaxBytes` | `256 KiB` | +| memory budget | tree setting `ariadne.limits.action_max_memory` | `4 MiB` by default; `0` = no limit | + +The memory budget is read on every submission. It bounds the action's state-tree writes and +also each native call's allocations: state_exec's per-step allocation limit takes what is left +of `max_memory` (developer guide §11, "Allocation limit"). A quick action completes within that drain. An action that `await`s, `sleep`s, or `msg-recv`s **parks** and resumes on a later frame — this is what makes the async arc diff --git a/docs/STATE_EXEC_DEVELOPER_GUIDE.md b/docs/STATE_EXEC_DEVELOPER_GUIDE.md index 605d1eace..393a689a9 100644 --- a/docs/STATE_EXEC_DEVELOPER_GUIDE.md +++ b/docs/STATE_EXEC_DEVELOPER_GUIDE.md @@ -132,6 +132,7 @@ The full syntax (literals, escapes, Unicode rules) is specified in **I/O:** `print` **Generator:** `generator`, `next`, `range`, `collect`, `gen-done?` **OOP:** `send` (method dispatch), `apply` +**Stdlib:** `string.*`, `bytes.*`, `math.*`, `collections.*`, always loaded ([§11](#11-standard-library)) #### Type Conversion @@ -888,6 +889,7 @@ sched.load_settings(); // read and apply settings | Setting | Per-scheduler path | Global path | Default | |---------|-------------------|-------------|---------| | `max_pending_messages` | `state_exec.schedulers..max_pending_messages` | `state_exec.defaults.max_pending_messages` | `1024` | +| `limits.max_alloc_bytes` | `state_exec.schedulers..limits.max_alloc_bytes` | `state_exec.defaults.limits.max_alloc_bytes` | `16777216` ([§11](#allocation-limit)) | ```cpp // Set a global default for all schedulers @@ -907,6 +909,7 @@ After `load_settings()`, the scheduler publishes its effective configuration back to the state tree: - `state_exec.schedulers..max_pending_messages` — the resolved cap +- `state_exec.schedulers..limits.max_alloc_bytes` — the resolved per-call allocation limit - `state_exec.schedulers..policy` — `"round_robin"`, `"priority"`, or `"priority_rr"` --- @@ -1290,75 +1293,181 @@ coord.set_resource_policy(policy); ## 11. Standard Library -The `stdlib_registry` provides additional functions organized into modules. -Import a module into a process's environment to make its functions available: +Four modules of native functions sit beside the builtins: `string`, `bytes`, `math` and +`collections`. Every environment made by `builtins::make_default_environment()` reaches them, +so every host does without an import step: + +- the sync and async schedulers' default environment (`cvc exec`, `app::exec_scheduler()`); +- Ariadne's `init:`, action (`on:`) and resident lanes; +- pycvc's `Exec`. + +```lisp +(string.length "café") ;; => 4 +(math.sqrt 16) ;; => 4 +(collections.reverse (list 1 2 3)) ;; => (3 2 1) +``` + +Every name carries its module prefix, so none clashes with a builtin: `length`, `range` and +`slice` are still the builtins, and `string.length` or `collections.range` are the stdlib. + +### How the modules are loaded + +`stdlib_registry::shared_scope()` builds one scope holding every function of +`stdlib_registry::standard()`, once per process. `make_default_environment()` links it as its +outer scope, so a lookup that misses the builtins falls through to it. Nothing is copied into +each environment. + +The shared scope is frozen. `set`, `defun` and `environment::set_existing()` never write into +it: a program that rebinds a stdlib name shadows it in its own scope, and other programs keep +the real function. + +```lisp +(set string.length 42) +string.length ;; => 42 in this program only +``` + +Downstream C++ code can build its own `stdlib_registry`, add modules with +`register_function()`, and import them into an environment it owns: ```cpp #include -stdlib_registry stdlib; +stdlib_registry mine; +mine.register_function("geo", "geo.area", area_fn); +mine.import_module("geo", env); // every function of the module +mine.import_module("geo", env, {"geo.area"}); // or a subset +``` + +### Allocation limit -// Import all functions from a module -stdlib.import_module("string", env); -stdlib.import_module("math", env); -stdlib.import_module("collections", env); +A native call builds its whole result inside one evaluator step. The per-process +`max_memory` check runs between steps and counts only state-tree writes, so it cannot see +`(collections.range 1000000000)`. Each function whose result can be larger than all of its +arguments therefore estimates the result's size and fails before allocating when it is over +the limit: -// Import specific functions -stdlib.import_module("math", env, {"math.sqrt", "math.abs"}); +- `string.split`, `string.join`, `string.replace`, `bytes.concat`; +- `collections.range`, `collections.zip`, `collections.flatten`, `collections.dict-merge`; +- the builtins `str-concat` and `collect`. + +The limit for a step is the tighter of two values: + +1. the tree setting `limits.max_alloc_bytes` (table below; `0` means no limit of its own); +2. what is left of the process's `max_memory`, when it has one. + +| Setting | Per-scheduler path | Global path | Default | +|---------|-------------------|-------------|---------| +| `limits.max_alloc_bytes` | `state_exec.schedulers..limits.max_alloc_bytes` | `state_exec.defaults.limits.max_alloc_bytes` | `16777216` (16 MiB) | + +The sync scheduler resolves it in `load_settings()` and publishes the effective value under +`state_exec.schedulers..limits.max_alloc_bytes`. The async scheduler (the app's exec +scheduler, and the one inside pycvc's `Exec`) reads the global path on every submission. A +bare evaluator, or a native called directly from C++, gets the 16 MiB default. + +A call over the limit is a runtime error naming the function. The async scheduler kills the +process with it as the exit error; the sync scheduler's `run()` throws it: + +```lisp +(collections.range 1000000000) +;; error: collections.range: the result needs bytes, over the allocation limit of +;; 16777216 bytes (limits.max_alloc_bytes, or what is left of max_memory) ``` -### Module: `string` (12 functions) +`collections.unique` and `collections.dict-merge` are quadratic, so they also poll the +evaluation deadline (`max_time`) as they go. + +### In Ariadne predicates + +A per-frame predicate (`visible_when`, `bind`, `fmt`, …) runs in a reduced environment. It +gets the stdlib functions whose result is a scalar or no bigger than their argument: + +- `string.length`, `string.byte-length`, `string.char-at`, `string.substring`, + `string.upper`, `string.lower`, `string.trim`, `string.starts-with`, `string.ends-with`, + `string.contains`; +- `bytes.length`, `bytes.byte-at`; +- every `math.*` function except `math.random`, which would make a predicate + nondeterministic. + +The rest (`string.join`/`replace`/`split`/`encode`, `bytes.slice`/`concat`/`decode` and all of +`collections.*`) are unbound in a predicate until they run as scheduled work, a slice per +step. The `init:` and action lanes get every function. + +### Module: `string` (14 functions) + +Indices and lengths count code points, not bytes (`UNICODE_SUPPORT_ROADMAP.md` SE-1). ```lisp -(string.split "a,b,c" ",") ;; => ("a" "b" "c") +(string.split "a,b,c" ",") ;; => ("a" "b" "c") +(string.split "né" "") ;; => ("n" "é") one element per code point (string.join (list "a" "b" "c") ",") ;; => "a,b,c" -(string.replace "hello" "l" "r") ;; => "herro" -(string.trim " hello ") ;; => "hello" -(string.upper "hello") ;; => "HELLO" -(string.lower "HELLO") ;; => "hello" -(string.starts-with "hello" "he") ;; => true -(string.ends-with "hello" "lo") ;; => true -(string.contains "hello" "ell") ;; => true -(string.substring "hello" 1 3) ;; => "el" -(string.char-at "hello" 0) ;; => "h" -(string.length "hello") ;; => 5 +(string.replace "hello" "l" "r") ;; => "herro" +(string.trim " hello ") ;; => "hello" +(string.upper "hello") ;; => "HELLO" ASCII letters only +(string.lower "HELLO") ;; => "hello" +(string.starts-with "hello" "he") ;; => #t +(string.ends-with "hello" "lo") ;; => #t +(string.contains "hello" "ell") ;; => #t +(string.substring "hello" 1 3) ;; => "ell" (start, length) +(string.char-at "café" 3) ;; => "é" +(string.length "café") ;; => 4 +(string.byte-length "café") ;; => 5 +(string.encode "café") ;; => b"caf\xc3\xa9" utf-8 only +``` + +### Module: `bytes` (5 functions) + +Byte semantics throughout; `string.encode` and `bytes.decode` are the only crossings between +text and bytes. + +```lisp +(bytes.length b"\x01\x02") ;; => 2 +(bytes.byte-at b"\x01\xff" 1) ;; => 255 +(bytes.slice b"abcd" 1 2) ;; => b"bc" (start, length) +(bytes.concat b"ab" b"cd") ;; => b"abcd" +(bytes.decode b"caf\xc3\xa9") ;; => "café" invalid UTF-8 is an error ``` ### Module: `math` (16 functions) ```lisp -(math.sqrt 16) ;; => 4.0 +(math.sqrt 16) ;; => 4 (math.abs -5) ;; => 5 -(math.floor 3.7) ;; => 3.0 -(math.ceil 3.2) ;; => 4.0 -(math.round 3.5) ;; => 4.0 -(math.pow 2 10) ;; => 1024.0 -(math.log 2.718) ;; => ~1.0 -(math.sin 0) ;; => 0.0 -(math.cos 0) ;; => 1.0 -(math.min 3 7 1 5) ;; => 1 -(math.max 3 7 1 5) ;; => 7 -(math.clamp 15 0 10) ;; => 10 -(math.pi) ;; => 3.14159... -(math.e) ;; => 2.71828... -(math.random) ;; => random double in [0,1) +(math.floor 3.7) ;; => 3 an integer +(math.ceil 3.2) ;; => 4 an integer +(math.round 3.5) ;; => 4 an integer +(math.pow 2 10) ;; => 1024 +(math.log 1) ;; => 0 +(math.sin 0) ;; => 0 +(math.cos 0) ;; => 1 +(math.tan 0) ;; => 0 +(math.min 3 7 1 5) ;; => 1 a float +(math.max 3 7 1 5) ;; => 7 a float +(math.clamp 15 0 10) ;; => 10 a float +(math.pi) ;; => 3.14159 +(math.e) ;; => 2.71828 +(math.random) ;; => a float in [0, 1) ``` ### Module: `collections` (12 functions) +`collections.map`, `filter` and `reduce` take the list first, then a native function: a +builtin or a stdlib function. A DSL `lambda` is not accepted yet; it needs the scheduled-work +mechanism above. + ```lisp -(collections.map (lambda (x) (* x x)) (list 1 2 3)) ;; => (1 4 9) -(collections.filter (lambda (x) (> x 2)) (list 1 2 3 4)) ;; => (3 4) -(collections.reduce (lambda (a b) (+ a b)) (list 1 2 3) 0) ;; => 6 -(collections.zip (list 1 2) (list "a" "b")) ;; => ((1 "a") (2 "b")) -(collections.flatten (list (list 1 2) (list 3 4))) ;; => (1 2 3 4) -(collections.sort (list 3 1 2)) ;; => (1 2 3) -(collections.reverse (list 1 2 3)) ;; => (3 2 1) -(collections.range 1 5) ;; => (1 2 3 4) -(collections.unique (list 1 2 2 3 3)) ;; => (1 2 3) -(collections.dict-keys (dict "a" 1 "b" 2)) ;; => ("a" "b") -(collections.dict-values (dict "a" 1 "b" 2)) ;; => (1 2) -(collections.dict-merge (dict "a" 1) (dict "b" 2)) ;; => {a: 1, b: 2} +(collections.map (list 1 4 9) math.sqrt) ;; => (1 2 3) +(collections.filter (list 1 "a" 2) is-int) ;; => (1 2) +(collections.reduce (list 1 2 3) + 0) ;; => 6 +(collections.zip (list 1 2) (list "a" "b")) ;; => ((1 "a") (2 "b")) +(collections.flatten (list (list 1 2) 3 (list 4))) ;; => (1 2 3 4) one level +(collections.sort (list 3 1 2)) ;; => (1 2 3) +(collections.reverse (list 1 2 3)) ;; => (3 2 1) +(collections.range 1 5) ;; => (1 2 3 4) +(collections.range 5 0 -2) ;; => (5 3 1) +(collections.unique (list 1 2 2 3 3)) ;; => (1 2 3) +(collections.dict-keys (dict "a" 1 "b" 2)) ;; => ("a" "b") +(collections.dict-values (dict "a" 1 "b" 2)) ;; => (1 2) +(collections.dict-merge (dict "a" 1) (dict "b" 2)) ;; => {"a": 1, "b": 2} ``` --- @@ -1557,7 +1666,7 @@ unique socket paths (incorporating PID or a UUID) and ├─────────────────────────────────────────────────────────────────┤ │ parser.h → value_t AST │ │ builtins.h → +,-,*,/,list,dict,str,... │ -│ stdlib.h → string.*,math.*,collections.* │ +│ stdlib.h → string.*,bytes.*,math.*,collections.* │ ├─────────────────────────────────────────────────────────────────┤ │ evaluator.h → sync recursive (local only) │ │ stackless_evaluator.h → sync stackless (serializable) │ @@ -1607,6 +1716,6 @@ types.h ← parser.h ← builtins.h ← evaluator.h task.h ← async_evaluator.h ← async_stackless_evaluator.h resource_policy.h ← exec_coordinator.h - stdlib.h (standalone) + stdlib.h ← builtins.cpp (the shared stdlib scope) memory_tracker.h ← scheduler.h ``` diff --git a/inc/cvc/state/state_exec/process.h b/inc/cvc/state/state_exec/process.h index 21b2b5a3a..39053c693 100644 --- a/inc/cvc/state/state_exec/process.h +++ b/inc/cvc/state/state_exec/process.h @@ -69,6 +69,9 @@ struct process { // pump's own per-drain cap bounds it; this cap is the backstop for an in-STEP loop the pump can't // interrupt.) double max_step_time = 0.0; + // The most a single native call may allocate (0 = no limit of its own): the tree setting + // limits.max_alloc_bytes, resolved when the process is submitted. See alloc_limit(). + uint64_t max_alloc_bytes = kDefaultMaxAllocBytes; // Evaluator state (stackless — serializable) evaluator_state state; @@ -142,6 +145,11 @@ struct process { /// Seconds of max_time left (nullopt when unlimited): the budget of the next step's deadline. std::optional time_left() const; + /// The allocation limit for the next step's eval_alloc_guard (nullopt when unlimited): the + /// tighter of max_alloc_bytes and what is left of max_memory once `used_bytes` (the memory + /// tracker's count for this process) is taken out. + std::optional alloc_limit(std::size_t used_bytes) const; + /// Step count from evaluator stats. uint64_t step_count() const { return state.stats.get_step_count(); } }; diff --git a/inc/cvc/state/state_exec/scheduler.h b/inc/cvc/state/state_exec/scheduler.h index 4bb9e5989..67a30a72a 100644 --- a/inc/cvc/state/state_exec/scheduler.h +++ b/inc/cvc/state/state_exec/scheduler.h @@ -187,8 +187,13 @@ class scheduler : public scheduler_base { static constexpr const char *defaults_prefix = "state_exec.defaults"; static constexpr const char *sched_prefix = "state_exec.schedulers"; + /// The effective limits.max_alloc_bytes (kDefaultMaxAllocBytes until load_settings() reads + /// the tree); each process submitted afterwards takes it as its max_alloc_bytes. + uint64_t max_alloc_bytes() const { return max_alloc_bytes_; } + private: std::string id_; + uint64_t max_alloc_bytes_ = kDefaultMaxAllocBytes; scheduling_policy policy_; int next_pid_ = 1; int rr_index_ = 0; @@ -241,6 +246,14 @@ class scheduler : public scheduler_base { process_info make_info(const process &proc) const; }; +/// limits.max_alloc_bytes from the state tree under `root`: +/// `state_exec.schedulers..limits.max_alloc_bytes` (when scheduler_id is not +/// empty), then `state_exec.defaults.limits.max_alloc_bytes`, then `base`. A missing or +/// unparsable value is skipped. 0 means no limit of its own; the process's max_memory still +/// bounds each allocation. +uint64_t max_alloc_bytes_from_state(cvc::state &root, const std::string &scheduler_id, + uint64_t base = kDefaultMaxAllocBytes); + } // namespace cvc::state_exec #endif // CVC_STATE_EXEC_SCHEDULER_H diff --git a/inc/cvc/state/state_exec/stdlib.h b/inc/cvc/state/state_exec/stdlib.h index e1b4864ae..e3cf76c27 100644 --- a/inc/cvc/state/state_exec/stdlib.h +++ b/inc/cvc/state/state_exec/stdlib.h @@ -2,9 +2,11 @@ * @file stdlib.h * @brief Standard library modules for the state_exec DSL. * - * Provides loadable modules (string, math, collections) containing - * native utility functions. Modules are registered via stdlib_registry - * and imported into a DSL program's environment with (import "module"). + * Provides modules (string, bytes, math, collections) of native utility functions, named + * with their module prefix (string.length, math.sqrt, ...). Every environment from + * builtins::make_default_environment() reaches all of them through one shared, frozen scope + * (stdlib_registry::shared_scope()), so a DSL program calls (string.length "café") without + * importing anything. */ #ifndef CVC_STATE_EXEC_STDLIB_H #define CVC_STATE_EXEC_STDLIB_H @@ -19,14 +21,22 @@ namespace cvc::state_exec { /// Registry for standard library modules. /// -/// Modules are collections of named native functions that can be imported -/// into a DSL environment via (require "module"). Built-in modules -/// (string, math, collections) are registered at construction; downstream -/// code can add custom modules via register_function(). +/// Modules are collections of named native functions. Built-in modules (string, bytes, math, +/// collections) are registered at construction; downstream code can build its own registry +/// and add modules via register_function(), then import_module() them into an environment. class stdlib_registry { public: stdlib_registry(); + /// The built-in modules, built once on first use and shared read-only by every caller. + static const stdlib_registry &standard(); + + /// A frozen scope holding every function of standard(), built once on first use. + /// builtins::make_default_environment() links it as its outer scope, so the functions are + /// shared, never copied. A program's (set string.length ...) shadows the name in its own + /// scope; nothing writes the shared scope, so threads may read it concurrently. + static environment_ptr shared_scope(); + /// Register a native function into a module. void register_function(const std::string &module, const std::string &func_name, native_fn fn); diff --git a/inc/cvc/state/state_exec/types.h b/inc/cvc/state/state_exec/types.h index 0ce3556cb..b7e784043 100644 --- a/inc/cvc/state/state_exec/types.h +++ b/inc/cvc/state/state_exec/types.h @@ -179,6 +179,10 @@ struct closure { struct environment { std::unordered_map bindings; std::shared_ptr outer; + /// A frozen scope is shared read-only, possibly across threads (the stdlib scope that every + /// default environment links to). set() refuses it, and set_existing() never writes into it: a + /// name bound there is shadowed in the outermost unfrozen scope instead. + bool frozen = false; /// Drops `outer` through detail::drop_owner, so a long scope chain is not freed recursively. ~environment(); @@ -187,11 +191,11 @@ struct environment { value_t *lookup(const std::string &name); const value_t *lookup(const std::string &name) const; - /// Set a binding in this scope. + /// Set a binding in this scope. Throws std::logic_error when this scope is frozen. void set(const std::string &name, value_t val); - /// Set a binding in the nearest enclosing scope that has it, - /// or in this scope if not found. + /// Set a binding in the nearest enclosing unfrozen scope that has it, or in the outermost + /// unfrozen scope if none has it. Throws std::logic_error when called on a frozen scope. void set_existing(const std::string &name, value_t val); /// Create a new child scope. @@ -333,6 +337,49 @@ class eval_deadline_guard { /// long-running builtins call this in their loops so they too honour the budget. bool eval_deadline_expired(); +// -- per-call allocation limit ---------------------------------------------- +// +// A native call builds its whole result inside one evaluator step, where the per-process +// memory limit (checked between steps, and counting only state-tree writes) cannot see it: +// (collections.range 1000000000) would allocate tens of GB in one call. Every builtin or +// stdlib function whose result can be larger than all of its arguments estimates the result's +// size first and calls eval_alloc_check() before allocating. +// +// The schedulers arm a guard around each step with the tighter of the tree setting +// limits.max_alloc_bytes (state_exec.schedulers., then state_exec.defaults) and what is +// left of the process's max_memory. With no guard armed (a bare evaluator, a direct native +// call) the limit is kDefaultMaxAllocBytes. + +/// The limit when no guard is armed, and the default of limits.max_alloc_bytes: 16 MiB. +inline constexpr std::size_t kDefaultMaxAllocBytes = std::size_t{16} << 20; + +/// RAII: bound what a single native call may allocate on this thread while the guard lives +/// (std::nullopt: no bound). A guard nested inside another only tightens it. The schedulers +/// arm one per step. +class eval_alloc_guard { +public: + explicit eval_alloc_guard(std::optional max_bytes); + ~eval_alloc_guard(); + eval_alloc_guard(const eval_alloc_guard &) = delete; + eval_alloc_guard &operator=(const eval_alloc_guard &) = delete; + +private: + friend std::optional eval_alloc_limit(); + std::optional limit_; // the effective limit: this guard's, tightened by outer ones + eval_alloc_guard *outer_ = nullptr; +}; + +/// The allocation limit in force on this thread (std::nullopt: unbounded). +std::optional eval_alloc_limit(); + +/// Throw std::runtime_error naming `who` when `bytes` is over eval_alloc_limit(). +void eval_alloc_check(std::size_t bytes, const char *who); + +/// `count * each`, saturating at SIZE_MAX, for size estimates that must not wrap. +inline std::size_t alloc_bytes(std::size_t count, std::size_t each) { + return each != 0 && count > SIZE_MAX / each ? SIZE_MAX : count * each; +} + } // namespace cvc::state_exec #endif // CVC_STATE_EXEC_TYPES_H diff --git a/src/cvc/ariadne/ariadne.cpp b/src/cvc/ariadne/ariadne.cpp index 0660e8b38..99a9919f3 100644 --- a/src/cvc/ariadne/ariadne.cpp +++ b/src/cvc/ariadne/ariadne.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -259,6 +260,23 @@ void clear_action_intrinsics() { action_intrinsics_registry().clear(); } +namespace { +// §7.4 an action's memory budget: the tree setting ariadne.limits.action_max_memory (bytes; +// 0 = no limit), read on every submission so a host can tune it live. It also bounds each +// native call's allocations in the action (state_exec's per-step allocation limit takes what +// is left of max_memory). A missing or unparsable value keeps the 4 MiB default. +uint64_t action_max_memory(cvc::state &root) { + constexpr uint64_t kDefault = uint64_t{4} << 20; + cvc::state *node = root.findDescendant("ariadne.limits.action_max_memory"); + if (!node) + return kDefault; + const std::string v = node->value(); + uint64_t bytes = 0; + const auto [end, ec] = std::from_chars(v.data(), v.data() + v.size(), bytes); + return !v.empty() && ec == std::errc() && end == v.data() + v.size() ? bytes : kDefault; +} +} // namespace + namespace { // Run `script` as a FULL-ENV state_exec program chrooted to `prefix`, bounded by (max_steps, // max_seconds, max_bytes). Returns "" on a clean, normal finish, else a human diagnostic (no @@ -404,7 +422,16 @@ class ReactiveEngine { "list", "cons", "car", "cdr", "nth", "length", "dict", "get-attr", // side-effect-free state readers (state-data-get deep-copies; no aliasing, no callables) "state-get", "state-exists", "state-children", "state-data-get", "state-root-path", - "state-has-expiry", "state-is-expired"}; + "state-has-expiry", "state-is-expired", + // the pure, bounded stdlib: a scalar, or a result no bigger than its argument. The + // materializers (string.join/replace/split/encode, bytes.slice/concat/decode, + // collections.*) stay out until they run as scheduled, sliced work; math.random stays + // out because a predicate must be deterministic. + "string.length", "string.byte-length", "string.char-at", "string.substring", "string.upper", + "string.lower", "string.trim", "string.starts-with", "string.ends-with", "string.contains", + "bytes.length", "bytes.byte-at", "math.sqrt", "math.abs", "math.floor", "math.ceil", + "math.round", "math.pow", "math.log", "math.sin", "math.cos", "math.tan", "math.min", + "math.max", "math.clamp", "math.pi", "math.e"}; for (const char *name : kAllowed) if (const se::value_t *v = full->lookup(name)) env_->set(name, *v); @@ -1396,7 +1423,6 @@ void Runtime::Impl::submit_action(const QueuedAction &action) { // cap. Full env, writes chrooted to the action's mount prefix. static constexpr uint64_t kActionMaxSteps = 5000; static constexpr double kActionMaxSeconds = 0.1; - static constexpr uint64_t kActionMaxBytes = 262144; try { auto &sched = app.exec_scheduler(); used_scheduler_ = true; @@ -1404,6 +1430,7 @@ void Runtime::Impl::submit_action(const QueuedAction &action) { ac->proc = se::make_process(); // placeholder for ictx.proc; the RUNNING process is the ac->proc->status = se::process_status::ready; // scheduler's own, reached via current_process() cvc::state &root = cvc::state::instance(app); + const uint64_t max_bytes = action_max_memory(root); ac->ictx.sched = &sched; ac->ictx.tracker = &ac->tracker; ac->ictx.proc = ac->proc; @@ -1432,7 +1459,7 @@ void Runtime::Impl::submit_action(const QueuedAction &action) { opts.owner = owner_; // so ~Impl's kill_owner reaps exactly this document's processes opts.max_steps = kActionMaxSteps; opts.max_time = kActionMaxSeconds; - opts.max_memory = kActionMaxBytes; + opts.max_memory = max_bytes; const int pid = sched.execute(script, opts); // throws se::parse_error on a syntax error live_actions_[pid] = std::move(ac); // keep the context alive until the process ends } catch (const std::exception &e) { diff --git a/src/cvc/state/state_exec/async_scheduler.cpp b/src/cvc/state/state_exec/async_scheduler.cpp index 8895ed041..f4069fa6f 100644 --- a/src/cvc/state/state_exec/async_scheduler.cpp +++ b/src/cvc/state/state_exec/async_scheduler.cpp @@ -3,6 +3,7 @@ #include #include #include +#include // max_alloc_bytes_from_state namespace cvc::state_exec { @@ -34,6 +35,9 @@ int async_scheduler::execute(const std::string &script, const execute_options &o proc.max_memory = opts.max_memory; proc.max_messages = opts.max_messages; proc.max_message_bytes = opts.max_message_bytes; + // No load_settings() here: read the allocation limit from the tree on every submission. + proc.max_alloc_bytes = + watch_root_ ? max_alloc_bytes_from_state(*watch_root_, {}) : kDefaultMaxAllocBytes; proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); @@ -60,6 +64,9 @@ int async_scheduler::execute(const value_t &expr, const execute_options &opts) { proc.max_memory = opts.max_memory; proc.max_messages = opts.max_messages; proc.max_message_bytes = opts.max_message_bytes; + // No load_settings() here: read the allocation limit from the tree on every submission. + proc.max_alloc_bytes = + watch_root_ ? max_alloc_bytes_from_state(*watch_root_, {}) : kDefaultMaxAllocBytes; proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); @@ -160,6 +167,12 @@ void async_scheduler::execute_process_step(process &proc) { if (proc.max_step_time > 0.0) step_budget = step_budget ? std::min(*step_budget, proc.max_step_time) : proc.max_step_time; eval_deadline_guard step_deadline(step_budget); + // Bound what one native call in this step may allocate (see scheduler.cpp). + // Armed only when it changes something (see scheduler.cpp). + std::optional step_alloc; + if (proc.max_memory > 0 || proc.max_alloc_bytes != kDefaultMaxAllocBytes) + step_alloc.emplace( + proc.alloc_limit(proc.max_memory > 0 ? mem_tracker_.current_bytes(proc.pid) : 0)); const bool charge_cpu = proc.charges_cpu_time(); const double cpu0 = charge_cpu ? thread_cpu_seconds() : 0.0; // Drive one async_stackless_evaluator step to completion at the leaf. The evaluator's step() @@ -686,6 +699,7 @@ int async_scheduler::fork(int pid) { child->max_time = parent.max_time; child->max_step_time = parent.max_step_time; child->max_memory = parent.max_memory; + child->max_alloc_bytes = parent.max_alloc_bytes; child->max_messages = parent.max_messages; child->max_message_bytes = parent.max_message_bytes; child->signal_handlers = parent.signal_handlers; diff --git a/src/cvc/state/state_exec/builtins.cpp b/src/cvc/state/state_exec/builtins.cpp index cd93f3da9..ab0bc020e 100644 --- a/src/cvc/state/state_exec/builtins.cpp +++ b/src/cvc/state/state_exec/builtins.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include // SE-1: (length "...") counts codepoints for a string #include #include @@ -287,12 +288,20 @@ value_t builtin_ne(std::span args) { // --------------------------------------------------------------------------- value_t builtin_str_concat(std::span args) { + // The same string may be passed many times, so the result can outgrow every argument: check + // each part against the allocation limit before appending it. std::string result; + const auto add = [&result](const std::string &part) { + eval_alloc_check(result.size() > SIZE_MAX - part.size() ? SIZE_MAX + : result.size() + part.size(), + "str-concat"); + result += part; + }; for (auto &a : args) { if (auto *s = std::get_if(&a.v)) - result += *s; // raw string, no quoting + add(*s); // raw string, no quoting else - result += to_string(a); + add(to_string(a)); } return value_t{std::move(result)}; } @@ -741,6 +750,7 @@ value_t builtin_collect(std::span args) { // budget instead of hanging/OOMing (a no-op when no deadline is armed). if (eval_deadline_expired()) throw std::runtime_error("collect: evaluation exceeded time budget"); + eval_alloc_check(alloc_bytes(result.size() + 1, sizeof(value_t)), "collect"); result.push_back(std::move(*next)); } return make_list(std::move(result)); @@ -779,6 +789,9 @@ void builtins::register_fn(environment_ptr env, const std::string &name, native_ environment_ptr builtins::make_default_environment() { auto env = std::make_shared(); + // The stdlib modules (string.*, bytes.*, math.*, collections.*) are reached through the + // shared, frozen stdlib scope rather than copied into every environment. + env->outer = stdlib_registry::shared_scope(); // Arithmetic register_fn(env, "+", builtin_add); diff --git a/src/cvc/state/state_exec/process.cpp b/src/cvc/state/state_exec/process.cpp index 4465b96a3..afa45bfe4 100644 --- a/src/cvc/state/state_exec/process.cpp +++ b/src/cvc/state/state_exec/process.cpp @@ -55,4 +55,16 @@ std::optional process::time_left() const { return std::max(0.0, left); } +std::optional process::alloc_limit(std::size_t used_bytes) const { + std::optional limit; + if (max_alloc_bytes > 0) + limit = static_cast(std::min(max_alloc_bytes, SIZE_MAX)); + if (max_memory > 0) { + const uint64_t left = max_memory > used_bytes ? max_memory - used_bytes : 0; + const std::size_t left_bytes = static_cast(std::min(left, SIZE_MAX)); + limit = limit ? std::min(*limit, left_bytes) : left_bytes; + } + return limit; +} + } // namespace cvc::state_exec diff --git a/src/cvc/state/state_exec/scheduler.cpp b/src/cvc/state/state_exec/scheduler.cpp index a62aacca0..fbf35360d 100644 --- a/src/cvc/state/state_exec/scheduler.cpp +++ b/src/cvc/state/state_exec/scheduler.cpp @@ -55,10 +55,14 @@ void scheduler::load_settings() { max_pending_messages = effective; + // --- limits.max_alloc_bytes (same resolution order; see max_alloc_bytes_from_state) --- + max_alloc_bytes_ = max_alloc_bytes_from_state(*watch_root_, id_); + // Publish effective values to per-scheduler subtree if (!id_.empty()) { std::string base = std::string(sched_prefix) + "." + id_; (*watch_root_)(base + ".max_pending_messages").value(std::to_string(max_pending_messages)); + (*watch_root_)(base + ".limits.max_alloc_bytes").value(std::to_string(max_alloc_bytes_)); (*watch_root_)(base + ".policy") .value(policy_ == scheduling_policy::round_robin ? "round_robin" : policy_ == scheduling_policy::priority ? "priority" @@ -66,6 +70,19 @@ void scheduler::load_settings() { } } +uint64_t max_alloc_bytes_from_state(cvc::state &root, const std::string &scheduler_id, + uint64_t base) { + constexpr const char *suffix = ".limits.max_alloc_bytes"; + uint64_t effective = base; + if (auto v = read_size_setting(&root, std::string(scheduler::defaults_prefix) + suffix)) + effective = *v; + if (!scheduler_id.empty()) + if (auto v = read_size_setting(&root, std::string(scheduler::sched_prefix) + "." + + scheduler_id + suffix)) + effective = *v; + return effective; +} + // --------------------------------------------------------------------------- // Process submission // --------------------------------------------------------------------------- @@ -86,6 +103,7 @@ int scheduler::execute(const std::string &script, const execute_options &opts) { proc.max_memory = opts.max_memory; proc.max_messages = opts.max_messages; proc.max_message_bytes = opts.max_message_bytes; + proc.max_alloc_bytes = max_alloc_bytes_; proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); @@ -111,6 +129,7 @@ int scheduler::execute(const value_t &expr, const execute_options &opts) { proc.max_memory = opts.max_memory; proc.max_messages = opts.max_messages; proc.max_message_bytes = opts.max_message_bytes; + proc.max_alloc_bytes = max_alloc_bytes_; proc.signal_handlers = opts.signal_handlers; proc.on_complete = opts.on_complete; proc.create_time = std::chrono::steady_clock::now(); @@ -214,6 +233,14 @@ void scheduler::execute_process_step(process &proc) { // cooperative deadline for this step lets that inner work abort at the budget too — the // same guarantee the read-lane's run() already has. max_time <= 0 means unlimited (nullopt). eval_deadline_guard step_deadline(proc.time_left()); + // Bound what one native call in this step may allocate: limits.max_alloc_bytes, tightened to + // what is left of max_memory (which, between steps, only sees state-tree writes). + // With no budget and the default cap, an armed guard would change nothing (the default applies + // unarmed), so it is armed only when it does: it costs thread-local writes every step. + std::optional step_alloc; + if (proc.max_memory > 0 || proc.max_alloc_bytes != kDefaultMaxAllocBytes) + step_alloc.emplace( + proc.alloc_limit(proc.max_memory > 0 ? mem_tracker_.current_bytes(proc.pid) : 0)); const bool charge_cpu = proc.charges_cpu_time(); const double cpu0 = charge_cpu ? thread_cpu_seconds() : 0.0; bool done = evaluator_.step(proc.state); @@ -674,6 +701,7 @@ int scheduler::fork(int pid) { child->max_steps = parent.max_steps; child->max_time = parent.max_time; child->max_memory = parent.max_memory; + child->max_alloc_bytes = parent.max_alloc_bytes; child->max_messages = parent.max_messages; child->max_message_bytes = parent.max_message_bytes; child->signal_handlers = parent.signal_handlers; diff --git a/src/cvc/state/state_exec/stdlib.cpp b/src/cvc/state/state_exec/stdlib.cpp index 63506b773..40c9c7136 100644 --- a/src/cvc/state/state_exec/stdlib.cpp +++ b/src/cvc/state/state_exec/stdlib.cpp @@ -61,6 +61,21 @@ const std::vector &as_list(const value_t &v, const char *name) { throw std::runtime_error(std::string(name) + ": expected list, got " + v.type_name()); } +// Allocation estimates for eval_alloc_check(), saturating so a huge count never wraps small. +std::size_t sat_add(std::size_t a, std::size_t b) { return a > SIZE_MAX - b ? SIZE_MAX : a + b; } + +// One list of n elements: the vector, make_list's shared control block, and the elements. +std::size_t list_bytes(std::size_t n) { + return sat_add(sizeof(std::vector) + 2 * sizeof(void *), + alloc_bytes(n, sizeof(value_t))); +} + +// Long loops inside one native call honour the evaluation deadline (see collect in builtins.cpp). +void poll_deadline(const char *name) { + if (eval_deadline_expired()) + throw std::runtime_error(std::string(name) + ": evaluation exceeded time budget"); +} + } // anonymous namespace // --------------------------------------------------------------------------- @@ -74,7 +89,17 @@ void stdlib_registry::register_string_module() { std::string delim = " "; if (args.size() > 1) delim = as_string(args[1], "string.split"); + // Every part costs a list element, so many short parts outgrow the string: count first. + std::size_t count = 1; + if (delim.empty()) + count = utf8::count(s); + else + for (auto pos = s.find(delim); pos != std::string::npos; + pos = s.find(delim, pos + delim.size())) + ++count; + eval_alloc_check(sat_add(list_bytes(count), s.size()), "string.split"); std::vector parts; + parts.reserve(count); if (delim.empty()) { // SE-1 (A3): split into CODEPOINTS, not bytes — each element is one whole UTF-8 codepoint, // so a multibyte character is never shredded into invalid fragments. @@ -105,7 +130,12 @@ void stdlib_registry::register_string_module() { expect_exact(args, 2, "string.join"); auto &lst = as_list(args[0], "string.join"); auto &sep = as_string(args[1], "string.join"); + std::size_t total = lst.empty() ? 0 : alloc_bytes(lst.size() - 1, sep.size()); + for (const value_t &item : lst) + total = sat_add(total, as_string(item, "string.join").size()); + eval_alloc_check(total, "string.join"); std::string result; + result.reserve(total); for (size_t i = 0; i < lst.size(); ++i) { if (i > 0) result += sep; @@ -116,17 +146,29 @@ void stdlib_registry::register_string_module() { register_function("string", "string.replace", [](std::span args) -> value_t { expect_exact(args, 3, "string.replace"); - auto s = as_string(args[0], "string.replace"); + auto &s = as_string(args[0], "string.replace"); auto &from = as_string(args[1], "string.replace"); auto &to = as_string(args[2], "string.replace"); if (from.empty()) - return value_t(std::move(s)); - size_t pos = 0; - while ((pos = s.find(from, pos)) != std::string::npos) { - s.replace(pos, from.size(), to); - pos += to.size(); + return value_t(s); + // Non-overlapping matches, left to right. A longer `to` multiplies the size: count first. + std::size_t count = 0; + for (auto pos = s.find(from); pos != std::string::npos; pos = s.find(from, pos + from.size())) + ++count; + if (count == 0) + return value_t(s); + const std::size_t size = sat_add(s.size() - count * from.size(), alloc_bytes(count, to.size())); + eval_alloc_check(size, "string.replace"); + std::string out; + out.reserve(size); + std::size_t start = 0; + for (auto pos = s.find(from); pos != std::string::npos; pos = s.find(from, start)) { + out.append(s, start, pos - start); + out += to; + start = pos + from.size(); } - return value_t(std::move(s)); + out.append(s, start); + return value_t(std::move(out)); }); register_function("string", "string.trim", [](std::span args) -> value_t { @@ -294,7 +336,12 @@ void stdlib_registry::register_bytes_module() { }); register_function("bytes", "bytes.concat", [](std::span args) -> value_t { + std::size_t total = 0; + for (const value_t &a : args) + total = sat_add(total, as_bytes(a, "bytes.concat").size()); + eval_alloc_check(total, "bytes.concat"); std::string out; + out.reserve(total); for (const value_t &a : args) out += as_bytes(a, "bytes.concat"); return make_bytes(std::move(out)); @@ -470,6 +517,7 @@ void stdlib_registry::register_collections_module() { auto &a = as_list(args[0], "collections.zip"); auto &b = as_list(args[1], "collections.zip"); size_t n = std::min(a.size(), b.size()); + eval_alloc_check(sat_add(list_bytes(n), alloc_bytes(n, list_bytes(2))), "collections.zip"); std::vector result; result.reserve(n); for (size_t i = 0; i < n; ++i) { @@ -482,7 +530,16 @@ void stdlib_registry::register_collections_module() { [](std::span args) -> value_t { expect_exact(args, 1, "collections.flatten"); auto &lst = as_list(args[0], "collections.flatten"); + // The same sublist may appear many times, so the result can outgrow the + // argument: count the elements first. + std::size_t total = 0; + for (auto &item : lst) { + auto *sub = std::get_if(&item.v); + total = sat_add(total, sub ? (**sub).size() : 1); + } + eval_alloc_check(list_bytes(total), "collections.flatten"); std::vector result; + result.reserve(total); for (auto &item : lst) { if (auto *sub = std::get_if(&item.v)) { for (auto &x : **sub) @@ -529,30 +586,38 @@ void stdlib_registry::register_collections_module() { return make_list(std::move(elems)); }); - register_function("collections", "collections.range", - [](std::span args) -> value_t { - expect_min(args, 1, "collections.range"); - int64_t start = 0, stop, step = 1; - if (args.size() == 1) { - stop = as_int(args[0], "collections.range"); - } else { - start = as_int(args[0], "collections.range"); - stop = as_int(args[1], "collections.range"); - if (args.size() > 2) - step = as_int(args[2], "collections.range"); - } - if (step == 0) - throw std::runtime_error("collections.range: step cannot be zero"); - std::vector result; - if (step > 0) { - for (int64_t i = start; i < stop; i += step) - result.emplace_back(i); - } else { - for (int64_t i = start; i > stop; i += step) - result.emplace_back(i); - } - return make_list(std::move(result)); - }); + register_function( + "collections", "collections.range", [](std::span args) -> value_t { + expect_min(args, 1, "collections.range"); + int64_t start = 0, stop, step = 1; + if (args.size() == 1) { + stop = as_int(args[0], "collections.range"); + } else { + start = as_int(args[0], "collections.range"); + stop = as_int(args[1], "collections.range"); + if (args.size() > 2) + step = as_int(args[2], "collections.range"); + } + if (step == 0) + throw std::runtime_error("collections.range: step cannot be zero"); + // Count the elements in unsigned arithmetic (no signed overflow at the + // int64 ends), check the size, then build: element k is start + k*step. + const uint64_t ustart = static_cast(start); + const uint64_t ustop = static_cast(stop); + const uint64_t ustep = static_cast(step); + uint64_t count = 0; + if (step > 0 && start < stop) + count = (ustop - ustart - 1) / ustep + 1; + else if (step < 0 && start > stop) + count = (ustart - ustop - 1) / (0 - ustep) + 1; + eval_alloc_check(count > SIZE_MAX ? SIZE_MAX : list_bytes(static_cast(count)), + "collections.range"); + std::vector result; + result.reserve(static_cast(count)); + for (uint64_t k = 0; k < count; ++k) + result.emplace_back(static_cast(ustart + k * ustep)); + return make_list(std::move(result)); + }); register_function("collections", "collections.unique", [](std::span args) -> value_t { @@ -560,6 +625,7 @@ void stdlib_registry::register_collections_module() { auto &lst = as_list(args[0], "collections.unique"); std::vector result; for (auto &item : lst) { + poll_deadline("collections.unique"); // quadratic in the list length bool found = false; for (auto &r : result) { if (values_equal(r, item)) { @@ -606,10 +672,14 @@ void stdlib_registry::register_collections_module() { auto *d2 = std::get_if(&args[1].v); if (!d1 || !d2) throw std::runtime_error("collections.dict-merge: expected two dicts"); + eval_alloc_check( + alloc_bytes((**d1).size() + (**d2).size(), sizeof(std::pair)), + "collections.dict-merge"); // Start with copy of d1 auto result = std::make_shared>>(**d1); // Merge d2: update existing keys, append new ones for (auto &[k, v] : **d2) { + poll_deadline("collections.dict-merge"); // quadratic in the dict sizes bool found = false; for (auto &[rk, rv] : *result) { if (rk == k) { @@ -636,6 +706,25 @@ stdlib_registry::stdlib_registry() { register_collections_module(); } +const stdlib_registry &stdlib_registry::standard() { + static const stdlib_registry registry; + return registry; +} + +environment_ptr stdlib_registry::shared_scope() { + // Built once (thread-safe static init) and never written again: frozen, so set_existing() + // shadows a stdlib name in the caller's scope instead of writing here. + static const environment_ptr scope = [] { + auto env = std::make_shared(); + const stdlib_registry ® = standard(); + for (const std::string &module : reg.list_modules()) + reg.import_module(module, env); + env->frozen = true; + return env; + }(); + return scope; +} + void stdlib_registry::register_function(const std::string &module, const std::string &func_name, native_fn fn) { modules_[module].functions[func_name] = std::move(fn); diff --git a/src/cvc/state/state_exec/types.cpp b/src/cvc/state/state_exec/types.cpp index 00865238c..709a28304 100644 --- a/src/cvc/state/state_exec/types.cpp +++ b/src/cvc/state/state_exec/types.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -207,6 +208,35 @@ void detail::release_nested(std::shared_ptr owner) noexcept { t_release_queue = nullptr; } +// -- per-call allocation limit ---------------------------------------------- + +namespace { +thread_local eval_alloc_guard *t_alloc_innermost = nullptr; +} // namespace + +eval_alloc_guard::eval_alloc_guard(std::optional max_bytes) + : limit_(max_bytes), outer_(t_alloc_innermost) { + if (outer_ && outer_->limit_) + limit_ = limit_ ? std::min(*limit_, *outer_->limit_) : outer_->limit_; + t_alloc_innermost = this; +} + +eval_alloc_guard::~eval_alloc_guard() { t_alloc_innermost = outer_; } + +std::optional eval_alloc_limit() { + if (!t_alloc_innermost) + return kDefaultMaxAllocBytes; + return t_alloc_innermost->limit_; +} + +void eval_alloc_check(std::size_t bytes, const char *who) { + const std::optional limit = eval_alloc_limit(); + if (limit && bytes > *limit) + throw std::runtime_error(std::string(who) + ": the result needs " + std::to_string(bytes) + + " bytes, over the allocation limit of " + std::to_string(*limit) + + " bytes (limits.max_alloc_bytes, or what is left of max_memory)"); +} + // -- value_tag --------------------------------------------------------------- std::string value_tag::type_name() const { @@ -267,18 +297,27 @@ const value_t *environment::lookup(const std::string &name) const { return nullptr; } -void environment::set(const std::string &name, value_t val) { bindings[name] = std::move(val); } +void environment::set(const std::string &name, value_t val) { + if (frozen) + throw std::logic_error("set: '" + name + "' in a frozen scope"); + bindings[name] = std::move(val); +} void environment::set_existing(const std::string &name, value_t val) { - for (environment *e = this; e; e = e->outer.get()) { + if (frozen) + throw std::logic_error("set_existing: '" + name + "' in a frozen scope"); + environment *outermost = this; // the outermost unfrozen scope walked so far + for (environment *e = this; e && !e->frozen; e = e->outer.get()) { auto it = e->bindings.find(name); if (it != e->bindings.end()) { it->second = std::move(val); return; } + outermost = e; } - // Not found anywhere — set in this scope - bindings[name] = std::move(val); + // Not bound in an unfrozen scope: bind it in the outermost one, which shadows a frozen + // scope's binding of the same name rather than writing into the shared scope. + outermost->bindings[name] = std::move(val); } environment_ptr environment::extend(environment_ptr parent) { diff --git a/src/cvc/tests/CMakeLists.txt b/src/cvc/tests/CMakeLists.txt index d2a71f1c1..d4d270430 100644 --- a/src/cvc/tests/CMakeLists.txt +++ b/src/cvc/tests/CMakeLists.txt @@ -66,6 +66,7 @@ add_executable(state_exec_evaluator_test state_exec_evaluator_test.cpp) add_executable(state_exec_stackless_evaluator_test state_exec_stackless_evaluator_test.cpp) add_executable(state_exec_codec_test state_exec_codec_test.cpp) add_executable(state_exec_unicode_test state_exec_unicode_test.cpp) +add_executable(state_exec_stdlib_test state_exec_stdlib_test.cpp) add_executable(state_exec_grammar_test state_exec_grammar_test.cpp) add_executable(state_exec_corpus_test state_exec_corpus_test.cpp) add_executable(state_exec_pegtl_test state_exec_pegtl_test.cpp) @@ -344,6 +345,7 @@ list(APPEND TEST_TARGETS state_exec_stackless_evaluator_test state_exec_codec_test state_exec_unicode_test + state_exec_stdlib_test state_exec_grammar_test state_exec_corpus_test state_exec_pegtl_test @@ -1271,6 +1273,12 @@ target_link_libraries(state_exec_unicode_test GTest::gtest GTest::gtest_main ) +target_link_libraries(state_exec_stdlib_test + PRIVATE + cvc + GTest::gtest + GTest::gtest_main +) target_link_libraries(state_exec_grammar_test PRIVATE cvc @@ -1659,6 +1667,7 @@ target_compile_features(state_exec_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_stackless_evaluator_test PRIVATE cxx_std_20) target_compile_features(state_exec_codec_test PRIVATE cxx_std_20) target_compile_features(state_exec_unicode_test PRIVATE cxx_std_20) +target_compile_features(state_exec_stdlib_test PRIVATE cxx_std_20) target_compile_features(state_exec_grammar_test PRIVATE cxx_std_20) target_compile_features(state_exec_corpus_test PRIVATE cxx_std_20) target_compile_features(state_exec_pegtl_test PRIVATE cxx_std_20) @@ -2078,6 +2087,7 @@ cvc_discover_tests(state_exec_evaluator_test) cvc_discover_tests(state_exec_stackless_evaluator_test) cvc_discover_tests(state_exec_codec_test) cvc_discover_tests(state_exec_unicode_test) +cvc_discover_tests(state_exec_stdlib_test) cvc_discover_tests(state_exec_grammar_test) cvc_discover_tests(state_exec_corpus_test) cvc_discover_tests(state_exec_pegtl_test) diff --git a/src/cvc/tests/ariadne_runtime_test.cpp b/src/cvc/tests/ariadne_runtime_test.cpp index 970158b3a..00388e839 100644 --- a/src/cvc/tests/ariadne_runtime_test.cpp +++ b/src/cvc/tests/ariadne_runtime_test.cpp @@ -1575,6 +1575,132 @@ TEST(AriadneReactive, CompoundsAndAllowedFormsNowWork) { EXPECT_TRUE(rt.take_reactive_warnings().empty()); } +// The stdlib is in every program lane's environment, and a predicate gets its pure, bounded +// subset (a scalar, or a result no bigger than its argument). +TEST(AriadneStdlib, PredicateGetsThePureBoundedSubset) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + ASSERT_TRUE(run_init(app, "", "(state-set \"name\" \"caf\xC3\xA9\")", nullptr)); + Widget len = text("len"); + len.visible_when = "(= (string.length (state-get \"name\")) 4)"; // codepoints -> shown + Widget up = text("up"); + up.visible_when = "(string.starts-with (string.upper (state-get \"name\")) \"CAF\")"; + Widget math = text("math"); + math.visible_when = "(= (math.floor (math.sqrt 17)) 4)"; + Widget bytes = text("bytes"); + bytes.visible_when = "(= (bytes.byte-at b\"\\x01\\xff\" 1) 255)"; + rt.set_root(group({len, up, math, bytes})); + rt.render(); + EXPECT_TRUE(mb.saw("text_line:len")); + EXPECT_TRUE(mb.saw("text_line:up")); + EXPECT_TRUE(mb.saw("text_line:math")); + EXPECT_TRUE(mb.saw("text_line:bytes")); + EXPECT_TRUE(rt.take_reactive_warnings().empty()); +} + +TEST(AriadneStdlib, PredicateExcludesStdlibMaterializersAndRandom) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + // Results that can outgrow their arguments wait for scheduled, sliced versions; math.random + // would make a predicate nondeterministic. Each is an unbound symbol -> fail-safe hidden. + Widget join = text("join"); + join.visible_when = "(is-string (string.join (list \"a\") \",\"))"; + Widget rep = text("rep"); + rep.visible_when = "(is-string (string.replace \"a\" \"a\" \"aa\"))"; + Widget range = text("range"); + range.visible_when = "(is-list (collections.range 3))"; + Widget rnd = text("rnd"); + rnd.visible_when = "(< (math.random) 2)"; + rt.set_root(group({join, rep, range, rnd})); + rt.render(); + EXPECT_FALSE(mb.saw("text_line:join")); + EXPECT_FALSE(mb.saw("text_line:rep")); + EXPECT_FALSE(mb.saw("text_line:range")); + EXPECT_FALSE(mb.saw("text_line:rnd")); + EXPECT_FALSE(rt.take_reactive_warnings().empty()); +} + +TEST(AriadneStdlib, InitAndActionLanesReachTheWholeStdlib) { + cvc::app app; + std::vector errs; + ASSERT_TRUE(run_init(app, "ui", + "(state-set \"words\" (string.join (collections.reverse " + "(string.split \"a b c\" \" \")) \",\"))", + &errs)) + << (errs.empty() ? std::string("(no error)") : errs[0]); + EXPECT_EQ(cvc::state::instance(app)("ui.words").value(), "c,b,a"); + + Runtime rt(app, "ui"); + MockBackend mb; + rt.set_backend(&mb); + rt.set_root(group({button("Go", "(state-set \"shout\" (string.upper (state-get \"words\")))")})); + mb.button_click = true; + rt.render(); + rt.drain(); + EXPECT_EQ(cvc::state::instance(app)("ui.shout").value(), "C,B,A"); + EXPECT_TRUE(rt.take_reactive_warnings().empty()); +} + +TEST(AriadneStdlib, ActionHitsTheAllocationLimitAndIsReported) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + // The app's exec scheduler reads limits.max_alloc_bytes on every submission. + cvc::state::instance(app)("state_exec.defaults.limits.max_alloc_bytes").value("1024"); + rt.set_root(group({button("Big", "(state-set \"n\" (length (collections.range 1000)))")})); + mb.button_click = true; + rt.render(); + rt.drain(); + EXPECT_FALSE(cvc::state::instance(app)("n").value() == "1000"); // killed before the write + const auto warns = rt.take_reactive_warnings(); + ASSERT_FALSE(warns.empty()); + EXPECT_NE(warns[0].find("did not complete"), std::string::npos) << warns[0]; + + // A limit that fits lets the same action finish. + cvc::state::instance(app)("state_exec.defaults.limits.max_alloc_bytes").value("1048576"); + mb.button_click = true; + rt.render(); + rt.drain(); + EXPECT_EQ(cvc::state::instance(app)("n").value(), "1000"); +} + +// An action's memory budget is the tree setting ariadne.limits.action_max_memory (default +// 4 MiB, 0 = none), read on every submission; it also bounds each native call's allocations. +TEST(AriadneStdlib, ActionMemoryBudgetIsATreeSetting) { + cvc::app app; + Runtime rt(app, ""); + MockBackend mb; + rt.set_backend(&mb); + auto &root = cvc::state::instance(app); + // 10000 elements are well over the old fixed 256 KiB and well under 4 MiB. + rt.set_root(group({button("Big", "(state-set \"n\" (length (collections.range 10000)))")})); + const auto click = [&] { + root("n").value(std::string("-")); + mb.button_click = true; + rt.render(); + rt.drain(); + return root("n").value(); + }; + EXPECT_EQ(click(), "10000"); // the 4 MiB default + EXPECT_TRUE(rt.take_reactive_warnings().empty()); + + root("ariadne.limits.action_max_memory").value(std::string("65536")); + EXPECT_EQ(click(), "-"); // over the budget: killed before the write + const auto warns = rt.take_reactive_warnings(); + ASSERT_FALSE(warns.empty()); + EXPECT_NE(warns[0].find("did not complete"), std::string::npos) << warns[0]; + + root("ariadne.limits.action_max_memory").value(std::string("0")); // no limit + EXPECT_EQ(click(), "10000"); + root("ariadne.limits.action_max_memory").value(std::string("lots")); // unparsable: default + EXPECT_EQ(click(), "10000"); +} + TEST(AriadneReactive, DeniedSpecialFormsAreRejectedNotRun) { cvc::app app; Runtime rt(app, ""); diff --git a/src/cvc/tests/state_exec_intrinsics_test.cpp b/src/cvc/tests/state_exec_intrinsics_test.cpp index 574e6e1e8..f32523709 100644 --- a/src/cvc/tests/state_exec_intrinsics_test.cpp +++ b/src/cvc/tests/state_exec_intrinsics_test.cpp @@ -1057,8 +1057,10 @@ TEST_F(StdlibRegistryTest, ListEmpty) { EXPECT_TRUE(fns.empty()); } +// A bare environment: a default one already reaches every stdlib function through the shared +// stdlib scope, so it can't show what import_module() adds. TEST_F(StdlibRegistryTest, ImportAll) { - auto env = builtins::make_default_environment(); + auto env = std::make_shared(); registry.import_module("string", env); EXPECT_NE(env->lookup("string.split"), nullptr); EXPECT_NE(env->lookup("string.join"), nullptr); @@ -1066,7 +1068,7 @@ TEST_F(StdlibRegistryTest, ImportAll) { } TEST_F(StdlibRegistryTest, ImportSpecific) { - auto env = builtins::make_default_environment(); + auto env = std::make_shared(); registry.import_module("math", env, {"math.sqrt", "math.abs"}); EXPECT_NE(env->lookup("math.sqrt"), nullptr); EXPECT_NE(env->lookup("math.abs"), nullptr); diff --git a/src/cvc/tests/state_exec_stdlib_test.cpp b/src/cvc/tests/state_exec_stdlib_test.cpp new file mode 100644 index 000000000..6e93cd8a9 --- /dev/null +++ b/src/cvc/tests/state_exec_stdlib_test.cpp @@ -0,0 +1,361 @@ +/// @file state_exec_stdlib_test.cpp +/// @brief The stdlib modules (string, bytes, math, collections) reached from DSL programs, the +/// shared frozen scope they live in, and the per-call allocation limit. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace cvc::state_exec; + +namespace { + +// A result as text: a string bare, anything else as to_string() prints it. +std::string show(const std::optional &r) { + if (!r) + return ""; + if (auto *s = std::get_if(&r->v)) + return *s; + return to_string(*r); +} + +// Run a script to completion on the sync scheduler with its default environment. A runtime +// error propagates out of run(). +std::string run_sync(const std::string &script, execute_options opts = {}, + cvc::state *root = nullptr, const std::string &id = {}) { + scheduler sched; + if (root) { + sched.set_watch_root(root); + sched.set_id(id); + sched.load_settings(); + } + const int pid = sched.execute(script, opts); + sched.run(); + return show(sched.get_result(pid)); +} + +// Same on the async scheduler (the app's exec scheduler type). A runtime error kills the +// process; it comes back through `error`. +std::string run_async(const std::string &script, std::string *error, execute_options opts = {}, + cvc::state *root = nullptr) { + async_scheduler sched; + if (root) + sched.set_watch_root(root); + const int pid = sched.execute(script, opts); + sched.sync_run(); + *error = sched.get_exit_error(pid).value_or(""); + return show(sched.get_result(pid)); +} + +// The message of the runtime_error `f` throws, or "" when it doesn't throw. +template std::string thrown(F &&f) { + try { + f(); + } catch (const std::runtime_error &e) { + return e.what(); + } + return ""; +} + +// A list just over the 16 MiB default, as a collections.range length. +std::string over_default_elements() { + return std::to_string(kDefaultMaxAllocBytes / sizeof(value_t) + 1); +} + +value_t call(const char *fn, std::vector args) { + const native_fn *f = stdlib_registry::standard().lookup_qualified(fn); + if (!f) + throw std::logic_error(std::string("no stdlib function ") + fn); + return (*f)(std::span(args.data(), args.size())); +} + +} // namespace + +// --- reachable from programs --------------------------------------------------------------- + +TEST(StdlibReachTest, SyncSchedulerCallsStringLength) { + EXPECT_EQ(run_sync("(string.length \"x\")"), "1"); + EXPECT_EQ(run_sync("(string.length \"caf\xC3\xA9\")"), "4"); // codepoints, not bytes +} + +TEST(StdlibReachTest, AsyncSchedulerCallsStringLength) { + std::string err; + EXPECT_EQ(run_async("(string.length \"caf\xC3\xA9\")", &err), "4"); + EXPECT_EQ(err, ""); +} + +TEST(StdlibReachTest, EveryModuleIsReachable) { + EXPECT_EQ(run_sync("(string.upper \"go\")"), "GO"); + EXPECT_EQ(run_sync("(bytes.length (string.encode \"caf\xC3\xA9\"))"), "5"); + EXPECT_EQ(run_sync("(bytes.decode (string.encode \"ok\"))"), "ok"); + EXPECT_EQ(run_sync("(math.floor 3.7)"), "3"); + EXPECT_EQ(run_sync("(collections.reverse (list 1 2 3))"), "(3 2 1)"); + // map/filter/reduce take a native function (a builtin or a stdlib function). + EXPECT_EQ(run_sync("(collections.reduce (list 1 2 3) + 0)"), "6"); + EXPECT_EQ(run_sync("(collections.map (list -1 2 -3) math.abs)"), "(1 2 3)"); +} + +TEST(StdlibReachTest, BuiltinsKeepTheirUnqualifiedNames) { + // The stdlib names are all module-qualified, so the builtins of the same short name stay. + EXPECT_EQ(run_sync("(length (list 1 2 3))"), "3"); + EXPECT_EQ(run_sync("(collect (range 3))"), "(0 1 2)"); +} + +TEST(StdlibReachTest, ExplicitEnvironmentFromMakeDefaultEnvironment) { + // Hosts that build their own environment (Ariadne, pycvc) start from + // make_default_environment(), so they reach the stdlib the same way. + scheduler sched; + execute_options opts; + opts.env = builtins::make_default_environment(); + const int pid = sched.execute(std::string("(string.substring \"caf\xC3\xA9s\" 3 1)"), opts); + sched.run(); + EXPECT_EQ(show(sched.get_result(pid)), "\xC3\xA9"); +} + +// --- one shared, frozen scope --------------------------------------------------------------- + +TEST(StdlibSharedScopeTest, DefaultEnvironmentsLinkOneScope) { + const environment_ptr scope = stdlib_registry::shared_scope(); + ASSERT_TRUE(scope); + EXPECT_TRUE(scope->frozen); + EXPECT_EQ(stdlib_registry::shared_scope(), scope); // built once + + auto a = builtins::make_default_environment(); + auto b = builtins::make_default_environment(); + EXPECT_EQ(a->outer, scope); + EXPECT_EQ(b->outer, scope); + // Not copied: neither environment binds the name itself, both find the one shared value. + EXPECT_EQ(a->bindings.count("string.length"), 0u); + ASSERT_NE(a->lookup("string.length"), nullptr); + EXPECT_EQ(a->lookup("string.length"), b->lookup("string.length")); + + // Every function of every module is in it. + const stdlib_registry ® = stdlib_registry::standard(); + for (const std::string &module : {"string", "bytes", "math", "collections"}) + for (const std::string &fn : reg.list_functions(module)) + EXPECT_NE(scope->lookup(fn), nullptr) << fn; +} + +TEST(StdlibSharedScopeTest, SetShadowsInsteadOfWritingTheSharedScope) { + EXPECT_EQ(run_sync("(begin (set string.length 42) string.length)"), "42"); + // Another program (another scheduler) still sees the real function. + EXPECT_EQ(run_sync("(string.length \"ab\")"), "2"); + ASSERT_NE(stdlib_registry::shared_scope()->lookup("string.length"), nullptr); + EXPECT_TRUE(std::holds_alternative( + stdlib_registry::shared_scope()->lookup("string.length")->v)); +} + +TEST(StdlibSharedScopeTest, DefunShadowsAStdlibName) { + EXPECT_EQ(run_sync("(begin (defun string.upper (s) \"mine\") (string.upper \"x\"))"), "mine"); + EXPECT_EQ(run_sync("(string.upper \"x\")"), "X"); +} + +TEST(StdlibSharedScopeTest, SetExistingShadowsInOutermostUnfrozenScope) { + auto env = builtins::make_default_environment(); + auto inner = environment::extend(env); + inner->set_existing("math.pi", value_t{int64_t{3}}); + EXPECT_EQ(inner->bindings.count("math.pi"), 0u); // not in the scope it was called on + ASSERT_EQ(env->bindings.count("math.pi"), 1u); // shadowed one level below the shared scope + EXPECT_EQ(std::get(env->bindings["math.pi"].v), 3); + EXPECT_TRUE( + std::holds_alternative(stdlib_registry::shared_scope()->lookup("math.pi")->v)); + // A name bound in an unfrozen scope is still updated where it is. + inner->set("x", value_t{int64_t{1}}); + inner->set_existing("x", value_t{int64_t{2}}); + EXPECT_EQ(std::get(inner->bindings["x"].v), 2); +} + +TEST(StdlibSharedScopeTest, WritingAFrozenScopeThrows) { + EXPECT_THROW(stdlib_registry::shared_scope()->set_existing("math.pi", value_t{}), + std::logic_error); + EXPECT_THROW(stdlib_registry::shared_scope()->set("math.pi", value_t{}), std::logic_error); + EXPECT_THROW(stdlib_registry::standard().import_module("math", stdlib_registry::shared_scope()), + std::logic_error); +} + +// --- per-call allocation limit -------------------------------------------------------------- + +TEST(StdlibAllocLimitTest, DefaultLimitStopsAHugeRange) { + const std::string err = thrown([] { run_sync("(collections.range 1000000000)"); }); + EXPECT_NE(err.find("collections.range"), std::string::npos) << err; + EXPECT_NE(err.find("allocation limit"), std::string::npos) << err; + + std::string async_err; + run_async("(collections.range 1000000000)", &async_err); + EXPECT_NE(async_err.find("collections.range"), std::string::npos) << async_err; + + // Just over the default fails, a normal size works. + EXPECT_NE(thrown([] { run_sync("(collections.range " + over_default_elements() + ")"); }), ""); + EXPECT_EQ(run_sync("(collections.range 5)"), "(0 1 2 3 4)"); +} + +TEST(StdlibAllocLimitTest, CheckedAgainstWhatIsLeftOfMaxMemory) { + execute_options opts; + opts.max_memory = 64 * 1024; // the whole budget is under the size of 10000 elements + const std::string err = thrown([&] { run_sync("(collections.range 10000)", opts); }); + EXPECT_NE(err.find("allocation limit"), std::string::npos) << err; + std::string async_err; + run_async("(collections.range 10000)", &async_err, opts); + EXPECT_NE(async_err.find("allocation limit"), std::string::npos) << async_err; + EXPECT_EQ(run_sync("(collections.range 3)", opts), "(0 1 2)"); +} + +TEST(StdlibAllocLimitTest, TreeSettingResolvesPerSchedulerThenDefaults) { + cvc::app app; + auto &root = cvc::state::instance(app); + root("state_exec.defaults.limits.max_alloc_bytes").value("4096"); + EXPECT_EQ(max_alloc_bytes_from_state(root, "s1"), 4096u); + root("state_exec.schedulers.s1.limits.max_alloc_bytes").value("8192"); + EXPECT_EQ(max_alloc_bytes_from_state(root, "s1"), 8192u); + EXPECT_EQ(max_alloc_bytes_from_state(root, "s2"), 4096u); + root("state_exec.schedulers.s2.limits.max_alloc_bytes").value("lots"); // skipped + EXPECT_EQ(max_alloc_bytes_from_state(root, "s2"), 4096u); + + // The sync scheduler reads it in load_settings() and publishes the effective value. + scheduler sched; + sched.set_watch_root(&root); + sched.set_id("s1"); + sched.load_settings(); + EXPECT_EQ(sched.max_alloc_bytes(), 8192u); + EXPECT_EQ(root("state_exec.schedulers.s1.limits.max_alloc_bytes").value(), "8192"); + EXPECT_NE(thrown([&] { run_sync("(collections.range 1000)", {}, &root, "s1"); }), ""); + + // The async scheduler reads the defaults key on every submission. + std::string err; + run_async("(collections.range 1000)", &err, {}, &root); + EXPECT_NE(err.find("allocation limit"), std::string::npos) << err; + root("state_exec.defaults.limits.max_alloc_bytes").value(std::to_string(1 << 20)); + EXPECT_EQ(run_async("(length (collections.range 1000))", &err, {}, &root), "1000"); + EXPECT_EQ(err, ""); +} + +TEST(StdlibAllocLimitTest, ZeroMeansNoLimitOfItsOwn) { + cvc::app app; + auto &root = cvc::state::instance(app); + root("state_exec.defaults.limits.max_alloc_bytes").value("0"); + const std::string n = over_default_elements(); + EXPECT_EQ(run_sync("(length (collections.range " + n + "))", {}, &root), n); + std::string err; + EXPECT_EQ(run_async("(length (collections.range " + n + "))", &err, {}, &root), n); + EXPECT_EQ(err, ""); + // max_memory still bounds it. + execute_options opts; + opts.max_memory = 1 << 20; + EXPECT_NE(thrown([&] { run_sync("(collections.range " + n + ")", opts, &root); }), ""); +} + +TEST(StdlibAllocLimitTest, GuardsNestAndOnlyTighten) { + EXPECT_EQ(eval_alloc_limit(), std::optional(kDefaultMaxAllocBytes)); + { + eval_alloc_guard unlimited(std::nullopt); + EXPECT_EQ(eval_alloc_limit(), std::nullopt); + { + eval_alloc_guard outer(100); + EXPECT_EQ(eval_alloc_limit(), std::optional(100)); + eval_alloc_guard looser(1000); + EXPECT_EQ(eval_alloc_limit(), std::optional(100)); + eval_alloc_guard none(std::nullopt); + EXPECT_EQ(eval_alloc_limit(), std::optional(100)); + } + EXPECT_EQ(eval_alloc_limit(), std::nullopt); + } + EXPECT_EQ(eval_alloc_limit(), std::optional(kDefaultMaxAllocBytes)); +} + +TEST(StdlibAllocLimitTest, EveryGrowingResultIsChecked) { + // Each of these builds a result bigger than any one argument; under a 64-byte limit each + // throws naming itself, before allocating. + const value_t s(std::string(40, 'a')); + const value_t big(std::string(100, 'a')); + const value_t three = make_list({int64_t(1), int64_t(2), int64_t(3)}); + const value_t nested = make_list({three, three, three}); + auto d1 = std::make_shared>>(); + d1->emplace_back("a", value_t(int64_t(1))); + auto d2 = std::make_shared>>(); + d2->emplace_back("b", value_t(int64_t(2))); + const auto concat = [](std::vector args) { + return builtin_str_concat(std::span(args.data(), args.size())); + }; + + eval_alloc_guard tight(64); + EXPECT_NE( + thrown([&] { call("string.split", {big, value_t(std::string(""))}); }).find("string.split"), + std::string::npos); + EXPECT_NE(thrown([&] { + call("string.join", {make_list({s, s}), value_t(std::string(""))}); + }).find("string.join"), + std::string::npos); + EXPECT_NE(thrown([&] { + call("string.replace", {s, value_t(std::string("a")), s}); + }).find("string.replace"), + std::string::npos); + EXPECT_NE(thrown([&] { + call("bytes.concat", + {make_bytes(std::string(40, 'x')), make_bytes(std::string(40, 'x'))}); + }).find("bytes.concat"), + std::string::npos); + EXPECT_NE( + thrown([&] { call("collections.range", {value_t(int64_t(100))}); }).find("collections.range"), + std::string::npos); + EXPECT_NE(thrown([&] { call("collections.zip", {three, three}); }).find("collections.zip"), + std::string::npos); + EXPECT_NE(thrown([&] { call("collections.flatten", {nested}); }).find("collections.flatten"), + std::string::npos); + EXPECT_NE(thrown([&] { + call("collections.dict-merge", {value_t(d1), value_t(d2)}); + }).find("collections.dict-merge"), + std::string::npos); + EXPECT_NE(thrown([&] { concat({s, s}); }).find("str-concat"), std::string::npos); + const auto env = builtins::make_default_environment(); + const value_t hundred(int64_t(100)); + const value_t gen = std::get(env->lookup("range")->v)(std::span(&hundred, 1)); + const native_fn &collect = std::get(env->lookup("collect")->v); + EXPECT_NE(thrown([&] { collect(std::span(&gen, 1)); }).find("collect"), std::string::npos); + + // Results that fit still work under the same limit. + EXPECT_EQ(show(call("string.replace", {value_t(std::string("hello")), value_t(std::string("l")), + value_t(std::string("r"))})), + "herro"); + EXPECT_EQ(show(concat({value_t(std::string("ab")), value_t(std::string("cd"))})), "abcd"); +} + +// --- behaviour kept or fixed by the rewrites ------------------------------------------------- + +TEST(StdlibBehaviourTest, ReplaceIsNonOverlappingLeftToRight) { + EXPECT_EQ(run_sync("(string.replace \"hello\" \"l\" \"r\")"), "herro"); + EXPECT_EQ(run_sync("(string.replace \"aaa\" \"a\" \"bb\")"), "bbbbbb"); + EXPECT_EQ(run_sync("(string.replace \"aaaa\" \"aa\" \"b\")"), "bb"); + EXPECT_EQ(run_sync("(string.replace \"abc\" \"x\" \"y\")"), "abc"); + EXPECT_EQ(run_sync("(string.replace \"abc\" \"\" \"y\")"), "abc"); +} + +TEST(StdlibBehaviourTest, SplitAndJoinRoundTrip) { + EXPECT_EQ(run_sync("(string.split \"a,b,,c\" \",\")"), "(\"a\" \"b\" \"\" \"c\")"); + EXPECT_EQ(run_sync("(string.join (string.split \"a,b,,c\" \",\") \"-\")"), "a-b--c"); + EXPECT_EQ(run_sync("(length (string.split \"caf\xC3\xA9\" \"\"))"), "4"); +} + +TEST(StdlibBehaviourTest, RangeCountsWithoutOverflow) { + EXPECT_EQ(run_sync("(collections.range 2 5)"), "(2 3 4)"); + EXPECT_EQ(run_sync("(collections.range 5 0 -2)"), "(5 3 1)"); + EXPECT_EQ(run_sync("(collections.range 0 10 3)"), "(0 3 6 9)"); + EXPECT_EQ(run_sync("(collections.range 5 5)"), "()"); + EXPECT_EQ(run_sync("(collections.range 5 0)"), "()"); + // The old loop overflowed past INT64_MAX here; the count-based one stops. + EXPECT_EQ(run_sync("(collections.range 9223372036854775806 9223372036854775807 10)"), + "(9223372036854775806)"); + EXPECT_EQ(run_sync("(collections.range -9223372036854775807 -9223372036854775808 -5)"), + "(-9223372036854775807)"); +} + +TEST(StdlibBehaviourTest, FlattenAndZip) { + EXPECT_EQ(run_sync("(collections.flatten (list (list 1 2) 3 (list 4)))"), "(1 2 3 4)"); + EXPECT_EQ(run_sync("(collections.zip (list 1 2 3) (list \"a\" \"b\"))"), "((1 \"a\") (2 \"b\"))"); +}