From a20b970fbb3b2c536306a093047cea0a46352dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 8 Oct 2026 23:02:00 +0200 Subject: [PATCH 1/4] Adapt subtree selector work to indexed candidate size --- benchmark/README.md | 6 + benchmark/adaptive_selectors.jl | 328 ++++++++ benchmark/adaptive_selectors.md | 174 ++++ benchmark/adaptive_selectors_comparison.json | 826 +++++++++++++++++++ src/composite_model/compilation.jl | 35 +- src/composite_model/selectors.jl | 304 ++++--- test/Project.toml | 3 + test/runtests.jl | 7 + test/test-model-adaptive-selectors.jl | 575 +++++++++++++ test/test-model-api-stabilization.jl | 11 +- 10 files changed, 2128 insertions(+), 141 deletions(-) create mode 100644 benchmark/adaptive_selectors.jl create mode 100644 benchmark/adaptive_selectors.md create mode 100644 benchmark/adaptive_selectors_comparison.json create mode 100644 test/test-model-adaptive-selectors.jl diff --git a/benchmark/README.md b/benchmark/README.md index 364a66532..d697a6619 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -25,3 +25,9 @@ The flag enables the downstream suite; it does not install its dependencies. `benchmark/test/runtests.jl` uses the same default. Passing a test-name pattern explicitly selects those tests, as the dedicated downstream CI jobs do. The pinned release comparisons in `release_baselines/` use separate projects. + +The standalone [adaptive selector comparison](adaptive_selectors.md) measures +common-label plant subtrees and huge subtrees with rare matches. It separates +construction, selector resolution, compilation, and ordinary timestep work, +and runs outside the routinely tuned suite because large baseline scenes are +expensive. diff --git a/benchmark/adaptive_selectors.jl b/benchmark/adaptive_selectors.jl new file mode 100644 index 000000000..98c92e0de --- /dev/null +++ b/benchmark/adaptive_selectors.jl @@ -0,0 +1,328 @@ +module AdaptiveSelectorBenchmarks + +using Dates +using PlantSimEngine + +# This driver uses only the package and standard libraries so exactly the same +# file can be included in Kaimon sessions for the base and candidate revisions. +# It is intentionally separate from the AirspeedVelocity suite: full scene +# compilation at the largest density is too costly for routine CI tuning. + +PlantSimEngine.@process "adaptive_selector_benchmark_source" verbose = false +struct Source{V} <: AbstractAdaptive_Selector_Benchmark_SourceModel end +PlantSimEngine.inputs_(::Source) = NamedTuple() +PlantSimEngine.outputs_(::Source{V}) where {V} = NamedTuple{(V,)}((0.0,)) +function PlantSimEngine.run!(::Source{V}, status, environment, constants, context) where {V} + setproperty!(status, V, getproperty(status, V) + 1.0) + return nothing +end + +const SOURCE_VARIABLES = (:area, :A, :T_l, :lambdaE) +const INPUT_VARIABLES = (:areas, :assimilations, :temperatures, :evaporations) +const SOURCE_APPLICATIONS = (:leaf_area, :leaf_A, :leaf_T_l, :leaf_lambdaE) + +PlantSimEngine.@process "adaptive_selector_benchmark_consumer" verbose = false +struct Consumer{N} <: AbstractAdaptive_Selector_Benchmark_ConsumerModel end +PlantSimEngine.inputs_(::Consumer{N}) where {N} = + NamedTuple{INPUT_VARIABLES[1:N]}(ntuple(_ -> Required(Vector{Float64}), N)) +PlantSimEngine.outputs_(::Consumer) = (total=0.0,) +function PlantSimEngine.run!(::Consumer{N}, status, environment, constants, context) where {N} + total = 0.0 + for input in INPUT_VARIABLES[1:N] + total += sum(getproperty(status, input)) + end + status.total = total + return nothing +end + +function applications(ninputs::Int) + 1 <= ninputs <= length(INPUT_VARIABLES) || + throw(ArgumentError("`ninputs` must be between 1 and 4.")) + producers = ntuple(ninputs) do index + ModelSpec( + Source{SOURCE_VARIABLES[index]}(); + name=SOURCE_APPLICATIONS[index], + on=Many(scale=:Leaf, kind=:leaf, species=:crop), + ) + end + inputs = NamedTuple{INPUT_VARIABLES[1:ninputs]}(ntuple(ninputs) do index + Many( + Subtree(); + scale=:Leaf, + kind=:leaf, + species=:crop, + application=SOURCE_APPLICATIONS[index], + var=SOURCE_VARIABLES[index], + ) + end) + consumer = ModelSpec( + Consumer{ninputs}(); + name=:plant_total, + on=Many(scale=:Plant), + inputs=inputs, + ) + return (producers..., consumer) +end + +function fixture(objects, roots, expected; ninputs::Int) + model = CompositeModel( + objects...; + applications=applications(ninputs), + environment=(duration=Hour(1),), + ) + selectors = Tuple(values(PlantSimEngine.value_inputs(last(model.applications)))) + matchers = map(selector -> PlantSimEngine._compile_selector_matcher(model, selector), selectors) + return (; model, roots, expected, selectors, matchers, ninputs) +end + +""" + common_fixture(nplants; ninputs=4) + +Create many bounded subtrees sharing global label buckets. Each plant has +exactly 196 objects: one plant, 47 branches, 47 matching leaf objects, and 101 +other geometry objects. Four inputs select the same leaves but explicitly use +different producer applications and variables. These are generic compiler +fixtures, not an AgriPV geometry or scientific simulation. +""" +function common_fixture(nplants::Int; ninputs::Int=4) + nplants > 0 || throw(ArgumentError("`nplants` must be positive.")) + objects = Object[Object(0; scale=:Scene, kind=:scene)] + roots = ObjectId[] + expected = Vector{ObjectId}[] + for plant in 1:nplants + root = 196 * (plant - 1) + 1 + push!(roots, ObjectId(root)) + push!(objects, Object(root; scale=:Plant, kind=:plant, species=:crop, parent=0)) + for branch in 1:47 + push!(objects, Object(root + branch; scale=:Branch, kind=:branch, species=:crop, parent=root)) + push!(objects, Object(root + 47 + branch; scale=:Leaf, kind=:leaf, species=:crop, parent=root + branch)) + end + for geometry in 1:101 + push!(objects, Object(root + 94 + geometry; scale=:Geometry, kind=:geometry, species=:crop, parent=root + mod1(geometry, 47))) + end + push!(expected, ObjectId[ObjectId(root + 47 + leaf) for leaf in 1:47]) + end + return fixture(objects, roots, expected; ninputs) +end + +""" + sparse_fixture(nobjects=100_000; nleaves=64, ninputs=4) + +One huge shallow subtree has a rare matching label. Scanning the smallest +global bucket is preferable to traversing every descendant in this case. +""" +function sparse_fixture(nobjects::Int=100_000; nleaves::Int=64, ninputs::Int=4) + 1 <= nleaves < nobjects || + throw(ArgumentError("Require `1 <= nleaves < nobjects`.")) + objects = Object[ + Object(0; scale=:Scene, kind=:scene), + Object(1; scale=:Plant, kind=:plant, species=:crop, parent=0), + ] + for index in 2:nobjects + leaf = index <= nleaves + 1 + push!(objects, Object(index; scale=leaf ? :Leaf : :Geometry, kind=leaf ? :leaf : :geometry, species=:crop, parent=1)) + end + return fixture(objects, ObjectId[ObjectId(1)], [ObjectId[ObjectId(index) for index in 2:(nleaves + 1)]]; ninputs) +end + +function resolve(data, index, root) + return PlantSimEngine._resolve_object_ids( + data.model, + data.selectors[index], + data.matchers[index]; + context=root, + ) +end + +"""Resolve every input's selector for every plant; return a compact checksum.""" +function selector_sweep(data) + count = 0 + for root in data.roots, index in eachindex(data.selectors) + count += length(resolve(data, index, root)) + end + return count +end + +""" +Candidate-only query-work comparison. Each consumer receives a fresh cache, +matching the compiler's ownership/invalidation boundary. This measures query +resolution, dictionary lookups, and safe result copies, not full compilation. +""" +function cached_selector_sweep(data) + count = 0 + for root in data.roots + cache = Dict{Any,Vector{ObjectId}}() + for index in eachindex(data.selectors) + count += length(PlantSimEngine._cached_dependency_object_ids( + cache, data.model, data.selectors[index], data.matchers[index], root, + )) + end + end + return count +end + +"""Candidate-only gate for ordered IDs and independently owned cache results.""" +function check_cached_selectors(data) + for (root, expected) in zip(data.roots, data.expected) + cache = Dict{Any,Vector{ObjectId}}() + for index in eachindex(data.selectors) + ids = PlantSimEngine._cached_dependency_object_ids( + cache, data.model, data.selectors[index], data.matchers[index], root, + ) + @assert ids == expected + @assert length(cache) == 1 + @assert ids !== only(values(cache)) + ids[1] = ObjectId(-1) + restored = PlantSimEngine._cached_dependency_object_ids( + cache, data.model, data.selectors[index], data.matchers[index], root, + ) + @assert restored == expected + @assert restored !== ids && restored !== only(values(cache)) + end + end + return (; nplants=length(data.roots), ninputs=data.ninputs, safely_owned_results=true) +end + +"""Check the ordered IDs before measuring either revision.""" +function check_selectors(data) + for (root, expected) in zip(data.roots, data.expected), index in eachindex(data.selectors) + @assert resolve(data, index, root) == expected + end + return (; nplants=length(data.roots), ninputs=data.ninputs, matching_ids=sum(length, data.expected)) +end + +"""Check input-specific routing and live carriers outside the timed regions.""" +function check_bindings(data) + compiled = Advanced.refresh_bindings!(data.model) + expected = Dict(root.value => [id.value for id in ids] for (root, ids) in zip(data.roots, data.expected)) + records = Diagnostics.explain_bindings(compiled) + @assert length(records) == length(data.roots) * data.ninputs + for record in records + index = findfirst(==(record.input), INPUT_VARIABLES) + @assert !isnothing(index) && index <= data.ninputs + @assert record.application_id == :plant_total + @assert record.source_ids == expected[record.consumer_id] + @assert all(==(SOURCE_APPLICATIONS[index]), record.source_application_ids) + @assert record.source_var == SOURCE_VARIABLES[index] + @assert record.carrier_kind == :ref_vector + @assert record.copy_semantics == :live_references + end + for binding in compiled.input_bindings + source = model_status(data.model, first(binding.source_ids)) + original = getproperty(source, binding.source_var) + try + setproperty!(source, binding.source_var, original + 11.0) + @assert first(Diagnostics.input_value(binding)) == original + 11.0 + finally + setproperty!(source, binding.source_var, original) + end + end + @assert Advanced.compiled_bindings(data.model) === compiled + return length(records) +end + +function median_value(values) + ordered = sort(values) + middle = length(ordered) ÷ 2 + return isodd(length(ordered)) ? ordered[middle + 1] : (ordered[middle] + ordered[middle + 1]) / 2 +end + +""" +Warm `operation` once, then measure independent GC-clean samples. Allocation +bytes are cumulative work, not retained/peak process memory. `reachable_bytes` +uses `Base.summarysize` on the final returned value; it is neither RSS nor a +peak-memory measurement and does not include code or allocator slack. +""" +function measure(operation; samples::Int=3, warmup::Bool=true, measure_reachable::Bool=true) + samples > 0 || throw(ArgumentError("`samples` must be positive.")) + warmup && operation() + seconds = Float64[] + allocated_bytes = Int[] + result = nothing + for sample in 1:samples + GC.gc() + timing = @timed operation() + result = timing.value + push!(seconds, timing.time) + push!(allocated_bytes, timing.bytes) + end + return ( + samples=samples, + warmup=warmup, + median_seconds=median_value(seconds), + minimum_seconds=minimum(seconds), + median_allocated_bytes=median_value(allocated_bytes), + minimum_allocated_bytes=minimum(allocated_bytes), + reachable_bytes=measure_reachable ? Base.summarysize(result) : nothing, + ) +end + +function measure_fixture(factory; samples::Int=3, timestep_steps::Int=10, warmup::Bool=true, measure_reachable::Bool=true, measure_selectors::Bool=true) + construction = measure(factory; samples, warmup, measure_reachable) + data = factory() + correctness = if measure_selectors + check_selectors(data) + else + (; nplants=length(data.roots), ninputs=data.ninputs, matching_ids=sum(length, data.expected)) + end + selector = measure_selectors ? measure(() -> selector_sweep(data); samples, warmup, measure_reachable) : nothing + # Force a full compiler run and retain its result in the model. This avoids + # a cached no-op and also avoids a redundant full compile when starting the + # untimed simulation before measuring ordinary timesteps. + compilation = measure(() -> Advanced.refresh_bindings!(data.model; force=true); samples, warmup, measure_reachable) + simulation = run!(data.model; steps=1, outputs=:none) + binding_checks = check_bindings(data) + previous_cache = Advanced.compiled_bindings(data.model) + timestep = measure(() -> continue!(simulation; steps=timestep_steps); samples, warmup, measure_reachable) + @assert Advanced.compiled_bindings(data.model) === previous_cache + # Every leaf producer increments once per step, so this aggregate checks + # execution after the exact routing and live-reference assertions above. + executed_steps = 1 + (samples + Int(warmup)) * timestep_steps + for (root, expected) in zip(data.roots, data.expected) + total = model_status(data.model, root).total + @assert total == data.ninputs * length(expected) * executed_steps + end + return ( + correctness=correctness, + binding_checks=binding_checks, + selector_sweep_measured=measure_selectors, + construction=construction, + selector=selector, + compilation=compilation, + timestep=(; steps_per_sample=timestep_steps, timestep...), + model_reachable_bytes=measure_reachable ? Base.summarysize(data.model) : nothing, + ordinary_steps_reuse_compiled_bindings=true, + ) +end + +""" + run(; densities=(64, 304, 1376), sparse_objects=100_000, samples=3) + +Run through Kaimon for each compared checkout, including this same driver. +Results separate object/registry construction, the raw selector sweep, full +compiler work, and already-compiled ordinary timesteps. Comparing one versus +four inputs exposes the value of sharing object-query work without sharing +input-specific producer filtering. There are no CI wall-clock assertions. +""" +function run(; densities=(64, 304, 1376), sparse_objects::Int=100_000, samples::Int=3) + results = Pair{String,Any}[] + for nplants in densities, ninputs in (1, 4) + name = "common_$(nplants)_inputs_$(ninputs)" + @info "Adaptive selector benchmark" name + push!(results, name => measure_fixture(() -> common_fixture(nplants; ninputs); samples, measure_reachable=nplants <= 304)) + end + for ninputs in (1, 4) + name = "sparse_$(sparse_objects)_inputs_$(ninputs)" + @info "Adaptive selector benchmark" name + push!(results, name => measure_fixture(() -> sparse_fixture(sparse_objects; ninputs); samples, measure_reachable=sparse_objects <= 60_000)) + end + return ( + package_path=Base.pathof(PlantSimEngine), + package_version=string(Base.pkgversion(PlantSimEngine)), + active_project=Base.active_project(), + julia_version=string(VERSION), + rows=Dict(results), + ) +end + +end # module diff --git a/benchmark/adaptive_selectors.md b/benchmark/adaptive_selectors.md new file mode 100644 index 000000000..d500e32a9 --- /dev/null +++ b/benchmark/adaptive_selectors.md @@ -0,0 +1,174 @@ +# Adaptive selector comparison + +`adaptive_selectors.jl` is a standalone, generic compiler benchmark. It does +not add dependencies or join the routinely tuned AirspeedVelocity suite. +Run it through Kaimon sessions for each compared PlantSimEngine checkout. + +For an initial smoke run: + +```julia +include(joinpath(pkgdir(PlantSimEngine), "benchmark", "adaptive_selectors.jl")) +bench = AdaptiveSelectorBenchmarks +bench.measure_fixture(() -> bench.common_fixture(2); samples=1, timestep_steps=2) +bench.measure_fixture(() -> bench.sparse_fixture(1_000); samples=1, timestep_steps=2) +``` + +Use the same driver file in both checkout snapshots. The aggregate comparison +is `bench.run(densities=(64, 304, 1376), sparse_objects=100_000, samples=3)`; +large baseline compilation is expensive, so individual `measure_fixture` +calls and `samples=1` are suitable for bounded exploratory measurements. +After warming the same method signatures on the smaller sizes, passing +`warmup=false` skips a redundant full-scene operation at the largest size. +The returned measurement records whether it performed its own warmup. +Set `measure_reachable=false` for large cases: `Base.summarysize` can itself +be expensive on large typed status/batch tuples. The aggregate driver omits +reachable-graph estimates above 304 plants or 60,000 sparse objects. Omitted +estimates are `nothing`; cumulative allocation bytes remain measured. +For a bounded compiler comparison, `measure_selectors=false` omits the raw +selector sweep while still checking exact ordered source IDs, producers, and +live references through the compiled bindings. Its `selector` measurement is +`nothing`; `selector_sweep_measured` records this omission. +On a candidate with query reuse, `cached_selector_sweep(data)` and +`check_cached_selectors(data)` isolate its query-work benefit and verify that +mutating a returned ID vector cannot modify the cached snapshot. Use a fresh +cache per consumer, as the compiler does. Comparing this sweep with +`selector_sweep(data)` does not measure a full compiler speedup. +Record the source revision alongside the returned package path, version, +active project, and Julia version. + +The common-label fixture contains exactly 196 objects per plant, including 47 +matching leaf objects under 47 branches. Increasing the number of plants +increases the shared global label buckets while preserving each plant's +local query size. One and four input variants select identical object sets +but use distinct producer applications and variables. This comparison can +show whether sharing the object-query work is material after adaptive +resolution. The sparse fixture has one huge subtree with only 64 matching +leaf objects, where a rare index bucket should remain preferable to a full +descendant walk. + +Each operation receives an untimed warmup. Object and registry construction, +raw selector sweeps, fresh full compilation, and ordinary compiled timesteps +are measured separately, with garbage collection outside each sample. The +compiler operation uses `Advanced.refresh_bindings!(model; force=true)` so that +a valid binding cache cannot turn it into a no-op. Retaining that compiled +result avoids another full compile when starting the untimed simulation. +Timesteps use no retained +output streams and verify that compiled bindings remain identical across +ordinary continuation. + +`median_allocated_bytes` measures cumulative allocations during the operation. +`reachable_bytes` and `model_reachable_bytes` use `Base.summarysize`; these +describe reachable object graphs, not resident memory, peak memory, compiled +code, or allocator slack. Medians and minima are reported without CI timing +thresholds. Untimed gates check ordered resolved IDs, each input's producer +application and source variable, live reference carriers, and aggregate values. +The reference gate mutates and restores a source value while confirming that +the binding cache remains unchanged. Lifecycle, cardinality, and coverage +regressions belong to the package tests. + +These fixtures isolate generic PlantSimEngine work. They are not scientific +AgriPV validation and do not measure the separate geometry-ID selector or +ArchimedLight indexing changes. + +The resolver estimates global candidates from the smallest usable registry +criterion. Scalar label buckets are borrowed directly; label collections borrow +their buckets and use the sum of their lengths as a conservative estimate. +Repeated collection entries can overestimate the budget but never change the +matched set. A criterion containing an unindexed `nothing` label cannot supply +a complete index, so another criterion or the registry supplies candidates. +An empty usable bucket provides a zero budget and an empty match. + +Subtree, named scope, plant/ancestor scope, and descendant-relation queries try +a local walk with that candidate count as the budget, capped at scene size. +The walk stores only depth-sized child cursors and stops before exceeding its +budget. A completed walk is filtered locally; an incomplete walk falls back to +the borrowed smallest bucket and checks all labels and topology. There is no +empirical multiplier in this implementation. The common and sparse fixtures +exercise both choices without assuming that every subtree should be traversed. + +Within one consumer's input compilation, repeated `Many` object queries share +an owned, unfiltered ID snapshot. The key contains normalized object matcher +fields, excluding variable, process, and application filters; context and +scene are fixed by the cache's local lifetime. Each input owns a separate vector +before writer filtering. The cache is created only for multiple declared Many +inputs and discarded when that consumer's bindings are assembled. It also +applies when lifecycle refresh compiles new consumers or performs a full +recompile. Existing incremental membership updates and fallback re-resolution +of old bindings remain unchanged. + +Ordinary simulation steps already retain compiled bindings. Topology changes +use the existing addition/removal/reparent refresh paths; metadata label edits +through the graph editor reindex objects and request full binding refresh. +Consequently fresh resolutions see current bucket sizes, and no strategy cache +needs to be invalidated or reconsidered on each timestep. + +## Measured comparison (2026-10-08) + +The base is commit `7f3293644c408a1197131a57ab5301c700ed3140`; the candidate +is the working tree on `codex/adaptive-subtree-selectors` before the PR commit. +Both ran through Kaimon with Julia 1.13.1 and PlantSimEngine 0.15.0 on macOS +aarch64, with sequential timing runs and matched external dependency versions +and tree hashes. The [comparison JSON](adaptive_selectors_comparison.json) +contains package/project paths, source SHA-256 fingerprints and exact values. + +Full coupling compilation uses `refresh_bindings!(force=true)`. Allocations +below are cumulative per operation; GB and MB use decimal units. One-input +and four-input cases use distinct producer applications and variables. + +| Scene | Inputs | Base compilation (s) | Candidate compilation (s) | Base allocations | Candidate allocations | Time ratio | +|---|---:|---:|---:|---:|---:|---:| +| 64 plants | 1 | 0.098942 | 0.023189 | 117.03 MB | 11.85 MB | 4.3x | +| 64 plants | 4 | 0.414017 | 0.078134 | 471.51 MB | 45.25 MB | 5.3x | +| 304 plants | 1 | 2.635 | 0.111883 | 2.318 GB | 54.79 MB | 23.6x | +| 304 plants | 4 | 11.465 | 0.400852 | 9.292 GB | 211.23 MB | 28.6x | +| 1376 plants | 1 | 100.007 | 0.729261 | 45.509 GB | 324.07 MB | 137.1x | +| 1376 plants | 4 | 370.556 | 2.208 | 181.959 GB | 1.090 GB | 167.8x | +| 100,000-object sparse subtree | 1 | 0.013656 | 0.000994 | 12.22 MB | 1.07 MB | 13.7x | +| 100,000-object sparse subtree | 4 | 0.037373 | 0.002305 | 46.49 MB | 1.85 MB | 16.2x | + +The common fixtures contain 12,545, 59,585 and 269,697 total objects for 64, +304 and 1376 plants, with 3,008, 14,288 and 64,672 matching leaves. The sparse +fixture contains 100,001 total objects, including its scene object, with 64 +matching leaves. At 1376 plants and four inputs, compilation fell from +370.556 s / 181.959 GB allocated to 2.208 s / 1.090 GB allocated. Both revisions +passed exact ordered IDs, producers, variables and live-reference checks for +all 5504 bindings. The smaller baseline rows also passed ordered raw-selector +and aggregate execution gates; the revised reference gate passed separately +on the baseline tiny fixture. + +Raw selector work illustrates the scaling independently of full compilation: +at 304 plants with four inputs it fell from 10.651 s / 9.064 GB to +0.050535 s / 37.47 MB. At 1376 plants with one input it fell from +100.428 s / 45.196 GB to 0.086715 s / 42.10 MB. The largest four-input raw +sweep was omitted on both revisions to keep the baseline workload bounded. + +Construction and ordinary execution were measured separately: + +| Scene, four inputs | Base construction (s) | Candidate construction (s) | Base 10 steps (ms) | Candidate 10 steps (ms) | +|---|---:|---:|---:|---:| +| 64 plants | 0.025 | 0.029 | 0.688 | 0.856 | +| 304 plants | 0.181 | 0.198 | 6.341 | 6.924 | +| 1376 plants | 1.999 | 1.904 | 41.562 | 44.480 | +| 100,000-object sparse subtree | 5.830 | 5.882 | 0.035 | 0.041 | + +Ordinary timesteps retained the same compiled-binding object. The largest +four-input 10-step operation allocated 2,655,520 bytes on both revisions. +Timestep time varies in these measurements; no timestep speedup is claimed. + +The candidate-only query reuse comparison at 304 plants and four inputs +measured 0.051452 s / 37.47 MB without sharing versus 0.017047 s / 10.06 MB +with a fresh per-consumer cache. Returned IDs and independently owned result +vectors matched exactly. This measures query resolution, cache lookup and +result copying; it is not a full compiler ablation. + +The 64/304-plant and sparse timings are medians of three samples after warmup. +The largest cases have one timed sample after warming the same compiler and +selector method signatures on smaller fixtures; their construction can +include size-specific Julia compilation. Largest four-input selector sweeps +and large reachable-memory estimates are explicitly unmeasured, except for +the completed baseline largest one-input graph estimate. Base summary-size +accounting became expensive outside the timed operations, so it was omitted +for the remaining large cases. Root and temporary test manifests differ in +Julia-bundled stdlib/JLL metadata, while both execute the same Julia 1.13.1 +runtime. These machine-local, generic fixtures do not establish scientific +AgriPV validity or downstream full-scene performance. diff --git a/benchmark/adaptive_selectors_comparison.json b/benchmark/adaptive_selectors_comparison.json new file mode 100644 index 000000000..bc980ff7f --- /dev/null +++ b/benchmark/adaptive_selectors_comparison.json @@ -0,0 +1,826 @@ +{ + "schema_version": 1, + "measured_on": "2026-10-08", + "timezone": "Europe/Paris", + "platform": { + "system": "Darwin", + "architecture": "arm64" + }, + "sources": { + "base_revision": "7f3293644c408a1197131a57ab5301c700ed3140", + "candidate_branch": "codex/adaptive-subtree-selectors", + "candidate_state": "working tree before PR commit", + "candidate_sha256": { + "src/composite_model/selectors.jl": "9f68fd0b4fcc0929344cfe9c52c05460331aa7144adcf062192d9c58f08e3930", + "src/composite_model/compilation.jl": "99ed6533242b6891ff5f023d1bca724791b95065362cae891f0a3898e35559a4", + "benchmark/adaptive_selectors.jl": "12f68cf03103784bda9beca0414e88ce4341f8eb1c55aa3082f5eae49a25f04f" + } + }, + "methodology": { + "execution": "Kaimon-controlled baseline test subprocess and exact-project candidate session; timing runs sequential", + "julia_version": "1.13.1", + "package_version": "0.15.0", + "dependencies": "External package versions and git-tree hashes match. Baseline instantiate rewrote Julia-bundled stdlib/JLL manifest entries to the same Julia 1.13.1 runtime used by the candidate; candidate root Manifest contains older stdlib metadata.", + "common_scene": "One scene plus 196 objects per plant; 47 matching leaves, 47 branches, 101 other geometry objects and one plant. Four distinct producer applications/variables share label/topology queries.", + "sparse_scene": "One scene plus a 100000-object subtree with 64 matching leaves.", + "samples": "Three timed samples after warmup for 64/304 plants and sparse scene. Largest cases use one sample after signatures warmed on smaller fixtures; no repeated largest operation warmup.", + "operations": "construction; raw selector sweep when measured; refresh_bindings!(force=true) full compilation; 10 ordinary timesteps with outputs=:none", + "validation": "Exact ordered source IDs, producer applications/variables and live reference carriers checked; source mutation restored; ordinary timesteps retain identical compiled bindings. Largest four-input case verifies 5504 bindings.", + "missing": "1376-plant four-input raw selector sweep was omitted on both revisions. Large graph-size estimates are unmeasured/null except the already-completed baseline 1376-plant one-input row.", + "allocation_units": "Cumulative allocated bytes per operation, not retained or peak RAM. Non-null graph estimates use Base.summarysize and are neither RSS nor peak memory.", + "query_reuse_ablation": "Candidate-only 304-plant four-input raw query sweep compares fresh per-consumer caches with unshared resolution. It does not isolate the full compiler-cache speedup.", + "scope": "Generic engine fixtures; no scientific AgriPV validation, downstream full performance run, geometry-ID selector fix, or ArchimedLight indexing change." + }, + "base": { + "revision": "7f329364", + "package_path": "/private/tmp/pse-adaptive-base/src/PlantSimEngine.jl", + "active_project": "/private/tmp/pse-adaptive-base/test/Project.toml", + "julia_version": "1.13.1" + }, + "candidate": { + "package_path": "/Users/rvezy/Documents/dev/PlantSimEngine/src/PlantSimEngine.jl", + "package_version": "0.15.0", + "active_project": "/Users/rvezy/Documents/dev/PlantSimEngine/Project.toml", + "julia_version": "1.13.1" + }, + "rows": { + "common_64_inputs_1": { + "base": { + "correctness": { + "nplants": 64, + "ninputs": 1, + "matching_ids": 3008 + }, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.024775583, + "minimum_seconds": 0.024539, + "median_allocated_bytes": 23257248, + "minimum_allocated_bytes": 23257248, + "reachable_bytes": 9447488 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.081458709, + "minimum_seconds": 0.080675208, + "median_allocated_bytes": 105666624, + "minimum_allocated_bytes": 105666624, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.098942, + "minimum_seconds": 0.098690084, + "median_allocated_bytes": 117032560, + "minimum_allocated_bytes": 117032560, + "reachable_bytes": 11226488 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000138833, + "minimum_seconds": 0.000136625, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": 14104736 + }, + "model_reachable_bytes": 12645256, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 64, + "ninputs": 1, + "matching_ids": 3008 + }, + "binding_checks": 64, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.029082334, + "minimum_seconds": 0.028632125, + "median_allocated_bytes": 23257264, + "minimum_allocated_bytes": 23257264, + "reachable_bytes": 9447488 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.002906625, + "minimum_seconds": 0.00289375, + "median_allocated_bytes": 1957888, + "minimum_allocated_bytes": 1957888, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.023188875, + "minimum_seconds": 0.02280075, + "median_allocated_bytes": 11851920, + "minimum_allocated_bytes": 11849664, + "reachable_bytes": 11177848 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000141667, + "minimum_seconds": 0.000136416, + "median_allocated_bytes": 5936, + "minimum_allocated_bytes": 5440, + "reachable_bytes": 14056096 + }, + "model_reachable_bytes": 12596616, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "common_64_inputs_4": { + "base": { + "correctness": { + "nplants": 64, + "ninputs": 4, + "matching_ids": 3008 + }, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.025326958, + "minimum_seconds": 0.025297083, + "median_allocated_bytes": 23277584, + "minimum_allocated_bytes": 23277584, + "reachable_bytes": 9447968 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.339618542, + "minimum_seconds": 0.338225375, + "median_allocated_bytes": 422723840, + "minimum_allocated_bytes": 422723840, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.414017042, + "minimum_seconds": 0.410043, + "median_allocated_bytes": 471510912, + "minimum_allocated_bytes": 471510912, + "reachable_bytes": 13967144 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000687666, + "minimum_seconds": 0.000675125, + "median_allocated_bytes": 136480, + "minimum_allocated_bytes": 136480, + "reachable_bytes": 29331384 + }, + "model_reachable_bytes": 23014312, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 64, + "ninputs": 4, + "matching_ids": 3008 + }, + "binding_checks": 256, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.028663583, + "minimum_seconds": 0.027962792, + "median_allocated_bytes": 23277600, + "minimum_allocated_bytes": 23277600, + "reachable_bytes": 9447968 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.010085791, + "minimum_seconds": 0.009932208, + "median_allocated_bytes": 7888896, + "minimum_allocated_bytes": 7888896, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.078134, + "minimum_seconds": 0.078095583, + "median_allocated_bytes": 45253904, + "minimum_allocated_bytes": 45253488, + "reachable_bytes": 13927720 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000855834, + "minimum_seconds": 0.000766542, + "median_allocated_bytes": 136480, + "minimum_allocated_bytes": 136480, + "reachable_bytes": 29291960 + }, + "model_reachable_bytes": 22974888, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "common_304_inputs_1": { + "base": { + "correctness": { + "nplants": 304, + "ninputs": 1, + "matching_ids": 14288 + }, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.192112375, + "minimum_seconds": 0.174021667, + "median_allocated_bytes": 107566176, + "minimum_allocated_bytes": 107566176, + "reachable_bytes": 42231912 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 2.725255834, + "minimum_seconds": 2.682604791, + "median_allocated_bytes": 2265946176, + "minimum_allocated_bytes": 2265946176, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 2.635327083, + "minimum_seconds": 2.470813333, + "median_allocated_bytes": 2318314784, + "minimum_allocated_bytes": 2318314784, + "reachable_bytes": 49359336 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000726917, + "minimum_seconds": 0.000672125, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": 61294272 + }, + "model_reachable_bytes": 54391208, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 304, + "ninputs": 1, + "matching_ids": 14288 + }, + "binding_checks": 304, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.199102584, + "minimum_seconds": 0.190593041, + "median_allocated_bytes": 107568448, + "minimum_allocated_bytes": 107568448, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.016413459, + "minimum_seconds": 0.014935833, + "median_allocated_bytes": 9299968, + "minimum_allocated_bytes": 9299968, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.111883167, + "minimum_seconds": 0.110292334, + "median_allocated_bytes": 54794032, + "minimum_allocated_bytes": 54794032, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.000705333, + "minimum_seconds": 0.000657834, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "common_304_inputs_4": { + "base": { + "correctness": { + "nplants": 304, + "ninputs": 4, + "matching_ids": 14288 + }, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.180666166, + "minimum_seconds": 0.171142833, + "median_allocated_bytes": 107586512, + "minimum_allocated_bytes": 107586512, + "reachable_bytes": 42232392 + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 10.650859667, + "minimum_seconds": 10.407101958, + "median_allocated_bytes": 9064057088, + "minimum_allocated_bytes": 9064057088, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 11.465299375, + "minimum_seconds": 10.901285166, + "median_allocated_bytes": 9291568624, + "minimum_allocated_bytes": 9291568624, + "reachable_bytes": 61672696 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.006341333, + "minimum_seconds": 0.006296083, + "median_allocated_bytes": 597280, + "minimum_allocated_bytes": 597280, + "reachable_bytes": 128340960 + }, + "model_reachable_bytes": 98423200, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 304, + "ninputs": 4, + "matching_ids": 14288 + }, + "binding_checks": 1216, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 0.198386625, + "minimum_seconds": 0.198005709, + "median_allocated_bytes": 107588784, + "minimum_allocated_bytes": 107588784, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.050535417, + "minimum_seconds": 0.049924042, + "median_allocated_bytes": 37472256, + "minimum_allocated_bytes": 37472256, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.400851708, + "minimum_seconds": 0.396143583, + "median_allocated_bytes": 211225616, + "minimum_allocated_bytes": 211225616, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 0.006924291, + "minimum_seconds": 0.005669334, + "median_allocated_bytes": 597280, + "minimum_allocated_bytes": 597280, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "common_1376_inputs_1": { + "base": { + "correctness": { + "nplants": 1376, + "ninputs": 1, + "matching_ids": 64672 + }, + "construction": { + "samples": 1, + "warmup": false, + "median_seconds": 2.002655333, + "minimum_seconds": 2.002655333, + "median_allocated_bytes": 446204528, + "minimum_allocated_bytes": 446204528, + "reachable_bytes": 158501032 + }, + "selector": { + "samples": 1, + "warmup": false, + "median_seconds": 100.4283965, + "minimum_seconds": 100.4283965, + "median_allocated_bytes": 45195881536, + "minimum_allocated_bytes": 45195881536, + "reachable_bytes": 8 + }, + "compilation": { + "samples": 1, + "warmup": false, + "median_seconds": 100.007437834, + "minimum_seconds": 100.007437834, + "median_allocated_bytes": 45509188256, + "minimum_allocated_bytes": 45509188256, + "reachable_bytes": 189868984 + }, + "timestep": { + "steps_per_sample": 10, + "samples": 1, + "warmup": false, + "median_seconds": 0.006164, + "minimum_seconds": 0.006164, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": 238654472 + }, + "model_reachable_bytes": 207468784, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 1376, + "ninputs": 1, + "matching_ids": 64672 + }, + "binding_checks": 1376, + "selector_sweep_measured": true, + "construction": { + "samples": 1, + "warmup": false, + "median_seconds": 2.050062042, + "minimum_seconds": 2.050062042, + "median_allocated_bytes": 445970448, + "minimum_allocated_bytes": 445970448, + "reachable_bytes": null + }, + "selector": { + "samples": 1, + "warmup": false, + "median_seconds": 0.086714958, + "minimum_seconds": 0.086714958, + "median_allocated_bytes": 42096848, + "minimum_allocated_bytes": 42096848, + "reachable_bytes": null + }, + "compilation": { + "samples": 1, + "warmup": false, + "median_seconds": 0.729260833, + "minimum_seconds": 0.729260833, + "median_allocated_bytes": 324074368, + "minimum_allocated_bytes": 324074368, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 1, + "warmup": false, + "median_seconds": 0.006123125, + "minimum_seconds": 0.006123125, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "common_1376_inputs_4": { + "base": { + "correctness": { + "nplants": 1376, + "ninputs": 4, + "matching_ids": 64672 + }, + "binding_checks": 5504, + "selector_sweep_measured": false, + "construction": { + "samples": 1, + "warmup": false, + "median_seconds": 1.999367958, + "minimum_seconds": 1.999367958, + "median_allocated_bytes": 444914960, + "minimum_allocated_bytes": 444914960, + "reachable_bytes": null + }, + "selector": null, + "compilation": { + "samples": 1, + "warmup": false, + "median_seconds": 370.55618375, + "minimum_seconds": 370.55618375, + "median_allocated_bytes": 181959382816, + "minimum_allocated_bytes": 181959382816, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 1, + "warmup": false, + "median_seconds": 0.041562083, + "minimum_seconds": 0.041562083, + "median_allocated_bytes": 2655520, + "minimum_allocated_bytes": 2655520, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 1376, + "ninputs": 4, + "matching_ids": 64672 + }, + "binding_checks": 5504, + "selector_sweep_measured": false, + "construction": { + "samples": 1, + "warmup": false, + "median_seconds": 1.904139417, + "minimum_seconds": 1.904139417, + "median_allocated_bytes": 426052256, + "minimum_allocated_bytes": 426052256, + "reachable_bytes": null + }, + "selector": null, + "compilation": { + "samples": 1, + "warmup": false, + "median_seconds": 2.207682, + "minimum_seconds": 2.207682, + "median_allocated_bytes": 1089764992, + "minimum_allocated_bytes": 1089764992, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 1, + "warmup": false, + "median_seconds": 0.044480208, + "minimum_seconds": 0.044480208, + "median_allocated_bytes": 2655520, + "minimum_allocated_bytes": 2655520, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "sparse_100000_inputs_1": { + "base": { + "correctness": { + "nplants": 1, + "ninputs": 1, + "matching_ids": 64 + }, + "binding_checks": 1, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 5.867201167, + "minimum_seconds": 5.488262084, + "median_allocated_bytes": 153092192, + "minimum_allocated_bytes": 153092192, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.007872, + "minimum_seconds": 0.007222125, + "median_allocated_bytes": 5592768, + "minimum_allocated_bytes": 5592768, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.013656042, + "minimum_seconds": 0.013107125, + "median_allocated_bytes": 12216632, + "minimum_allocated_bytes": 12216632, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 1.175e-05, + "minimum_seconds": 1.0833e-05, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 1, + "ninputs": 1, + "matching_ids": 64 + }, + "binding_checks": 1, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 5.740721833, + "minimum_seconds": 5.653062292, + "median_allocated_bytes": 153169760, + "minimum_allocated_bytes": 153168416, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 9.6834e-05, + "minimum_seconds": 8.975e-05, + "median_allocated_bytes": 14112, + "minimum_allocated_bytes": 14112, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.000993542, + "minimum_seconds": 0.00092775, + "median_allocated_bytes": 1065032, + "minimum_allocated_bytes": 1065032, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 1.8667e-05, + "minimum_seconds": 1.05e-05, + "median_allocated_bytes": 5440, + "minimum_allocated_bytes": 5440, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + }, + "sparse_100000_inputs_4": { + "base": { + "correctness": { + "nplants": 1, + "ninputs": 4, + "matching_ids": 64 + }, + "binding_checks": 4, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 5.830173167, + "minimum_seconds": 5.498936667, + "median_allocated_bytes": 153112528, + "minimum_allocated_bytes": 153112528, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.019328292, + "minimum_seconds": 0.017674583, + "median_allocated_bytes": 22371968, + "minimum_allocated_bytes": 22371968, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.0373725, + "minimum_seconds": 0.036395458, + "median_allocated_bytes": 46486776, + "minimum_allocated_bytes": 46486776, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 3.5084e-05, + "minimum_seconds": 3.1167e-05, + "median_allocated_bytes": 15520, + "minimum_allocated_bytes": 15520, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + }, + "candidate": { + "correctness": { + "nplants": 1, + "ninputs": 4, + "matching_ids": 64 + }, + "binding_checks": 4, + "selector_sweep_measured": true, + "construction": { + "samples": 3, + "warmup": true, + "median_seconds": 5.882271125, + "minimum_seconds": 5.730991, + "median_allocated_bytes": 153190096, + "minimum_allocated_bytes": 153187840, + "reachable_bytes": null + }, + "selector": { + "samples": 3, + "warmup": true, + "median_seconds": 0.000167917, + "minimum_seconds": 0.00015525, + "median_allocated_bytes": 57344, + "minimum_allocated_bytes": 57344, + "reachable_bytes": null + }, + "compilation": { + "samples": 3, + "warmup": true, + "median_seconds": 0.002304875, + "minimum_seconds": 0.002218959, + "median_allocated_bytes": 1849304, + "minimum_allocated_bytes": 1849304, + "reachable_bytes": null + }, + "timestep": { + "steps_per_sample": 10, + "samples": 3, + "warmup": true, + "median_seconds": 4.075e-05, + "minimum_seconds": 4.025e-05, + "median_allocated_bytes": 16016, + "minimum_allocated_bytes": 15600, + "reachable_bytes": null + }, + "model_reachable_bytes": null, + "ordinary_steps_reuse_compiled_bindings": true + } + } + }, + "query_reuse": { + "unshared": { + "samples": 3, + "warmup": true, + "median_seconds": 0.051451875, + "minimum_seconds": 0.049885834, + "median_allocated_bytes": 37474528, + "minimum_allocated_bytes": 37472272, + "reachable_bytes": null + }, + "shared": { + "samples": 3, + "warmup": true, + "median_seconds": 0.017047, + "minimum_seconds": 0.015888625, + "median_allocated_bytes": 10058768, + "minimum_allocated_bytes": 10058768, + "reachable_bytes": null + } + } +} diff --git a/src/composite_model/compilation.jl b/src/composite_model/compilation.jl index 6d9a2dccf..d3351da70 100644 --- a/src/composite_model/compilation.jl +++ b/src/composite_model/compilation.jl @@ -2076,6 +2076,14 @@ Base.@nospecializeinfer function _compile_added_consumer_bindings!( ) # Cold assembly must not specialize on each application × input-plan tuple. @nospecialize application input_plans + # One consumer's inputs often select the same objects for different + # variables/producers. Keep this cache local to this invocation so neither + # scene identity nor lifecycle invalidation needs persistent cache state. + share_object_queries = count( + plan -> plan.origin != :inferred_same_object && plan.selector isa Many, + input_plans, + ) > 1 + object_query_cache = share_object_queries ? Dict{Any,Vector{ObjectId}}() : nothing for plan in input_plans plan.origin == :inferred_same_object && continue if !isnothing(many_binding_cache) @@ -2104,6 +2112,7 @@ Base.@nospecializeinfer function _compile_added_consumer_bindings!( applications_by_id, nothing, distributed_outputs, + ; object_query_cache, ) end application.id in manual_application_ids && return bindings @@ -4977,6 +4986,28 @@ function _dependency_object_ids( ) end +function _cached_dependency_object_ids( + cache, + model::CompositeModel, + selector::AbstractObjectMultiplicity, + matcher::CompiledSelectorMatcher, + context::ObjectId, +) + (isnothing(cache) || !(selector isa Many)) && + return _dependency_object_ids(model, selector, matcher, context) + key = ( + matcher.scope, matcher.relation, matcher.scale, matcher.kind, + matcher.species, matcher.id, matcher.defaults_to_context, + ) + cached_ids = get(cache, key, nothing) + isnothing(cached_ids) || return copy(cached_ids) + ids = _dependency_object_ids(model, selector, matcher, context) + # Writer filtering and lifecycle carriers own mutable vectors. The cache + # retains an unfiltered snapshot and never lends it to a binding. + cache[key] = copy(ids) + return ids +end + function _carrier_hint(selector::AbstractObjectMultiplicity, policy, window) !isnothing(window) && return :temporal_stream policy isa HoldLast || return :temporal_stream @@ -7140,6 +7171,7 @@ Base.@nospecializeinfer function _push_model_input_binding!( applications_by_id, source_ids_override=nothing, distributed_outputs=NoCompiledDistributedOutputs(), + ; object_query_cache=nothing, ) @nospecialize application plan input_sym = plan.input @@ -7148,7 +7180,8 @@ Base.@nospecializeinfer function _push_model_input_binding!( process_filter = plan.process application_filter = plan.application source_ids = isnothing(source_ids_override) ? - _dependency_object_ids( + _cached_dependency_object_ids( + object_query_cache, model, selector, plan.matcher, diff --git a/src/composite_model/selectors.jl b/src/composite_model/selectors.jl index 92cfa6af5..e034d0f6a 100644 --- a/src/composite_model/selectors.jl +++ b/src/composite_model/selectors.jl @@ -515,36 +515,37 @@ function _descendant_ids(model::CompositeModel, root_id::ObjectId) return _append_descendant_ids!(ObjectId[], model, root_id) end -# Object-relative selectors created with a newborn organ commonly cover only a -# handful of objects. Probe at most 32 objects before falling back to the -# registry-index heuristic so that those tiny scopes avoid copying a scene-wide -# label index without making large plant subtrees pay for a full extra walk. -const _SCOPE_FIRST_DESCENDANT_LIMIT = 32 - -function _append_descendant_ids_up_to!( - ids::Vector{ObjectId}, - model::CompositeModel, - root_id::ObjectId, - limit::Int, -) - length(ids) >= limit && return false - push!(ids, root_id) - object = _model_object(model, root_id) - for child_id in object.children - _append_descendant_ids_up_to!(ids, model, child_id, limit) || - return false - end - return true -end - function _descendant_ids_up_to( model::CompositeModel, root_id::ObjectId, limit::Int, ) ids = ObjectId[] - complete = _append_descendant_ids_up_to!(ids, model, root_id, limit) - return complete ? ids : nothing + # Borrow child vectors and retain only a depth-sized cursor stack. Pushing + # every child onto a worklist would allocate for a huge subtree even when + # a sparse index gives this traversal a budget of only a few objects. + cursors = Tuple{Vector{ObjectId},Int}[] + current_id = root_id + while true + length(ids) >= limit && return nothing + push!(ids, current_id) + children = _model_object(model, current_id).children + if !isempty(children) + push!(cursors, (children, 2)) + current_id = first(children) + continue + end + found_next = false + while !isempty(cursors) + siblings, next_index = pop!(cursors) + next_index > length(siblings) && continue + push!(cursors, (siblings, next_index + 1)) + current_id = siblings[next_index] + found_next = true + break + end + found_next || return ids + end end function _ancestor_id( @@ -834,39 +835,77 @@ function _scope_object_ids(model::CompositeModel, scope, context) error("Unsupported object scope selector `$(scope)` of type `$(typeof(scope))`.") end -function _registry_selector_ids(index, requested) +function _registry_selector_candidates(index, requested) isnothing(requested) && return nothing - requested_values = requested isa Tuple ? requested : (requested,) - ids = Set{ObjectId}() - for value in requested_values - union!(ids, get(index, Symbol(value), Set{ObjectId}())) + if !(requested isa Tuple) + bucket = get(index, requested, ()) + return (ids=bucket, estimate=length(bucket)) end - return ids -end - -function _indexed_object_ids(model::CompositeModel; scale=nothing, kind=nothing, species=nothing, id=nothing) - candidate_sets = Set{ObjectId}[] - for ids in ( - _registry_selector_ids(model.registry.by_scale, scale), - _registry_selector_ids(model.registry.by_kind, kind), - _registry_selector_ids(model.registry.by_species, species), + requested_values = requested + # Unlabelled objects have no index bucket. Another criterion may still + # provide a complete candidate source; otherwise scan the registry. + any(isnothing, requested_values) && return nothing + buckets = map(value -> get(index, value, ()), requested_values) + return (ids=Iterators.flatten(buckets), estimate=sum(length, buckets; init=0)) +end + +function _smallest_indexed_candidates(model::CompositeModel; scale=nothing, kind=nothing, species=nothing, id=nothing) + smallest = nothing + for candidates in ( + _registry_selector_candidates(model.registry.by_scale, scale), + _registry_selector_candidates(model.registry.by_kind, kind), + _registry_selector_candidates(model.registry.by_species, species), ) - isnothing(ids) || push!(candidate_sets, ids) + isnothing(candidates) && continue + if isnothing(smallest) || candidates.estimate < smallest.estimate + smallest = candidates + end end if !isnothing(id) requested_ids = id isa Tuple ? id : (id,) - push!(candidate_sets, Set{ObjectId}( - candidate for candidate in requested_ids - if haskey(model.registry.objects, candidate) - )) + if isnothing(smallest) || length(requested_ids) < smallest.estimate + smallest = (ids=requested_ids, estimate=length(requested_ids)) + end + end + return smallest +end + +function _indexed_object_ids(model::CompositeModel; scale=nothing, kind=nothing, species=nothing, id=nothing) + candidates = _smallest_indexed_candidates(model; scale, kind, species, id) + if isnothing(candidates) && all(isnothing, (scale, kind, species, id)) + return nothing end - isempty(candidate_sets) && return nothing - sort!(candidate_sets; by=length) - ids = copy(first(candidate_sets)) - for candidates in Iterators.drop(candidate_sets, 1) - intersect!(ids, candidates) + candidate_ids = isnothing(candidates) ? keys(model.registry.objects) : candidates.ids + return unique!(ObjectId[ + candidate for candidate in candidate_ids + if haskey(model.registry.objects, candidate) && + _matches_object_criteria(_model_object(model, candidate); scale, kind, species, id) + ]) +end + +# Resolve topology once, before choosing between a bounded local walk and a +# borrowed index bucket. The budget follows this query's global candidate count, +# rather than a fixed cutoff that penalizes many medium-sized plant subtrees. +function _selector_subtree_root(model::CompositeModel, scope, context) + if scope isa Union{Subtree,SelfPlant,Ancestor} + current_id = _object_id_from_context(context) + isnothing(current_id) && error("`$(scope)` selectors require a current object context.") + _model_object(model, current_id) + scope isa Subtree && return current_id + root = scope isa SelfPlant ? + _ancestor_id(model, current_id; scale=:Plant) : + _ancestor_id(model, current_id; scale=scope.scale, include_self=false) + if isnothing(root) + scope isa SelfPlant && error("No `scale=:Plant` ancestor found for object `$(current_id.value)`.") + error("No matching ancestor found for object `$(current_id.value)` and selector `$(scope)`.") + end + return root + elseif scope isa Scope + return _named_scope_root_id(model, scope) + elseif scope isa CompiledNamedScope + return scope.root_id end - return ObjectId[ids...] + return nothing end function _object_is_in_subtree(model::CompositeModel, object_id::ObjectId, root_id::ObjectId) @@ -1001,6 +1040,9 @@ end function _near_symbol_matches(requested, available) isnothing(requested) && return Symbol[] + requested isa Tuple && return unique!(reduce( + vcat, (_near_symbol_matches(value, available) for value in requested); init=Symbol[], + )) requested_symbol = Symbol(requested) threshold = max(2, cld(length(string(requested_symbol)), 3)) ranked = Tuple{Int,Symbol}[ @@ -1200,6 +1242,27 @@ function _resolve_object_ids( ) end +function _matching_selector_ids( + model::CompositeModel, + matcher::CompiledSelectorMatcher, + candidates, + context, + default_to_context::Bool, + default_scope, +) + # The adaptive candidate source may be a local vector, an index iterator, + # or registry keys. Specialize this loop at that boundary, rather than + # dispatching dynamically for every object in a heterogeneous query. + ids = ObjectId[] + for id in candidates + haskey(model.registry.objects, id) || continue + _selector_matches_object_id( + model, matcher, id; context, default_to_context, default_scope, + ) && push!(ids, id) + end + return _sort_object_ids!(unique!(ids)) +end + function _resolve_object_ids( model::CompositeModel, selector::AbstractObjectMultiplicity, @@ -1230,108 +1293,79 @@ function _resolve_object_ids( return ObjectId[_object_id_from_context(context)] end - has_indexed_criteria = !isnothing(scale) || - !isnothing(kind) || - !isnothing(species) || - !isnothing(requested_id) - scope_candidate_ids = if isnothing(relation) && has_indexed_criteria - if scope isa Self - _scope_object_ids(model, scope, context) - elseif scope isa Subtree - current_id = _object_id_from_context(context) - isnothing(current_id) && error( - "`Subtree()` selectors require a current object context.", - ) - _descendant_ids_up_to( - model, - current_id, - _SCOPE_FIRST_DESCENDANT_LIMIT, - ) - else - nothing - end - else + indexed = _smallest_indexed_candidates( + model; scale, kind, species, id=requested_id, + ) + nearest_ancestor = isnothing(relation) && scope isa Ancestor && + !isnothing(scale) && scale == scope.scale + subtree_root = if nearest_ancestor nothing - end - indexed_ids = if isnothing(scope_candidate_ids) - _indexed_object_ids( - model; - scale=scale, - kind=kind, - species=species, - id=requested_id, + elseif relation == :descendants + current_id = _object_id_from_context(context) + isnothing(current_id) && error( + "`Relation(:descendants)` selectors require a current object context.", ) + _model_object(model, current_id) + current_id + elseif isnothing(relation) + _selector_subtree_root(model, scope, context) else nothing end - candidate_ids = if !isnothing(scope_candidate_ids) - scope_candidate_ids - elseif isnothing(relation) - if scope isa Ancestor && !isnothing(scale) && scale == scope.scale - current_id = _object_id_from_context(context) - isnothing(current_id) && error( - "`Ancestor(...)` selectors require a current object context." - ) - ancestor_id = _ancestor_id( - model, - current_id; - scale=scope.scale, - include_self=false, - ) - isnothing(ancestor_id) ? ObjectId[] : ObjectId[ancestor_id] - elseif isnothing(indexed_ids) - _scope_object_ids(model, scope, context) - elseif isnothing(scope) || scope isa SceneScope - indexed_ids - elseif length(indexed_ids) * 4 <= length(model.registry.objects) - ObjectId[ - id for id in indexed_ids - if _scope_contains_object(model, scope, context, id) - ] + local_ids = if !isnothing(subtree_root) + if !haskey(model.registry.objects, subtree_root) + ObjectId[] else - scoped_ids = Set(_scope_object_ids(model, scope, context)) - ObjectId[id for id in indexed_ids if id in scoped_ids] + budget = isnothing(indexed) ? length(model.registry.objects) : + min(indexed.estimate, length(model.registry.objects)) + _descendant_ids_up_to(model, subtree_root, budget) end else - related_ids = _relation_object_ids(model, relation, context) - if isnothing(explicit_scope) - related_ids - else - scoped_ids = Set(_scope_object_ids(model, explicit_scope, context)) - ObjectId[id for id in related_ids if id in scoped_ids] - end + nothing end - ids = ObjectId[ - id for id in candidate_ids - if _matches_object_criteria( - _model_object(model, id); - scale=scale, - kind=kind, - species=species, - id=requested_id, + candidate_ids = if nearest_ancestor + current_id = _object_id_from_context(context) + isnothing(current_id) && error( + "`Ancestor(...)` selectors require a current object context.", ) - ] - _sort_object_ids!(ids) - - diagnostic_candidate_ids = if !isnothing(scope_candidate_ids) - # Indexed candidates already satisfy the requested labels. Preserve - # that diagnostic contract for ambiguous singular matches, while an - # empty match still reports every label available inside the scope. - isempty(ids) ? scope_candidate_ids : ids - elseif isempty(ids) && !isnothing(indexed_ids) + ancestor_id = _ancestor_id(model, current_id; scale=scope.scale, include_self=false) + isnothing(ancestor_id) ? ObjectId[] : ObjectId[ancestor_id] + elseif !isnothing(local_ids) + local_ids + elseif !isnothing(relation) && relation != :descendants + _relation_object_ids(model, relation, context) + elseif scope isa Self && isnothing(relation) + self_ids = _scope_object_ids(model, scope, context) + _model_object(model, only(self_ids)) + self_ids + elseif !isnothing(indexed) + indexed.ids + elseif isnothing(relation) + _scope_object_ids(model, scope, context) + else + _relation_object_ids(model, relation, context) + end + ids = _matching_selector_ids( + model, matcher, candidate_ids, context, default_to_context, default_scope, + ) + + # Cardinality diagnostics may need all available labels in the topology + # scope. Materialize them only on an error, never for empty Many queries. + invalid_cardinality = (selector isa One && length(ids) != 1) || + (selector isa OptionalOne && length(ids) > 1) + diagnostic_candidate_ids = if invalid_cardinality && + (isempty(ids) || !isnothing(relation)) if isnothing(relation) _scope_object_ids(model, scope, context) else related_ids = _relation_object_ids(model, relation, context) - if isnothing(explicit_scope) - related_ids - else - scoped_ids = Set(_scope_object_ids(model, explicit_scope, context)) - ObjectId[id for id in related_ids if id in scoped_ids] - end + isnothing(explicit_scope) ? related_ids : ObjectId[ + id for id in related_ids + if _scope_contains_object(model, explicit_scope, context, id) + ] end else - candidate_ids + ids end if selector isa One && length(ids) != 1 diff --git a/test/Project.toml b/test/Project.toml index 7fc926474..6cc748a13 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -13,3 +13,6 @@ PlantSimEngine = "9a576370-710b-4269-adf9-4f603a9c6423" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[sources] +PlantSimEngine = {path = ".."} diff --git a/test/runtests.jl b/test/runtests.jl index 7ccd7a6f5..fafb554af 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,7 @@ using PlantSimEngine +realpath(pkgdir(PlantSimEngine)) == realpath(joinpath(@__DIR__, "..")) || error( + "Tests must load this PlantSimEngine checkout; loaded $(pkgdir(PlantSimEngine)).", +) using PlantSimEngine.Diagnostics using PlantSimEngine.GraphEditor using PlantSimEngine.EnvironmentAPI @@ -146,6 +149,10 @@ else include("test-model-binding-inference.jl") end + @testset "Adaptive subtree selectors" begin + include("test-model-adaptive-selectors.jl") + end + @testset "Identity-aware Many inputs" begin include("test-model-bound-many.jl") end diff --git a/test/test-model-adaptive-selectors.jl b/test/test-model-adaptive-selectors.jl new file mode 100644 index 000000000..0501418c1 --- /dev/null +++ b/test/test-model-adaptive-selectors.jl @@ -0,0 +1,575 @@ +using PlantSimEngine +using PlantSimEngine.Diagnostics +using PlantSimEngine.GraphEditor +using Test + +# This oracle deliberately scans objects and walks parent links. It must not use +# registry label indexes, compiled matchers, or the optimized selector resolver. +function adaptive_reference_subtree(model, id, root) + current = ObjectId(id) + while true + current == ObjectId(root) && return true + parent = model_object(model, current).parent + isnothing(parent) && return false + current = parent + end +end + +function adaptive_reference_label(value, requested) + isnothing(requested) && return true + requested isa Union{Tuple,AbstractVector} && return value in requested + return value == requested +end + +function adaptive_reference_ids( + model; + root=nothing, + context=nothing, + relation=nothing, + scale=nothing, + kind=nothing, + species=nothing, + id=nothing, +) + requested_ids = isnothing(id) ? nothing : + id isa Union{Tuple,AbstractVector} ? ObjectId.(id) : ObjectId(id) + context_id = isnothing(context) ? nothing : ObjectId(context) + ids = ObjectId[] + for object in values(model.registry.objects) + adaptive_reference_label(object.scale, scale) || continue + adaptive_reference_label(object.kind, kind) || continue + adaptive_reference_label(object.species, species) || continue + adaptive_reference_label(object.id, requested_ids) || continue + isnothing(root) || adaptive_reference_subtree(model, object.id, root) || continue + if !isnothing(relation) + current = model_object(model, context_id) + matches = if relation == :self + object.id == context_id + elseif relation == :parent + object.id == current.parent + elseif relation == :children + object.parent == context_id + elseif relation == :ancestors + object.id != context_id && + adaptive_reference_subtree(model, context_id, object.id) + elseif relation == :descendants + object.id != context_id && + adaptive_reference_subtree(model, object.id, context_id) + elseif relation == :siblings + object.id != context_id && !isnothing(current.parent) && + object.parent == current.parent + else + error("Unsupported reference relation") + end + matches || continue + end + push!(ids, object.id) + end + # All fixture IDs are symbols, whose specified natural ordering is stable. + return sort!(ids; by=id -> id.value) +end + +function adaptive_selector_scene() + objects = Object[Object(:scene; scale=:Scene, kind=:scene)] + for plant in 1:8 + plant_id = Symbol(:plant_, plant) + axis_id = Symbol(:axis_, plant) + push!(objects, Object(plant_id; scale=:Plant, kind=:plant, parent=:scene)) + push!(objects, Object(axis_id; scale=:Axis, kind=:axis, parent=plant_id)) + for leaf in 1:72 + push!(objects, Object( + Symbol(:leaf_, plant, :_, leaf); + scale=:Leaf, + kind=isodd(leaf) ? :active : :dormant, + species=leaf % 3 == 0 ? :alternate : :common, + parent=isodd(leaf) ? plant_id : axis_id, + )) + end + end + push!(objects, Object(:nested_plant; scale=:Plant, kind=:plant, parent=:plant_1)) + push!(objects, Object(:nested_leaf; scale=:Leaf, kind=:active, species=:common, + parent=:nested_plant)) + push!(objects, Object(:unlabelled; parent=:plant_1)) + push!(objects, Object(:unlabelled_leaf; scale=:Leaf, parent=:plant_1)) + push!(objects, Object(:large; scale=:Plant, kind=:plant, parent=:scene)) + for leaf in 1:1024 + push!(objects, Object( + Symbol(:large_leaf_, leaf); + scale=:Leaf, + kind=:active, + species=leaf in (2, 512, 1024) ? :rare : :common, + parent=:large, + )) + end + return CompositeModel(objects...) +end + +@testset "adaptive selector IDs and deterministic ordering" begin + model = adaptive_selector_scene() + for plant in 1:8 + context = Symbol(:plant_, plant) + for labels in ( + (scale=:Leaf,), + (scale=:Leaf, kind=:active), + (scale=(:Leaf, :Axis), kind=(:active, :axis), species=(:common, :alternate)), + (scale=[:Leaf, :Leaf], kind=[:active, :dormant]), + (scale=:Leaf, species=:missing), + (scale=(),), + (scale=:Leaf, species=(:missing, :common, :common)), + (species=(nothing, :common),), + (scale=(nothing, :Leaf), species=(nothing, :common)), + (scale=:Leaf, kind=(nothing, :active)), + (scale=(nothing, :Leaf),), + ) + normalized = (; (key => (value isa AbstractVector ? Tuple(value) : value) + for (key, value) in pairs(labels))...) + @test resolve_object_ids(model, Many(Subtree(); labels...); context) == + adaptive_reference_ids(model; root=context, normalized...) + @test resolve_object_ids(model, Many(Relation(:descendants); labels...); context) == + adaptive_reference_ids(model; context, relation=:descendants, normalized...) + end + end + + for species in (:rare, :common, :missing, (:rare, :missing)) + @test resolve_object_ids(model, Many(Subtree(); scale=:Leaf, species); context=:large) == + adaptive_reference_ids(model; root=:large, scale=:Leaf, species) + @test resolve_object_ids(model, Many(Scope(:large); scale=:Leaf, species)) == + adaptive_reference_ids(model; root=:large, scale=:Leaf, species) + end + @test object_ids(model; scale=(nothing, :Leaf)) == + adaptive_reference_ids(model; scale=(nothing, :Leaf)) + for selector in (Many(), Many(SceneScope()), Many(Subtree())) + context = selector isa Many && any(x -> x isa Subtree, selector.criteria.selectors) ? + :plant_1 : nothing + @test resolve_object_ids(model, selector; context) == + adaptive_reference_ids(model; root=context) + end + @test resolve_object_ids(model, Many(Subtree(); id=(:leaf_1_2, :missing, :leaf_1_1, + :leaf_1_2, :leaf_2_1)); context=:plant_1) == + adaptive_reference_ids(model; root=:plant_1, + id=(:leaf_1_2, :missing, :leaf_1_1, :leaf_1_2, :leaf_2_1)) + @test resolve_object_ids(model, Many(Subtree(); id=:missing); context=:plant_1) == ObjectId[] + @test resolve_object_ids(model, Many(Subtree(); id=()); context=:plant_1) == ObjectId[] + + mixed_ids = CompositeModel( + Object(:scene; scale=:Scene), + Object(10; scale=:Leaf, parent=:scene), + Object(2; scale=:Leaf, parent=:scene), + Object(Symbol("2"); scale=:Leaf, parent=:scene), + ) + @test resolve_object_ids(mixed_ids, Many(Subtree(); scale=:Leaf); context=:scene) == + ObjectId[ObjectId(2), ObjectId(10), ObjectId(Symbol("2"))] + @test resolve_object_ids(mixed_ids, Many(Subtree(); id=(Symbol("2"), 10, 2, 2)); + context=:scene) == ObjectId[ObjectId(2), ObjectId(10), ObjectId(Symbol("2"))] + + # None of the resolver paths may mutate their source label buckets. + original_indexes = deepcopy((model.registry.by_scale, model.registry.by_kind, + model.registry.by_species)) + resolve_object_ids(model, Many(Subtree(); scale=(:Leaf, :Axis), kind=:active); + context=:plant_1) + @test (model.registry.by_scale, model.registry.by_kind, model.registry.by_species) == + original_indexes +end + +PlantSimEngine.@process "adaptive_output_action" verbose = false +struct AdaptiveOutputAction{F} <: AbstractAdaptive_Output_ActionModel + action::F +end +PlantSimEngine.inputs_(::AdaptiveOutputAction) = NamedTuple() +PlantSimEngine.outputs_(::AdaptiveOutputAction) = (flux=Distributed(Default(0.0)),) +function PlantSimEngine.run!(model::AdaptiveOutputAction, status, environment, constants, context) + model.action(context) + return nothing +end + +function adaptive_output_scene(action) + objects = Object[Object(:scene; scale=:Scene)] + for plant in (:plant_a, :plant_b) + push!(objects, Object(plant; scale=:Plant, parent=:scene)) + for leaf in 1:72 + push!(objects, Object(Symbol(plant, :_leaf_, leaf); + scale=:Leaf, kind=isodd(leaf) ? :active : :dormant, parent=plant)) + end + end + return CompositeModel(objects...; + applications=(ModelSpec(AdaptiveOutputAction(action); + name=:distributed, on=Many(scale=:Plant), + outputs_to=(OutputTo(Many(Subtree(); scale=:Leaf, kind=:active); + vars=(:flux,)),)),)) +end + +@testset "adaptive distributed destinations retain ordering and exact coverage" begin + seen = Dict{ObjectId,Vector{ObjectId}}() + action = context -> begin + targets = output_targets(context, (:flux,)) + ids = collect(object_ids(targets)) + seen[object_id(context)] = ids + # The external table may return IDs in another order. Coverage and + # assignment must still correspond to the selected object identities. + assign_outputs!(targets, + (object_id=reverse(ids), flux=reverse(Float64.(eachindex(ids)))); + id=:object_id) + end + model = adaptive_output_scene(action) + simulation = run!(model; outputs=:all) + @test Set(keys(outputs(simulation))) == Set( + (:distributed, id, :flux) for id in + adaptive_reference_ids(model; scale=:Leaf, kind=:active)) + for plant in (:plant_a, :plant_b) + ids = adaptive_reference_ids(model; root=plant, scale=:Leaf, kind=:active) + @test seen[ObjectId(plant)] == ids + @test [model_status(model, id).flux for id in ids] == Float64.(eachindex(ids)) + end + register_object!(model, Object(:aaa_new_destination; scale=:Leaf, kind=:active, + parent=:plant_a)) + continue!(simulation) + @test seen[ObjectId(:plant_a)] == + adaptive_reference_ids(model; root=:plant_a, scale=:Leaf, kind=:active) + @test first(seen[ObjectId(:plant_a)]) == ObjectId(:aaa_new_destination) + @test haskey(outputs(simulation), (:distributed, ObjectId(:aaa_new_destination), :flux)) + + invalid = adaptive_output_scene(context -> begin + targets = output_targets(context, (:flux,)) + ids = collect(object_ids(targets))[1:end-1] + assign_outputs!(targets, (object_id=ids, flux=fill(99.0, length(ids))); + id=:object_id) + end) + @test_throws "missing destination ID" run!(invalid; outputs=:none) + @test all(model_status(invalid, id).flux == 0.0 + for id in adaptive_reference_ids(invalid; scale=:Leaf, kind=:active)) +end + +@testset "nested scopes, relations, and cardinality" begin + model = adaptive_selector_scene() + for (scope, context, root) in ( + (Subtree(), :plant_1, :plant_1), + (SelfPlant(), :leaf_1_2, :plant_1), + (SelfPlant(), :nested_leaf, :nested_plant), + (Ancestor(scale=:Plant), :leaf_1_2, :plant_1), + (Scope(ObjectId(:plant_1)), nothing, :plant_1), + (Scope(:nested_plant), nothing, :nested_plant), + ) + @test resolve_object_ids(model, Many(scope; scale=:Leaf); context) == + adaptive_reference_ids(model; root, scale=:Leaf) + @test resolve_object_ids(model, Many(; within=scope, scale=:Leaf); context) == + adaptive_reference_ids(model; root, scale=:Leaf) + end + @test resolve_object_ids(model, One(Self(); scale=:Leaf); context=:nested_leaf) == + ObjectId[ObjectId(:nested_leaf)] + @test resolve_object_ids(model, One(Ancestor(scale=:Plant); scale=:Plant); + context=:nested_leaf) == ObjectId[ObjectId(:nested_plant)] + for relation in (:self, :parent, :children, :ancestors, :descendants, :siblings) + for context in (:scene, :plant_1, :axis_1, :leaf_1_2, :nested_leaf) + @test resolve_object_ids(model, Many(Relation(relation)); context) == + adaptive_reference_ids(model; context, relation) + @test resolve_object_ids(model, Many(Scope(:plant_1), Relation(relation); + scale=(:Leaf, :Axis, :Plant)); context) == + adaptive_reference_ids(model; root=:plant_1, context, relation, + scale=(:Leaf, :Axis, :Plant)) + end + end + @test resolve_object_ids(model, One(Subtree(); scale=:Leaf); context=:nested_plant) == + ObjectId[ObjectId(:nested_leaf)] + @test resolve_object_ids(model, OptionalOne(Subtree(); species=:missing); + context=:plant_1) == ObjectId[] + @test_throws "Expected exactly one object" resolve_object_ids(model, + One(Subtree(); scale=:Leaf); context=:plant_1) + @test_throws "Expected zero or one object" resolve_object_ids(model, + OptionalOne(Subtree(); scale=:Leaf); context=:plant_1) + @test_throws "Expected exactly one object" resolve_object_ids(model, + One(Subtree(); species=:missing); context=:large) + @test_throws "Expected exactly one object" resolve_object_ids(model, + One(Subtree(); scale=(:missing, :absent)); context=:large) + @test_throws "Expected zero or one object" resolve_object_ids(model, + OptionalOne(Subtree(); scale=(:Leaf, :Axis)); context=:plant_1) + @test_throws "Expected zero or one object" resolve_object_ids(model, + OptionalOne(Subtree(); species=:rare); context=:large) + @test resolve_object_ids(model, One(Subtree(); species=:rare, id=:large_leaf_512); + context=:large) == ObjectId[ObjectId(:large_leaf_512)] + @test_throws "require a current object" resolve_object_ids(model, + Many(Subtree(); scale=:Leaf)) + + # Instance names take precedence over equal object IDs; explicit ObjectId + # continues to select the object subtree instead. + template = CompositeModelTemplate() + scoped = CompositeModel( + Object(:scene; scale=:Scene), + Object(:named; scale=:Plant, parent=:scene), + Object(:outside_leaf; scale=:Leaf, parent=:named), + ObjectInstance(:named, template; + root=Object(:instance_root; scale=:Plant, parent=:scene), + objects=(Object(:inside_leaf; scale=:Leaf, parent=:instance_root),)), + ) + @test resolve_object_ids(scoped, Many(Scope(:named); scale=:Leaf)) == + ObjectId[ObjectId(:inside_leaf)] + @test resolve_object_ids(scoped, Many(Scope(ObjectId(:named)); scale=:Leaf)) == + ObjectId[ObjectId(:outside_leaf)] + + for scope in (Self(), Subtree(), Relation(:descendants)) + @test_throws ErrorException resolve_object_ids(model, Many(scope); context=:missing) + end + removable = CompositeModel(Object(:scene; scale=:Scene), + Object(:branch; scale=:Axis, parent=:scene), + Object(:leaf; scale=:Leaf, parent=:branch)) + selector = Many(Scope(:branch); scale=:Leaf) + matcher = PlantSimEngine._compile_selector_matcher(removable, selector) + remove_object!(removable, :branch) + @test PlantSimEngine._resolve_object_ids(removable, selector, matcher) == ObjectId[] +end + +@testset "deep adaptive local walk" begin + objects = Object[Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene)] + parent = :plant + for index in 1:512 + id = Symbol(:deep_, index) + push!(objects, Object(id; scale=:Axis, parent)) + parent = id + end + push!(objects, Object(:tip; scale=:Leaf, species=:rare, parent)) + for index in 1:1024 + push!(objects, Object(Symbol(:outside_, index); scale=:Axis, parent=:scene)) + end + model = CompositeModel(objects...) + @test resolve_object_ids(model, Many(Subtree(); scale=:Axis); context=:plant) == + adaptive_reference_ids(model; root=:plant, scale=:Axis) + @test resolve_object_ids(model, Many(Subtree(); species=:rare); context=:plant) == + ObjectId[ObjectId(:tip)] +end + +PlantSimEngine.@process "adaptive_leaf_a" verbose = false +PlantSimEngine.@process "adaptive_leaf_b" verbose = false +PlantSimEngine.@process "adaptive_plant_sum" verbose = false +PlantSimEngine.@process "adaptive_optional" verbose = false + +struct AdaptiveLeafA <: AbstractAdaptive_Leaf_AModel end +struct AdaptiveLeafB <: AbstractAdaptive_Leaf_BModel end +struct AdaptivePlantSum <: AbstractAdaptive_Plant_SumModel end +struct AdaptiveOptional <: AbstractAdaptive_OptionalModel end + +PlantSimEngine.inputs_(::AdaptiveLeafA) = NamedTuple() +PlantSimEngine.inputs_(::AdaptiveLeafB) = NamedTuple() +PlantSimEngine.outputs_(::AdaptiveLeafA) = (a=0.0,) +PlantSimEngine.outputs_(::AdaptiveLeafB) = (b=0.0,) +PlantSimEngine.run!(::AdaptiveLeafA, status, environment, constants, context) = nothing +PlantSimEngine.run!(::AdaptiveLeafB, status, environment, constants, context) = nothing +PlantSimEngine.inputs_(::AdaptivePlantSum) = ( + values_a=Required(Vector{Float64}), values_b=Required(Vector{Float64}), +) +PlantSimEngine.outputs_(::AdaptivePlantSum) = (total_a=0.0, total_b=0.0, seen_ids=ObjectId[]) +function PlantSimEngine.run!(::AdaptivePlantSum, status, environment, constants, context) + values_a = bound_input(context, :values_a) + values_b = bound_input(context, :values_b) + @assert object_ids(values_a) == object_ids(values_b) + status.seen_ids = collect(object_ids(values_a)) + status.total_a = sum(values_a; init=0.0) + status.total_b = sum(values_b; init=0.0) + return nothing +end +PlantSimEngine.inputs_(::AdaptiveOptional) = (optional=Default(7.0),) +PlantSimEngine.outputs_(::AdaptiveOptional) = (observed=0.0,) +function PlantSimEngine.run!(::AdaptiveOptional, status, environment, constants, context) + status.observed = status.optional + return nothing +end + +function adaptive_binding_scene() + objects = Object[Object(:scene; scale=:Scene), + Object(:plant_a; scale=:Plant, parent=:scene), + Object(:plant_b; scale=:Plant, parent=:scene)] + for plant in (:plant_a, :plant_b), leaf in 1:72 + push!(objects, Object(Symbol(plant, :_leaf_, leaf); + scale=:Leaf, kind=:active, parent=plant, + status=Status(a=Float64(leaf), b=Float64(10leaf)))) + end + return CompositeModel(objects...; + applications=( + ModelSpec(AdaptiveLeafA(); name=:leaf_a, on=Many(scale=:Leaf)), + ModelSpec(AdaptiveLeafB(); name=:leaf_b, on=Many(scale=:Leaf)), + ModelSpec(AdaptivePlantSum(); name=:plant_sum, on=Many(scale=:Plant), + inputs=( + values_a=Many(Subtree(); scale=:Leaf, kind=:active, application=:leaf_a, var=:a), + values_b=Many(Subtree(); scale=:Leaf, kind=:active, application=:leaf_b, var=:b), + )), + ModelSpec(AdaptiveOptional(); name=:optional, on=Many(scale=:Plant), + inputs=(optional=OptionalOne(Subtree(); species=:missing, + var=:a, from_status=true),)), + ), + ) +end + +function adaptive_binding(compiled, plant, input) + return only(binding for binding in compiled.input_bindings_by_target[ + (:plant_sum, ObjectId(plant))] if binding.input == input) +end + +function adaptive_check_bindings(model, compiled; check_runtime=false) + for plant in (:plant_a, :plant_b) + ids = adaptive_reference_ids(model; root=plant, scale=:Leaf, kind=:active) + for (input, variable, producer) in ((:values_a, :a, :leaf_a), (:values_b, :b, :leaf_b)) + binding = adaptive_binding(compiled, plant, input) + @test binding.source_ids == ids + @test binding.source_application_ids == [producer] + carrier = input_carrier(binding) + @test carrier isa PlantSimEngine.RefVector + @test collect(carrier) == [getproperty(model_status(model, id), variable) for id in ids] + @test all(reference === PlantSimEngine.refvalue(model_status(model, id), variable) + for (id, reference) in zip(ids, parent(carrier))) + end + if check_runtime + status = model_status(model, plant) + @test status.seen_ids == ids + @test status.total_a == sum((model_status(model, id).a for id in ids); init=0.0) + @test status.total_b == sum((model_status(model, id).b for id in ids); init=0.0) + end + end +end + +@testset "adaptive bindings preserve producers, references, and lifecycle membership" begin + model = adaptive_binding_scene() + simulation = run!(model; outputs=:none) + original = simulation.compiled + adaptive_check_bindings(model, original; check_runtime=true) + @test all(model_status(model, plant).observed == 7.0 for plant in (:plant_a, :plant_b)) + @test all(model_status(model, plant).seen_ids == + adaptive_reference_ids(model; root=plant, scale=:Leaf, kind=:active) + for plant in (:plant_a, :plant_b)) + + bindings = [adaptive_binding(original, plant, input) + for plant in (:plant_a, :plant_b) for input in (:values_a, :values_b)] + carriers = input_carrier.(bindings) + model_status(model, :plant_a_leaf_1).a = 123.0 + model_status(model, :plant_a_leaf_1).b = 456.0 + @test Advanced.refresh_bindings!(model) === original + continue!(simulation) + @test simulation.compiled === original + @test all(adaptive_binding(simulation.compiled, plant, input) === binding + for ((plant, input), binding) in zip( + [(plant, input) for plant in (:plant_a, :plant_b) for input in (:values_a, :values_b)], bindings)) + @test all(input_carrier(binding) === carrier for (binding, carrier) in zip(bindings, carriers)) + adaptive_check_bindings(model, simulation.compiled; check_runtime=true) + + # New IDs sort before old ones; refresh must preserve deterministic order + # even when the monotonic append shortcut cannot apply. + register_object!(model, Object(:aaa_new_leaf; scale=:Leaf, kind=:active, + parent=:plant_a, status=Status(a=2.0, b=20.0))) + continue!(simulation) + adaptive_check_bindings(model, simulation.compiled; check_runtime=true) + @test first(adaptive_binding(simulation.compiled, :plant_a, :values_a).source_ids) == + ObjectId(:aaa_new_leaf) + + reparent_object!(model, :plant_a_leaf_2, :plant_b) + continue!(simulation) + adaptive_check_bindings(model, simulation.compiled; check_runtime=true) + @test ObjectId(:plant_a_leaf_2) ∉ adaptive_binding(simulation.compiled, :plant_a, :values_a).source_ids + @test ObjectId(:plant_a_leaf_2) ∈ adaptive_binding(simulation.compiled, :plant_b, :values_a).source_ids + + remove_object!(model, :aaa_new_leaf) + continue!(simulation) + adaptive_check_bindings(model, simulation.compiled; check_runtime=true) + @test ObjectId(:aaa_new_leaf) ∉ adaptive_binding(simulation.compiled, :plant_a, :values_a).source_ids + + # The graph editor's metadata edit returns an independently owned model, + # reindexes labels and requests a full binding refresh for the edited scene. + relabeled = apply_model_graph_edit(model, + SetModelObjectMetadata(:plant_a_leaf_3; kind=:dormant)) + relabeled_compiled = Advanced.refresh_bindings!(relabeled) + adaptive_check_bindings(relabeled, relabeled_compiled) + @test ObjectId(:plant_a_leaf_3) ∉ adaptive_binding(relabeled_compiled, :plant_a, :values_a).source_ids + @test ObjectId(:plant_a_leaf_3) ∈ adaptive_binding(simulation.compiled, :plant_a, :values_a).source_ids + + # A separately owned scene with equal object IDs must never reuse results + # from the previous scene's different topology or labels. + independent = adaptive_binding_scene() + independent_compiled = Advanced.refresh_bindings!(independent) + adaptive_check_bindings(independent, independent_compiled) + @test ObjectId(:plant_a_leaf_3) ∈ adaptive_binding(independent_compiled, :plant_a, :values_a).source_ids + @test ObjectId(:plant_a_leaf_2) ∈ adaptive_binding(independent_compiled, :plant_a, :values_a).source_ids +end + +PlantSimEngine.@process "adaptive_partition_sum" verbose = false +struct AdaptivePartitionSum <: AbstractAdaptive_Partition_SumModel end +PlantSimEngine.inputs_(::AdaptivePartitionSum) = ( + active_values=Required(Vector{Float64}), + dormant_values=Required(Vector{Float64}), + supplied_values=Required(Vector{Float64}), +) +PlantSimEngine.outputs_(::AdaptivePartitionSum) = ( + total_active=0.0, total_dormant=0.0, total_supplied=0.0, +) +function PlantSimEngine.run!(::AdaptivePartitionSum, status, environment, constants, context) + status.total_active = sum(bound_input(context, :active_values); init=0.0) + status.total_dormant = sum(bound_input(context, :dormant_values); init=0.0) + status.total_supplied = sum(bound_input(context, :supplied_values); init=0.0) + return nothing +end + +function adaptive_partition_scene(reverse_inputs) + objects = Object[Object(:scene; scale=:Scene), + Object(:plant; scale=:Plant, parent=:scene)] + for index in 1:72 + push!(objects, Object(Symbol(:leaf_, index); scale=:Leaf, + kind=isodd(index) ? :active : :dormant, parent=:plant, + status=Status(a=Float64(index), b=Float64(10index), supplied=1.0))) + end + inputs = ( + active_values=Many(Subtree(); scale=:Leaf, application=:active, var=:a), + dormant_values=Many(Subtree(); scale=:Leaf, application=:dormant, var=:b), + supplied_values=Many(Subtree(); scale=:Leaf, var=:supplied, from_status=true), + ) + reverse_inputs && (inputs = ( + supplied_values=inputs.supplied_values, + dormant_values=inputs.dormant_values, + active_values=inputs.active_values, + )) + return CompositeModel(objects...; applications=( + ModelSpec(AdaptiveLeafB(); name=:dormant, on=Many(scale=:Leaf, kind=:dormant)), + ModelSpec(AdaptiveLeafA(); name=:active, on=Many(scale=:Leaf, kind=:active)), + ModelSpec(AdaptivePartitionSum(); name=:partition, on=One(id=:plant), inputs), + )) +end + +@testset "shared object queries keep input-specific producer filtering and ownership" begin + for reverse_inputs in (false, true) + model = adaptive_partition_scene(reverse_inputs) + simulation = run!(model; outputs=:none) + bindings = Dict(binding.input => binding for binding in + simulation.compiled.input_bindings_by_target[(:partition, ObjectId(:plant))]) + for (input, kind, variable, producer, output) in ( + (:active_values, :active, :a, :active, :total_active), + (:dormant_values, :dormant, :b, :dormant, :total_dormant), + (:supplied_values, nothing, :supplied, nothing, :total_supplied), + ) + ids = adaptive_reference_ids(model; root=:plant, scale=:Leaf, kind) + binding = bindings[input] + @test binding.source_ids == ids + @test binding.source_application_ids == (isnothing(producer) ? Symbol[] : [producer]) + @test all(reference === PlantSimEngine.refvalue(model_status(model, id), variable) + for (id, reference) in zip(ids, parent(input_carrier(binding)))) + @test getproperty(model_status(model, :plant), output) == + sum((getproperty(model_status(model, id), variable) for id in ids); init=0.0) + end + + # The cache is deliberately scoped to one consumer assembly. A cached + # unfiltered result must remain owned separately from every returned + # vector, because writer filtering mutates those returned vectors. + cache = Dict{Any,Vector{ObjectId}}() + selectors = (bindings[:active_values].selector, bindings[:dormant_values].selector, + bindings[:supplied_values].selector) + expected = adaptive_reference_ids(model; root=:plant, scale=:Leaf) + first_ids = PlantSimEngine._cached_dependency_object_ids(cache, model, + selectors[1], PlantSimEngine._compile_selector_matcher(model, selectors[1]), ObjectId(:plant)) + @test first_ids == expected + empty!(first_ids) + second_ids = PlantSimEngine._cached_dependency_object_ids(cache, model, + selectors[2], PlantSimEngine._compile_selector_matcher(model, selectors[2]), ObjectId(:plant)) + @test second_ids == expected + pop!(second_ids) + third_ids = PlantSimEngine._cached_dependency_object_ids(cache, model, + selectors[3], PlantSimEngine._compile_selector_matcher(model, selectors[3]), ObjectId(:plant)) + @test third_ids == expected + @test length(cache) == 1 + @test only(values(cache)) == expected + @test only(values(cache)) !== third_ids + end +end diff --git a/test/test-model-api-stabilization.jl b/test/test-model-api-stabilization.jl index e0a65cd15..f049a4c6c 100644 --- a/test/test-model-api-stabilization.jl +++ b/test/test-model-api-stabilization.jl @@ -1019,32 +1019,33 @@ end parent_id = object_id end long_scope_model = CompositeModel(long_scope_objects...) + traversal_budget = 32 # Explicit budget tests the bounded-walk contract. @test length( PlantSimEngine._descendant_ids_up_to( long_scope_model, ObjectId(:long_scope_10), - PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + traversal_budget, ), ) == 31 @test length( PlantSimEngine._descendant_ids_up_to( long_scope_model, ObjectId(:long_scope_9), - PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + traversal_budget, ), - ) == PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT + ) == traversal_budget @test isnothing( PlantSimEngine._descendant_ids_up_to( long_scope_model, ObjectId(:long_scope_8), - PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + traversal_budget, ), ) @test isnothing( PlantSimEngine._descendant_ids_up_to( long_scope_model, ObjectId(:long_root), - PlantSimEngine._SCOPE_FIRST_DESCENDANT_LIMIT, + traversal_budget, ), ) @test Set( From 41de32f248a268e6e476162dbe6ea028af5364ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 14 Oct 2026 11:27:34 +0200 Subject: [PATCH 2/4] Skip redundant membership checks for resolved relations --- src/composite_model/selectors.jl | 33 +++++++----- test/test-model-adaptive-selectors.jl | 73 +++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/src/composite_model/selectors.jl b/src/composite_model/selectors.jl index e034d0f6a..e733207cf 100644 --- a/src/composite_model/selectors.jl +++ b/src/composite_model/selectors.jl @@ -1119,6 +1119,7 @@ function _selector_matches_object_id( context=nothing, default_to_context::Bool=false, default_scope=nothing, + relation_resolved::Bool=false, ) if default_to_context && matcher.defaults_to_context && !isnothing(context) return object_id == _object_id_from_context(context) @@ -1149,16 +1150,18 @@ function _selector_matches_object_id( end return _scope_contains_object(model, scope, context, object_id) end - current_id = _object_id_from_context(context) - isnothing(current_id) && error( - "`Relation(:$(_compiled_relation_symbol(matcher.relation)))` selectors require a current object context.", - ) - _relation_contains_object( - model, - matcher.relation, - current_id, - object_id, - ) || return false + if !relation_resolved + current_id = _object_id_from_context(context) + isnothing(current_id) && error( + "`Relation(:$(_compiled_relation_symbol(matcher.relation)))` selectors require a current object context.", + ) + _relation_contains_object( + model, + matcher.relation, + current_id, + object_id, + ) || return false + end return isnothing(matcher.scope) || _scope_contains_object(model, matcher.scope, context, object_id) end @@ -1249,6 +1252,7 @@ function _matching_selector_ids( context, default_to_context::Bool, default_scope, + relation_resolved::Bool, ) # The adaptive candidate source may be a local vector, an index iterator, # or registry keys. Specialize this loop at that boundary, rather than @@ -1257,7 +1261,7 @@ function _matching_selector_ids( for id in candidates haskey(model.registry.objects, id) || continue _selector_matches_object_id( - model, matcher, id; context, default_to_context, default_scope, + model, matcher, id; context, default_to_context, default_scope, relation_resolved, ) && push!(ids, id) end return _sort_object_ids!(unique!(ids)) @@ -1323,6 +1327,9 @@ function _resolve_object_ids( else nothing end + # Only the relation resolver guarantees membership. Borrowed index buckets + # still need relation checks, and a completed descendant walk includes self. + relation_resolved = false candidate_ids = if nearest_ancestor current_id = _object_id_from_context(context) isnothing(current_id) && error( @@ -1333,6 +1340,7 @@ function _resolve_object_ids( elseif !isnothing(local_ids) local_ids elseif !isnothing(relation) && relation != :descendants + relation_resolved = true _relation_object_ids(model, relation, context) elseif scope isa Self && isnothing(relation) self_ids = _scope_object_ids(model, scope, context) @@ -1343,10 +1351,11 @@ function _resolve_object_ids( elseif isnothing(relation) _scope_object_ids(model, scope, context) else + relation_resolved = true _relation_object_ids(model, relation, context) end ids = _matching_selector_ids( - model, matcher, candidate_ids, context, default_to_context, default_scope, + model, matcher, candidate_ids, context, default_to_context, default_scope, relation_resolved, ) # Cardinality diagnostics may need all available labels in the topology diff --git a/test/test-model-adaptive-selectors.jl b/test/test-model-adaptive-selectors.jl index 0501418c1..952f5243d 100644 --- a/test/test-model-adaptive-selectors.jl +++ b/test/test-model-adaptive-selectors.jl @@ -337,6 +337,79 @@ end ObjectId[ObjectId(:tip)] end +# Count membership comparisons without counting dictionary lookup or sorting. +# This catches quadratic ancestry scans without a wall-time or allocation bound. +struct AdaptiveAncestorId + index::Int +end +const ADAPTIVE_ANCESTOR_ID_COMPARISONS = Ref(0) +function Base.:(==)(left::ObjectId{AdaptiveAncestorId}, right::ObjectId{AdaptiveAncestorId}) + ADAPTIVE_ANCESTOR_ID_COMPARISONS[] += 1 + return left.value.index == right.value.index +end +Base.isequal(left::ObjectId{AdaptiveAncestorId}, right::ObjectId{AdaptiveAncestorId}) = + left.value.index == right.value.index +Base.hash(id::ObjectId{AdaptiveAncestorId}, seed::UInt) = + hash(id.value.index, hash(AdaptiveAncestorId, seed)) +Base.isless(left::AdaptiveAncestorId, right::AdaptiveAncestorId) = left.index < right.index + +function adaptive_ancestor_chain(depth) + objects = Object[Object(AdaptiveAncestorId(0); scale=:Scene)] + for index in 1:depth + push!(objects, Object(AdaptiveAncestorId(index); + scale=:Axis, kind=isodd(index) ? :active : :dormant, + species=index % 3 == 0 ? :alternate : :common, + parent=AdaptiveAncestorId(index - 1))) + end + outside = AdaptiveAncestorId(depth + 1) + push!(objects, Object(outside; scale=:Axis, kind=:active, species=:common)) + return CompositeModel(objects...), AdaptiveAncestorId(depth), outside +end + +@testset "resolved ancestors avoid quadratic membership checks on deep chains" begin + for depth in (64, 256) + model, tip, outside = adaptive_ancestor_chain(depth) + selector = Many(Relation(:ancestors)) + ADAPTIVE_ANCESTOR_ID_COMPARISONS[] = 0 + ids = resolve_object_ids(model, selector; context=tip) + comparisons = ADAPTIVE_ANCESTOR_ID_COMPARISONS[] + @test comparisons <= 4depth + @test ids == ObjectId[ObjectId(AdaptiveAncestorId(index)) for index in 0:(depth - 1)] + + # Relation queries ignore the dependency's default scope, but still + # intersect explicit scopes and apply all label and identity criteria. + @test PlantSimEngine._resolve_object_ids(model, selector; + context=tip, default_scope=Self()) == ids + middle = depth ÷ 2 + scope = Scope(ObjectId(AdaptiveAncestorId(middle))) + @test resolve_object_ids(model, Many(scope, Relation(:ancestors)); context=tip) == + ObjectId[ObjectId(AdaptiveAncestorId(index)) for index in middle:(depth - 1)] + @test resolve_object_ids(model, Many(Scope(ObjectId(outside)), Relation(:ancestors)); + context=tip) == ObjectId[] + @test resolve_object_ids(model, Many(scope, Relation(:ancestors); + scale=(:Axis, :Leaf), kind=(:active, :dormant), species=:common); context=tip) == + ObjectId[ObjectId(AdaptiveAncestorId(index)) + for index in middle:(depth - 1) if index % 3 != 0] + requested = (AdaptiveAncestorId(8), AdaptiveAncestorId(middle), tip, outside) + @test resolve_object_ids(model, Many(Relation(:ancestors); id=requested); context=tip) == + ObjectId[ObjectId(AdaptiveAncestorId(8)), ObjectId(AdaptiveAncestorId(middle))] + @test resolve_object_ids(model, One(Relation(:ancestors); id=AdaptiveAncestorId(middle)); + context=tip) == ObjectId[ObjectId(AdaptiveAncestorId(middle))] + @test resolve_object_ids(model, OptionalOne(Relation(:ancestors); id=outside); + context=tip) == ObjectId[] + @test resolve_object_ids(model, selector; context=AdaptiveAncestorId(0)) == ObjectId[] + + # Local descendant walks include self; borrowed ID candidates can also + # contain self and unrelated objects. Both paths still need filtering. + @test resolve_object_ids(model, Many(Relation(:descendants)); + context=AdaptiveAncestorId(middle)) == + ObjectId[ObjectId(AdaptiveAncestorId(index)) for index in (middle + 1):depth] + @test resolve_object_ids(model, Many(Relation(:descendants); + id=(AdaptiveAncestorId(8), AdaptiveAncestorId(middle), tip, outside)); + context=AdaptiveAncestorId(middle)) == ObjectId[ObjectId(tip)] + end +end + PlantSimEngine.@process "adaptive_leaf_a" verbose = false PlantSimEngine.@process "adaptive_leaf_b" verbose = false PlantSimEngine.@process "adaptive_plant_sum" verbose = false From 6863c97ae884ece0f702beb7806663e09c2bf4e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Wed, 14 Oct 2026 17:04:55 +0200 Subject: [PATCH 3/4] Update skill (simplified, as per latest recommendations) + update docs --- docs/make.jl | 3 +- docs/src/agent_skill.md | 180 +++++++++--- docs/src/guides/data/forcing_observations.md | 50 +++- docs/src/guides/data/outputs_plotting.md | 40 +++ docs/src/guides/modelers/from_paper.md | 97 +++++++ docs/src/index.md | 26 +- docs/src/working_with_data/fitting.md | 136 ++++++++- skills/plantsimengine/SKILL.md | 261 +++++++----------- skills/plantsimengine/evaluations/README.md | 16 +- .../references/data-calibration-evaluation.md | 131 +++++++++ .../plantsimengine/references/diagnostics.md | 39 ++- .../references/model-authoring.md | 24 +- .../references/paper-to-model.md | 81 ++++++ .../references/scenario-coupling.md | 20 +- test/test-agent-skill.jl | 8 - 15 files changed, 839 insertions(+), 273 deletions(-) create mode 100644 docs/src/guides/modelers/from_paper.md create mode 100644 skills/plantsimengine/references/data-calibration-evaluation.md create mode 100644 skills/plantsimengine/references/paper-to-model.md diff --git a/docs/make.jl b/docs/make.jl index 9e251e60c..3dfb7f2fb 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -95,6 +95,7 @@ makedocs(; "Write models" => [ "New process or new hypothesis?" => "step_by_step/implement_a_process.md", "Write and test a first model" => "journeys/modelers/basic_model.md", + "Implement a model from a paper" => "guides/modelers/from_paper.md", "Keep equations readable" => "guides/modelers/port_existing_model.md", "Organize and test your models" => "guides/modelers/repository_and_tests.md", "Cross-object values" => "journeys/modelers/cross_object_values.md", @@ -128,7 +129,7 @@ makedocs(; "Inspect a simulation" => "troubleshooting/runtime_contracts.md", "Dependency cycles" => "troubleshooting/dependency_cycles.md", "Numerical reliability" => "guides/data/numerical_reliability.md", - "Parameter fitting" => "working_with_data/fitting.md", + "Calibration and evaluation" => "working_with_data/fitting.md", ], "Reference and tools" => [ "Julia language basics" => "prerequisites/julia_basics.md", diff --git a/docs/src/agent_skill.md b/docs/src/agent_skill.md index c8c8a4da2..728a4fd93 100644 --- a/docs/src/agent_skill.md +++ b/docs/src/agent_skill.md @@ -1,10 +1,14 @@ # AI agent skill An **AI coding agent** can read and edit code and run tests with the tools you -make available. PlantSimEngine provides an optional skill with instructions -and executable examples for implementing process models and assembling -simulations. A **skill** is a folder of instructions and examples that your -agent can read while working. +make available. The examples on this page assume a recent frontier coding or +reasoning model with access to your project, source documents, data, and Julia +execution tools. Choose a model suitable for the complexity of your scientific +task; a chat session without those tools cannot execute or validate the code. + +PlantSimEngine ships an optional **skill**: a folder of instructions, +references, and executable examples that an agent can read as needed. It covers +model authoring, coupling, debugging, observations, calibration, and evaluation. You choose the scientific question, assumptions, references, and validation evidence. The agent can help turn those choices into code and check that the @@ -24,50 +28,130 @@ skill_root = joinpath(package_root, "skills", "plantsimengine") Copy or link that complete directory into your coding agent's skill directory, following its installation instructions. Keep its `SKILL.md`, `references`, -`assets`, and `scripts` together. Don't forget to tell your agent to upgrade its skill -whenever you update PlantSimEngine: different versions can describe -different APIs. +`assets`, `scripts`, and `evaluations` together. Update the skill when you update +PlantSimEngine so its instructions describe the API you actually use. The skill uses the Julia tools available in your agent's environment. It does -not require a particular editor or connector, but you can use e.g. [Kaimon](https://kahliburke.github.io/Kaimon.jl) to help your agents further. - -## Ask it to assemble existing models - -You can ask your agents anything, for example you can ask it to assemble existing models into a simulation: - -> Use the PlantSimEngine skill shipped with my loaded package. I want to -> compare two existing models for the same process. First identify available -> alternatives and explain their inputs, outputs, units, and assumptions. -> Show what data each needs and flag anything that is missing. Build the -> smallest simulation that can compare them with the same input data. Explain -> their connections, run the tests, and show the results. Keep any required -> physical conversion explicit. +not require a particular editor or connector. We recommend installing +[Kaimon](https://github.com/kahliburke/Kaimon.jl#quick-start) as an optional way +to give your agent a live Julia session for execution, inspection, and debugging. +Its Quick Start explains installation and connecting your coding agent. + +## Describe the result and provide the evidence -Useful inputs to provide are the package names, the process of interest, -your input data and its units, and the outputs you want to compare. The -[Model catalog and related packages](@ref) and [Model compatibility and replacement](@ref) -show the corresponding manual workflow. +Start with the scientific goal, relevant files, constraints, and the result you +want to inspect. Let the agent choose how to investigate and implement it. +Detailed instructions help when a scientific protocol requires them or an +observed mistake needs correcting; a long sequence of generic reasoning or +verification instructions is rarely a useful starting point. + +Current [OpenAI guidance on skills and prompts](https://developers.openai.com/blog/rethinking-skills-and-prompts-for-gpt-6-astra) +and [Anthropic's skill authoring guidance](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices) +recommend focused instructions and references loaded as needed. Check their +effect on your own tasks; scientific requirements still need to be explicit. + +Tell the agent to use the skill shipped with your loaded PlantSimEngine package. +The following prompts are starting points to adapt, with square brackets +marking information from your study. + +### Implement a model from a paper + +Provide the paper and relevant supplements, the formulation you want, and any +reference implementation or known results. State the intended object scale +and time interval. + +> Implement [formulation] from [paper and supplements] as a reusable +> PlantSimEngine model for [scale and interval]. Keep equations and parameter +> sources traceable to their pages, equation numbers, or tables. Identify +> unresolved scientific choices without inventing values. Deliver readable +> model code, a minimal scenario, and checks against [reference results], +> with the assumptions and remaining limitations. + +See [Implement a model from a paper](guides/modelers/from_paper.md), +[Implement a basic model](journeys/modelers/basic_model.md), and +[Port an existing model](@ref). The skill's `references/paper-to-model.md` +supplies the corresponding scientific guidance. + +### Couple component models + +Provide the model packages, objects and topology, weather or spatial sources, +units, and the outputs of interest. Mention any known conversions or timing +requirements. + +> Couple [component models] for [objects and topology] using [input data] +> at [cadences]. Explain the selected sources, units, physical bases, and time +> meanings of the connections. Keep conversions explicit and flag missing +> inputs or incompatible interfaces. Deliver the scenario and evidence that +> it connects the intended objects and preserves [expected relationship]. + +See [Coupling models](@ref), [Model catalog and related packages](@ref), and +[Model compatibility and replacement](@ref). The skill includes +`references/scenario-coupling.md` and runnable coupling examples. + +### Debug an unexpected result + +Provide a runnable scenario, its inputs, the error or unexpected output, and +what you expected and why. Include the relevant observation or reference +result, rather than only a plot showing disagreement. + +> Investigate [error or discrepancy] in [scenario] with [inputs]. The expected +> result is [result and evidence]. Find where the discrepancy first appears +> and distinguish configuration, numerical, and scientific causes. Show the +> evidence for your diagnosis and a verified correction where supported; +> leave unsupported hypotheses explicit. Keep parameter tuning separate. + +See [Inspect a simulation](troubleshooting/runtime_contracts.md), +[Common Errors](@ref), and +[Debugging Growth And Resource Ordering](@ref). The skill's +`references/diagnostics.md` maps symptoms to the public inspection tools. + +### Force a model with observations + +Identify which measured variable should replace a predicted input, its units, +object identity, timestamps, sampling interval, and missing-data rules. State +which downstream results should still be predicted. + +> Use measured [variable] from [dataset] to drive [consumer models] instead +> of predicting that input. Respect [units, object IDs, and sampling policy] +> and document any time alignment or conversions. Show the values supplied +> at model updates and the downstream predictions. Keep the imposed values +> distinct from results used to assess model performance. + +See [Use Observed Values In A Simulation](@ref). Supplying observations may +require removing the model that ordinarily predicts the same input; choosing +two writers is a different experiment. + +### Calibrate selected parameters + +Provide the calibration data and its protocol, target variable, parameter +names and bounds, objective, and the data reserved for evaluation. Make clear +whether other parameters and initial conditions are fixed. + +> Calibrate [parameters within bounds] against [target observations] using +> [calibration subset and objective]. Keep [fixed parameters and initial +> conditions] fixed and reserve [evaluation subset]. Report the parameter +> estimates, fitting diagnostics, objective value, and limitations. Save +> the configuration and data identifiers needed to reproduce the fit. + +See [Parameter Fitting](@ref). PlantSimEngine's `Evaluation.fit` is an extension +point; the model package or study supplies its fitting method and optimizer. -Please give as much contest a possible to your agent, for example the data should be documented properly with protocols, units, etc... +### Evaluate on independent observations + +Provide frozen parameter sets, the evaluation subset, the prediction target, +matching rules, and meaningful metrics. Keep any model selection or tuning +data separate from the independent evaluation data. + +> Evaluate [frozen baseline and calibrated models] on [independent dataset] +> for [target, scale, and interval] without retuning. Match predictions to +> observations using [matching rules]. Report [metrics], residual patterns, +> and differences by [site or treatment], with sample counts and exclusions. +> Explain which conclusions the evaluation supports and where uncertainty +> or missing coverage limits them. -## Ask it to implement an equation - -Provide the equation or source paper, definitions of the variables, -parameter values and units, and at least one expected result: - -> Use the PlantSimEngine skill shipped with my loaded package to implement -> the equation and variable definitions I provide. Check whether an existing -> process already represents this question. Keep the equation readable and -> declare each input and output, including its units and physical meaning. -> Test one calculation against my reference result, then use the model in -> a simulation of one object. -> If any assumption, physical conversion, parameter, or validation criterion -> is unspecified, identify it before choosing one. Show the code, test -> results, and remaining scientific questions. - -The [basic model tutorial](journeys/modelers/basic_model.md) follows this -sequence without an agent. For an existing implementation, also use -[Port an existing model](@ref). +See [Collecting And Plotting Outputs](@ref) and [Parameter Fitting](@ref). +The skill's `references/data-calibration-evaluation.md` covers observation +roles, data alignment, calibration, and independent evaluation. ## Review what the agent returns @@ -80,11 +164,17 @@ Expect a concrete result you can inspect: match, and the models receive values from the intended sources; - tests actually run, their results, and any remaining limitations. +For a data-based study, also expect the observation roles and alignment rules, +the calibration/evaluation split, parameter provenance, and reproducible +metrics. An optimizer converging or a declaration check passing does not +establish predictive accuracy. + PlantSimEngine's `Authoring` reports describe models and compare interfaces. `Diagnostics` explains inputs, connections, timing, and execution. The reports show which information comes from the model author, which was found by -examining the code, and which is missing. These checks help find implementation and coupling errors; scientific -validation still needs suitable observations or reference results. +examining the code, and which is missing. These checks help find implementation +and coupling errors; scientific validation still needs suitable observations +or reference results. ## Check the packaged examples diff --git a/docs/src/guides/data/forcing_observations.md b/docs/src/guides/data/forcing_observations.md index b67a19a9e..9b9c14da2 100644 --- a/docs/src/guides/data/forcing_observations.md +++ b/docs/src/guides/data/forcing_observations.md @@ -24,7 +24,11 @@ final_state(fixed_simulation).aPPFD ``` If you add a model that calculates LAI, it will replace this supplied value -when it runs. To keep the observed LAI fixed, use it without an LAI model, or with an LAI model that just outputs the constant value. +when it runs. To keep the observed LAI fixed, leave out that predictive model, +or replace it with a model that supplies the observed value. In a composed +scenario, the observation model should own the supplied output. Adding it +alongside another writer of the same variable can create a writer conflict; +`Diagnostics.explain_writers(model)` helps check ownership. ## A sequence of observations @@ -71,15 +75,35 @@ The complete [source file](observed_lai.jl) is short: Main.DocsSources.section("docs/src/guides/data/observed_lai.jl", "struct ObservedLAI") ``` -Before using real measurements, check their units, whether they refer to -leaf or ground area, their time stamps, and any missing values. This example -has one observation per day. If your measurements are less frequent, decide -how to fill the gaps: for example, interpolate between measurements or keep -the last measured value until the next one. Document that choice before -using the data. This approach supplies a measured input directly. It does -not perform data assimilation, which would use observations to estimate or -correct the model state and could account for measurement uncertainty. - -See [Collect and plot results](outputs_plotting.md) for displaying predictions -alongside observations and [Parameter fitting](../../working_with_data/fitting.md) -when the objective is to estimate model parameters. +## Match measurements to the simulation + +Before using real measurements, check their source, units, whether they refer +to leaf or ground area, their time stamps or measurement intervals, and any +missing or rejected values. Associate measurements with stable object IDs; +display names need not be unique. A plot-scale observation may also need an +explicit aggregation or conversion before it can supply an organ-scale model. + +This example has one observation per simulation day and one canopy. Its +environment rows are already in simulation order; the observation model does +not look up values by date or object ID. Prepare that alignment before using +it. For several objects, arrange the observation source and bindings so each +object receives its own measurement. + +If measurements are less frequent than model execution, choose how to fill +the gaps: for example, interpolate between measurements or keep the last +measured value until the next one. State the interval over which each value +applies and the policy for gaps or values outside the measured period. +Document that choice before using the data. Filling a gap does not create +a new independent observation for evaluation. + +This approach supplies a measured input directly. It does not perform data +assimilation, which would use observations to estimate or correct the model +state and could account for measurement uncertainty. It also changes the +question being tested: predicted light absorption under measured LAI checks +the light component while treating canopy development as supplied data. + +See [Collect and plot results](outputs_plotting.md) for retaining outputs and +matching predictions with measurements, and [Parameter fitting](../../working_with_data/fitting.md) +for calibration followed by independent evaluation. The observation-forcing +prompt in [Work with an AI coding agent](../../agent_skill.md) can help you +adapt this pattern to documented measurements and your intended objects. diff --git a/docs/src/guides/data/outputs_plotting.md b/docs/src/guides/data/outputs_plotting.md index d2729dfca..a75223719 100644 --- a/docs/src/guides/data/outputs_plotting.md +++ b/docs/src/guides/data/outputs_plotting.md @@ -157,3 +157,43 @@ Saved values, including arrays, record the result at that time. Later changes do not rewrite earlier rows, and removing an organ does not delete its past results. Compatible numerical types are preserved, including values with attached physical units when your model uses them. + +## Match predictions with observations + +Before plotting measurements beside predictions, select the intended +application and variable. Match stable object IDs and measurement times, +rather than assuming that two tables have the same row order. Confirm that +units, area basis, and sampling intervals describe the same quantity. If an +observation covers a daily interval but predictions are hourly fluxes, +aggregate the predictions over that interval before matching them. Use the +same time representation in both tables, including the convention for whether +a timestamp labels the start or end of an interval. + +For your own observation table, adapt the following snippet after preparing +an `observations` DataFrame with `object_id`, `datetime`, and `observed` +columns. Here `light` is the one-canopy result from the first example, and +both tables contain one value per object and time: + +```julia +predictions = select(light, :object_id, :datetime, :value => :simulated) +match_columns = [:object_id, :datetime] +matched = innerjoin(observations, predictions; on=match_columns, validate=(true, true)) +unmatched_observations = antijoin(observations, predictions; on=match_columns) +unmatched_predictions = antijoin(predictions, observations; on=match_columns) +matched.residual = matched.simulated .- matched.observed +``` + +`validate=(true, true)` rejects duplicate matching keys, which might otherwise +multiply rows and distort scores. Repeated measurements require an explicit +replicate or aggregation policy. `innerjoin` retains only matching rows, so +inspect and count both unmatched tables before reporting results. Keep the +source and reason for any excluded observation. Define how to handle missing +or non-finite values before computing residuals or metrics. + +Plot `simulated` as a line and `observed` as points using their matched +`datetime` values, keeping objects separate as above and sorting by time +within each object before drawing lines. A residual plot can reveal a +time-dependent bias that the time-series overlay hides. Follow +[Parameter fitting](../../working_with_data/fitting.md) to compare baseline +and fitted predictions on calibration and independent evaluation datasets, +or use the evaluation prompt in [Work with an AI coding agent](../../agent_skill.md). diff --git a/docs/src/guides/modelers/from_paper.md b/docs/src/guides/modelers/from_paper.md new file mode 100644 index 000000000..d20601038 --- /dev/null +++ b/docs/src/guides/modelers/from_paper.md @@ -0,0 +1,97 @@ +# Implement a model from a paper + +Use this guide when you have a scientific formulation and want a reusable +PlantSimEngine model. The result should make it possible to trace the code +back to the source and to check a calculation independently. An AI agent can +help with extraction and implementation; the [AI agent guide](../../agent_skill.md) +provides a starting prompt. + +## Establish what the source specifies + +Keep the paper, supplements, and any reference implementation together with +their identifiers and versions. Identify which formulation you are reproducing: +a paper may describe several alternatives or change equations between +experiments. Cite pages, equation numbers, tables, or code locations beside +the relevant implementation notes. + +A compact record can capture the information needed to write the model: + +| Source item | Record | +|---|---| +| Equations | Formula, source location, and the meaning of each symbol | +| Variables | Input, output, state, forcing, or temporary quantity; units and physical basis | +| Parameters | Value, units, source, and whether published, measured, assumed, or fitted | +| Time | Rate or amount, update interval, integration rule, and temporal lag | +| Conditions | Initialization, boundary conditions, valid domain, and special cases | +| Reference results | Inputs, expected outputs, source, and appropriate comparison tolerance | +| Unresolved information | Missing definitions, conflicting passages, and choices needed to proceed | + +Pay attention to conventions that can change a calculation: carbon versus dry +matter, leaf versus ground area, one organ versus a whole plant, temperature +units, and rates versus interval totals. State updates also need an explicit +initial value and a rule for carrying a result into the next step. + +When the source is incomplete, distinguish what it states from what you +propose. An assumption made to complete an implementation should have its own +justification. Keep contradictory definitions visible until resolved. A missing +parameter, unit, or reference is not a reason to invent a plausible one. + +## Choose the model boundary + +Use [New process or new model?](@ref) to look for the existing scientific process. +An alternative equation usually belongs to that process, with its own concrete +model type. Sharing a process does not make alternatives interchangeable; +compare their ports, physical meanings, dependencies, and time requirements +before substituting one. + +Keep fixed parameters in the model and each object's changing state in its +`Status`. Put environmental forcing in the environment interface. Declare +units, basis, and temporal meaning with `VariableContract`, and keep conversions +in explicit [adapter models](../coupling.md). + +Split a formulation into components when users need to couple, replace, +schedule, or evaluate those calculations independently. For example, separating +light interception from biomass production allows observed intercepted light +to drive a test of the growth equation. A short equation does not need several +components merely because it contains several arithmetic operations. + +[Implement a basic model](../../journeys/modelers/basic_model.md) shows the +declarations and kernel. [Port an existing model](@ref) shows how to preserve +a calculation from reference code. Document the hypothesis and provenance with +`Authoring.model_metadata` and known parameter facts with +`Authoring.parameter_metadata`. + +## Check implementation and scientific agreement separately + +Start with a source-backed calculation, an independently derived numerical +example, or a trusted reference implementation. Compare the direct kernel +result with the same inputs before checking a composed simulation. Include +special cases relevant to the published equations, such as zero input or a +transition between branches. Choose tolerances justified by numerical precision +or the resolution of the reference, rather than by the discrepancy you obtain. + +`Authoring.validate_model` checks declarations. A minimal scenario and +`Diagnostics` can establish where inputs come from and when the model runs. +These checks establish computational behavior. They do not demonstrate that +the equations predict observations accurately. + +Reproducing a published figure may require weather, initial conditions, +parameter sets, or code that the paper does not provide. Record which information +is available and what remains unresolved. Digitized plot values are approximate +reference values; keep their extraction method and uncertainty visible. Do not +change published parameters merely to make a reproduction look convincing. + +Keep parameter estimation separate from reproduction of a published +parameterization. For fitting and independent prediction checks, continue with +[Parameter Fitting](@ref) and [Collecting And Plotting Outputs](@ref). Observations +used to drive a component are imposed inputs; agreement with those same imposed +values does not evaluate its predictive ability. + +## Leave a reviewable result + +Alongside the readable model and scenario, retain the source-to-code record, +parameter provenance, checks actually run, and unresolved scientific choices. +State the PlantSimEngine version used and whether the implementation has only +structural checks, reproduces reference calculations, was calibrated, or was +evaluated independently. See [Model repository layout and tests](@ref) for +organizing that evidence in a reusable package. diff --git a/docs/src/index.md b/docs/src/index.md index dad9fc440..e3a8ad9b6 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -163,14 +163,24 @@ alternatives and check whether their inputs require different connections. ## Work with an AI coding agent An **AI coding agent** is software that can read and edit code and run tests -with your development tools. Think OpenAI's ChatGPT App (formerly Codex), Anthropic's Claude Code, SpaceXAI's Grok Bot, or Mistral's Vibe. PlantSimEngine is designed to support this way -of working, so you can just make new models, compare hypothesis, make model calibration or anything you'd like with just a prompt, such as: - -> Replace the radiation-use efficiency approach I'm using at the moment in my model by the FvCB, Medlyn and Monteith models from PlantBiophysics. Apply them at an hourly rate first, then at half-hourly, and compare the impact on yield of the three approaches. Report the results through dynamic graphs and statistics. - -For optimum results, ask your agent to [install the **agent skill**](agent_skill.md). It'll give your agent concrete information to implement, inspect and check models. Then, it will have access to every detail of the simulation, so you'll be able to ask it complex prompts such as: - -> Yield simulation is too low compared to data in 2025 and 2026, can you check why? +with your development tools. With a recent frontier coding or reasoning model, +access to Julia, and the relevant papers and data, you can ask it to implement +scientific models, couple components, investigate unexpected results, or +prepare a calibration and independent evaluation. + +> Compare my radiation-use efficiency model with the FvCB, Medlyn, and +> Monteith components from PlantBiophysics using [scenario and input data]. +> Explain the required changes in inputs, units, and timing, then compare +> their predictions against [independent observations]. Keep any additional +> scientific assumptions explicit. + +Install the versioned [AI agent skill](agent_skill.md) to provide the +PlantSimEngine API, scientific contracts, diagnostics, and executable examples. +The guide includes concise prompts for paper implementation, coupling, +debugging, observation forcing, calibration, and evaluation. Describe the +result you want and supply its evidence; the agent can choose its approach. +You remain responsible for the scientific question and the interpretation +of the results. ## Choose your starting point diff --git a/docs/src/working_with_data/fitting.md b/docs/src/working_with_data/fitting.md index 78c2b5417..e169d9828 100644 --- a/docs/src/working_with_data/fitting.md +++ b/docs/src/working_with_data/fitting.md @@ -1,11 +1,17 @@ # Parameter Fitting Parameter fitting means finding parameter values that make a model agree -with observations. PlantSimEngine provides `Evaluation.fit` as a common +with observations. Independent evaluation asks how well those values predict +observations that were kept out of fitting. This page starts with a small fitting example, +then explains how to keep those two uses of data separate. + +PlantSimEngine provides `Evaluation.fit` as a common function name; each model package supplies the fitting method for its own models. Pass the model type first and a table of observations second. The table can be a DataFrame or another format supported by Tables.jl. +## Fit the example light model + For the `Beer` model, we can calculate the light-extinction coefficient `k` from LAI, incoming light, and absorbed light: @@ -61,5 +67,129 @@ PlantSimEngine.Evaluation.fit(Beer, data) ``` This example recovers the parameter used to generate the synthetic -observation. Real calibration methods can use any optimizer or uncertainty -framework and may return additional diagnostics. +observation. It checks that the fit and simulation agree algebraically; +it provides no evidence of predictive performance on measured data. + +The result contains parameters, rather than a modified simulation. Construct +the fitted component and run a new scenario. Using the variables from the +example above, the following illustrates that step: + +```julia +parameters = PlantSimEngine.Evaluation.fit(Beer, data) +fitted_model = CompositeModel( + Beer(parameters.k); + status=(LAI=2.0,), id=:plant, scale=:Plant, environment=meteo, +) +fitted_simulation = run!(fitted_model; outputs=:all) +fitted_rows = collect_outputs(fitted_simulation; sink=DataFrame) +``` + +For real data, keep the same scenario construction, environmental inputs, +initial states, and observation-matching rules when comparing the original +and fitted parameterizations. A new `run!` starts a fresh timeline. For a +stateful model, reconstruct its initial state for each trial so one candidate's +final state does not become the next candidate's starting state. + +## Prepare measurements and define the split + +Record where the measurements came from and what each column means: units, +leaf or ground area basis, object identity, measurement time or interval, +protocol, and missing-value or quality flags. Identify the simulated quantity +that corresponds to each observation. For example, an hourly mean flux and a +daily accumulated amount require different comparisons even when both refer +to absorbed light. The [output guide](../guides/data/outputs_plotting.md) +shows how to match object identities and times before comparing values. + +Choose the calibration and evaluation datasets before fitting. Preserve a +split specified by the experiment or project. Where you need to define a +split, its unit should match the intended prediction: a new plant, site, +treatment, or period. Closely related measurements from the same plant or +period may not provide an independent check when split between the two sets. +Record the membership and rationale so the comparison can be reproduced. + +Use only calibration data to select parameters, model alternatives, bounds, +or other fitting choices. Freeze those choices before evaluating the held-out +data. If evaluation results lead you to change the model or fitting method, +report that iteration and obtain a new independent check before describing +the revised result as held-out performance. + +[Use observed values](../guides/data/forcing_observations.md) when you want to +isolate a component: for example, supply measured LAI while fitting the light +model. State which quantities are measured inputs and which are predictions. +Agreement in absorbed light under measured LAI evaluates that conditional +light calculation; it does not evaluate the ability to predict LAI. + +## Choose a fitting method for the scientific problem + +`Evaluation.fit` is an extension point, rather than a general optimizer for +every `CompositeModel`. A model package can implement a closed-form fit, +an optimization loop, or another estimation method and return named +parameters with additional diagnostics. Check what the package's method +actually estimates and which data it requires. + +For an optimization-based calibration, define an objective using predictions +matched to the calibration observations. Explain the residual scale and any +weights, especially when combining variables with different units or sampling +frequencies. Choose parameter bounds and transformations from the model's +meaning and evidence; the optimizer and stopping criteria should suit that +objective. Record starting values, fitted values, convergence information, +and a random seed when the method uses randomness. + +The Beer example averages values obtained by inverting each observation. +That estimator need not be the same as minimizing squared errors in absorbed +light, particularly with noisy measurements. Choose the estimator according +to the measurement process and scientific question. + +## Compare baseline and fitted predictions + +Run both parameterizations on the calibration and evaluation scenarios. +Align predictions with observations before computing scores, and use the +same accepted observations for both parameterizations. Report missing or +excluded pairs and the number of retained pairs for each dataset. + +Suppose `comparison` is your aligned DataFrame with columns `dataset` +(`:calibration` or `:evaluation`), `configuration` (`:baseline` or `:fitted`), +`object_id`, `datetime`, `observed`, and `simulated`. Each row is one valid, +finite observation-prediction pair, with consistent units and sampling. +The following keeps the four comparisons separate: + +```julia +using Statistics + +scores = [ + ( + dataset=first(pairs.dataset), + configuration=first(pairs.configuration), + n=nrow(pairs), + rmse=PlantSimEngine.Evaluation.RMSE(pairs.observed, pairs.simulated), + bias=Statistics.mean(pairs.simulated .- pairs.observed), + ) + for pairs in groupby(comparison, [:dataset, :configuration]) +] +comparison.residual = comparison.simulated .- comparison.observed +``` + +RMSE has the observed variable's units; the bias above is positive when +predictions are too high. `Evaluation.NRMSE` divides RMSE by the observed +range, and `Evaluation.EF` computes Nash–Sutcliffe efficiency. Their +denominators vanish for constant observations, so they cannot provide a +useful score for that case. Check the [evaluation API](../API/API_public.md) +for the available metrics and their definitions. + +Inspect predictions alongside observations and residuals against time, +predicted values, and relevant treatments or objects. An overall score can +hide a systematic error for one site or condition. Report calibration +agreement and independent evaluation performance separately, with the +unchanged baseline as a reference. + +A good fit does not establish that each parameter is identifiable: several +parameter combinations can produce similar predictions. Interpret fitted +values within the observed conditions and report uncertainty or sensitivity +when needed for the intended use. The [uncertainty guide](../guides/data/uncertainty_propagation.md) +shows how to propagate specified uncertainty; choosing illustrative parameter +distributions is not an estimate of calibration uncertainty. + +For help carrying out this workflow, see the calibration and evaluation +prompts in [Work with an AI coding agent](../agent_skill.md). Provide the data +documentation, the intended prediction, and the dataset split; the shipped +skill supplies the package-specific guidance. diff --git a/skills/plantsimengine/SKILL.md b/skills/plantsimengine/SKILL.md index de7edde36..69961c5fc 100644 --- a/skills/plantsimengine/SKILL.md +++ b/skills/plantsimengine/SKILL.md @@ -1,177 +1,108 @@ --- name: plantsimengine -description: Create, couple, run, inspect, diagnose, and test PlantSimEngine.jl models and CompositeModel/Object scenarios. Use for model implementations in PlantSimEngine-dependent packages, scientific process alternatives, multiscale coupling, lifecycle behavior, environments, hard calls, multirate execution, or PlantSimEngine diagnostics. +description: Implement, couple, debug, force with observations, calibrate, and evaluate PlantSimEngine.jl models and CompositeModel/Object scenarios. Use for PlantSimEngine-dependent model packages and scientific process alternatives. --- # PlantSimEngine -Use the unified `CompositeModel`/`Object` API. Treat the Julia model code as the -readable scientific source and the compiler, contracts, and diagnostics as the -structured evidence. Do not recreate mapping-era APIs or add compatibility -wrappers unless the user explicitly requests compatibility work. +Assume a recent frontier coding and reasoning model with general Julia and +scientific programming competence. This skill supplies package-specific +constraints, references, and executable examples. Choose the approach that +fits the task; the references are resources, not a required sequence. Use the +unified `CompositeModel`/`Object` API. -## Verify the loaded package first +## Package and execution context -Use the Julia execution tools available in the host environment for the exact -target project. Follow the user's and repository's execution requirements; -when they require Kaimon, load `kaimon-julia` and use a Kaimon session. This -package skill does not otherwise require a particular editor, connector, or -additional skill. Before reading an API from memory, editing Julia, or running -an example, evaluate: +For source-only work, identify the checkout inspected. Before executing or +interpreting runtime evidence, verify the loaded package and active project: ```julia -using PlantSimEngine, Pkg -( - path = Base.pathof(PlantSimEngine), - root = pkgdir(PlantSimEngine), - version = Base.pkgversion(PlantSimEngine), - active_project = Base.active_project(), -) +using PlantSimEngine +(path=Base.pathof(PlantSimEngine), root=pkgdir(PlantSimEngine), + version=Base.pkgversion(PlantSimEngine), project=Base.active_project()) ``` -Use these values in the result you report. A loaded depot copy, another -checkout, a stale Codex skill, and the current worktree are different sources. -The package-owned skill is: - -```julia -joinpath(pkgdir(PlantSimEngine), "skills", "plantsimengine") -``` - -Resolve package documentation and assets from `pkgdir(PlantSimEngine)`, not -from the current working directory. Never overwrite an installed skill copy -automatically. Compare it with the package copy and update it only after an -explicit user request. - -When checking an installed copy, first resolve both directories, then use a -read-only recursive diff and report which one matches the loaded package -version. If the user explicitly asks to synchronize it, replace the installed -`plantsimengine` directory from the package-owned directory as one operation; -do not merge the two copies or maintain installed-only instructions. - -## Route the task - -Read only the references needed for the request: - -- Creating or wrapping a model, choosing a process, declaring ports and - contracts, implementing alternatives, or testing a kernel: read - [references/model-authoring.md](references/model-authoring.md). -- Building a `CompositeModel`, selecting objects, coupling values, configuring - time/environment, using hard calls, or adding an adapter: read - [references/scenario-coupling.md](references/scenario-coupling.md). -- Creating or reorganizing a package of models: read - [references/repository-layout.md](references/repository-layout.md). -- Adding/removing/reparenting objects, MTG organ creation, initializers, - geometry changes, or distributed outputs: read - [references/dynamic-models.md](references/dynamic-models.md). -- Inspecting a model, comparing alternatives, explaining compilation, or - diagnosing a failure: read - [references/diagnostics.md](references/diagnostics.md). - -Executable examples live in `assets/`. Prefer adapting them over inventing a -new skeleton: - -- [assets/minimal-model.jl](assets/minimal-model.jl): one generic model, direct - kernel call, scientific contracts, and one-object composition; -- [assets/alternative-model.jl](assets/alternative-model.jl): two drop-in - alternatives and one non-drop-in alternative of the same process; -- [assets/adapter-model.jl](assets/adapter-model.jl): explicit conversion of a - physical basis in a cross-object scenario; -- [assets/coupling-patterns.jl](assets/coupling-patterns.jl): explicit `One`, - unresolved `OptionalOne`, renamed values, and `Many` over `Subtree()`; -- [assets/runtime-patterns.jl](assets/runtime-patterns.jl): `HoldLast`, - `PreviousTimeStep`, and parent-owned hard-call publication; -- [assets/lifecycle-output.jl](assets/lifecycle-output.jl): lifecycle - registration with a one-shot initializer and identity-aware `outputs_to`; -- [assets/model-tests.jl](assets/model-tests.jl): executable tests for all - examples. - -Run `scripts/check-examples.jl` through the verified Julia environment after -changing the skill or its assets. - -Fresh-agent cases and their structured trace oracle live in -[evaluations/manifest.jl](evaluations/manifest.jl) and -[evaluations/harness.jl](evaluations/harness.jl); follow -[evaluations/README.md](evaluations/README.md). Static repository tests prove -that this matrix is complete and internally consistent, not that an agent -passed it. Run each case in a context-free agent task before claiming -behavioral evidence. - -## Non-negotiable model rules - -1. **The process carries the scientific meaning.** Implement a new concrete - model for a new hypothesis of an existing process. Declare a new process - only when the biological or physical question is genuinely different. -2. **Same process does not imply substitutability.** A direct replacement also - requires compatible inputs, outputs, environment ports, variable contracts, - dependencies, and relevant traits. Use `Authoring.compare_models` rather - than assuming compatibility from `process(model)`. -3. **Make scientific conversions explicit.** A change of unit, basis, - temporal meaning, aggregation, or extent belongs in a named adapter model, - never in an unexplained binding or hidden factor. -4. **Keep `run!` readable from top to bottom.** The kernel should show the - scientific calculation. Extract helpers for named equations, genuinely - reused/tested units, or isolated optimization machinery—not mechanically - for every line. -5. **Keep values generic.** Do not force `Float64`; preserve parameter, status, - carrier, and output types. Return `nothing` from `run!`. -6. **Do not invent science.** If units, basis, defaults, hypotheses, parameter - bounds, references, or validation criteria are unknown, leave the decision - explicit and ask or report it as unresolved. -7. **Use public evidence.** Prefer `Authoring` and `Diagnostics`; do not inspect - compiled fields or use `Advanced` unless the task is specifically compiler - integration or cache behavior. - -## Canonical authoring workflow - -1. Verify package path, version, and active project. -2. Inspect the package and `Authoring.available_processes()` before declaring - a process. Inspect candidate models with `Authoring.available_models(...)` - and `Authoring.describe_model(...)`. -3. Make the process-versus-hypothesis decision explicit. -4. Create the concrete model type and declare its fixed parameters. -5. Declare `inputs_`, `outputs_`, `environment_inputs_`, and - `environment_outputs_`. Keep local output values raw; wrap outputs destined - for other objects as `Distributed(Default(value))` or - `Distributed(Required(T))`. Bind them with anonymous `OutputTo` entries and - request target views by variable tuple, never by destination group name. -6. Declare complete `variable_contracts_` beside the corresponding ports. -7. Add `dep`, hard calls, initializers, temporal/output traits, or environment - hints only when they are intrinsic to the model. -8. Write a continuous, numerically generic `run!` that returns `nothing`. -9. Test the kernel directly with a minimal `Status` and environment. -10. Compose the model on one object, then add same-object and explicit - cross-object bindings as required. -11. Validate with `Authoring.validate_model`, - `Authoring.validate_scenario`, and public diagnostics for initialization, - bindings, calls, writers, schedule, execution plan, and environment before - a full simulation. -12. Test another numeric type and every introduced coupling or lifecycle - behavior. When adding or replacing an alternative of an existing process, - use `Authoring.compare_models` to establish whether it is interchangeable. - A second hypothesis is needed only when the task calls for one. -13. Document the hypothesis, units, domain of validity, reference status, - maturity, and scientific validation level. Use `model_metadata` and - `parameter_metadata` for structured facts, and state explicitly when an - example is pedagogical, non-calibrated, or has no scientific reference. - -When an `Authoring` report marks information as inferred or best effort, do -not present it as exact instance evidence. When the loaded package predates a -documented API, stop using that API and report the version mismatch; do not -silently substitute historical syntax. - -## Completion evidence - -For a new model or coupling, do not claim completion without evidence for: - -- process identity and declared ports; -- complete scientific contracts for every contracted connection; -- a direct kernel test; -- a compiled minimal scenario; -- intended bindings, calls, writers, schedule, and environment sources; -- generic numeric behavior where scientifically meaningful; -- targeted lifecycle or multirate behavior when used; -- the exact package path/version under which validation ran. - -Keep structural correctness separate from scientific validation. Passing -PlantSimEngine tests proves the declared computational contract, not that an -equation, parameterization, or dataset is scientifically valid. +Resolve version-specific documentation, assets, and the package-owned skill +from `pkgdir(PlantSimEngine)`; the skill is at `skills/plantsimengine`. +A checkout, installed skill copy, and loaded depot package may differ. Report +relevant mismatches rather than combining incompatible APIs. + +Use the Julia tools available in the host environment. Kaimon is optional for +this package skill; when the user or repository requires it, honor that +requirement. If execution is unavailable, report source checks as source checks. + +## References by task + +- Model from a paper: [paper-to-model.md](references/paper-to-model.md). +- Model boundary, process choice, contracts, kernel, or wrapper: + [model-authoring.md](references/model-authoring.md). +- Component coupling, selectors, time, environment, adapters, or hard calls: + [scenario-coupling.md](references/scenario-coupling.md). +- Debugging, model compatibility, or compiled/runtime explanations: + [diagnostics.md](references/diagnostics.md). +- Observation forcing, parameter fitting, or empirical evaluation: + [data-calibration-evaluation.md](references/data-calibration-evaluation.md). +- Topology, MTG growth, initializers, or distributed outputs: + [dynamic-models.md](references/dynamic-models.md). +- Reusable model package organization: + [repository-layout.md](references/repository-layout.md). + +## Domain and API constraints + +- The process identifies the scientific operation; concrete models represent + its hypotheses. Reuse an existing process when appropriate. Shared process + identity alone does not establish replacement compatibility. +- Declare ports and complete known `VariableContract`s. Unit, basis, temporal + meaning, aggregation, or extent conversions belong in explicit adapters. + Never invent metadata or remove contracts to make a connection compile. +- Preserve generic parameter/status types. Keep the scientific equations + readable in `run!`, use public context APIs, and return `nothing`. +- Declare distributed outputs as `Distributed(Default(value))` or + `Distributed(Required(T))`; use anonymous `OutputTo` entries and request + target views by output-variable tuple. Ordinary outputs remain local. +- Preserve canonical writers, object identities, lifecycle refresh, and + accepted-state publication. The task references explain these contracts. +- Prefer `Authoring` and `Diagnostics` for supported interface and runtime + facts. Preserve exact, inferred, and missing-information distinctions. +- Keep source facts, scientific assumptions, calibration results, held-out + evaluation, and structural tests distinct. Missing scientific information + remains explicit; passing a test does not validate a hypothesis. + +## Outcomes proportional to the task + +| Task | Useful completion evidence | +| --- | --- | +| New model | Traceable equations and assumptions, declared boundary/contracts, a reference calculation or stated lack of one, and a minimal compiled composition. | +| Coupling change | Intended source/destination identities, compatible contracts or tested adapters, and behavior of the changed bindings. | +| Debugging | A reproducible failure or bounded diagnosis, evidence for its cause, and a focused check of the correction. | +| Observation forcing | Identified measured inputs, ownership and sampling policy, and evidence that intended observations reach consumers. | +| Calibration | Reproducible fitting data/objective/parameter choices, fitted values, diagnostics, and the declared limits of the fit. | +| Empirical evaluation | Frozen configuration, independent evaluation data where claimed, explicit prediction/observation alignment, and appropriate metrics and plots. | + +Exercise numeric types, time policies, hard calls, lifecycle, or output +retention when the change affects them. Use narrow public reports that answer +the current question; all diagnostic reports need not run on every task. +Report what was actually executed, the runtime path/version when applicable, +and unresolved scientific or implementation limitations. + +## Executable examples + +Adapt these when they fit the task: + +- [minimal-model.jl](assets/minimal-model.jl): model boundary, direct kernel, + and one-object scenario. +- [alternative-model.jl](assets/alternative-model.jl): compatible and + incompatible alternatives of one process. +- [adapter-model.jl](assets/adapter-model.jl) and + [coupling-patterns.jl](assets/coupling-patterns.jl): conversion, `One`, + `OptionalOne`, renamed inputs, and `Many`/`Subtree()`. +- [runtime-patterns.jl](assets/runtime-patterns.jl): `HoldLast`, + `PreviousTimeStep`, and hard-call publication. +- [lifecycle-output.jl](assets/lifecycle-output.jl): newborn initialization + and distributed outputs; [model-tests.jl](assets/model-tests.jl) checks assets. + +After changing executable assets or their API contracts, run +`scripts/check-examples.jl` in the verified Julia environment when available. +For skill behavior evaluation, see [evaluations/README.md](evaluations/README.md). +Static checks and example tests are not evidence that an independent agent +successfully completed a task. diff --git a/skills/plantsimengine/evaluations/README.md b/skills/plantsimengine/evaluations/README.md index e97bcfcc4..faf6532da 100644 --- a/skills/plantsimengine/evaluations/README.md +++ b/skills/plantsimengine/evaluations/README.md @@ -28,6 +28,16 @@ oracles against the loaded PlantSimEngine package. These executable oracles establish the starting state and expected technical outcome; they still do not replace fresh-agent behavioral runs. The trace score -and transcript are the evidence that an agent selected the skill, read only the -routed references, and followed the workflow rather than merely reaching the -same runtime result. +and transcript record skill selection, reference routing, and the declared +actions. `assess_trace` scores those behaviors, not overall scientific or task +success. An equally correct implementation can use a different route. + +When comparing skill versions, assess the produced model/scenario, scientific +fidelity, executable results, and truthful evidence separately from the trace +score. Use the same task inputs and model configuration, and record cost or +runtime where relevant. Add guidance for demonstrated failures rather than +requiring every task to follow the manifest's action list. + +The current cases cover technical authoring, coupling, and runtime contracts. +They do not establish success at extracting a model from a paper, forcing real +observations, calibration, or independent empirical evaluation. diff --git a/skills/plantsimengine/references/data-calibration-evaluation.md b/skills/plantsimengine/references/data-calibration-evaluation.md new file mode 100644 index 000000000..69eec865e --- /dev/null +++ b/skills/plantsimengine/references/data-calibration-evaluation.md @@ -0,0 +1,131 @@ +# Observation forcing, calibration, and empirical evaluation + +These tasks answer different questions. Observation forcing supplies measured +drivers or state values to a simulation. Calibration estimates parameters. +Evaluation assesses predictions against observations under a declared scope. +Use only the sections relevant to the request; forcing is not parameter +fitting or data assimilation, and calibration agreement is not independent +evaluation. + +## Data meaning and alignment + +Identify source/protocol, column meanings and units, object/site/treatment +identities, timestamps, sampling intervals, missing values, and uncertainty +where available. Preserve raw observations and make transformations explicit. +The prediction/observation comparison must use the same physical basis, +spatial extent, and temporal support. For example, a daily integrated flux +cannot be compared directly with an instantaneous sample or a rate per second. + +Use stable object identities and explicit time alignment; row order and display +names are not sufficient identifiers. Decide how to handle missing values, +duplicate records, gaps, and interval boundaries according to the measurement +protocol and task. Record exclusions and conversions so retained sample counts +and their scientific interpretation can be checked. + +## Force observations into a scenario + +For a fixed measured input, supply status and omit the model that would +predict/write that variable. An initial status value alone does not keep it +fixed when a scheduled producer overwrites it. + +For a changing observation sequence, use an explicit observation-provider +model that reads the measured driver from the environment and writes the +required status output. The package's +`docs/src/guides/data/forcing_observations.md` and companion `observed_lai.jl` +demonstrate `ObservedLAI()` supplying `LAI` to `Beer` on the same object. +Resolve these paths from the verified package root rather than an installed +skill directory. This provider copies values; it performs no fitting, unit +conversion, interpolation, or uncertainty-based state correction. + +For other objects or variable names, bind the consumer explicitly. Match port +contracts and establish canonical writer ownership. Retain any alternative +prediction under a distinct output or stream when the comparison needs it; +do not let two producers silently compete for the measured field. + +Sparse measurements require a scientific sampling policy. Hold-last, +interpolation, or aggregation can mean different things for a state, rate, +or interval total. Document the choice and its initialization/gap behavior; +temporal policies do not change `VariableContract` metadata. See +[scenario-coupling.md](scenario-coupling.md) for supported policies and adapters. + +Useful evidence is a small trace showing observation identity/time/value, +the provider output, and the value received by the intended consumer. +Distinguish a subsystem evaluated with measured upstream inputs from the full +model evaluated with predicted inputs. Forcing an observed target into the +model and comparing that copied target to itself is not predictive validation. + +## Calibrate parameters + +PlantSimEngine provides `Evaluation.fit` as an extension point. Model packages +implement fitting methods; it is not a generic optimizer for arbitrary +`CompositeModel`s. The documented example is +`PlantSimEngine.Evaluation.fit(Beer, data)`, returning named parameter values +such as `(k=k,)`. See `docs/src/working_with_data/fitting.md` under the verified +package root. + +For a model without a fitting method, use an appropriate optimizer or inference +framework around a reproducible simulation/objective. Choose it according to +the formulation and task rather than assuming the package supplies a fitting, +likelihood, assimilation, or uncertainty API. Distinguish estimated parameters +from fixed values, imposed inputs, and initial conditions; changing the model +structure is a separate scientific choice. + +Make the fitting scope explicit: dataset split, estimated parameters and their +units/bounds, objective or likelihood, weighting, observation alignment, and +failure handling. Bounds need a scientific or documented numerical basis. +Check sensitivity or identifiability when the data may not constrain the +requested parameters. A returned optimum alone does not establish uniqueness +or parameter uncertainty. + +Each objective evaluation must start from the intended initial state. Use a +fresh scenario or verified reset; `continue!` advances an existing simulation +and would carry state between trials. Keep stochastic seeds and replicate +policy reproducible where used. Apply fitted values by constructing the model +with those values and verify the resulting simulation, not only the optimizer +return value. + +Report the data/objective/configuration, fitted values, convergence and +boundary diagnostics, and calibration residuals or plots. Recovering a known +parameter from synthetic data is a useful implementation check, not evidence +that the model is calibrated for a real population or environment. + +## Evaluate predictions + +Preserve an existing calibration/evaluation split unless the user changes it. +Choose a new split to reflect the intended generalization, such as unseen +dates, individuals, sites, years, or treatments; correlated observations may +need group/block separation. Record the split before fitting or model choice. +Keep evaluation targets out of parameter estimation, data-driven +preprocessing, model selection, and repeated tuning. If evaluation results +drive a revision, identify that reuse and reserve new evidence for subsequent +independent evaluation. + +Freeze fitted parameters, model structure, forcing policy, and initialization +for the evaluation run. Preserve declared boundaries such as warm-up periods, +validity domains, and earlier holdouts. Show separately which quantities were +measured/imposed, estimated, and predicted. State whether results test a +component with measured inputs, a coupled system, or transfer to a new scope. + +Use explicit output retention and align collected predictions with +observations by identity and time/support. Summarize errors in relevant units +and include sample counts, exclusions, residual structure, and useful plots. +Bias and RMSE/MAE often describe different failures; choose metrics for the +scientific question and avoid percentage metrics near zero without a clear +rationale. An aggregate score can hide site, treatment, or temporal failures. + +If uncertainty is requested, distinguish observation uncertainty, fitted +parameter uncertainty, stochastic variability, and structural uncertainty. +A descriptive min-max band is not a confidence or prediction interval. +With no independent observations, report calibration fit or reference/structural +checks; do not relabel them as held-out evaluation. + +## Example requests + +> Supply these measured values to the requested component. Preserve their +> units, identities, and temporal meaning; explain the writer and sampling +> policy and verify the values actually received by the component. + +> Calibrate these parameters using the declared training data and bounds. +> Make the objective and simulation reproducible, report what the data +> constrain, then evaluate the frozen model on the existing holdout. Keep +> imposed inputs, fitted quantities, and predictions distinct. diff --git a/skills/plantsimengine/references/diagnostics.md b/skills/plantsimengine/references/diagnostics.md index d5f9187e1..d570889b3 100644 --- a/skills/plantsimengine/references/diagnostics.md +++ b/skills/plantsimengine/references/diagnostics.md @@ -4,7 +4,7 @@ Read this reference when discovering processes/models, reviewing a model boundary, comparing alternatives, validating a scenario, or investigating a compiler/runtime failure. -## Use the neutral authoring API first +## Public authoring evidence `PlantSimEngine.Authoring` is the public machine-readable entry point: @@ -21,8 +21,10 @@ validate_model(model; strict=true) validate_scenario(composite; strict=false) ``` -Do not parse `show`, generated documentation, exception prose, or compiler -fields when one of these functions represents the information directly. +Prefer these structured reports for information they represent directly. +Stack traces, exception messages, source inspection, and runtime probes can +complement them when investigating a failure they do not explain. Compiler +fields are implementation details rather than a stable model-facing contract. An instance descriptor is exact for its parameter values and declared schemas. A type-only descriptor may be best effort when the type cannot be constructed @@ -32,7 +34,7 @@ reports and do not present inferred values as exact. ## Process identity versus replacement compatibility Use `compare_models(a, b)` before `Override` or another direct replacement. -Interpret the report in this order: +The report should distinguish: 1. Are both values concrete PlantSimEngine models? 2. Do they implement the same process? @@ -63,11 +65,11 @@ with `to_dict` or `to_json`. when the package or task has enough scientific metadata to satisfy it without inventing values. -`validate_scenario` should be used before `run!`. A scenario under construction -may return a partial report with structured diagnostics rather than a single -exception. Consume diagnostic codes, severity, application/object identities, -variables, and suggestions. Keep the report's `schema_version` when -serializing it. +`validate_scenario` is useful for checking a composition before a full run. +A scenario under construction may return a partial report with structured +diagnostics rather than a single exception. Diagnostic codes, severity, +application/object identities, variables, and suggestions help locate the +problem. Keep the report's `schema_version` when serializing it. Structural validation is exhaustive only for declared/compiled structure. Execution probes may depend on data values and branch paths; do not claim that @@ -112,16 +114,18 @@ This source shows the resolved application order, targets, bindings, calls, environment sampling, and invoked kernels. It is a view of the normal compiled plan, not a second scheduler and not a replacement for ordinary `run!`. -Only when the user requests a file artifact, write it explicitly: +When a file helps review or debug the composition, it can be generated with: ```julia Authoring.write_compiled_model_source("compiled_scenario.jl", scenario) ``` -Prefer `compiled_model_source` in memory for inspection and review. Treat the -written file as generated evidence and do not edit it as the source scenario. +Treat this as generated evidence and do not edit it as the source scenario. -## Common diagnosis sequence +## Common failure mechanisms + +The following questions guide investigation; choose the relevant mechanism +and evidence rather than applying every sequence to each failure. ### Missing or ambiguous input @@ -166,7 +170,7 @@ written file as generated evidence and do not edit it as the source scenario. ## Reporting evidence -Always separate: +Distinguish the evidence relevant to the result: - freshly executed validation under the reported path/version; - facts read from source or a structured descriptor; @@ -177,3 +181,10 @@ Always separate: Do not describe a UI view, old generated report, or different checkout as the current runtime result. + +## Example request + +> Diagnose this failure in the smallest useful reproducer. Identify the +> responsible equation, binding, writer, environment source, or time policy +> from evidence; apply a focused correction and verify the behavior it changes. +> Report any scientific assumption or unresolved limitation separately. diff --git a/skills/plantsimengine/references/model-authoring.md b/skills/plantsimengine/references/model-authoring.md index 6e39fe45c..34820ca93 100644 --- a/skills/plantsimengine/references/model-authoring.md +++ b/skills/plantsimengine/references/model-authoring.md @@ -3,8 +3,11 @@ Read this reference when implementing or wrapping a model. Also read `scenario-coupling.md` when the task includes a nontrivial scenario, and `repository-layout.md` when creating files in a model package. +For a scientific source paper, use [paper-to-model.md](paper-to-model.md) to +keep extraction, assumptions, and source-to-code evidence traceable. The +sections below describe model decisions, not a fixed execution sequence. -## 1. Decide whether the process already exists +## Decide whether the process already exists The process is the scientific operation; the concrete model is one hypothesis or numerical formulation of that operation. @@ -26,7 +29,7 @@ Two concrete types may share `process(model)` while exposing different ports. They are then alternatives of the same scientific process, but not drop-in replacements. Use `Authoring.compare_models(a, b)` to establish compatibility. -## 2. Declare the complete model boundary +## Declare the complete model boundary Keep fixed parameters in an immutable, parametrically typed struct. Keep timestep-varying state in `Status`. @@ -101,7 +104,7 @@ Rules: - Use `NamedTuple()` for an empty schema. - Do not store mutable simulation state in the model object. -## 3. Declare scientific contracts beside the ports +## Declare scientific contracts beside the ports Use `VariableContract` for unit and meaning, not only a prose comment: @@ -145,7 +148,7 @@ Do not invent a unit or basis to make compilation pass. If the producer and consumer meanings differ, create a named adapter with one input contract and one output contract. See `../assets/adapter-model.jl`. -## 4. Add dependencies only when they belong to the model +## Add dependencies only when they belong to the model Use `dep` for model-author defaults that remain valid across scenarios: @@ -172,7 +175,7 @@ routing, or initializer declarations only when they are intrinsic to the implementation. Do not bake one scenario's scheduling preference into a generic model. -## 5. Write `run!` as the scientific narrative +## Write `run!` as the scientific narrative The required kernel is: @@ -206,9 +209,9 @@ input/output, environmental forcing, constant, or internal temporary. Split an external routine into several PlantSimEngine models only when users need to couple, replace, schedule, or validate those subprocesses independently. -## 6. Test from the kernel outward +## Evidence for the implementation -Start with a direct call, which isolates the equation from the compiler: +A direct call isolates the equation from the compiler: ```julia model = MyModel(0.02f0, 2.0f0, 20.0f0) @@ -217,7 +220,7 @@ environment = (T=25.0f0,) PlantSimEngine.run!(model, status, environment, nothing, nothing) ``` -Then test, in proportion to the feature: +Choose evidence in proportion to the new or changed behavior: 1. `inputs_`, `outputs_`, environment schemas, contracts, and process identity; 2. direct `run!` values and a generic type such as `Float32`; @@ -228,8 +231,9 @@ Then test, in proportion to the feature: model; 7. a package-level scenario or empirical regression when available. -Use `Authoring.validate_model(model)` before scenario work. Validation proves -the declared model boundary; it does not prove the scientific hypothesis. +`Authoring.validate_model(model)` checks the declared model boundary. It does +not prove the scientific hypothesis. Existing kernels need new direct tests +when their behavior changes, rather than whenever their scenario is rewired. ## Complete executable examples diff --git a/skills/plantsimengine/references/paper-to-model.md b/skills/plantsimengine/references/paper-to-model.md new file mode 100644 index 000000000..41f846b30 --- /dev/null +++ b/skills/plantsimengine/references/paper-to-model.md @@ -0,0 +1,81 @@ +# Implement a model from a paper + +Use this reference for translating scientific source material into a +PlantSimEngine component. The outcome is a traceable implementation of the +requested hypothesis, with unresolved choices visible. See +[model-authoring.md](model-authoring.md) for the model API and +[scenario-coupling.md](scenario-coupling.md) when composing it. + +## Scientific information that changes the implementation + +Read the relevant paper, supplement, and supplied data or code. Record exact +source locations for equations, variable definitions, parameter values, and +boundary/initial conditions. A page, equation, table, or supplement identifier +is more useful than a citation alone. Distinguish the original formulation +from later corrections, adaptations, or choices made for this implementation. + +Resolve, where the source supports it: + +- the scientific process and hypothesis; +- parameter meanings, units, admissible ranges, and how values were obtained; +- input/output units and physical basis, spatial extent, and temporal meaning; +- state variables, initial conditions, and whether an equation defines a + rate, interval amount, algebraic relation, or differential equation; +- environmental drivers, validity range, and reported limitations; +- numerical methods and the observations or reference results used to assess + the formulation. + +These are scientific facts to extract, not fields to fill by guessing. If the +source is inaccessible, a symbol is undefined, or the evidence conflicts, +report the specific gap. Proceed with independent implementation work where +possible; choices that change the scientific result need an explicit basis. + +## Mapping into PlantSimEngine + +Classify fixed values as model parameters, changing state/inputs as `Status`, +drivers as environment inputs, and constants as constants. Reuse a process +type when the new formulation answers the same scientific question. +Separate subprocesses when they need independent coupling, replacement, +scheduling, or validation; a paper's section structure need not become a +package hierarchy. + +Declare the scientifically known port contracts. Keep conversions visible in +named adapters, including rate-to-amount and leaf/plant/ground-area changes. +Discretizing a continuous equation is an implementation choice: document the +integration method and initial conditions rather than presenting the +discretization as the published equation. Scenario cadence should remain a +scenario choice unless the scientific formulation requires a particular step. + +Keep equations readable and connect code/docstrings to their source locations. +Use `Authoring.model_metadata` and `Authoring.parameter_metadata` for known +structured facts; the model documentation carries the scientific explanation, +provenance, adaptations, and unresolved decisions. A second hypothesis is +useful for a requested comparison, not a prerequisite for implementing one. + +## Evidence for the result + +Prefer a direct calculation against a published numerical example, analytical +limit, or supplied reference result, followed by a minimal composed scenario. +Check scientifically relevant boundaries and any introduced numerical method. +If the paper gives only a figure, distinguish a visual or digitized comparison +from exact numerical reproduction and retain extraction uncertainty. + +For dynamic equations, conservation, positivity, equilibrium behavior, and +timestep sensitivity may supply stronger evidence than one endpoint. Choose +checks relevant to the formulation. If no independent reference result is +available, say which properties were checked and which claims remain untested. + +The reviewable result includes the model, relevant tests/scenario, and a short +source-to-code record: equation/source location, implemented quantity, and +any deviation or unresolved choice. Keep agreement with an equation separate +from empirical validity. Fitting published or local parameters does not +constitute independent evaluation; use +[data-calibration-evaluation.md](data-calibration-evaluation.md) for that work. + +## Example request + +> Implement the hypothesis in this paper as a reusable PlantSimEngine model. +> Preserve the equations, units, state definitions, and assumptions; trace +> implementation choices to source locations and flag missing information. +> Show evidence for a reference calculation and a minimal composition, and +> state what remains scientifically unvalidated. diff --git a/skills/plantsimengine/references/scenario-coupling.md b/skills/plantsimengine/references/scenario-coupling.md index 60ea931e8..8b08d1896 100644 --- a/skills/plantsimengine/references/scenario-coupling.md +++ b/skills/plantsimengine/references/scenario-coupling.md @@ -252,9 +252,12 @@ form to execute every target. Use `call_targets` and individual calls only for selective ordering, per-target environment values, or explicit target status inspection. Call-target-only applications are not run by the root scheduler. -## Validate before running the full scenario +## Evidence for the changed composition -Use `Authoring.validate_scenario(model)` and inspect: +`Authoring.validate_scenario(model)` checks declared scenario structure. Choose +the public reports that answer questions about the changed connections, +ownership, timing, or execution; these are available examples, not a required +bundle: ```julia Diagnostics.explain_initialization(model) @@ -266,7 +269,7 @@ Diagnostics.explain_schedule(model) Diagnostics.explain_execution_plan(model) ``` -Then run with explicit output retention: +For a focused execution check, retain only the outputs needed by the task: ```julia simulation = run!(model; steps=10, outputs=:none) @@ -276,3 +279,14 @@ latest = final_state(simulation, One(scale=:Plant)) Use `outputs=:all` or `OutputRequest` only for streams needed by the task. `collect_outputs` materializes rows and should not be the default internal representation. Use `continue!` or `step!` to advance the same timeline. + +For observed upstream values, parameter fitting, or scientific comparison, +see [data-calibration-evaluation.md](data-calibration-evaluation.md). These +tasks require data alignment and scientific evidence beyond compiled bindings. + +## Example request + +> Couple these component models into the requested scenario. Preserve units, +> physical bases, object identities, and time meaning. Make needed conversions +> explicit and show evidence that each changed input receives its intended +> source, with a focused execution check of the composition. diff --git a/test/test-agent-skill.jl b/test/test-agent-skill.jl index 257815dcb..fb1536993 100644 --- a/test/test-agent-skill.jl +++ b/test/test-agent-skill.jl @@ -77,14 +77,6 @@ missing_verification = PlantSimEngineSkillEvaluations.assess_trace( @test !missing_verification.passed @test missing_verification.missing_actions == [:verify_loaded_package] -skill_source = read( - joinpath(@__DIR__, "..", "skills", "plantsimengine", "SKILL.md"), - String, -) -@test occursin("13. Document the hypothesis", skill_source) -@test occursin("model_metadata", skill_source) -@test occursin("parameter_metadata", skill_source) - alternative_source = read( joinpath( @__DIR__, From 25f5acefe9afae613de208e32ac913bc996d2fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Vezy?= Date: Thu, 15 Oct 2026 15:06:04 +0200 Subject: [PATCH 4/4] Use the composite model directly for tests on PBP --- benchmark/test-plantbiophysics.jl | 5 ++++- benchmark/test/runtests.jl | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/benchmark/test-plantbiophysics.jl b/benchmark/test-plantbiophysics.jl index 5e685df3c..dece3875e 100644 --- a/benchmark/test-plantbiophysics.jl +++ b/benchmark/test-plantbiophysics.jl @@ -48,7 +48,7 @@ function _plantbiophysics_leaf_scene( row; environment=_plantbiophysics_atmosphere(row), ) - return PlantBiophysics.leaf_scene( + return CompositeModel( Monteith(), Fvcb( VcMaxRef=row.VcMaxRef, @@ -63,6 +63,9 @@ function _plantbiophysics_leaf_scene( aPPFD=row.Ra_SW_f * 0.48 * 4.57, d=row.d, ), + id=:leaf, + scale=:Leaf, + kind=:plant, environment=environment, ) end diff --git a/benchmark/test/runtests.jl b/benchmark/test/runtests.jl index f285e380c..2d7ef6a91 100644 --- a/benchmark/test/runtests.jl +++ b/benchmark/test/runtests.jl @@ -989,6 +989,19 @@ if benchmark_test_enabled("PlantBiophysics benchmark API smoke") outputs=:none, ) @test current_step(simulation) == nsteps + state = final_state(simulation) + @test all(isfinite, (state.A, state.Gₛ, state.Tₗ, state.λE)) + retained_model, retained_steps = setup_plantbiophysics_multistep(; nsteps=2) + retained = benchmark_plantbiophysics_multistep( + retained_model, + retained_steps; + outputs=:all, + ) + retained_state = final_state(retained) + @test retained_state.A ≈ state.A + @test retained_state.Gₛ ≈ state.Gₛ + @test retained_state.Tₗ ≈ state.Tₗ + @test retained_state.λE ≈ state.λE records = run_plantbiophysics_performance_profile(; nsteps=2, fanout_scenes=2,