From 78526452005e3b1dc2d657527b0038e6ae47f62e Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Fri, 13 Nov 2026 18:25:21 +0400 Subject: [PATCH 1/2] Count a data source's attributes in its identity when looking it up and when keeping the latest version get_data_source() now looks for the source without attributes, which is the one it creates, so a sibling recording attributes under the same name, model, version and type no longer makes it raise MultipleResultsFound. keep_latest_version() now groups sources by their attributes as well as their name, type and model, so sources differing only by attributes each keep their data. The members a partial aggregate names as missing do not count, so a complete aggregate still replaces its partial one. Fixes #2505 Signed-off-by: Sankalp Thakur --- documentation/changelog.rst | 1 + flexmeasures/data/models/data_sources.py | 28 +++- flexmeasures/data/tests/test_data_source.py | 138 +++++++++++++++++++- flexmeasures/data/utils.py | 5 + 4 files changed, 166 insertions(+), 6 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e20ed660b8..41fe502304 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,6 +21,7 @@ Infrastructure / Support Bugfixes ----------- +* Stop a lookup of a script's data source from failing once a source with the same name, model, version and type records attributes, and stop reading the most recent beliefs from hiding the data of sources that differ only by their attributes, such as a scheduler recording its parameters, keeping only the latest version of each [see `PR #2505 `_] v1.1.1 | November 11, 2026 diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 8c09be5351..ceff8d5cdb 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -717,13 +717,35 @@ def source_priorities(sources) -> dict: return {source: rank for rank, source in enumerate(priority_order)} +def attributes_variant(source) -> bytes: + """Hash what a source's attributes say about which data it records, leaving out the members it says it missed. + + Sources sharing a name, type, model and version but recording different attributes are distinct sources, + whose data may have to coexist, as when a scheduler records its parameters in its source's attributes. + The members a partial aggregate names as missing are the exception (see `DataSource.missing_members`): + such a partial aggregate is meant to be replaced by the complete one, so it should end up in the same group. + """ + attributes = deepcopy(dict(source.attributes or {})) + data_generator = attributes.get("data_generator") + if isinstance(data_generator, dict): + config = data_generator.get("config") + if isinstance(config, dict): + config.pop("missing-members", None) + if not config: + del data_generator["config"] + if not data_generator: + del attributes["data_generator"] + return DataSource.hash_attributes(attributes) + + def keep_latest_version( bdf: tb.BeliefsDataFrame, one_deterministic_belief_per_event: bool = False, ) -> tb.BeliefsDataFrame: """Filters the BeliefsDataFrame to keep the latest version of each source per event or per belief. - Sources are grouped by (name, type, model); within each group, only data from + Sources are grouped by (name, type, model, attributes), where the members a partial aggregate names as missing + do not count towards the attributes (see `attributes_variant`); within each group, only data from the latest version (breaking ties in favour of a source naming no missing members, and then by highest id) is kept, per event or per belief. Probabilistic beliefs are kept intact. @@ -740,12 +762,12 @@ def keep_latest_version( # Sanitize BeliefsDataFrame by removing duplicate indices bdf = bdf.loc[~bdf.index.duplicated(keep="first"), :] - # Group unique sources by (name, type, model) + # Group unique sources by (name, type, model, attributes) source_codes, unique_sources = pd.factorize(bdf.index.get_level_values("source")) group_keys: dict = {} group_per_source = np.empty(len(unique_sources), dtype=np.int64) for i, source in enumerate(unique_sources): - key = (source.name, source.type, source.model) + key = (source.name, source.type, source.model, attributes_variant(source)) group_per_source[i] = group_keys.setdefault(key, len(group_keys)) if len(group_keys) == len(unique_sources): # Every source is the sole member of its group, so nothing can be dropped diff --git a/flexmeasures/data/tests/test_data_source.py b/flexmeasures/data/tests/test_data_source.py index 6afeb35149..703f929f39 100644 --- a/flexmeasures/data/tests/test_data_source.py +++ b/flexmeasures/data/tests/test_data_source.py @@ -15,6 +15,7 @@ from flexmeasures.data.models.data_sources import ( DATA_SOURCE_UNIQUE_INDEX, + attributes_variant, keep_latest_version, DataSource, ) @@ -300,6 +301,77 @@ def test_keep_latest_version_prefers_a_complete_aggregate_of_the_same_belief_tim assert kept["event_value"].values == pytest.approx([0.12]) +def test_keep_latest_version_keeps_sources_differing_only_by_attributes(): + """Sources that share a name, type, model and version, but record different attributes, are not versions of one another. + + Each keeps its data, rather than the one with the highest id hiding the others. + Versions are still compared within one set of attributes, and a partial aggregate of one still gives way to its complete aggregate. + """ + sensor = tb.Sensor("schedule", event_resolution=timedelta(hours=1)) + + def scheduler(id: int, version: str, attributes: dict | None = None): + return DataSource( + id=id, + name="Seita", + model="CommunityScheduler", + type="scheduler", + version=version, + attributes=attributes, + ) + + plain = scheduler(1, "2.4.0") + run_a = scheduler(2, "2.4.0", {"data_generator": {"parameters": {"run": "a"}}}) + run_b = scheduler(3, "2.4.0", {"data_generator": {"parameters": {"run": "b"}}}) + run_a_old = scheduler(4, "2.3.0", {"data_generator": {"parameters": {"run": "a"}}}) + run_b_partial = scheduler( + 5, + "2.4.0", + { + "data_generator": { + "parameters": {"run": "b"}, + "config": {"missing-members": [3]}, + } + }, + ) + bdf = tb.BeliefsDataFrame( + [ + tb.TimedBelief( + sensor=sensor, + source=source, + event_start="2022-12-01T23:30+00", + belief_horizon="PT0H", + event_value=value, + ) + for value, source in enumerate( + (plain, run_a, run_b, run_a_old, run_b_partial) + ) + ] + ) + for one_deterministic in (False, True): + kept = keep_latest_version( + bdf, one_deterministic_belief_per_event=one_deterministic + ) + assert set(kept.sources) == {plain, run_a, run_b} + + +def test_attributes_variant_leaves_out_only_the_missing_members(): + complete = DataSource(name="Seita", type="reporter") + partial = DataSource( + name="Seita", + type="reporter", + attributes={"data_generator": {"config": {"missing-members": [3]}}}, + ) + other_config = DataSource( + name="Seita", + type="reporter", + attributes={"data_generator": {"config": {"resolved-members": [1, 2]}}}, + ) + assert attributes_variant(partial) == attributes_variant(complete) + assert attributes_variant(other_config) != attributes_variant(complete) + # Working out the variant leaves the source's own attributes alone + assert partial.missing_members == [3] + + def test_keep_latest_version_preserves_probabilistic_splits(): sensor = tb.Sensor("X", event_resolution=timedelta(hours=1)) s1v1 = DataSource(name="s1", model="model 1", type="forecaster", version="0.1.0") @@ -444,6 +516,43 @@ def test_get_data_source_finds_a_source_with_an_empty_model_and_version(db, app) assert get_data_source("test-script-source").id == existing.id +def test_get_data_source_takes_the_source_without_attributes(db, app): + """A sibling recording attributes under the same name, model, version and type is a different source. + + Finding it as well would leave two sources where one was expected. + """ + from flexmeasures.data.utils import get_data_source + + sibling = DataSource( + name="test-script-source", + type="scheduler", + model="CommunityScheduler", + version="2.4.0", + attributes={"data_generator": {"parameters": {"run": "a"}}}, + ) + db.session.add(sibling) + db.session.flush() + + # Only the sibling exists, so a source without attributes is created next to it + created = get_data_source( + "test-script-source", + data_source_model="CommunityScheduler", + data_source_version="2.4.0", + data_source_type="scheduler", + ) + assert created.id not in (None, sibling.id) + assert created.attributes_hash is None + + # Now both exist, and the one without attributes is found + found = get_data_source( + "test-script-source", + data_source_model="CommunityScheduler", + data_source_version="2.4.0", + data_source_type="scheduler", + ) + assert found.id == created.id + + def test_get_or_create_source_finds_a_source_without_a_model_and_version_when_asked_for_empty_ones( db, app ): @@ -707,7 +816,13 @@ def naive_keep_latest_version( def group_key(index_tuple): source = index_tuple[source_level] - key = (index_tuple[event_level], source.name, source.type, source.model) + key = ( + index_tuple[event_level], + source.name, + source.type, + source.model, + attributes_variant(source), + ) if not one_deterministic_belief_per_event: key += (index_tuple[belief_level],) return key @@ -745,7 +860,7 @@ def group_key(index_tuple): ) for i in range(6) ] - # a partial aggregate of each, which ranks behind it despite its higher id + # a sibling of each recording other attributes, which is a different source sources += [ DataSource( id=source.id + 6 if source.id is not None else None, @@ -753,7 +868,24 @@ def group_key(index_tuple): model=source.model, type=source.type, version=source.version, - attributes={"data_generator": {"config": {"missing-members": [3]}}}, + attributes={"data_generator": {"parameters": {"run": "b"}}}, + ) + for source in sources + ] + # a partial aggregate of each, which ranks behind it despite its higher id + sources += [ + DataSource( + id=source.id + 12 if source.id is not None else None, + name=source.name, + model=source.model, + type=source.type, + version=source.version, + attributes={ + "data_generator": { + **(source.attributes or {}).get("data_generator", {}), + "config": {"missing-members": [3]}, + } + }, ) for source in sources ] diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 0a53959aef..93c0dd331e 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -170,6 +170,10 @@ def get_data_source( naming none asks for a source belonging to none, which is what a script the host runs for everyone records under. Leaving it out of the lookup would both hand a script another organisation's source and, where two organisations run the same script, find two sources where one was expected. + + The source it looks for, and creates, has no attributes. + A sibling recording attributes under the same name, model, version and type is a different source, + so leaving the attributes out of the lookup would find several sources where one was expected. """ # Imported here to avoid a circular import at module load time. @@ -183,6 +187,7 @@ def get_data_source( func.coalesce(DataSource.version, "") == (data_source_version or ""), DataSource.type == data_source_type, DataSource.account_id == (account.id if account is not None else None), + DataSource.attributes_hash.is_(None), ) data_source = db.session.execute(query).scalar_one_or_none() if data_source is None: From e3cf63e9f9ea7ea66d34e39dd011d0b2f5a7cc60 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Fri, 13 Nov 2026 18:30:03 +0400 Subject: [PATCH 2/2] Point the changelog entry to PR #2867 Signed-off-by: Sankalp Thakur --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 41fe502304..55188b0550 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,7 +21,7 @@ Infrastructure / Support Bugfixes ----------- -* Stop a lookup of a script's data source from failing once a source with the same name, model, version and type records attributes, and stop reading the most recent beliefs from hiding the data of sources that differ only by their attributes, such as a scheduler recording its parameters, keeping only the latest version of each [see `PR #2505 `_] +* Fix ``get_data_source()`` raising ``MultipleResultsFound`` when a source with the same name, model, version and type has attributes, and stop ``keep_latest_version()`` from hiding the beliefs of sources that differ only by their attributes [see `PR #2867 `_] v1.1.1 | November 11, 2026