diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9f0ccb3aa9..8d2776886d 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -22,6 +22,7 @@ Infrastructure / Support Bugfixes ----------- * Saving beliefs that overlap stored ones in the hour that an autumn clock change repeats no longer fails with "Categorical categories must be unique" or keeps unchanged beliefs as new, when the new beliefs are in local time [see `PR #2869 `_] +* Fix ``get_data_source()`` raising ``MultipleResultsFound`` when a source with the same name, model, version and type has attributes, and stop ``keep_latest_version()`` from hiding the beliefs of sources that differ only by their attributes [see `PR #2867 `_] v1.1.1 | November 11, 2026 diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 8c09be5351..ceff8d5cdb 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -717,13 +717,35 @@ def source_priorities(sources) -> dict: return {source: rank for rank, source in enumerate(priority_order)} +def attributes_variant(source) -> bytes: + """Hash what a source's attributes say about which data it records, leaving out the members it says it missed. + + Sources sharing a name, type, model and version but recording different attributes are distinct sources, + whose data may have to coexist, as when a scheduler records its parameters in its source's attributes. + The members a partial aggregate names as missing are the exception (see `DataSource.missing_members`): + such a partial aggregate is meant to be replaced by the complete one, so it should end up in the same group. + """ + attributes = deepcopy(dict(source.attributes or {})) + data_generator = attributes.get("data_generator") + if isinstance(data_generator, dict): + config = data_generator.get("config") + if isinstance(config, dict): + config.pop("missing-members", None) + if not config: + del data_generator["config"] + if not data_generator: + del attributes["data_generator"] + return DataSource.hash_attributes(attributes) + + def keep_latest_version( bdf: tb.BeliefsDataFrame, one_deterministic_belief_per_event: bool = False, ) -> tb.BeliefsDataFrame: """Filters the BeliefsDataFrame to keep the latest version of each source per event or per belief. - Sources are grouped by (name, type, model); within each group, only data from + Sources are grouped by (name, type, model, attributes), where the members a partial aggregate names as missing + do not count towards the attributes (see `attributes_variant`); within each group, only data from the latest version (breaking ties in favour of a source naming no missing members, and then by highest id) is kept, per event or per belief. Probabilistic beliefs are kept intact. @@ -740,12 +762,12 @@ def keep_latest_version( # Sanitize BeliefsDataFrame by removing duplicate indices bdf = bdf.loc[~bdf.index.duplicated(keep="first"), :] - # Group unique sources by (name, type, model) + # Group unique sources by (name, type, model, attributes) source_codes, unique_sources = pd.factorize(bdf.index.get_level_values("source")) group_keys: dict = {} group_per_source = np.empty(len(unique_sources), dtype=np.int64) for i, source in enumerate(unique_sources): - key = (source.name, source.type, source.model) + key = (source.name, source.type, source.model, attributes_variant(source)) group_per_source[i] = group_keys.setdefault(key, len(group_keys)) if len(group_keys) == len(unique_sources): # Every source is the sole member of its group, so nothing can be dropped diff --git a/flexmeasures/data/tests/test_data_source.py b/flexmeasures/data/tests/test_data_source.py index 6afeb35149..703f929f39 100644 --- a/flexmeasures/data/tests/test_data_source.py +++ b/flexmeasures/data/tests/test_data_source.py @@ -15,6 +15,7 @@ from flexmeasures.data.models.data_sources import ( DATA_SOURCE_UNIQUE_INDEX, + attributes_variant, keep_latest_version, DataSource, ) @@ -300,6 +301,77 @@ def test_keep_latest_version_prefers_a_complete_aggregate_of_the_same_belief_tim assert kept["event_value"].values == pytest.approx([0.12]) +def test_keep_latest_version_keeps_sources_differing_only_by_attributes(): + """Sources that share a name, type, model and version, but record different attributes, are not versions of one another. + + Each keeps its data, rather than the one with the highest id hiding the others. + Versions are still compared within one set of attributes, and a partial aggregate of one still gives way to its complete aggregate. + """ + sensor = tb.Sensor("schedule", event_resolution=timedelta(hours=1)) + + def scheduler(id: int, version: str, attributes: dict | None = None): + return DataSource( + id=id, + name="Seita", + model="CommunityScheduler", + type="scheduler", + version=version, + attributes=attributes, + ) + + plain = scheduler(1, "2.4.0") + run_a = scheduler(2, "2.4.0", {"data_generator": {"parameters": {"run": "a"}}}) + run_b = scheduler(3, "2.4.0", {"data_generator": {"parameters": {"run": "b"}}}) + run_a_old = scheduler(4, "2.3.0", {"data_generator": {"parameters": {"run": "a"}}}) + run_b_partial = scheduler( + 5, + "2.4.0", + { + "data_generator": { + "parameters": {"run": "b"}, + "config": {"missing-members": [3]}, + } + }, + ) + bdf = tb.BeliefsDataFrame( + [ + tb.TimedBelief( + sensor=sensor, + source=source, + event_start="2022-12-01T23:30+00", + belief_horizon="PT0H", + event_value=value, + ) + for value, source in enumerate( + (plain, run_a, run_b, run_a_old, run_b_partial) + ) + ] + ) + for one_deterministic in (False, True): + kept = keep_latest_version( + bdf, one_deterministic_belief_per_event=one_deterministic + ) + assert set(kept.sources) == {plain, run_a, run_b} + + +def test_attributes_variant_leaves_out_only_the_missing_members(): + complete = DataSource(name="Seita", type="reporter") + partial = DataSource( + name="Seita", + type="reporter", + attributes={"data_generator": {"config": {"missing-members": [3]}}}, + ) + other_config = DataSource( + name="Seita", + type="reporter", + attributes={"data_generator": {"config": {"resolved-members": [1, 2]}}}, + ) + assert attributes_variant(partial) == attributes_variant(complete) + assert attributes_variant(other_config) != attributes_variant(complete) + # Working out the variant leaves the source's own attributes alone + assert partial.missing_members == [3] + + def test_keep_latest_version_preserves_probabilistic_splits(): sensor = tb.Sensor("X", event_resolution=timedelta(hours=1)) s1v1 = DataSource(name="s1", model="model 1", type="forecaster", version="0.1.0") @@ -444,6 +516,43 @@ def test_get_data_source_finds_a_source_with_an_empty_model_and_version(db, app) assert get_data_source("test-script-source").id == existing.id +def test_get_data_source_takes_the_source_without_attributes(db, app): + """A sibling recording attributes under the same name, model, version and type is a different source. + + Finding it as well would leave two sources where one was expected. + """ + from flexmeasures.data.utils import get_data_source + + sibling = DataSource( + name="test-script-source", + type="scheduler", + model="CommunityScheduler", + version="2.4.0", + attributes={"data_generator": {"parameters": {"run": "a"}}}, + ) + db.session.add(sibling) + db.session.flush() + + # Only the sibling exists, so a source without attributes is created next to it + created = get_data_source( + "test-script-source", + data_source_model="CommunityScheduler", + data_source_version="2.4.0", + data_source_type="scheduler", + ) + assert created.id not in (None, sibling.id) + assert created.attributes_hash is None + + # Now both exist, and the one without attributes is found + found = get_data_source( + "test-script-source", + data_source_model="CommunityScheduler", + data_source_version="2.4.0", + data_source_type="scheduler", + ) + assert found.id == created.id + + def test_get_or_create_source_finds_a_source_without_a_model_and_version_when_asked_for_empty_ones( db, app ): @@ -707,7 +816,13 @@ def naive_keep_latest_version( def group_key(index_tuple): source = index_tuple[source_level] - key = (index_tuple[event_level], source.name, source.type, source.model) + key = ( + index_tuple[event_level], + source.name, + source.type, + source.model, + attributes_variant(source), + ) if not one_deterministic_belief_per_event: key += (index_tuple[belief_level],) return key @@ -745,7 +860,7 @@ def group_key(index_tuple): ) for i in range(6) ] - # a partial aggregate of each, which ranks behind it despite its higher id + # a sibling of each recording other attributes, which is a different source sources += [ DataSource( id=source.id + 6 if source.id is not None else None, @@ -753,7 +868,24 @@ def group_key(index_tuple): model=source.model, type=source.type, version=source.version, - attributes={"data_generator": {"config": {"missing-members": [3]}}}, + attributes={"data_generator": {"parameters": {"run": "b"}}}, + ) + for source in sources + ] + # a partial aggregate of each, which ranks behind it despite its higher id + sources += [ + DataSource( + id=source.id + 12 if source.id is not None else None, + name=source.name, + model=source.model, + type=source.type, + version=source.version, + attributes={ + "data_generator": { + **(source.attributes or {}).get("data_generator", {}), + "config": {"missing-members": [3]}, + } + }, ) for source in sources ] diff --git a/flexmeasures/data/utils.py b/flexmeasures/data/utils.py index 0a53959aef..93c0dd331e 100644 --- a/flexmeasures/data/utils.py +++ b/flexmeasures/data/utils.py @@ -170,6 +170,10 @@ def get_data_source( naming none asks for a source belonging to none, which is what a script the host runs for everyone records under. Leaving it out of the lookup would both hand a script another organisation's source and, where two organisations run the same script, find two sources where one was expected. + + The source it looks for, and creates, has no attributes. + A sibling recording attributes under the same name, model, version and type is a different source, + so leaving the attributes out of the lookup would find several sources where one was expected. """ # Imported here to avoid a circular import at module load time. @@ -183,6 +187,7 @@ def get_data_source( func.coalesce(DataSource.version, "") == (data_source_version or ""), DataSource.type == data_source_type, DataSource.account_id == (account.id if account is not None else None), + DataSource.attributes_hash.is_(None), ) data_source = db.session.execute(query).scalar_one_or_none() if data_source is None: