Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions documentation/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Infrastructure / Support
Bugfixes
-----------
* Saving beliefs that overlap stored ones in the hour that an autumn clock change repeats no longer fails with "Categorical categories must be unique" or keeps unchanged beliefs as new, when the new beliefs are in local time [see `PR #2869 <https://www.github.com/FlexMeasures/flexmeasures/pull/2869>`_]
* Fix ``get_data_source()`` raising ``MultipleResultsFound`` when a source with the same name, model, version and type has attributes, and stop ``keep_latest_version()`` from hiding the beliefs of sources that differ only by their attributes [see `PR #2867 <https://www.github.com/FlexMeasures/flexmeasures/pull/2867>`_]


v1.1.1 | November 11, 2026
Expand Down
28 changes: 25 additions & 3 deletions flexmeasures/data/models/data_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -717,13 +717,35 @@ def source_priorities(sources) -> dict:
return {source: rank for rank, source in enumerate(priority_order)}


def attributes_variant(source) -> bytes:
"""Hash what a source's attributes say about which data it records, leaving out the members it says it missed.

Sources sharing a name, type, model and version but recording different attributes are distinct sources,
whose data may have to coexist, as when a scheduler records its parameters in its source's attributes.
The members a partial aggregate names as missing are the exception (see `DataSource.missing_members`):
such a partial aggregate is meant to be replaced by the complete one, so it should end up in the same group.
"""
attributes = deepcopy(dict(source.attributes or {}))
data_generator = attributes.get("data_generator")
if isinstance(data_generator, dict):
config = data_generator.get("config")
if isinstance(config, dict):
config.pop("missing-members", None)
if not config:
del data_generator["config"]
if not data_generator:
del attributes["data_generator"]
return DataSource.hash_attributes(attributes)


def keep_latest_version(
bdf: tb.BeliefsDataFrame,
one_deterministic_belief_per_event: bool = False,
) -> tb.BeliefsDataFrame:
"""Filters the BeliefsDataFrame to keep the latest version of each source per event or per belief.

Sources are grouped by (name, type, model); within each group, only data from
Sources are grouped by (name, type, model, attributes), where the members a partial aggregate names as missing
do not count towards the attributes (see `attributes_variant`); within each group, only data from
the latest version (breaking ties in favour of a source naming no missing members, and then by highest id) is kept, per event or per belief.
Probabilistic beliefs are kept intact.

Expand All @@ -740,12 +762,12 @@ def keep_latest_version(
# Sanitize BeliefsDataFrame by removing duplicate indices
bdf = bdf.loc[~bdf.index.duplicated(keep="first"), :]

# Group unique sources by (name, type, model)
# Group unique sources by (name, type, model, attributes)
source_codes, unique_sources = pd.factorize(bdf.index.get_level_values("source"))
group_keys: dict = {}
group_per_source = np.empty(len(unique_sources), dtype=np.int64)
for i, source in enumerate(unique_sources):
key = (source.name, source.type, source.model)
key = (source.name, source.type, source.model, attributes_variant(source))
group_per_source[i] = group_keys.setdefault(key, len(group_keys))
if len(group_keys) == len(unique_sources):
# Every source is the sole member of its group, so nothing can be dropped
Expand Down
138 changes: 135 additions & 3 deletions flexmeasures/data/tests/test_data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from flexmeasures.data.models.data_sources import (
DATA_SOURCE_UNIQUE_INDEX,
attributes_variant,
keep_latest_version,
DataSource,
)
Expand Down Expand Up @@ -300,6 +301,77 @@ def test_keep_latest_version_prefers_a_complete_aggregate_of_the_same_belief_tim
assert kept["event_value"].values == pytest.approx([0.12])


def test_keep_latest_version_keeps_sources_differing_only_by_attributes():
"""Sources that share a name, type, model and version, but record different attributes, are not versions of one another.

Each keeps its data, rather than the one with the highest id hiding the others.
Versions are still compared within one set of attributes, and a partial aggregate of one still gives way to its complete aggregate.
"""
sensor = tb.Sensor("schedule", event_resolution=timedelta(hours=1))

def scheduler(id: int, version: str, attributes: dict | None = None):
return DataSource(
id=id,
name="Seita",
model="CommunityScheduler",
type="scheduler",
version=version,
attributes=attributes,
)

plain = scheduler(1, "2.4.0")
run_a = scheduler(2, "2.4.0", {"data_generator": {"parameters": {"run": "a"}}})
run_b = scheduler(3, "2.4.0", {"data_generator": {"parameters": {"run": "b"}}})
run_a_old = scheduler(4, "2.3.0", {"data_generator": {"parameters": {"run": "a"}}})
run_b_partial = scheduler(
5,
"2.4.0",
{
"data_generator": {
"parameters": {"run": "b"},
"config": {"missing-members": [3]},
}
},
)
bdf = tb.BeliefsDataFrame(
[
tb.TimedBelief(
sensor=sensor,
source=source,
event_start="2022-12-01T23:30+00",
belief_horizon="PT0H",
event_value=value,
)
for value, source in enumerate(
(plain, run_a, run_b, run_a_old, run_b_partial)
)
]
)
for one_deterministic in (False, True):
kept = keep_latest_version(
bdf, one_deterministic_belief_per_event=one_deterministic
)
assert set(kept.sources) == {plain, run_a, run_b}


def test_attributes_variant_leaves_out_only_the_missing_members():
complete = DataSource(name="Seita", type="reporter")
partial = DataSource(
name="Seita",
type="reporter",
attributes={"data_generator": {"config": {"missing-members": [3]}}},
)
other_config = DataSource(
name="Seita",
type="reporter",
attributes={"data_generator": {"config": {"resolved-members": [1, 2]}}},
)
assert attributes_variant(partial) == attributes_variant(complete)
assert attributes_variant(other_config) != attributes_variant(complete)
# Working out the variant leaves the source's own attributes alone
assert partial.missing_members == [3]


def test_keep_latest_version_preserves_probabilistic_splits():
sensor = tb.Sensor("X", event_resolution=timedelta(hours=1))
s1v1 = DataSource(name="s1", model="model 1", type="forecaster", version="0.1.0")
Expand Down Expand Up @@ -444,6 +516,43 @@ def test_get_data_source_finds_a_source_with_an_empty_model_and_version(db, app)
assert get_data_source("test-script-source").id == existing.id


def test_get_data_source_takes_the_source_without_attributes(db, app):
"""A sibling recording attributes under the same name, model, version and type is a different source.

Finding it as well would leave two sources where one was expected.
"""
from flexmeasures.data.utils import get_data_source

sibling = DataSource(
name="test-script-source",
type="scheduler",
model="CommunityScheduler",
version="2.4.0",
attributes={"data_generator": {"parameters": {"run": "a"}}},
)
db.session.add(sibling)
db.session.flush()

# Only the sibling exists, so a source without attributes is created next to it
created = get_data_source(
"test-script-source",
data_source_model="CommunityScheduler",
data_source_version="2.4.0",
data_source_type="scheduler",
)
assert created.id not in (None, sibling.id)
assert created.attributes_hash is None

# Now both exist, and the one without attributes is found
found = get_data_source(
"test-script-source",
data_source_model="CommunityScheduler",
data_source_version="2.4.0",
data_source_type="scheduler",
)
assert found.id == created.id


def test_get_or_create_source_finds_a_source_without_a_model_and_version_when_asked_for_empty_ones(
db, app
):
Expand Down Expand Up @@ -707,7 +816,13 @@ def naive_keep_latest_version(

def group_key(index_tuple):
source = index_tuple[source_level]
key = (index_tuple[event_level], source.name, source.type, source.model)
key = (
index_tuple[event_level],
source.name,
source.type,
source.model,
attributes_variant(source),
)
if not one_deterministic_belief_per_event:
key += (index_tuple[belief_level],)
return key
Expand Down Expand Up @@ -745,15 +860,32 @@ def group_key(index_tuple):
)
for i in range(6)
]
# a partial aggregate of each, which ranks behind it despite its higher id
# a sibling of each recording other attributes, which is a different source
sources += [
DataSource(
id=source.id + 6 if source.id is not None else None,
name=source.name,
model=source.model,
type=source.type,
version=source.version,
attributes={"data_generator": {"config": {"missing-members": [3]}}},
attributes={"data_generator": {"parameters": {"run": "b"}}},
)
for source in sources
]
# a partial aggregate of each, which ranks behind it despite its higher id
sources += [
DataSource(
id=source.id + 12 if source.id is not None else None,
name=source.name,
model=source.model,
type=source.type,
version=source.version,
attributes={
"data_generator": {
**(source.attributes or {}).get("data_generator", {}),
"config": {"missing-members": [3]},
}
},
)
for source in sources
]
Expand Down
5 changes: 5 additions & 0 deletions flexmeasures/data/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ def get_data_source(
naming none asks for a source belonging to none, which is what a script the host runs for everyone records under.
Leaving it out of the lookup would both hand a script another organisation's source and,
where two organisations run the same script, find two sources where one was expected.

The source it looks for, and creates, has no attributes.
A sibling recording attributes under the same name, model, version and type is a different source,
so leaving the attributes out of the lookup would find several sources where one was expected.
"""

# Imported here to avoid a circular import at module load time.
Expand All @@ -183,6 +187,7 @@ def get_data_source(
func.coalesce(DataSource.version, "") == (data_source_version or ""),
DataSource.type == data_source_type,
DataSource.account_id == (account.id if account is not None else None),
DataSource.attributes_hash.is_(None),
)
data_source = db.session.execute(query).scalar_one_or_none()
if data_source is None:
Expand Down
Loading