From 6c7798aef065afddc1adb14aa6942018e890be23 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:22 +0100 Subject: [PATCH 01/37] data/services/automations: record how schedule, report and plugin runs end Context: - Only forecast runs followed their jobs, so schedule, report and plugin runs stayed at execution state 'pending' whatever became of their jobs. Change: - record_automation_run_jobs records the jobs such a run queued, then takes the outcome of any job that already finished or failed from Redis; a run with no jobs has succeeded. - The record_outcome_on_automation_run decorator records a job's start, success or failure on its run, found through the job's RQ ID, and wraps make_schedule, the sequential wrap-up job, run_report_job and execute_automation_job. - Plugin jobs carry automation_run_id in their trigger. - A schedule that fell back counts as a failed run, as the fallback job is not one of the run's recorded jobs. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/automations.py | 17 ++- flexmeasures/data/services/automations.py | 139 +++++++++++++++++++++- flexmeasures/data/services/reporting.py | 2 + flexmeasures/data/services/scheduling.py | 3 + 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/automations.py b/flexmeasures/data/automations.py index dc97a37cc6..e6ea8a3909 100644 --- a/flexmeasures/data/automations.py +++ b/flexmeasures/data/automations.py @@ -15,6 +15,7 @@ from flexmeasures.data import db from flexmeasures.data.models.data_sources import DataGenerator +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.utils.job_utils import KNOWN_JOB_QUEUES @@ -180,8 +181,8 @@ def run(self, automation, automation_run=None, scheduled_at=None): A durable run (see `dispatch_automation_run`) carries the parameters and the jobs its run was planned with, which the built-in types read to pick up where an earlier attempt at the same run stopped. - A plugin type's run is dispatched with the parameters its run was planned with, but records no jobs of its own, - so such a run reports how far its dispatch got and leaves its execution state at 'pending'. + A plugin type's run is dispatched with the parameters its run was planned with, + and records its one job once queued, so that the run follows how the job ends. """ if self.generator_class is None: from flexmeasures.data.services.automations import ( @@ -212,6 +213,9 @@ def run(self, automation, automation_run=None, scheduled_at=None): parameters = planned_parameters db.session.commit() queue = current_app.queues[self.queue] + trigger = {"origin": "automation", "automation_id": automation.id} + if automation_run is not None: + trigger["automation_run_id"] = automation_run.id job = Job.create( execute_automation_job, kwargs={ @@ -232,7 +236,7 @@ def run(self, automation, automation_run=None, scheduled_at=None): ).total_seconds() ), meta={ - "trigger": {"origin": "automation", "automation_id": automation.id}, + "trigger": trigger, "data_source_info": {"id": source_id}, "asset_id": automation.asset_id, "asset_or_sensor": {"class": "Asset", "id": automation.asset_id}, @@ -245,6 +249,12 @@ def run(self, automation, automation_run=None, scheduled_at=None): queue=self.queue, asset_or_sensor_type="asset", ) + if automation_run is not None: + from flexmeasures.data.services.automations import ( + record_automation_run_jobs, + ) + + record_automation_run_jobs(automation_run.id, [("job", self.type_id, job)]) return {"job_id": job.id, "n_jobs": 1} @@ -395,6 +405,7 @@ def check_execution_access(automation, sensors): ) +@record_outcome_on_automation_run def execute_automation_job(automation_id: int, data_source_id: int, parameters: dict): """Compute and persist declared results with provenance in a plugin-enabled worker.""" from flexmeasures.data.models.automations import Automation diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 838dd4e649..4ccd0a07a6 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -5,6 +5,7 @@ from __future__ import annotations from contextlib import contextmanager +import functools from copy import copy, deepcopy from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -21,7 +22,8 @@ from isodate.isoerror import ISO8601Error from flask import current_app from marshmallow import ValidationError -from rq.job import Job +from rq import get_current_job +from rq.job import Job, JobStatus from sqlalchemy import func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import selectinload @@ -574,6 +576,116 @@ def reconcile_automation_job_intent(intent: AutomationRunJob) -> bool: return False +class AutomationJobFailed(Exception): + """Stands in for the error of a job that failed before its run had a record of it, as only its traceback is left.""" + + +def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> None: + """Record the jobs that a schedule, report or plugin run queued, so that the run follows what becomes of them. + + A forecast run records its jobs before queueing them, under IDs it derives from the run (see `ensure_automation_run_job_intents`). + The other types get their job IDs from RQ, and a sequential schedule queues each device job as it creates it, + so their jobs are recorded once they are queued. + A worker may have finished one of them in the meantime, when there was no record yet to report to, + so whatever Redis already knows to be over is taken from there. + A run which queued no jobs, such as a report with nothing new to report on, has nothing left to do, so it succeeded. + + :param jobs: the logical job key, the kind and the RQ job of each job, in the order they were queued. + """ + run = db.session.get(AutomationRun, run_id) + if run is None: + raise ValueError(f"Automation run {run_id} does not exist.") + now = _now_utc() + if not jobs: + run.execution_state = "succeeded" + run.execution_completed_at = now + db.session.commit() + return + run.plan = { + **dict(run.plan or {}), + "jobs": [ + {"logical_job_key": key, "kind": kind, "rq_job_id": job.id} + for key, kind, job in jobs + ], + } + for key, kind, job in jobs: + db.session.add( + AutomationRunJob( + run=run, + logical_job_key=key, + rq_job_id=job.id, + queue=job.origin, + kind=kind, + status="queued", + enqueued_at=now, + ) + ) + run.first_enqueued_at = run.first_enqueued_at or now + db.session.commit() + for key, _, job in jobs: + status = job.get_status(refresh=True) + if status == JobStatus.FINISHED: + record_automation_job_succeeded(run_id, key) + elif status == JobStatus.FAILED: + traceback_lines = (job.exc_info or "").strip().splitlines() + record_automation_job_failed( + run_id, + key, + AutomationJobFailed( + traceback_lines[-1] if traceback_lines else "unknown error" + ), + ) + + +def record_outcome_on_automation_run(job_function): + """Record a job's start and outcome on the automation run that queued it, if one did. + + Decorates the function a schedule, report or plugin job runs. + The job is looked up by its RQ ID among the jobs its run recorded (see `record_automation_run_jobs`), + so a job queued outside a run, or on behalf of a run that never recorded it (such as a fallback schedule job), is left alone. + A forecast job reports on its own, as its run hands it the key it was recorded under. + """ + + @functools.wraps(job_function) + def wrapper(*args, **kwargs): + rq_job = get_current_job() + run_id = ( + (rq_job.meta.get("trigger") or {}).get("automation_run_id") + if rq_job is not None + else None + ) + if run_id is None: + return job_function(*args, **kwargs) + # The run records its jobs once they are queued, so a job which starts right away may not find itself yet, + # in which case it looks again once it is done (or the run catches up from Redis, see `record_automation_run_jobs`). + logical_job_key = _logical_job_key(run_id, rq_job.id) + record_automation_job_started(run_id, logical_job_key) + try: + result = job_function(*args, **kwargs) + except Exception as exc: + # The job may have failed on the database itself, so get a usable session before looking anything up. + db.session.rollback() + record_automation_job_failed( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id), exc + ) + raise + record_automation_job_succeeded( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id) + ) + return result + + return wrapper + + +def _logical_job_key(run_id: int, rq_job_id: str) -> str | None: + """Return the key under which a run recorded the job with this RQ ID, if it did.""" + return db.session.scalar( + select(AutomationRunJob.logical_job_key).filter_by( + run_id=run_id, rq_job_id=rq_job_id + ) + ) + + # Fields naming a sensor that a scheduler records its results on, rather than reads from. # A scheduler hands its results to `make_schedule` as (sensor, data) pairs, and these are the fields that decide which sensors those are: # besides the power sensor of each device in the flex-model, its state of charge and its consumption and production sensors, @@ -2452,6 +2564,8 @@ def _run_report_automation( current_app.logger.info( f"Report automation {automation.id} has nothing new to report on, up to {parameters['end']}, so it queues no job." ) + if automation_run is not None: + record_automation_run_jobs(automation_run.id, []) return { "job_id": None, "n_jobs": 0, @@ -2472,7 +2586,13 @@ def _run_report_automation( automation_id=automation.id, automation_run_id=automation_run.id if automation_run is not None else None, ) - return reporter.compute(as_job=True, parameters=parameters) + returns = reporter.compute(as_job=True, parameters=parameters) + if automation_run is not None: + job = Job.fetch( + returns["job_id"], connection=current_app.queues["reporting"].connection + ) + record_automation_run_jobs(automation_run.id, [("report", "report", job)]) + return returns def _run_schedule_automation( @@ -2529,5 +2649,20 @@ def _run_schedule_automation( trigger=trigger, **scheduler_kwargs, ) + if automation_run is not None: + if trigger_data["sequential"]: + # The returned job is the wrap-up job, which names the device jobs it waits for. + connection = current_app.queues["scheduling"].connection + run_jobs = [ + ( + f"device-{index}", + "schedule-device", + Job.fetch(job_id, connection=connection), + ) + for index, job_id in enumerate(job.args[0]) + ] + [("wrap-up", "schedule-wrap-up", job)] + else: + run_jobs = [("schedule", "schedule", job)] + record_automation_run_jobs(automation_run.id, run_jobs) n_jobs = len(job.args[0]) + 1 if trigger_data["sequential"] else 1 return {"job_id": job.id, "n_jobs": n_jobs} diff --git a/flexmeasures/data/services/reporting.py b/flexmeasures/data/services/reporting.py index e22230f99c..5fc2b54ed9 100644 --- a/flexmeasures/data/services/reporting.py +++ b/flexmeasures/data/services/reporting.py @@ -11,6 +11,7 @@ from flexmeasures.data import db from flexmeasures.data.schemas.reporting import ReporterParametersSchema +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.utils import save_to_db if TYPE_CHECKING: @@ -88,6 +89,7 @@ class ReportWritesUncheckedSensor(PermissionError): """Raised when a reporter returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def run_report_job(data_source_id: int, parameters: dict) -> list[dict]: """Compute and store a report in a reporting worker. diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index eaaee0a274..488e2cd94e 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -36,6 +36,7 @@ from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException from flexmeasures.data.models.planning.process import ProcessScheduler from flexmeasures.data.services.scheduling_result import SchedulingJobResult +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.generic_assets import GenericAsset as Asset from flexmeasures.data.models.data_sources import DataSource @@ -347,6 +348,7 @@ def create_scheduling_job( return job +@record_outcome_on_automation_run def cb_done_sequential_scheduling_job(jobs_ids: list[str]): """ TODO: maybe check if any of the subjobs used a fallback scheduler or accrued a relaxation penalty. @@ -912,6 +914,7 @@ class ScheduleWritesUncheckedSensor(PermissionError): """Raised when a scheduler returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def make_schedule( # noqa: C901 sensor_id: int | None = None, start: datetime | None = None, From 196ccc15fc2a95561c0052ea55982eaf8fe4f904 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 02/37] data/tests: cover how schedule, report and plugin runs end Context: - Schedule, report and plugin runs now record how their jobs ended. Change: - Schedule runs, single and sequential, succeed once their jobs did and fail with the job's error; a job queued outside a run is not recorded. - A plugin run records its job's outcome, also when the job finished before the run recorded it. - Report run tests check the run's outcome: succeeded, nothing to do, and refused. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_automations.py | 24 ++++ .../data/tests/test_automations_fresh_db.py | 130 ++++++++++++++++++ .../tests/test_plugin_automations_fresh_db.py | 77 +++++++++++ 3 files changed, 231 insertions(+) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index fdc0a35ac9..2f782b8fe6 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1724,6 +1724,14 @@ def test_run_report_automation( ) assert (stored_report.values.T == [1, 2 + 3, 4 + 5, 6 + 7, 8 + 9]).all() + # the run followed its job to the end + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run.id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("report", "succeeded") + ] + assert run.execution_state == "succeeded" + # the successful job recorded the end of the report window as covered import pandas as pd @@ -1770,6 +1778,12 @@ def test_report_automation_with_nothing_new_to_report_queues_no_job( assert result.exit_code == 0, result.output assert "queued 0 reporting job(s)" in result.output, result.output assert len(app.queues["reporting"].jobs) == 0 + # A run with nothing to do is done. + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.job_intents == [] + assert run.execution_state == "succeeded" # Run on demand, it says why it queued nothing. result = runner.invoke(run_one_automation, ["--automation", str(automation.id)]) @@ -1836,9 +1850,19 @@ def test_report_automation_refuses_a_sensor_nobody_checked( mocker.patch( "flexmeasures.data.services.reporting.get_current_job", return_value=job ) + mocker.patch( + "flexmeasures.data.services.automations.get_current_job", return_value=job + ) with pytest.raises(ReportWritesUncheckedSensor, match=str(sensor1_id)): run_report_job(**job.kwargs) + # the refusal failed the run, saying why + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.execution_state == "failed" + assert run.last_error_type == "ReportWritesUncheckedSensor" + # a refused report covers nothing, so the next run still starts where the last successful one ended assert not app.redis_connection.get(f"automation-last-run:{automation.id}") diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index d479ad3c07..4813efecc8 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -5,6 +5,7 @@ import isodate import pytest from rq.job import Job +from sqlalchemy import select from sqlalchemy.exc import IntegrityError from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule @@ -223,6 +224,135 @@ def test_a_durable_run_schedules_what_it_was_planned_with( ) +def _planned_schedule_run(fresh_db, battery, sequential: bool): + """Set up a schedule automation on the battery, and the durable run a runner would have claimed for it.""" + message = message_for_trigger_schedule() + flex_model = message.pop("flex-model") + flex_model["sensor"] = battery.sensors[0].id + parameters = {**message, "flex-model": [flex_model], "sequential": sequential} + automation = build_schedule_automation( + battery, + name="Nightly schedules", + cronstr="0 0 * * *", + parameters=parameters, + ) + fresh_db.session.add(automation) + fresh_db.session.flush() + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=parameters, + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return automation, run + + +@pytest.mark.parametrize( + "sequential, expected_jobs", + [(False, ["schedule"]), (True, ["device-0", "wrap-up"])], +) +def test_a_schedule_run_records_how_its_jobs_ended( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + sequential, + expected_jobs, +): + """A schedule run records each job it queued, and succeeds once the workers finished all of them.""" + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + run = fresh_db.session.get(AutomationRun, run_id) + assert [intent.logical_job_key for intent in run.job_intents] == sorted( + expected_jobs + ) + assert {intent.status for intent in run.job_intents} == {"queued"} + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert {intent.status for intent in run.job_intents} == {"succeeded"} + assert run.execution_state == "succeeded" + assert run.execution_completed_at is not None + + +@pytest.mark.parametrize("sequential", [False, True]) +def test_a_schedule_run_records_a_failed_job( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, + sequential, +): + """A schedule run whose job fails is failed, with the job's error, and over once nothing of it runs any more. + + A failed device job of a sequential schedule leaves its wrap-up job waiting for good, which must not keep the run going. + """ + from flexmeasures.data.services import scheduling + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_to_db", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "failed" + assert run.last_error_type == "ValueError" + assert run.last_error_message == "The database is read-only today." + assert run.execution_completed_at is not None + + +def test_a_schedule_job_queued_outside_a_run_is_not_recorded( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, +): + """Running an automation on demand has no run to record its job on, and its job runs as it always did.""" + from flexmeasures.data.models.automations import AutomationRunJob + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, _ = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + returns = run_automation(automation) + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + job = Job.fetch(returns["job_id"], connection=app.queues["scheduling"].connection) + assert job.is_finished + assert fresh_db.session.scalars(select(AutomationRunJob)).all() == [] + + def test_run_day_ahead_schedule_automation( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 77ff20b5bb..c2ecd8ed54 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -417,3 +417,80 @@ def test_a_source_that_is_not_the_users_to_work_with_is_refused_for_a_plugin_typ assert "not yours to work with" in refusal.value.api_message finally: logout_user() + + +def _claimed_run(fresh_db, automation): + """Return the durable run a runner would have claimed for this automation.""" + from datetime import datetime, timezone + + from flexmeasures.data.models.automations import AutomationRun + + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=dict(automation.parameters), + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return run + + +def test_a_plugin_run_records_how_its_job_ended( + fresh_db, app, ingestion_plugin, ingestion_assets +): + """A plugin run names itself on its job, records the job, and succeeds once a worker finished it.""" + from flexmeasures.data.models.automations import AutomationRun + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + + result = run_automation(automation, automation_run=run) + job = Job.fetch(result["job_id"], connection=app.redis_connection) + assert job.meta["trigger"]["automation_run_id"] == run_id + work_on_rq(app.queues["ingestion"], job=job) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("job", "succeeded") + ] + assert run.execution_state == "succeeded" + + +def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A worker may finish a job before the run that queued it has recorded it, so the run takes the outcome from Redis.""" + from flexmeasures.data.models.automations import AutomationRun + from flexmeasures.data.services import automations as automation_services + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + record_automation_run_jobs = automation_services.record_automation_run_jobs + held_back = [] + monkeypatch.setattr( + automation_services, + "record_automation_run_jobs", + lambda *args: held_back.append(args), + ) + result = run_automation(automation, automation_run=run) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + record_automation_run_jobs(*held_back[0]) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "succeeded" From da14723a3c893c01f64788505f1981e6345d3a1b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 03/37] docs: every automation run records how its jobs ended Context: - Schedule, report and plugin runs now record how their jobs ended, as forecast runs did. Change: - Say which jobs each type of run records, that a run without jobs has succeeded, and that a schedule which fell back counts as failed. Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f8a6d3feb7..737fd4b2de 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -268,8 +268,11 @@ and the run's record says what went wrong on each attempt. Retrying a failed dispatch this way is what a forecast run does. A schedule run is recorded, claimed and reported in just the same way, but is left where it failed rather than dispatched again, because its jobs get a fresh ID on every dispatch, so a retry could not tell an already queued schedule from a missing one. -A forecast run also records each job it created, and how that job ended, which is what its *execution* state describes. -A schedule or report run records its dispatch in the same way, while its execution state stays ``pending``, because the jobs of those runs are not recorded individually yet. +A run also records each job it created, and how that job ended, which is what its *execution* state describes. +This holds for every type of automation: a schedule run records its scheduling job (or, for a sequential schedule, each device job and the job wrapping them up), +a report run its reporting job, and a plugin run the job its plugin queued. +A run which queued no jobs, such as a report run with nothing new to report on, has succeeded. +A job which a schedule queues to fall back on a simpler scheduler is not one of the run's jobs, so a run whose schedule fell back has failed, with the error that made it fall back. A run tracks two things separately: how far its *dispatch* got (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), and how its *execution* by the workers ended (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``). Each attempt to dispatch a run is recorded too, with the runner which made it, what it queued, and why it failed if it did. From 2441424fd5904df974475afaafc956ff1da69123 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:50:15 +0100 Subject: [PATCH 04/37] changelog: append PR #2670 to the durable automation runs entry Context: - Schedule, report and plugin runs now record how their jobs ended, not only forecast runs. Change: - Append PR #2670 to the #2457 entry, and drop "for its forecast jobs", which no longer holds. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4998df87c2..fcfa8d1415 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -112,7 +112,7 @@ Automations arrived over several pull requests. This is what each of them contri * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_] * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2299 `_] -* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and, for its forecast jobs, how they ended [see `PR #2457 `_] +* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and how its jobs ended [see `PR #2457 `_ and `PR #2670 `_] * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization [see `PR #2553 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] From 56e5f8300548dead50abb6cc039f2f2e2d4290d2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:41 +0100 Subject: [PATCH 05/37] data/models: record when an automation run was reported by monitoring Context: - #2519: a monitor command will email a digest of failed automation runs, and has to report each failure once, also when a monitoring run is missed. Change: - Add a nullable alerted_at column to AutomationRun, with a migration. - Add a partial index on the runs the monitor looks at (not reported yet, and failed, running or not fully dispatched), so that its hourly query does not scan the whole run history. Signed-off-by: Mohamed Belhsan Hmida --- ...1c362_add_alerted_at_to_automation_runs.py | 43 +++++++++++++++++++ flexmeasures/data/models/automations.py | 3 ++ 2 files changed, 46 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py new file mode 100644 index 0000000000..eac54bffce --- /dev/null +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -0,0 +1,43 @@ +"""Record when an automation run was reported by `flexmeasures monitor automations` + +Revision ID: d4b7e9a1c362 +Revises: c5e1a7b94d20 +Create Date: 2026-10-07 10:00:00.000000 + +The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. +Every run recorded before this upgrade has no mark, so the first monitoring run reports the failures already on record, collapsed per automation. + +The partial index holds only the runs the monitor has to look at: +those not reported yet which failed, are still running, or never finished their dispatch. +Every other run, which is nearly all of them, stays out of it. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "d4b7e9a1c362" +down_revision = "c5e1a7b94d20" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "automation_run", + sa.Column("alerted_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "automation_run_unalerted_idx", + "automation_run", + ["automation_id"], + postgresql_where=sa.text( + "alerted_at IS NULL" + " AND (execution_state = 'failed' OR execution_state = 'running' OR dispatch_completed_at IS NULL)" + ), + ) + + +def downgrade(): + op.drop_index("automation_run_unalerted_idx", table_name="automation_run") + op.drop_column("automation_run", "alerted_at") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index 78e275f533..2b419ee1fc 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -243,6 +243,8 @@ class AutomationRun(db.Model): execution_completed_at = db.Column(db.DateTime(timezone=True), nullable=True) last_error_type = db.Column(db.String(160), nullable=True) last_error_message = db.Column(db.Text, nullable=True) + # When `flexmeasures monitor automations` reported this run as failed or stuck, so that it is reported once. + alerted_at = db.Column(db.DateTime(timezone=True), nullable=True) parameters = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) plan = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) @@ -272,6 +274,7 @@ class AutomationRun(db.Model): "dispatch_completed_at", "execution_started_at", "execution_completed_at", + "alerted_at", ) def validate_datetime_is_aware( self, key: str, value: datetime | None From 39ca4a05d89b1c64612d21e4fd86a391b5d3ade1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:41 +0100 Subject: [PATCH 06/37] utils/config: add FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER Context: - #2519: a run whose worker died stays "running" for good, so monitoring needs a threshold after which it counts as stuck. Change: - Add the setting, defaulting to six hours, well above any job timeout. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/utils/config_defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index a163c09035..3f681a4a6f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -125,6 +125,8 @@ class Config(object): [] ) # Deprecated. Use FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS instead. FLEXMEASURES_MONITORING_MAIL_RECIPIENTS: list[str] = [] + # How long an automation run may execute before `flexmeasures monitor automations` reports it as stuck. + FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER: timedelta = timedelta(hours=6) FLEXMEASURES_PLATFORM_NAME: str | list[str | tuple[str, list[str]]] = "FlexMeasures" FLEXMEASURES_MODE: str = "" From ddfbe7ade0b000843924f9609c90934661ce7ccc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 07/37] data/services: find the automation runs that failed and were not reported yet Context: - #2519: monitoring reports runs whose execution failed, which are stuck, or whose dispatch failed for good. Change: - Add get_unalerted_failed_automation_runs, with one criterion for every automation type: only a forecast run with dispatch attempts to spare is left out, as it will be dispatched again, and so is a run that a runner still holds a live claim on. - Add group_failed_runs_by_automation, collapsing failures per automation with their counts, latest run and latest error. Signed-off-by: Mohamed Belhsan Hmida --- .../data/services/automation_monitoring.py | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 flexmeasures/data/services/automation_monitoring.py diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py new file mode 100644 index 0000000000..0b82190887 --- /dev/null +++ b/flexmeasures/data/services/automation_monitoring.py @@ -0,0 +1,150 @@ +"""Find the automation runs that failed, so that monitoring can report them.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from sqlalchemy import and_, not_, or_, select +from sqlalchemy.orm import selectinload + +from flexmeasures.data import db +from flexmeasures.data.models.automations import Automation, AutomationRun +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.services.automations import AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS +from flexmeasures.utils.time_utils import server_now + +EXECUTION_FAILED = "execution failed" +STUCK = "stuck" +DISPATCH_FAILED = "dispatch failed" + + +@dataclass(frozen=True) +class FailedAutomationRun: + """An automation run that failed, with the way it failed.""" + + run: AutomationRun + reason: str + + +@dataclass +class AutomationFailures: + """The failed runs of one automation, collapsed into what a digest says about them.""" + + automation: Automation + failed_runs: list[FailedAutomationRun] = field(default_factory=list) + + @property + def reason_counts(self) -> Counter: + """How many runs failed in each way.""" + return Counter(failed_run.reason for failed_run in self.failed_runs) + + @property + def last_run(self) -> AutomationRun: + """The failed run that was due most recently.""" + return max( + (failed_run.run for failed_run in self.failed_runs), + key=lambda run: (run.scheduled_at, run.id), + ) + + @property + def last_error(self) -> str | None: + """The error of the most recent failed run that recorded one, if any did.""" + runs_with_an_error = [ + failed_run.run + for failed_run in self.failed_runs + if failed_run.run.last_error_type or failed_run.run.last_error_message + ] + if not runs_with_an_error: + return None + run = max(runs_with_an_error, key=lambda run: (run.scheduled_at, run.id)) + return ": ".join( + part for part in (run.last_error_type, run.last_error_message) if part + ) + + +def _dispatch_failed_for_good(now: datetime): + """Return the criterion for a run whose dispatch did not finish and will not be attempted again. + + No runner may hold a live claim on it, or it is still being dispatched. + Only a forecast run is dispatched again, until its attempts are used up (see `get_dispatchable_automation_runs`), + so a run of any other type has failed for good after its first attempt. + """ + return and_( + AutomationRun.dispatch_completed_at.is_(None), + or_( + AutomationRun.claim_owner.is_(None), + AutomationRun.claim_expires_at.is_(None), + AutomationRun.claim_expires_at <= now, + ), + not_( + and_( + AutomationRun.automation_type == "forecasting", + AutomationRun.attempt_count < AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + ), + ) + + +def get_unalerted_failed_automation_runs( + stuck_after: timedelta, + now: datetime | None = None, +) -> list[FailedAutomationRun]: + """Return the automation runs that failed and were not reported yet, oldest first. + + A run failed when its execution failed, when its execution has been running for longer than ``stuck_after``, + or when its dispatch failed and will not be attempted again. + This holds for every automation type, although only the types that record their execution can fail on it. + + :param stuck_after: How long a run may execute before it counts as stuck. + :param now: The moment to judge by, which defaults to the server's current time. + """ + if now is None: + now = server_now() + now = now.astimezone(timezone.utc) + execution_failed = AutomationRun.execution_state == "failed" + stuck = and_( + AutomationRun.execution_state == "running", + AutomationRun.execution_started_at <= now - stuck_after, + ) + runs = db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.alerted_at.is_(None), + or_(execution_failed, stuck, _dispatch_failed_for_good(now)), + ) + .options( + selectinload(AutomationRun.automation) + .joinedload(Automation.asset) + .joinedload(GenericAsset.owner) + ) + .order_by( + AutomationRun.automation_id, AutomationRun.scheduled_at, AutomationRun.id + ) + ).all() + failed_runs = [] + for run in runs: + if run.execution_state == "failed": + reason = EXECUTION_FAILED + elif ( + run.execution_state == "running" + and run.execution_started_at <= now - stuck_after + ): + reason = STUCK + else: + reason = DISPATCH_FAILED + failed_runs.append(FailedAutomationRun(run=run, reason=reason)) + return failed_runs + + +def group_failed_runs_by_automation( + failed_runs: list[FailedAutomationRun], +) -> list[AutomationFailures]: + """Collapse failed runs per automation, keeping the order in which the automations first appear.""" + failures: dict[int, AutomationFailures] = {} + for failed_run in failed_runs: + automation = failed_run.run.automation + failures.setdefault(automation.id, AutomationFailures(automation=automation)) + failures[automation.id].failed_runs.append(failed_run) + return list(failures.values()) From 3cf9ef6449e8c5fb5351a4bed89a529da66fcc0c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 08/37] cli: add flexmeasures monitor automations Context: - #2519: automations fail silently unless someone watches their asset's Automations page. Change: - Email one digest of the failed runs to the monitoring recipients (or --recipient), collapsed per automation, and mark the runs as reported once it is sent. - Send nothing when nothing failed, and refuse to run without recipients, rather than marking runs as reported that nobody heard of. - --stuck-after-minutes overrides FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER. - Kept out of flexmeasures jobs run-automations, so a mail problem never holds up dispatching. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/monitor.py | 122 ++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index 2c4edebbe0..c255a3725b 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -21,6 +21,11 @@ from flexmeasures.data import db from flexmeasures.data.models.task_runs import LatestTaskRun +from flexmeasures.data.services.automation_monitoring import ( + AutomationFailures, + get_unalerted_failed_automation_runs, + group_failed_runs_by_automation, +) from flexmeasures.data.models.user import Account, User from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.api.common.schemas.users import UserIdField @@ -505,4 +510,121 @@ def monitor_last_seen( db.session.commit() +# Error messages can carry a whole traceback, which does not help a digest that lists many automations. +MAX_ERROR_LENGTH_IN_DIGEST = 500 + + +def _format_utc(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def _format_duration(duration: timedelta) -> str: + """Say how long a threshold is, exactly, which a rounded duration such as humanize's would not.""" + minutes = int(duration.total_seconds() // 60) + if minutes % 60 == 0: + hours = minutes // 60 + return "1 hour" if hours == 1 else f"{hours} hours" + return "1 minute" if minutes == 1 else f"{minutes} minutes" + + +def _times(count: int) -> str: + return "once" if count == 1 else f"{count} times" + + +def describe_automation_failures(failures: AutomationFailures) -> str: + """Describe the failed runs of one automation in a few lines, for the monitoring digest.""" + automation = failures.automation + asset = automation.asset + if asset.owner is None: + where = f"public asset {asset.id} '{asset.name}'" + else: + where = f"asset {asset.id} '{asset.name}' of organisation '{asset.owner.name}'" + reasons = ", ".join( + f"{reason}: {count}" for reason, count in failures.reason_counts.items() + ) + last_run = failures.last_run + lines = [ + f"Automation {automation.id} '{automation.name}' ({automation.type}) on {where}:", + f" failed {_times(len(failures.failed_runs))} ({reasons})," + f" last at {_format_utc(last_run.scheduled_at)} (run {last_run.id})", + ] + last_error = failures.last_error + if last_error: + if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: + last_error = last_error[:MAX_ERROR_LENGTH_IN_DIGEST] + " ..." + lines.append(f" last error: {last_error}") + return "\n".join(lines) + + +@fm_monitor.command("automations") +@with_appcontext +@click.option( + "--stuck-after-minutes", + type=click.IntRange(min=1), + default=None, + help="Report a run as stuck once it has been executing for this many minutes. Defaults to the FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER setting (6 hours).", +) +@click.option( + "--recipient", + type=str, + multiple=True, + help="User ID or email address to send the digest to. Use multiple times if needed. If not used, the default monitoring mail recipients are informed.", +) +def monitor_automations( + stuck_after_minutes: int | None = None, + recipient: tuple[str, ...] = (), +): + """ + Email a digest of the automation runs that failed since the last check. + + A run is reported when its execution failed, when it has been executing for too long (see --stuck-after-minutes), + or when its dispatch failed and will not be attempted again. + Each run is reported once, so a monitoring run that was missed is made up for by the next one. + Repeated failures are collapsed per automation, and no email is sent when nothing failed. + + Meant to be run regularly, for instance hourly from cron. + """ + email_recipients = get_monitoring_email_recipients(recipient) + if not email_recipients: + # Marking runs as reported without telling anyone would hide them from the next monitoring run. + raise click.UsageError( + "Nobody to send the digest to. Set FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS or use --recipient." + ) + if stuck_after_minutes is not None: + stuck_after = timedelta(minutes=stuck_after_minutes) + else: + stuck_after = app.config["FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER"] + + now = server_now() + failed_runs = get_unalerted_failed_automation_runs(stuck_after, now=now) + if not failed_runs: + click.secho( + "All good ― no automation runs failed since the last check.", + **MsgStyle.SUCCESS, + ) + return + failures = group_failed_runs_by_automation(failed_runs) + + subject = f"Failed runs of {len(failures)} automation(s)" + body = ( + f"These automations have runs that failed, or that have been executing for longer than {_format_duration(stuck_after)}, which were not reported before:\n\n" + + "\n\n".join(describe_automation_failures(f) for f in failures) + + "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." + ) + capture_message_for_sentry(subject) + email = Message(subject=subject, bcc=email_recipients) + email.body = body + app.mail.send(email) + app.logger.error(body) + + # Only once the digest is out, so that a failure to send it leaves the runs to the next monitoring run. + for failed_run in failed_runs: + failed_run.run.alerted_at = now + db.session.commit() + click.secho( + f"Reported {len(failed_runs)} failed run(s) of {len(failures)} automation(s) to {', '.join(email_recipients)}.", + **MsgStyle.WARN, + ) + + app.cli.add_command(fm_monitor) From 47e200c827a8452568eb348903f8e89906f7a1e2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 09/37] cli/tests: test flexmeasures monitor automations Context: - #2519: cover the new monitor command. Change: - Drive forecast runs to a failed execution, a stuck execution and an exhausted dispatch through the real runner and worker, and check each is reported once, and not before it counts. - Check which runs count as failed for other types, the threshold's config and CLI override, collapsing per automation, and refusing to run without recipients. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_monitor.py | 3 +- .../test_monitor_automations_fresh_db.py | 353 ++++++++++++++++++ 2 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 flexmeasures/cli/tests/test_monitor_automations_fresh_db.py diff --git a/flexmeasures/cli/tests/test_monitor.py b/flexmeasures/cli/tests/test_monitor.py index caa18fdd6f..ebcfc83da0 100644 --- a/flexmeasures/cli/tests/test_monitor.py +++ b/flexmeasures/cli/tests/test_monitor.py @@ -10,6 +10,7 @@ def test_monitor_help(app): from flexmeasures.cli.monitor import ( fm_monitor, + monitor_automations, monitor_last_seen, monitor_latest_run, ) @@ -19,7 +20,7 @@ def test_monitor_help(app): result = runner.invoke(fm_monitor, ["--help"]) assert result.exit_code == 0 - for command in (monitor_last_seen, monitor_latest_run): + for command in (monitor_last_seen, monitor_latest_run, monitor_automations): result = runner.invoke(command, ["--help"]) assert result.exit_code == 0 diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py new file mode 100644 index 0000000000..e112231904 --- /dev/null +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -0,0 +1,353 @@ +"""Tests for `flexmeasures monitor automations`, which emails a digest of failed automation runs.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from flexmeasures.cli.tests.utils import to_flags +from flexmeasures.data.models.automations import Automation, AutomationRun + +RECIPIENT = "ops@example.test" +DUE_AT = datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc) + + +@pytest.fixture(scope="function") +def clean_redis(app): + app.redis_connection.flushdb() + yield + app.redis_connection.flushdb() + + +@pytest.fixture() +def forecast_automation( + app, fresh_db, clean_redis, setup_fresh_test_forecast_data, freeze_server_now +): + """Create a forecast automation which is due at the frozen minute.""" + from flexmeasures.cli.data_add import add_automation + + freeze_server_now(DUE_AT - timedelta(minutes=2)) + sensor = setup_fresh_test_forecast_data["solar-sensor"] + result = app.test_cli_runner().invoke( + add_automation, + to_flags( + { + "asset": sensor.generic_asset_id, + "name": "Monitored forecasts", + "cron": "0 1 * * *", + "timezone": "UTC", + "sensor": sensor.id, + "duration": "PT2H", + "forecast-frequency": "PT1H", + "max-forecast-horizon": "PT2H", + "retrain-frequency": "PT1H", + } + ), + ) + assert result.exit_code == 0, result.output + freeze_server_now(DUE_AT) + return fresh_db.session.scalars(select(Automation)).one() + + +def run_automations(app, exit_code: int = 0): + from flexmeasures.cli.jobs import run_automations as run_automations_command + + result = app.test_cli_runner().invoke(run_automations_command) + assert result.exit_code == exit_code, result.output + + +def monitor(app, *args: str): + """Run the monitor, returning its result and the emails it sent.""" + from flexmeasures.cli.monitor import monitor_automations + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", RECIPIENT, *args] + ) + return result, outbox + + +def perform_job(app, intent): + from rq import SimpleWorker + + queue = app.queues["forecasting"] + worker = SimpleWorker([queue], connection=queue.connection) + worker.perform_job(queue.fetch_job(intent.rq_job_id), queue) + + +def add_run( + db, automation: Automation, scheduled_at: datetime, **columns +) -> AutomationRun: + """Record a run of the automation in whatever state the test needs, dispatched by default.""" + columns = { + "automation_type": automation.type, + "dispatch_state": "queued", + "dispatch_completed_at": scheduled_at, + "execution_state": "pending", + "attempt_count": 1, + **columns, + } + run = AutomationRun( + automation=automation, + scheduled_at=scheduled_at, + schedule_revision=automation.schedule_revision, + generator_id=automation.generator_id, + parameters={}, + plan={}, + **columns, + ) + db.session.add(run) + db.session.commit() + return run + + +def test_a_failed_forecast_is_reported_once(app, fresh_db, forecast_automation): + """A forecast run whose job failed on the worker is emailed about, and only the first time the monitor runs.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + queue = app.queues["forecasting"] + wrap_up = next(i for i in run.job_intents if i.kind == "forecast-wrap-up") + cycle = next(i for i in run.job_intents if i.kind == "forecast-cycle") + # Make the wrap-up job fail by taking away one of the cycle jobs it reports on. + queue.fetch_job(cycle.rq_job_id).delete() + perform_job(app, wrap_up) + fresh_db.session.refresh(run) + assert run.execution_state == "failed" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert outbox[0].bcc == [RECIPIENT] + assert outbox[0].subject == "Failed runs of 1 automation(s)" + body = outbox[0].body + assert ( + f"Automation {forecast_automation.id} 'Monitored forecasts' (forecasting)" + in body + ) + assert "failed once (execution failed: 1), last at 2026-08-05 01:00 UTC" in body + assert f"last error: {run.last_error_type}" in body + fresh_db.session.refresh(run) + assert run.alerted_at == DUE_AT + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + assert "All good" in result.output + + +def test_a_running_forecast_is_reported_once_it_is_stuck( + app, fresh_db, forecast_automation, freeze_server_now +): + """A forecast run which started executing is reported once it has been at it for longer than the configured threshold.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + # The wrap-up job succeeds while the cycle jobs have not run, so the run keeps executing. + perform_job(app, next(i for i in run.job_intents if i.kind == "forecast-wrap-up")) + fresh_db.session.refresh(run) + assert run.execution_state == "running" + + freeze_server_now(DUE_AT + timedelta(hours=5, minutes=59)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + freeze_server_now(DUE_AT + timedelta(hours=6)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (stuck: 1)" in outbox[0].body + assert "last error" not in outbox[0].body + + +def test_the_stuck_threshold_comes_from_the_config_or_the_command_line( + app, fresh_db, forecast_automation, freeze_server_now, monkeypatch +): + """The command-line option overrides the configured threshold, which overrides the default of six hours.""" + stuck_since = DUE_AT - timedelta(hours=2) + add_run( + fresh_db, + forecast_automation, + stuck_since, + execution_state="running", + execution_started_at=stuck_since, + ) + + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER", timedelta(hours=1) + ) + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "longer than 1 hour" in outbox[0].body + + +def test_a_forecast_dispatch_is_reported_once_its_attempts_are_used_up( + app, fresh_db, forecast_automation, freeze_server_now, mocker +): + """A forecast run whose dispatch failed is retried, so it is only reported once it will not be tried again.""" + from flexmeasures.data.services.automations import ( + AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + + mocker.patch.object( + app.queues["forecasting"], + "enqueue_job", + side_effect=RuntimeError("redis unavailable"), + ) + run_automations(app, exit_code=1) + run = fresh_db.session.scalars(select(AutomationRun)).one() + assert run.dispatch_state == "failed" + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + # Each attempt waits longer than the last, so stepping an hour ahead each time takes them all. + for attempt_no in range(2, AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + 1): + freeze_server_now(DUE_AT + timedelta(hours=attempt_no)) + result, outbox = monitor(app) + assert outbox == [], f"reported before attempt {attempt_no}" + run_automations(app, exit_code=1) + fresh_db.session.refresh(run) + assert run.attempt_count == AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (dispatch failed: 1)" in outbox[0].body + assert "last error: RuntimeError: redis unavailable" in outbox[0].body + + +def test_which_runs_count_as_failed(fresh_db, forecast_automation): + """Failures are judged the same way for every automation type, except that only a forecast run is dispatched again.""" + from flexmeasures.data.services.automation_monitoring import ( + get_unalerted_failed_automation_runs, + ) + + def add(minute: int, **columns) -> AutomationRun: + return add_run( + fresh_db, forecast_automation, DUE_AT - timedelta(minutes=minute), **columns + ) + + expected = { + add(1, execution_state="failed").id: "execution failed", + add( + 2, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=7), + ).id: "stuck", + # A schedule run is never dispatched again, so its first failed attempt is its last. + add( + 3, + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + claim_expires_at=DUE_AT + timedelta(minutes=1), + ).id: "dispatch failed", + # A runner which died mid-dispatch leaves its claim behind, until its lease runs out. + add( + 4, + automation_type="reporting", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:dead", + claim_expires_at=DUE_AT - timedelta(minutes=1), + ).id: "dispatch failed", + } + not_failed = [ + add(5, execution_state="succeeded"), + add( + 6, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=5), + ), + # A forecast run with attempts to spare will be dispatched again. + add(7, dispatch_state="failed", dispatch_completed_at=None), + # A runner is still dispatching this one. + add( + 8, + automation_type="scheduling", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:alive", + claim_expires_at=DUE_AT + timedelta(minutes=5), + ), + add(9, execution_state="failed", alerted_at=DUE_AT - timedelta(hours=1)), + ] + + failed_runs = get_unalerted_failed_automation_runs(timedelta(hours=6), now=DUE_AT) + + assert {f.run.id: f.reason for f in failed_runs} == expected + assert not {run.id for run in not_failed} & {f.run.id for f in failed_runs} + + +def test_repeated_failures_are_collapsed_per_automation( + app, fresh_db, forecast_automation +): + """An automation which failed several times takes up one entry, naming its latest failure and error.""" + for hours_ago, error in ((3, "first"), (1, "latest"), (2, "middle")): + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=hours_ago), + execution_state="failed", + last_error_type="ValueError", + last_error_message=error, + ) + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(minutes=30), + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert body.count(f"Automation {forecast_automation.id} ") == 1 + assert ( + "failed 4 times (execution failed: 3, dispatch failed: 1), last at 2026-08-05 00:30 UTC" + in body + ) + # The dispatch failure recorded no error, so the latest error is that of the latest failed execution. + assert "last error: ValueError: latest" in body + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + +def test_without_recipients_nothing_is_marked_as_reported( + app, fresh_db, forecast_automation, monkeypatch +): + """Without anyone to email, the monitor refuses to run, rather than marking runs as reported that nobody heard of.""" + from flexmeasures.cli.monitor import monitor_automations + + monkeypatch.setitem( + app.config, "FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS", [] + ) + monkeypatch.setitem(app.config, "FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", []) + run = add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result = app.test_cli_runner().invoke(monitor_automations) + + assert result.exit_code == 2 + assert "Nobody to send the digest to" in result.output + fresh_db.session.refresh(run) + assert run.alerted_at is None From 379a8b810fb66f9485f6dbf35f26156837ef7f88 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:43 +0100 Subject: [PATCH 10/37] docs: document flexmeasures monitor automations Context: - #2519: new monitor command and setting. Change: - Describe what the command reports in the automations feature docs, and point to it from the host's error monitoring page. - List the command, document the stuck threshold setting, and add a CLI changelog entry. Signed-off-by: Mohamed Belhsan Hmida --- documentation/cli/change_log.rst | 1 + documentation/cli/commands.rst | 1 + documentation/configuration.rst | 13 ++++++++++++- documentation/features/automations.rst | 25 +++++++++++++++++++++++++ documentation/host/error-monitoring.rst | 11 +++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 2a8a3983a6..4a15790a58 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -7,6 +7,7 @@ FlexMeasures CLI Changelog since v1.1.0 | September XX, 2026 ================================= +* Add ``flexmeasures monitor automations`` to email a digest of the automation runs that failed since the last check: runs whose execution failed, which have been executing for too long (``--stuck-after-minutes``), or whose dispatch failed for good. Each run is reported once, and repeated failures are collapsed per automation. * Add ``flexmeasures show automations`` to list all automations, including inactive ones, with the IDs which the edit, delete and run commands expect, and ``flexmeasures show automations --id `` to show one automation in detail: its recurrence in words, timezone, cursor, data generator, parameters and the sensors it reads from and writes to. * Fix ``flexmeasures add schedule --dry-run`` saving a schedule when combined with ``--as-job``: the combination is now rejected, and a dry run reports the beliefs and events it would have saved. * Add ``flexmeasures add forecasts --dry-run`` to compute a forecast and see what it would record, without saving anything to the database. It cannot be combined with ``--as-job``. diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 370dbf6eb8..bcd4a13dc8 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -106,6 +106,7 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures monitor latest-run`` Check if the given task's last successful execution happened less than the allowed time ago. ``flexmeasures monitor last-seen`` Check if given users last contact (via a request) happened less than the allowed time ago. +``flexmeasures monitor automations`` Email a digest of the automation runs that failed since the last check. ================================================= ======================================= diff --git a/documentation/configuration.rst b/documentation/configuration.rst index e760f04622..bfeb155059 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -832,11 +832,22 @@ Default: ``None`` FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run`` and ``flexmeasures monitor last-seen`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. +E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run``, ``flexmeasures monitor last-seen`` and ``flexmeasures monitor automations`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. Default: ``[]`` +.. _monitor_automations_stuck_after: + +FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +How long an automation run may execute before ``flexmeasures monitor automations`` reports it as stuck, for instance because the worker running its job died. +Its ``--stuck-after-minutes`` option overrides this setting. + +Default: ``timedelta(hours=6)`` + + .. _redis-config: Redis diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f8a6d3feb7..1b3c4c68e3 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -278,6 +278,31 @@ This is what an operator needs to tell a run which failed before queueing anythi Editing an automation's cron string or timezone, or reactivating it, counts up its schedule revision. Runs of the old and the new schedule therefore stay distinct, even when they fall on the same scheduled UTC time. +.. _monitoring_automations: + +Monitoring automation runs +-------------------------- + +As a host, you can have FlexMeasures email you a digest of the automation runs that failed, by running ``flexmeasures monitor automations`` regularly, for instance every hour: + +.. code-block:: bash + + 0 * * * * flexmeasures monitor automations + +A run is reported when its execution failed, when it has been executing for longer than six hours, or when its dispatch failed and will not be attempted again. +A forecast run therefore counts once its five dispatch attempts are used up, and a run of any other type after its first failed attempt. +The six hours are set by :ref:`monitor_automations_stuck_after`, and ``--stuck-after-minutes`` overrides that setting. + +Each run is reported once, so if one monitoring run is missed, the next one still reports what failed in the meantime. +Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it last did and with which error. +No email is sent when nothing failed. + +The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). +The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. + +.. note:: Until schedule, report and plugin runs record how their jobs ended, only forecast runs can be reported for a failed or stuck execution. + A failed dispatch is reported for every type. + Running one automation on demand -------------------------------- diff --git a/documentation/host/error-monitoring.rst b/documentation/host/error-monitoring.rst index 1ff8d049cf..ac98a8ec1f 100644 --- a/documentation/host/error-monitoring.rst +++ b/documentation/host/error-monitoring.rst @@ -36,6 +36,17 @@ If you run distinct filters, such as separate checks per account, account group .. todo:: Adding roles and assigning them to accounts is not supported by the UI yet (user roles can be added in the UI). Account roles can be added with ``flexmeasures add account-role``. +Monitoring automations +----------------------- + +The CLI task ``flexmeasures monitor automations`` emails a digest of the automation runs that failed since its last check, with repeated failures collapsed per automation. +Run it from cron, for instance every hour; see :ref:`monitoring_automations` for what it reports. + +.. code-block:: bash + + $ flexmeasures monitor automations --recipient alerts@example.com + + Monitoring task runs --------------------- From 45696537a0cc62d3b21e8c854d249a9cf5be5786 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:21:00 +0100 Subject: [PATCH 11/37] data/models: let an automation opt out of monitoring Add a `monitored` flag to automations, true by default and for every existing automation. Signed-off-by: Mohamed Belhsan Hmida --- ...a3f6c2b917_add_monitored_to_automations.py | 31 +++++++++++++++++++ flexmeasures/data/models/automations.py | 4 +++ 2 files changed, 35 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py diff --git a/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py b/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py new file mode 100644 index 0000000000..6677918c52 --- /dev/null +++ b/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py @@ -0,0 +1,31 @@ +"""Let an automation opt out of monitoring + +Revision ID: e8a3f6c2b917 +Revises: d4b7e9a1c362 +Create Date: 2026-10-12 10:00:00.000000 + +`flexmeasures monitor automations` skips the runs of an automation whose ``monitored`` is false. +Every existing automation stays monitored. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "e8a3f6c2b917" +down_revision = "d4b7e9a1c362" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "automation", + sa.Column( + "monitored", sa.Boolean(), nullable=False, server_default=sa.text("true") + ), + ) + + +def downgrade(): + op.drop_column("automation", "monitored") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index 2b419ee1fc..826f41c1ce 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -95,6 +95,10 @@ class Automation(db.Model, AuthModelMixin): ) schedule_revision = db.Column(db.Integer, nullable=False, default=1) active = db.Column(db.Boolean, nullable=False, default=True) + # Whether `flexmeasures monitor automations` reports this automation's failed runs. + monitored = db.Column( + db.Boolean, nullable=False, default=True, server_default="true" + ) generator_id = db.Column( db.Integer, db.ForeignKey("data_source.id"), nullable=False ) From 57458d1beb756c0a9f9dc747228af4edf30f3e46 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:21:46 +0100 Subject: [PATCH 12/37] data/services: leave out the runs of automations that are not monitored Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/services/automation_monitoring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py index 0b82190887..64cad48444 100644 --- a/flexmeasures/data/services/automation_monitoring.py +++ b/flexmeasures/data/services/automation_monitoring.py @@ -96,6 +96,7 @@ def get_unalerted_failed_automation_runs( A run failed when its execution failed, when its execution has been running for longer than ``stuck_after``, or when its dispatch failed and will not be attempted again. This holds for every automation type, although only the types that record their execution can fail on it. + Runs of an automation that opted out of monitoring are left out, and stay unreported until it opts back in. :param stuck_after: How long a run may execute before it counts as stuck. :param now: The moment to judge by, which defaults to the server's current time. @@ -110,7 +111,9 @@ def get_unalerted_failed_automation_runs( ) runs = db.session.scalars( select(AutomationRun) + .join(AutomationRun.automation) .where( + Automation.monitored.is_(True), AutomationRun.alerted_at.is_(None), or_(execution_failed, stuck, _dispatch_failed_for_good(now)), ) From 956d4137f5fa0dbb596843fadad711121281b6ae Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:21:48 +0100 Subject: [PATCH 13/37] cli/tests: test that monitoring skips unmonitored automations until they opt back in Signed-off-by: Mohamed Belhsan Hmida --- .../test_monitor_automations_fresh_db.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py index e112231904..53cdc53a3f 100644 --- a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -351,3 +351,30 @@ def test_without_recipients_nothing_is_marked_as_reported( assert "Nobody to send the digest to" in result.output fresh_db.session.refresh(run) assert run.alerted_at is None + + +def test_an_unmonitored_automation_is_reported_once_it_is_monitored_again( + app, fresh_db, forecast_automation +): + """The failed runs of an automation that opted out of monitoring are left out, until it opts back in.""" + forecast_automation.monitored = False + fresh_db.session.commit() + run = add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + fresh_db.session.refresh(run) + assert run.alerted_at is None + + forecast_automation.monitored = True + fresh_db.session.commit() + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert f"Automation {forecast_automation.id} " in outbox[0].body + fresh_db.session.refresh(run) + assert run.alerted_at == DUE_AT From c0f2abaca4683582730327f7bf22a9c4e73283bf Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:23:54 +0100 Subject: [PATCH 14/37] data/services: set whether an automation is monitored when creating, editing or copying it A copy keeps the original's choice. It is set beside the type handler rather than by it, so that no handler, a plugin's included, needs to know about monitoring. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/services/automation_copies.py | 1 + flexmeasures/data/services/automations.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/services/automation_copies.py b/flexmeasures/data/services/automation_copies.py index 2d0eba027a..6a9cc63cd8 100644 --- a/flexmeasures/data/services/automation_copies.py +++ b/flexmeasures/data/services/automation_copies.py @@ -197,6 +197,7 @@ def _copy_automation( timezone=automation.timezone, # A copy starts inactive, so that it can be inspected and tested before it runs. active=False, + monitored=automation.monitored, generator_id=generator.id, parameters=parameters, # A fresh cursor, so that the copy does not inherit the original's run history. diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 838dd4e649..bd56eb5a5a 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -1851,6 +1851,7 @@ def create_automation( source: DataSource | None = None, origin: str = "API", check_permissions: bool = False, + monitored: bool = True, ) -> tuple[Automation, list[str]]: """Create an automation through its registered handler, without committing.""" from flask_security import current_user @@ -1874,6 +1875,8 @@ def create_automation( origin=origin, check_permissions=check_permissions, ) + # Set here rather than by the handler, so that no handler, a plugin's included, needs to know about monitoring. + automation.monitored = monitored if check_permissions: if current_user.is_anonymous: from werkzeug.exceptions import Unauthorized @@ -2062,9 +2065,10 @@ def update_automation( cronstr: str | None = None, timezone: str | None = None, active: bool | None = None, + monitored: bool | None = None, origin: str = "API", ) -> list[str]: - """Update an automation's name, cron string, timezone and/or activation status (not committed yet). + """Update an automation's name, cron string, timezone, activation status and/or monitoring (not committed yet). Anything that changes which runs are due, namely the recurrence, the timezone and reactivation, also resets the cursor to just before the minute of the change, so that runs from before it are not caught up on, @@ -2093,6 +2097,11 @@ def update_automation( if active: reset_cursor = True automation.active = active + if monitored is not None and monitored != automation.monitored: + changes.append( + "monitoring switched on" if monitored else "monitoring switched off" + ) + automation.monitored = monitored if reset_cursor: # A new revision keeps the durable runs of the old and the new schedule apart, # even where they fall on the same scheduled UTC time. From 8f9f6cbb5132d0d973380c4c32622c3d433cc04b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:24:20 +0100 Subject: [PATCH 15/37] api: accept and report whether an automation is monitored Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/assets.py | 14 +++++++++---- flexmeasures/data/schemas/automations.py | 21 ++++++++++++++++++- flexmeasures/ui/static/openapi-specs.json | 25 ++++++++++++++++++++--- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 86297e0f5b..5729ff2bab 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1468,7 +1468,7 @@ def get_automations( description: | The response will be a list of automations: recurring forecasting, scheduling, reporting or plugin-defined tasks defined on the asset. Each entry shows the automation's ID, when it was created, - its type, name, activation status, and its recurrence, both as a cron string + its type, name, activation status, whether it is monitored, and its recurrence, both as a cron string and described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted, and both its cursor and its next scheduled run as clock times in that same timezone (the next run is null while inactive). The next run excludes pending catch-up work. @@ -1515,6 +1515,7 @@ def get_automations( recurrence-description: "At 06:00" schedule-revision: 1 active: true + monitored: true job-stats: finished: 3 redis-connection-err: null @@ -1629,6 +1630,7 @@ def get_automation(self, id: int, automation_id: int, asset: GenericAsset): recurrence-description: "At 06:00" schedule-revision: 1 active: true + monitored: true parameters: sensor: 2092 source: @@ -1871,13 +1873,13 @@ def patch_automation( self, automation_data: dict, id: int, automation_id: int, asset: GenericAsset ): """ - .. :quickref: Assets; Update an automation's name, cron string, timezone or activation status. + .. :quickref: Assets; Update an automation's name, cron string, timezone, activation status or monitoring. --- patch: - summary: Update an automation's name, cron string, timezone or activation status. + summary: Update an automation's name, cron string, timezone, activation status or monitoring. description: | - Any subset of the fields `name`, `cronstr`, `timezone` and `active` can be sent. + Any subset of the fields `name`, `cronstr`, `timezone`, `active` and `monitored` can be sent. Changing the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change, so runs from before it are not caught up on, while a run due in that very minute still is. Other automation fields cannot be updated; instead, create a new automation. @@ -1906,6 +1908,10 @@ def patch_automation( summary: Deactivate the automation value: active: false + stop-monitoring: + summary: Stop reporting the automation's failed runs + value: + monitored: false responses: 200: description: PROCESSED diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py index 458e617413..e2d47d0721 100644 --- a/flexmeasures/data/schemas/automations.py +++ b/flexmeasures/data/schemas/automations.py @@ -102,6 +102,13 @@ class AutomationCreationSchema(Schema): }, ) active = fields.Bool(load_default=True) + monitored = fields.Bool( + load_default=True, + metadata={ + "description": "Whether `flexmeasures monitor automations` reports this automation's failed runs. Runs that failed while it was not monitored are reported once it is monitored again.", + "example": True, + }, + ) parameters = fields.Dict(keys=fields.Str(), load_default=dict) generator_class = fields.Str( data_key="data-generator", @@ -161,7 +168,7 @@ def validate_generator_is_named_once(self, data: dict, **kwargs): class AutomationUpdateSchema(Schema): - """Request schema for updating an automation's name, recurrence, timezone and/or activation status. + """Request schema for updating an automation's name, recurrence, timezone, activation status and/or monitoring. The parameters cannot be updated, so the sensors an automation involves stay the ones its creator was checked against. """ @@ -175,6 +182,12 @@ class AutomationUpdateSchema(Schema): } ) active = fields.Bool() + monitored = fields.Bool( + metadata={ + "description": "Whether `flexmeasures monitor automations` reports this automation's failed runs. Runs that failed while it was not monitored are reported once it is monitored again.", + "example": True, + }, + ) class AutomationSchema(ma.SQLAlchemySchema): @@ -221,6 +234,12 @@ class Meta: }, ) active = ma.auto_field() + monitored = ma.auto_field( + metadata={ + "description": "Whether `flexmeasures monitor automations` reports this automation's failed runs. Runs that failed while it was not monitored are reported once it is monitored again.", + "example": True, + }, + ) @staticmethod def _in_automation_timezone( diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 8c51b957f0..9b49b423fc 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3556,6 +3556,7 @@ "recurrence-description": "At 06:00", "schedule-revision": 1, "active": true, + "monitored": true, "parameters": { "sensor": 2092 }, @@ -3636,8 +3637,8 @@ ] }, "patch": { - "summary": "Update an automation's name, cron string, timezone or activation status.", - "description": "Any subset of the fields `name`, `cronstr`, `timezone` and `active` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", + "summary": "Update an automation's name, cron string, timezone, activation status or monitoring.", + "description": "Any subset of the fields `name`, `cronstr`, `timezone`, `active` and `monitored` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", "security": [ { "ApiKeyAuth": [] @@ -3675,6 +3676,12 @@ "value": { "active": false } + }, + "stop-monitoring": { + "summary": "Stop reporting the automation's failed runs", + "value": { + "monitored": false + } } } } @@ -3708,7 +3715,7 @@ "/api/v3_0/assets/{id}/automations": { "get": { "summary": "Get all automations defined on an asset.", - "description": "The response will be a list of automations: recurring forecasting, scheduling, reporting or plugin-defined tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, and its recurrence, both as a cron string\nand described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted,\nand both its cursor and its next scheduled run as clock times in that same timezone (the next run is null while inactive).\nThe next run excludes pending catch-up work.\n\nBy default, the automations of the assets below it are included as well, at any depth, so that a site asset reports everything that runs below it.\nPass `include-child-assets=false` to list only the automations defined on the asset itself.\nOnly the assets below it which you may read are included.\nEach entry names the asset it is defined on, in `asset` and `asset-name`.\n", + "description": "The response will be a list of automations: recurring forecasting, scheduling, reporting or plugin-defined tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, whether it is monitored, and its recurrence, both as a cron string\nand described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted,\nand both its cursor and its next scheduled run as clock times in that same timezone (the next run is null while inactive).\nThe next run excludes pending catch-up work.\n\nBy default, the automations of the assets below it are included as well, at any depth, so that a site asset reports everything that runs below it.\nPass `include-child-assets=false` to list only the automations defined on the asset itself.\nOnly the assets below it which you may read are included.\nEach entry names the asset it is defined on, in `asset` and `asset-name`.\n", "security": [ { "ApiKeyAuth": [] @@ -3758,6 +3765,7 @@ "recurrence-description": "At 06:00", "schedule-revision": 1, "active": true, + "monitored": true, "job-stats": { "finished": 3 } @@ -6812,6 +6820,12 @@ "type": "boolean", "default": true }, + "monitored": { + "type": "boolean", + "default": true, + "description": "Whether `flexmeasures monitor automations` reports this automation's failed runs. Runs that failed while it was not monitored are reported once it is monitored again.", + "example": true + }, "parameters": { "type": "object", "additionalProperties": {} @@ -6864,6 +6878,11 @@ }, "active": { "type": "boolean" + }, + "monitored": { + "type": "boolean", + "description": "Whether `flexmeasures monitor automations` reports this automation's failed runs. Runs that failed while it was not monitored are reported once it is monitored again.", + "example": true } }, "additionalProperties": false From 6095634cc1070b5eded74caf19e015be20777c77 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:24:28 +0100 Subject: [PATCH 16/37] api/tests: test opting an automation out of monitoring and back in, and copying that choice Signed-off-by: Mohamed Belhsan Hmida --- .../test_asset_copy_automations_fresh_db.py | 3 ++ .../api/v3_0/tests/test_automations_api.py | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py index 4fba308614..8d07f30dcd 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py @@ -164,6 +164,8 @@ def test_the_output_scoped_automation_types_are_supported(): def test_copy_copies_direct_and_descendant_automations(fresh_db, automated_site): """Automations on the copied asset and on its descendants are copied, inactive and without run history.""" + # One original opted out of monitoring, so that the copies show they keep that choice rather than the default. + automated_site["meter_automation"].monitored = False asset_copy = copy_asset(automated_site["site"]) assert asset_copy.skipped_automations == [] @@ -187,6 +189,7 @@ def test_copy_copies_direct_and_descendant_automations(fresh_db, automated_site) assert copied.type == original.type assert copied.cronstr == original.cronstr assert copied.timezone == original.timezone + assert copied.monitored == original.monitored # A copy waits to be switched on, and starts without the original's run history. assert copied.active is False assert copied.cursor > OLD_CURSOR diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py index d4aed194c5..d057ee38b4 100644 --- a/flexmeasures/api/v3_0/tests/test_automations_api.py +++ b/flexmeasures/api/v3_0/tests/test_automations_api.py @@ -306,6 +306,7 @@ def test_post_automation( if expected_status_code == 201: assert response.json["name"] == "Posted schedules" assert response.json["active"] is True + assert response.json["monitored"] is True assert response.json["recurrence-description"] == "At 00:00" automation = db.session.get(Automation, response.json["id"]) assert automation.parameters == {"duration": "PT12H"} @@ -552,6 +553,55 @@ def test_post_and_patch_automation_timezone( assert response.status_code == 422 +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user_2@seita.nl"], indirect=True +) +def test_post_and_patch_automation_monitoring( + app, + db, + add_battery_assets, + requesting_user, +): + """An automation can opt out of monitoring on creation, and opt back in afterwards.""" + from flexmeasures.data.models.audit_log import AssetAuditLog + + battery = add_battery_assets["Test battery"] + with app.test_client() as client: + response = client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Experimental schedules", + "cron": "0 0 * * *", + "type": "scheduling", + "parameters": {"duration": "PT12H"}, + "monitored": False, + }, + ) + assert response.status_code == 201, response.json + assert response.json["monitored"] is False + automation = db.session.get(Automation, response.json["id"]) + assert automation.monitored is False + + with app.test_client() as client: + response = client.patch( + url_for( + "AssetAPI:patch_automation", + id=battery.id, + automation_id=automation.id, + ), + json={"monitored": True}, + ) + assert response.status_code == 200, response.json + assert response.json["monitored"] is True + assert automation.monitored is True + assert db.session.scalars( + select(AssetAuditLog.event).where( + AssetAuditLog.affected_asset_id == battery.id, + AssetAuditLog.event.contains("monitoring switched on"), + ) + ).all() + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user_2@seita.nl"], indirect=True ) From f829681e956ff8825c23cfc0a489416102864164 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:27:32 +0100 Subject: [PATCH 17/37] cli: set and show whether an automation is monitored `flexmeasures add automation --unmonitored` and `flexmeasures edit automation --monitored/--unmonitored`, with a Monitored column and row in `flexmeasures show automations`. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/data_add.py | 8 ++++++++ flexmeasures/cli/data_edit.py | 10 +++++++++- flexmeasures/cli/data_show.py | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 83222ea254..8954cb63ee 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1792,6 +1792,12 @@ def add_forecast( # noqa: C901 is_flag=True, help="Add this flag to create the automation in deactivated state.", ) +@click.option( + "--unmonitored", + "unmonitored", + is_flag=True, + help="Add this flag to keep the automation's failed runs out of `flexmeasures monitor automations`.", +) @click.option( "--data-generator", "--forecaster", @@ -1868,6 +1874,7 @@ def add_automation( # noqa: C901 timezone: str, automation_type: str, inactive: bool = False, + unmonitored: bool = False, generator_class: str | None = None, source: DataSource | None = None, config_file: TextIOBase | None = None, @@ -1966,6 +1973,7 @@ def add_automation( # noqa: C901 timezone=timezone, automation_type=automation_type, active=not inactive, + monitored=not unmonitored, parameters=parameters, generator_class=generator_class, config=config, diff --git a/flexmeasures/cli/data_edit.py b/flexmeasures/cli/data_edit.py index 271076c999..2886102799 100644 --- a/flexmeasures/cli/data_edit.py +++ b/flexmeasures/cli/data_edit.py @@ -98,20 +98,28 @@ def fm_edit_data(): default=None, help="Activate or deactivate the automation.", ) +@click.option( + "--monitored/--unmonitored", + "monitored", + default=None, + help="Whether `flexmeasures monitor automations` reports the automation's failed runs.", +) def edit_automation( automation: Automation, name: str | None = None, cronstr: str | None = None, timezone: str | None = None, active: bool | None = None, + monitored: bool | None = None, ): - """Edit the name, recurrence, timezone or activation status of an automation.""" + """Edit the name, recurrence, timezone, activation status or monitoring of an automation.""" changes = update_automation( automation, name=name, cronstr=cronstr, timezone=timezone, active=active, + monitored=monitored, origin="CLI", ) if not changes: diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index 6b4a3fa380..aa2008697c 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -507,6 +507,7 @@ def list_automations( else f"{automation.type} (plugin unavailable)" ), "yes" if automation.active else "no", + "yes" if automation.monitored else "no", automation.cronstr, automation.timezone, ) @@ -518,6 +519,7 @@ def list_automations( "Name", "Type", "Active", + "Monitored", "Cron", "Timezone", ], @@ -539,6 +541,7 @@ def _show_automation(automation: Automation): ("Asset", f"{automation.asset.name} (ID: {automation.asset_id})"), ("Type", automation.type), ("Active", "yes" if automation.active else "no"), + ("Monitored", "yes" if automation.monitored else "no"), ( "Recurrence", f"{automation.cronstr} ({describe_cronstr(automation.cronstr)})", From 323582337a6ef0092002be1c762d4160894cbeb6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:27:34 +0100 Subject: [PATCH 18/37] cli/tests: test adding, editing and showing an unmonitored automation Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_automations.py | 46 +++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index fdc0a35ac9..9eab6efa50 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -2211,7 +2211,7 @@ def automation_row_cell(output: str, name: str, header: str) -> str: Substring checks on a whole row are not enough here, as a cell's value may also occur in the automation's own name. """ - headers = ["ID", "Asset", "Name", "Type", "Active", "Cron", "Timezone"] + headers = ["ID", "Asset", "Name", "Type", "Active", "Monitored", "Cron", "Timezone"] lines = output.splitlines() header_line = next(line for line in lines if all(h in line for h in headers)) starts = [header_line.index(h) for h in headers] @@ -2355,6 +2355,50 @@ def test_show_automations_includes_inactive_ones(app, setup_dummy_data_fresh_db) assert automation_row_cell(result.output, "Paused forecasts", "Active") == "no" +def test_add_edit_and_show_an_unmonitored_automation( + app, fresh_db, setup_dummy_data_fresh_db +): + """An automation can be created without monitoring and edited to be monitored, and both views say which it is.""" + from flexmeasures.cli.data_add import add_automation + from flexmeasures.cli.data_edit import edit_automation + from flexmeasures.cli.data_show import list_automations + + runner = app.test_cli_runner() + result = runner.invoke( + add_automation, + to_flags( + { + "asset": 1, + "name": "Experimental forecasts", + "sensor": setup_dummy_data_fresh_db[0], + } + ) + + ["--unmonitored"], + ) + assert "Successfully created" in result.output, result.output + automation = fresh_db.session.scalars(select(Automation)).one() + assert automation.monitored is False + + result = runner.invoke(list_automations, []) + assert result.exit_code == 0, result.output + assert ( + automation_row_cell(result.output, "Experimental forecasts", "Monitored") + == "no" + ) + result = runner.invoke(list_automations, ["--id", automation.id]) + assert result.exit_code == 0, result.output + assert re.search(r"^Monitored\s+no$", result.output, re.MULTILINE) + result = runner.invoke(list_automations, ["--id", automation.id, "--as-json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["monitored"] is False + + result = runner.invoke(edit_automation, ["--id", automation.id, "--monitored"]) + assert result.exit_code == 0, result.output + assert "monitoring switched on" in result.output + fresh_db.session.refresh(automation) + assert automation.monitored is True + + def test_show_automations_without_any_automations(app, setup_dummy_data_fresh_db): """Without automations, the command says so rather than printing an empty table.""" from flexmeasures.cli.data_show import list_automations From 72a5e11669f23bcd612d0dc02d7c995f2544bbf8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 01:29:20 +0100 Subject: [PATCH 19/37] ui: set whether an automation is monitored on the Automations page A Monitored checkbox in the add and edit forms, carried over by Copy, and a muted bell beside the activation status of an automation that is not monitored. Signed-off-by: Mohamed Belhsan Hmida --- .../templates/assets/asset_automations.html | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 3ddbb9285e..554c249dc5 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -243,6 +243,13 @@
Configuration
+
+ + +
+
+ + +
- + + {% if not user_can_change_monitoring %} +
Only admins can switch monitoring off or on.
+ {% endif %}
@@ -293,11 +296,14 @@
- + + {% if not user_can_change_monitoring %} +
Only admins can switch monitoring off or on.
+ {% endif %}
@@ -346,6 +352,8 @@ const userCanManage = {{ user_can_manage_automations | tojson }}; // Running an automation writes data under the asset, so the same permission gates the button and the endpoint. const userCanRunAutomations = {{ user_can_create_children | tojson }}; + // Switching monitoring off silences the digest for whoever receives it, so only admins may, and the forms leave the field out for everyone else. + const userCanChangeMonitoring = {{ user_can_change_monitoring | tojson }}; const automationTypes = {{ automation_types | tojson }}; const automationsById = new Map(); // What the details endpoint said about each automation, kept so that copying one does not have to ask again. @@ -524,7 +532,8 @@ $("#automationTimezone").val(values.timezone); $("#automationParameters").val(values.parameters); $("#automationActive").prop("checked", values.active); - $("#automationMonitored").prop("checked", values.monitored); + // Someone who may not switch monitoring off gets a monitored automation, also when copying an unmonitored one. + $("#automationMonitored").prop("checked", userCanChangeMonitoring ? values.monitored : true); } // The name column allows 80 characters, and so does the API, so a long name loses its tail rather than the suffix that marks the copy. @@ -1084,7 +1093,7 @@
Recently created jobs cron: $("#editAutomationCron").val(), timezone: $("#editAutomationTimezone").val(), active: $("#editAutomationActive").is(":checked"), - monitored: $("#editAutomationMonitored").is(":checked"), + ...(userCanChangeMonitoring ? { monitored: $("#editAutomationMonitored").is(":checked") } : {}), }), success: () => location.reload(), error: function (xhr) { @@ -1226,7 +1235,7 @@
Recently created jobs cron: $("#automationCron").val(), timezone: $("#automationTimezone").val(), active: $("#automationActive").is(":checked"), - monitored: $("#automationMonitored").is(":checked"), + ...(userCanChangeMonitoring ? { monitored: $("#automationMonitored").is(":checked") } : {}), parameters: parameters, }; if (reusesSource) { diff --git a/flexmeasures/ui/views/assets/views.py b/flexmeasures/ui/views/assets/views.py index d5809769f5..82834c6c28 100644 --- a/flexmeasures/ui/views/assets/views.py +++ b/flexmeasures/ui/views/assets/views.py @@ -11,6 +11,7 @@ from flexmeasures.data import db from flexmeasures.data.automations import get_automation_types +from flexmeasures.data.services.automations import user_may_change_monitoring from flexmeasures.auth.policy import check_access from flexmeasures.auth.error_handling import unauthorized_handler from flexmeasures.data.schemas import StartEndTimeSchema @@ -278,6 +279,7 @@ def automations(self, id: str): # Managing an automation is gated like running one, so both follow create-children. user_can_manage_automations=user_can_create_children(asset), user_can_create_children=user_can_create_children(asset), + user_can_change_monitoring=user_may_change_monitoring(current_user, asset), current_page="Automations", ) From dbca6420011f739d11f2283b06322ba6f89beb0b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 16 Nov 2026 00:08:34 +0100 Subject: [PATCH 34/37] cli/monitor: say which runs failed while the automation was not monitored Once monitoring is switched back on, the digest reports the runs that failed meanwhile, which can be thousands. It now names the first failure besides the last one, and says how many runs failed before monitoring was switched back on, and when that was. The automation records that moment in a new monitoring_resumed_at column. The docs also say who may switch monitoring, and that it only affects the digest. Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 7 +++- flexmeasures/cli/monitor.py | 14 ++++++- .../test_monitor_automations_fresh_db.py | 39 ++++++++++++++++++- ...a3f6c2b917_add_monitored_to_automations.py | 7 ++++ flexmeasures/data/models/automations.py | 2 + .../data/services/automation_monitoring.py | 23 +++++++++++ flexmeasures/data/services/automations.py | 2 + 7 files changed, 90 insertions(+), 4 deletions(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f4da5a1ba6..dbc2cf51f0 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -303,13 +303,16 @@ A forecast run therefore counts once its five dispatch attempts are used up, and The six hours are set by :ref:`monitor_automations_stuck_after`, and ``--stuck-after-minutes`` overrides that setting. Each run is reported once, so if one monitoring run is missed, the next one still reports what failed in the meantime. -Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it last did and with which error. +Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it first and last did, and with which error. No email is sent when nothing failed. Every automation is monitored, unless it opts out, for instance an experiment whose failures nobody needs to hear about. Untick *Monitored* on the Automations page, create it with ``flexmeasures add automation --unmonitored``, switch it with ``flexmeasures edit automation --monitored/--unmonitored``, or send ``"monitored": false`` to the API. Its runs are then left out of every digest. -They are not marked as reported, though, so once the automation is monitored again, the next digest reports the runs that failed in the meantime. +They are not marked as reported, though, so once the automation is monitored again, the next digest reports the runs that failed in the meantime, +saying how many of them failed while it was not monitored, and when monitoring was switched back on. +Only the host's admins and the account-admins of the asset's organisation may switch monitoring off or on, as it decides what the recipients of the digest hear about. +Switching it off only keeps the automation out of these digests; the automation runs as before. The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index f8fdaafe85..2efb4cb5be 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -542,11 +542,23 @@ def describe_automation_failures(failures: AutomationFailures) -> str: f"{reason}: {count}" for reason, count in failures.reason_counts.items() ) last_run = failures.last_run + # With more than one failure, the first one tells how long the automation has been failing. + first_at = ( + f" first at {_format_utc(failures.first_run.scheduled_at)}," + if len(failures.failed_runs) > 1 + else "" + ) lines = [ f"Automation {automation.id} '{automation.name}' ({automation.type}) on {where}:", - f" failed {_times(len(failures.failed_runs))} ({reasons})," + f" failed {_times(len(failures.failed_runs))} ({reasons}),{first_at}" f" last at {_format_utc(last_run.scheduled_at)} (run {last_run.id})", ] + unmonitored_runs = failures.runs_failed_while_unmonitored + if unmonitored_runs: + lines.append( + f" {len(unmonitored_runs)} of these failed while the automation was not monitored," + f" before monitoring was switched back on at {_format_utc(automation.monitoring_resumed_at)}" + ) last_error = failures.last_error if last_error: if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py index 9c44a3f445..bcc5953569 100644 --- a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -322,7 +322,7 @@ def test_repeated_failures_are_collapsed_per_automation( body = outbox[0].body assert body.count(f"Automation {forecast_automation.id} ") == 1 assert ( - "failed 4 times (execution failed: 3, dispatch failed: 1), last at 2026-08-05 00:30 UTC" + "failed 4 times (execution failed: 3, dispatch failed: 1), first at 2026-08-04 22:00 UTC, last at 2026-08-05 00:30 UTC" in body ) # The dispatch failure recorded no error, so the latest error is that of the latest failed execution. @@ -380,6 +380,43 @@ def test_an_unmonitored_automation_is_reported_once_it_is_monitored_again( assert run.alerted_at == DUE_AT +def test_the_digest_says_which_runs_failed_while_unmonitored( + app, fresh_db, forecast_automation, freeze_server_now +): + """Once monitoring is switched back on, the digest names the first failure and says how many runs failed while it was off.""" + from flexmeasures.data.services.automations import update_automation + + update_automation(forecast_automation, monitored=False, origin="test") + fresh_db.session.commit() + for hours_ago in (72, 48, 24): + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=hours_ago), + execution_state="failed", + ) + switched_on_at = DUE_AT - timedelta(hours=2) + freeze_server_now(switched_on_at) + update_automation(forecast_automation, monitored=True, origin="test") + fresh_db.session.commit() + freeze_server_now(DUE_AT) + # One more failure once it is monitored again. + add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + body = outbox[0].body + assert ( + "failed 4 times (execution failed: 4), first at 2026-08-02 01:00 UTC, last at 2026-08-05 01:00 UTC" + in body + ) + assert ( + " 3 of these failed while the automation was not monitored, before monitoring was switched back on at 2026-08-04 23:00 UTC" + in body + ) + + def test_the_digest_links_to_the_automation_once_the_base_url_is_set( app, fresh_db, forecast_automation, monkeypatch ): diff --git a/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py b/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py index 6677918c52..ff2f153321 100644 --- a/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py +++ b/flexmeasures/data/migrations/versions/e8a3f6c2b917_add_monitored_to_automations.py @@ -6,6 +6,8 @@ `flexmeasures monitor automations` skips the runs of an automation whose ``monitored`` is false. Every existing automation stays monitored. +``monitoring_resumed_at`` records when monitoring was last switched back on, +so that the digest can tell which of the runs it reports failed while the automation was not monitored. """ from alembic import op @@ -25,7 +27,12 @@ def upgrade(): "monitored", sa.Boolean(), nullable=False, server_default=sa.text("true") ), ) + op.add_column( + "automation", + sa.Column("monitoring_resumed_at", sa.DateTime(timezone=True), nullable=True), + ) def downgrade(): + op.drop_column("automation", "monitoring_resumed_at") op.drop_column("automation", "monitored") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index 8b950a81fa..cc344501a9 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -99,6 +99,8 @@ class Automation(db.Model, AuthModelMixin): monitored = db.Column( db.Boolean, nullable=False, default=True, server_default="true" ) + # When monitoring was last switched back on, which tells the digest which runs failed while it was off. + monitoring_resumed_at = db.Column(db.DateTime(timezone=True), nullable=True) generator_id = db.Column( db.Integer, db.ForeignKey("data_source.id"), nullable=False ) diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py index 64cad48444..10d06b7811 100644 --- a/flexmeasures/data/services/automation_monitoring.py +++ b/flexmeasures/data/services/automation_monitoring.py @@ -40,6 +40,29 @@ def reason_counts(self) -> Counter: """How many runs failed in each way.""" return Counter(failed_run.reason for failed_run in self.failed_runs) + @property + def first_run(self) -> AutomationRun: + """The failed run that was due first.""" + return min( + (failed_run.run for failed_run in self.failed_runs), + key=lambda run: (run.scheduled_at, run.id), + ) + + @property + def runs_failed_while_unmonitored(self) -> list[AutomationRun]: + """The failed runs that were due before monitoring was last switched back on. + + Had the automation been monitored then, an earlier digest would have reported them. + """ + resumed_at = self.automation.monitoring_resumed_at + if resumed_at is None: + return [] + return [ + failed_run.run + for failed_run in self.failed_runs + if failed_run.run.scheduled_at < resumed_at + ] + @property def last_run(self) -> AutomationRun: """The failed run that was due most recently.""" diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 9fbfdd409b..772ac77411 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -2299,6 +2299,8 @@ def update_automation( "monitoring switched on" if monitored else "monitoring switched off" ) automation.monitored = monitored + if monitored: + automation.monitoring_resumed_at = server_now() if reset_cursor: # A new revision keeps the durable runs of the old and the new schedule apart, # even where they fall on the same scheduled UTC time. From 67f40a0262531df78d33146790604aa619b9ab83 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 16 Nov 2026 00:10:19 +0100 Subject: [PATCH 35/37] ui/tests: check that only admins can switch monitoring in the automation form The page script now reads user_can_change_monitoring, so the test helpers render it, and the creation form is rendered through Jinja rather than patched by hand. Signed-off-by: Mohamed Belhsan Hmida --- .../ui/tests/js/test_automation_actions.py | 2 + .../ui/tests/js/test_automation_copy.py | 44 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/flexmeasures/ui/tests/js/test_automation_actions.py b/flexmeasures/ui/tests/js/test_automation_actions.py index 2bb030e9fd..a073bddee4 100644 --- a/flexmeasures/ui/tests/js/test_automation_actions.py +++ b/flexmeasures/ui/tests/js/test_automation_actions.py @@ -24,6 +24,7 @@ def automation_script( can_run: bool, has_children: bool = False, include_children: bool = True, + can_change_monitoring: bool = True, ) -> str: """Render inline JavaScript with representative registered automation types.""" template = TEMPLATE.read_text() @@ -37,6 +38,7 @@ def automation_script( include_child_assets=include_children, user_can_manage_automations=can_manage, user_can_create_children=can_run, + user_can_change_monitoring=can_change_monitoring, automation_types={ "forecasting": "Forecasts", "scheduling": "Schedules", diff --git a/flexmeasures/ui/tests/js/test_automation_copy.py b/flexmeasures/ui/tests/js/test_automation_copy.py index a4c89098d5..38bb891e93 100644 --- a/flexmeasures/ui/tests/js/test_automation_copy.py +++ b/flexmeasures/ui/tests/js/test_automation_copy.py @@ -3,6 +3,7 @@ import json import re +from jinja2 import Environment, StrictUndefined import pytest # The page's script is rendered by one helper, which asserts that it leaves no Jinja value behind. @@ -15,18 +16,26 @@ def setup_ui_test_data(): """The inline-script checks do not need the UI suite's database fixtures.""" -def copy_script(can_manage: bool = True) -> str: +def copy_script(can_manage: bool = True, can_change_monitoring: bool = True) -> str: """Render the page's inline JavaScript, as someone who may copy an automation sees it.""" - return automation_script(can_manage=can_manage, can_run=True) + return automation_script( + can_manage=can_manage, + can_run=True, + can_change_monitoring=can_change_monitoring, + ) -def creation_form_html() -> str: +def creation_form_html(can_change_monitoring: bool = True) -> str: """The creation form as the page renders it, so the checks run against the real fields.""" match = re.search( r'(
.*?
)', TEMPLATE.read_text(), re.DOTALL ) assert match is not None - return match.group(1).replace("{{ asset.timezone }}", "Europe/Amsterdam") + environment = Environment(autoescape=True, undefined=StrictUndefined) + return environment.from_string(match.group(1)).render( + asset={"timezone": "Europe/Amsterdam"}, + user_can_change_monitoring=can_change_monitoring, + ) # A stand-in for the handful of jQuery calls the form helpers make, backed by the real DOM. @@ -221,3 +230,30 @@ def test_copy_is_offered_to_managers_only(assert_js): check("someone who may not manage automations is not offered Copy", !viewerRow.actions.includes("automation-copy"), viewerRow.actions); """) + + +def test_only_admins_can_switch_monitoring_in_the_form(assert_js): + """Someone who may not switch monitoring off finds the checkbox disabled, and a copy of an unmonitored automation monitored.""" + assert_js(f""" + const holder = document.createElement("div"); + holder.innerHTML = {json.dumps(creation_form_html(can_change_monitoring=False))}; + document.body.appendChild(holder); + {JQUERY_STUB} + const page = new Function({json.dumps(copy_script(can_change_monitoring=False))} + + "\\nreturn {{ prefillNewAutomationForm, automationCopyValues }};")(); + + check("the checkbox is disabled", document.getElementById("automationMonitored").disabled, "enabled"); + check("and says why", holder.textContent.includes("Only admins can switch monitoring off or on."), holder.textContent); + + page.prefillNewAutomationForm(page.automationCopyValues( + {{id: 7, name: "Experiment", type: "forecasting", "cron": "0 6 * * *", timezone: "UTC", monitored: false}}, + {{parameters: {{}}, source: {{id: 6}}}})); + eq("a copy of an unmonitored automation is monitored", document.getElementById("automationMonitored").checked, true); + """) + assert_js(f""" + const holder = document.createElement("div"); + holder.innerHTML = {json.dumps(creation_form_html(can_change_monitoring=True))}; + document.body.appendChild(holder); + check("an admin's checkbox is enabled", !document.getElementById("automationMonitored").disabled, "disabled"); + check("and needs no explanation", !holder.textContent.includes("Only admins"), holder.textContent); + """) From 54789ab98cf1a9de063828ba3623b9490465e798 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 16 Nov 2026 00:10:35 +0100 Subject: [PATCH 36/37] ui/assets: filter the Automations page on monitoring Unmonitored automations are hard to find on an asset with many automations, so a filter lists all, only monitored or only unmonitored ones, all by default. The Active column also sorts unmonitored automations together, and the muted bell carries text for screen readers. Signed-off-by: Mohamed Belhsan Hmida --- .../templates/assets/asset_automations.html | 65 +++++++++++++++---- .../ui/tests/js/test_automation_copy.py | 17 +++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 26f53668ec..55af925616 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -143,6 +143,15 @@

{% endif %} +
+ + +
+ {% if user_can_manage_automations %}
@@ -362,6 +371,20 @@ // How often the listing catches up with the server, while the tab is in front and nothing is open. const REFRESH_INTERVAL_MS = 60000; let loadInFlight = false; + // Which automations to list by their monitoring: "all", "monitored" or "unmonitored". + let monitoringFilter = "all"; + // The last listing from the server, so that changing the filter does not need another request. + let lastListedAutomations = []; + + function matchesMonitoringFilter(automation) { + if (monitoringFilter === "monitored") { + return automation.monitored !== false; + } + if (monitoringFilter === "unmonitored") { + return automation.monitored === false; + } + return true; + } let lastRefreshedAt = null; // Only an asset with something below it can widen its scope, so only then is the Asset column worth a place. const assetHasChildren = {{ 'true' if asset.child_assets else 'false' }}; @@ -670,10 +693,13 @@ name: esc(automation.name), "asset-name": `${esc(automation["asset-name"] || "")}`, "created-at": createdAt(automation), + // Sorted by activity first, then with the unmonitored ones of each kind together. active: sortable( - automation.active ? 1 : 0, + (automation.active ? 2 : 0) + (automation.monitored === false ? 0 : 1), `${automation.active ? "🟢" : "⚪"}` - + (automation.monitored ? "" : ' ') + + (automation.monitored === false + ? ' Not monitored' + : ""), ), recurrence: sortable( automation["cron"], @@ -823,7 +849,9 @@
Recently created jobs ordering: true, order: [[createdAtColumn, "desc"]], // most recently created first language: { - emptyTable: `No automations of type '${esc(automationTypes[automationType] || automationType)}' have been defined ${scope} yet.` + emptyTable: monitoringFilter !== "all" + ? `No ${monitoringFilter} automations of type '${esc(automationTypes[automationType] || automationType)}' ${scope}.` + : `No automations of type '${esc(automationTypes[automationType] || automationType)}' have been defined ${scope} yet.` + (userCanManage ? (["forecasting", "scheduling", "reporting"].includes(automationType) ? " Create one with the 'New automation' button, or with the CLI command `flexmeasures add automation`." : " Create one with the CLI command `flexmeasures add automation`.") : ""), @@ -871,6 +899,20 @@
Recently created jobs } } + function renderAutomationsTables({ rebuild = false } = {}) { + for (const automationType of AUTOMATION_TYPES) { + $(`#automationsTable-${automationType}`).show(); + const ofThisType = lastListedAutomations.filter( + automation => automation.type === automationType && matchesMonitoringFilter(automation) + ); + if (rebuild) { + makeAutomationsTable(automationType, ofThisType); + } else { + updateAutomationsTable(automationType, ofThisType); + } + } + } + function loadAutomations({ rebuild = false } = {}) { // One refresh at a time: a slow response must not pile up behind the next tick. if (loadInFlight) { @@ -883,15 +925,8 @@
Recently created jobs success: function (res) { automationsById.clear(); res.automations.forEach(automation => automationsById.set(automation.id, automation)); - for (const automationType of AUTOMATION_TYPES) { - $(`#automationsTable-${automationType}`).show(); - const ofThisType = res.automations.filter(automation => automation.type === automationType); - if (rebuild) { - makeAutomationsTable(automationType, ofThisType); - } else { - updateAutomationsTable(automationType, ofThisType); - } - } + lastListedAutomations = res.automations; + renderAutomationsTables({ rebuild }); markAutomationsRefreshed(); openLinkedAutomation(); // The listing counts the jobs itself, so it is also where it says when Redis could not be reached. @@ -982,6 +1017,12 @@
Recently created jobs }); loadAutomations({ rebuild: true }); + // Rebuilt rather than redrawn, so that an empty table says what it is filtered on. + $("#monitoringFilter").on("change", function () { + monitoringFilter = this.value; + renderAutomationsTables({ rebuild: true }); + }); + // Widen or narrow what the page reports on, and keep that choice for the next visit. $("#includeChildAssets").on("change", function () { includeChildAssets = this.checked; diff --git a/flexmeasures/ui/tests/js/test_automation_copy.py b/flexmeasures/ui/tests/js/test_automation_copy.py index 38bb891e93..c831ca3e07 100644 --- a/flexmeasures/ui/tests/js/test_automation_copy.py +++ b/flexmeasures/ui/tests/js/test_automation_copy.py @@ -257,3 +257,20 @@ def test_only_admins_can_switch_monitoring_in_the_form(assert_js): check("an admin's checkbox is enabled", !document.getElementById("automationMonitored").disabled, "disabled"); check("and needs no explanation", !holder.textContent.includes("Only admins"), holder.textContent); """) + + +def test_the_monitoring_filter(assert_js): + """The listing can be narrowed to monitored or unmonitored automations, and shows all of them by default.""" + assert_js(f""" + window.$ = () => ({{ready: () => {{}}}}); + const page = new Function({json.dumps(copy_script())} + + "\\nreturn {{ matchesMonitoringFilter, setFilter: (value) => {{ monitoringFilter = value; }} }};")(); + + const monitored = {{id: 1, monitored: true}}; + const unmonitored = {{id: 2, monitored: false}}; + check("all are listed by default", page.matchesMonitoringFilter(monitored) && page.matchesMonitoringFilter(unmonitored), ""); + page.setFilter("monitored"); + check("monitored only keeps the monitored ones", page.matchesMonitoringFilter(monitored) && !page.matchesMonitoringFilter(unmonitored), ""); + page.setFilter("unmonitored"); + check("unmonitored only keeps the others", !page.matchesMonitoringFilter(monitored) && page.matchesMonitoringFilter(unmonitored), ""); + """) From 90d76efac3f809dd861c31a156e5f09d27e7f24b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 16 Nov 2026 00:10:54 +0100 Subject: [PATCH 37/37] documentation: add the monitored field and options to the API and CLI change logs Requested in review. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 1 + documentation/cli/change_log.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 378806dfcf..5c25b157d5 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -20,6 +20,7 @@ v3.0-40 | October XX, 2026 - ``GET /api/v3_0/sources`` and ``GET /api/v3_0/sources/`` now decide access per data source, rather than only by the organisation a source names. A source is readable when it belongs to an organisation the caller may read, when an automation they may read computes under it, or when it has recorded data on a sensor they may read. The listing holds only the first two, which are the sources the caller may reuse for an automation of their own, and ``attributes`` is returned only for those two as well, because a data generator's configuration names the sensors it runs on. A data source which names no organisation is no longer readable for that reason alone, where it used to be readable by every authenticated user. - Data sources returned by ``GET /api/v3_0/sources`` and ``GET /api/v3_0/sources/`` now carry the ``account_id`` of the organisation they record for, where only sources of type ``user`` used to. Source filters that select by organisation, such as ``source-account`` on ``GET /api/v3_0/sensors//data`` and inside sensor references in a flex-model or flex-context, therefore match data computed by forecasters, schedulers and reporters, where they used to match user-recorded data only. A source belonging to no organisation is one the host runs for everyone. - ``POST /api/v3_0/assets//automations`` now also refuses a ``source`` which is not the caller's to work with for automation types registered by a plugin, where the refusal used to cover the built-in types only. +- ``POST /api/v3_0/assets//automations`` and ``PATCH /api/v3_0/assets//automations/`` now accept ``monitored``, which says whether ``flexmeasures monitor automations`` reports the automation's failed runs (``true`` by default), and the automation endpoints return it. Only the host's admins and the account-admins of the asset's organisation may create an unmonitored automation or switch monitoring off or on; others get a ``403``. - ``GET /api/v3_0/sensors//status`` now returns ``missing-members`` with the status of reporter data whose most recent aggregate of a device group, old enough to count as stale, had to leave out members, because they did not report: the ``id``, ``name`` and ``asset-name`` of each of those sensors. The field is left out where no member is missing, as for most sensors. v3.0-39 | September 22, 2026 diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index b507afdd1f..a3403ce63c 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -26,6 +26,7 @@ since v1.1.0 | September XX, 2026 * Remove the ``--rate-limit-key`` option from ``flexmeasures add plan`` and ``flexmeasures edit plan`` (including ``--clear rate-limit-key``), and its column from ``flexmeasures show plans``, as triggers are now always counted per account. ``flexmeasures show account`` now displays the account's plan and effective default API and trigger rate limits, including server defaults. * ``flexmeasures add forecasts`` no longer fails when it runs again from the same ``--from-date``, and says how many of the forecast beliefs it computed were new and saved. * Add ``flexmeasures monitor automations`` to email a digest of the automation runs that failed since the last check: runs whose execution failed, which have been executing for too long (``--stuck-after-minutes``), or whose dispatch failed for good. Each run is reported once, and repeated failures are collapsed per automation. +* Add ``flexmeasures add automation --unmonitored`` and ``flexmeasures edit automation --monitored/--unmonitored`` to keep an automation's failed runs out of ``flexmeasures monitor automations``, or to report them again. ``flexmeasures show automations`` shows whether each automation is monitored. * Fix ``flexmeasures add schedule --dry-run`` saving a schedule when combined with ``--as-job``: the combination is now rejected, and a dry run reports the beliefs and events it would have saved. * Fix ``flexmeasures show beliefs`` ignoring its ``--timezone`` option, which is now called ``--export-timezone`` (``--timezone`` still works as a deprecated alias). By default, the CSV file now has its timestamps in the timezone of the first sensor with data, with a UTC offset, where it used to have them in UTC without an offset. An unknown timezone name is now refused, and the option warns when it is given without ``--to-file``, as the terminal plot always shows UTC. * ``flexmeasures delete sensor`` now warns which automations read from or write to a sensor before it is deleted, as an automation refers to its sensors by ID and would fail on its next run.