From 6c7798aef065afddc1adb14aa6942018e890be23 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:22 +0100 Subject: [PATCH 01/20] data/services/automations: record how schedule, report and plugin runs end Context: - Only forecast runs followed their jobs, so schedule, report and plugin runs stayed at execution state 'pending' whatever became of their jobs. Change: - record_automation_run_jobs records the jobs such a run queued, then takes the outcome of any job that already finished or failed from Redis; a run with no jobs has succeeded. - The record_outcome_on_automation_run decorator records a job's start, success or failure on its run, found through the job's RQ ID, and wraps make_schedule, the sequential wrap-up job, run_report_job and execute_automation_job. - Plugin jobs carry automation_run_id in their trigger. - A schedule that fell back counts as a failed run, as the fallback job is not one of the run's recorded jobs. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/automations.py | 17 ++- flexmeasures/data/services/automations.py | 139 +++++++++++++++++++++- flexmeasures/data/services/reporting.py | 2 + flexmeasures/data/services/scheduling.py | 3 + 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/automations.py b/flexmeasures/data/automations.py index dc97a37cc6..e6ea8a3909 100644 --- a/flexmeasures/data/automations.py +++ b/flexmeasures/data/automations.py @@ -15,6 +15,7 @@ from flexmeasures.data import db from flexmeasures.data.models.data_sources import DataGenerator +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.utils.job_utils import KNOWN_JOB_QUEUES @@ -180,8 +181,8 @@ def run(self, automation, automation_run=None, scheduled_at=None): A durable run (see `dispatch_automation_run`) carries the parameters and the jobs its run was planned with, which the built-in types read to pick up where an earlier attempt at the same run stopped. - A plugin type's run is dispatched with the parameters its run was planned with, but records no jobs of its own, - so such a run reports how far its dispatch got and leaves its execution state at 'pending'. + A plugin type's run is dispatched with the parameters its run was planned with, + and records its one job once queued, so that the run follows how the job ends. """ if self.generator_class is None: from flexmeasures.data.services.automations import ( @@ -212,6 +213,9 @@ def run(self, automation, automation_run=None, scheduled_at=None): parameters = planned_parameters db.session.commit() queue = current_app.queues[self.queue] + trigger = {"origin": "automation", "automation_id": automation.id} + if automation_run is not None: + trigger["automation_run_id"] = automation_run.id job = Job.create( execute_automation_job, kwargs={ @@ -232,7 +236,7 @@ def run(self, automation, automation_run=None, scheduled_at=None): ).total_seconds() ), meta={ - "trigger": {"origin": "automation", "automation_id": automation.id}, + "trigger": trigger, "data_source_info": {"id": source_id}, "asset_id": automation.asset_id, "asset_or_sensor": {"class": "Asset", "id": automation.asset_id}, @@ -245,6 +249,12 @@ def run(self, automation, automation_run=None, scheduled_at=None): queue=self.queue, asset_or_sensor_type="asset", ) + if automation_run is not None: + from flexmeasures.data.services.automations import ( + record_automation_run_jobs, + ) + + record_automation_run_jobs(automation_run.id, [("job", self.type_id, job)]) return {"job_id": job.id, "n_jobs": 1} @@ -395,6 +405,7 @@ def check_execution_access(automation, sensors): ) +@record_outcome_on_automation_run def execute_automation_job(automation_id: int, data_source_id: int, parameters: dict): """Compute and persist declared results with provenance in a plugin-enabled worker.""" from flexmeasures.data.models.automations import Automation diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 838dd4e649..4ccd0a07a6 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -5,6 +5,7 @@ from __future__ import annotations from contextlib import contextmanager +import functools from copy import copy, deepcopy from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -21,7 +22,8 @@ from isodate.isoerror import ISO8601Error from flask import current_app from marshmallow import ValidationError -from rq.job import Job +from rq import get_current_job +from rq.job import Job, JobStatus from sqlalchemy import func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import selectinload @@ -574,6 +576,116 @@ def reconcile_automation_job_intent(intent: AutomationRunJob) -> bool: return False +class AutomationJobFailed(Exception): + """Stands in for the error of a job that failed before its run had a record of it, as only its traceback is left.""" + + +def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> None: + """Record the jobs that a schedule, report or plugin run queued, so that the run follows what becomes of them. + + A forecast run records its jobs before queueing them, under IDs it derives from the run (see `ensure_automation_run_job_intents`). + The other types get their job IDs from RQ, and a sequential schedule queues each device job as it creates it, + so their jobs are recorded once they are queued. + A worker may have finished one of them in the meantime, when there was no record yet to report to, + so whatever Redis already knows to be over is taken from there. + A run which queued no jobs, such as a report with nothing new to report on, has nothing left to do, so it succeeded. + + :param jobs: the logical job key, the kind and the RQ job of each job, in the order they were queued. + """ + run = db.session.get(AutomationRun, run_id) + if run is None: + raise ValueError(f"Automation run {run_id} does not exist.") + now = _now_utc() + if not jobs: + run.execution_state = "succeeded" + run.execution_completed_at = now + db.session.commit() + return + run.plan = { + **dict(run.plan or {}), + "jobs": [ + {"logical_job_key": key, "kind": kind, "rq_job_id": job.id} + for key, kind, job in jobs + ], + } + for key, kind, job in jobs: + db.session.add( + AutomationRunJob( + run=run, + logical_job_key=key, + rq_job_id=job.id, + queue=job.origin, + kind=kind, + status="queued", + enqueued_at=now, + ) + ) + run.first_enqueued_at = run.first_enqueued_at or now + db.session.commit() + for key, _, job in jobs: + status = job.get_status(refresh=True) + if status == JobStatus.FINISHED: + record_automation_job_succeeded(run_id, key) + elif status == JobStatus.FAILED: + traceback_lines = (job.exc_info or "").strip().splitlines() + record_automation_job_failed( + run_id, + key, + AutomationJobFailed( + traceback_lines[-1] if traceback_lines else "unknown error" + ), + ) + + +def record_outcome_on_automation_run(job_function): + """Record a job's start and outcome on the automation run that queued it, if one did. + + Decorates the function a schedule, report or plugin job runs. + The job is looked up by its RQ ID among the jobs its run recorded (see `record_automation_run_jobs`), + so a job queued outside a run, or on behalf of a run that never recorded it (such as a fallback schedule job), is left alone. + A forecast job reports on its own, as its run hands it the key it was recorded under. + """ + + @functools.wraps(job_function) + def wrapper(*args, **kwargs): + rq_job = get_current_job() + run_id = ( + (rq_job.meta.get("trigger") or {}).get("automation_run_id") + if rq_job is not None + else None + ) + if run_id is None: + return job_function(*args, **kwargs) + # The run records its jobs once they are queued, so a job which starts right away may not find itself yet, + # in which case it looks again once it is done (or the run catches up from Redis, see `record_automation_run_jobs`). + logical_job_key = _logical_job_key(run_id, rq_job.id) + record_automation_job_started(run_id, logical_job_key) + try: + result = job_function(*args, **kwargs) + except Exception as exc: + # The job may have failed on the database itself, so get a usable session before looking anything up. + db.session.rollback() + record_automation_job_failed( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id), exc + ) + raise + record_automation_job_succeeded( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id) + ) + return result + + return wrapper + + +def _logical_job_key(run_id: int, rq_job_id: str) -> str | None: + """Return the key under which a run recorded the job with this RQ ID, if it did.""" + return db.session.scalar( + select(AutomationRunJob.logical_job_key).filter_by( + run_id=run_id, rq_job_id=rq_job_id + ) + ) + + # Fields naming a sensor that a scheduler records its results on, rather than reads from. # A scheduler hands its results to `make_schedule` as (sensor, data) pairs, and these are the fields that decide which sensors those are: # besides the power sensor of each device in the flex-model, its state of charge and its consumption and production sensors, @@ -2452,6 +2564,8 @@ def _run_report_automation( current_app.logger.info( f"Report automation {automation.id} has nothing new to report on, up to {parameters['end']}, so it queues no job." ) + if automation_run is not None: + record_automation_run_jobs(automation_run.id, []) return { "job_id": None, "n_jobs": 0, @@ -2472,7 +2586,13 @@ def _run_report_automation( automation_id=automation.id, automation_run_id=automation_run.id if automation_run is not None else None, ) - return reporter.compute(as_job=True, parameters=parameters) + returns = reporter.compute(as_job=True, parameters=parameters) + if automation_run is not None: + job = Job.fetch( + returns["job_id"], connection=current_app.queues["reporting"].connection + ) + record_automation_run_jobs(automation_run.id, [("report", "report", job)]) + return returns def _run_schedule_automation( @@ -2529,5 +2649,20 @@ def _run_schedule_automation( trigger=trigger, **scheduler_kwargs, ) + if automation_run is not None: + if trigger_data["sequential"]: + # The returned job is the wrap-up job, which names the device jobs it waits for. + connection = current_app.queues["scheduling"].connection + run_jobs = [ + ( + f"device-{index}", + "schedule-device", + Job.fetch(job_id, connection=connection), + ) + for index, job_id in enumerate(job.args[0]) + ] + [("wrap-up", "schedule-wrap-up", job)] + else: + run_jobs = [("schedule", "schedule", job)] + record_automation_run_jobs(automation_run.id, run_jobs) n_jobs = len(job.args[0]) + 1 if trigger_data["sequential"] else 1 return {"job_id": job.id, "n_jobs": n_jobs} diff --git a/flexmeasures/data/services/reporting.py b/flexmeasures/data/services/reporting.py index e22230f99c..5fc2b54ed9 100644 --- a/flexmeasures/data/services/reporting.py +++ b/flexmeasures/data/services/reporting.py @@ -11,6 +11,7 @@ from flexmeasures.data import db from flexmeasures.data.schemas.reporting import ReporterParametersSchema +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.utils import save_to_db if TYPE_CHECKING: @@ -88,6 +89,7 @@ class ReportWritesUncheckedSensor(PermissionError): """Raised when a reporter returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def run_report_job(data_source_id: int, parameters: dict) -> list[dict]: """Compute and store a report in a reporting worker. diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index eaaee0a274..488e2cd94e 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -36,6 +36,7 @@ from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException from flexmeasures.data.models.planning.process import ProcessScheduler from flexmeasures.data.services.scheduling_result import SchedulingJobResult +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.generic_assets import GenericAsset as Asset from flexmeasures.data.models.data_sources import DataSource @@ -347,6 +348,7 @@ def create_scheduling_job( return job +@record_outcome_on_automation_run def cb_done_sequential_scheduling_job(jobs_ids: list[str]): """ TODO: maybe check if any of the subjobs used a fallback scheduler or accrued a relaxation penalty. @@ -912,6 +914,7 @@ class ScheduleWritesUncheckedSensor(PermissionError): """Raised when a scheduler returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def make_schedule( # noqa: C901 sensor_id: int | None = None, start: datetime | None = None, From 196ccc15fc2a95561c0052ea55982eaf8fe4f904 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 02/20] data/tests: cover how schedule, report and plugin runs end Context: - Schedule, report and plugin runs now record how their jobs ended. Change: - Schedule runs, single and sequential, succeed once their jobs did and fail with the job's error; a job queued outside a run is not recorded. - A plugin run records its job's outcome, also when the job finished before the run recorded it. - Report run tests check the run's outcome: succeeded, nothing to do, and refused. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_automations.py | 24 ++++ .../data/tests/test_automations_fresh_db.py | 130 ++++++++++++++++++ .../tests/test_plugin_automations_fresh_db.py | 77 +++++++++++ 3 files changed, 231 insertions(+) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index fdc0a35ac9..2f782b8fe6 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1724,6 +1724,14 @@ def test_run_report_automation( ) assert (stored_report.values.T == [1, 2 + 3, 4 + 5, 6 + 7, 8 + 9]).all() + # the run followed its job to the end + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run.id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("report", "succeeded") + ] + assert run.execution_state == "succeeded" + # the successful job recorded the end of the report window as covered import pandas as pd @@ -1770,6 +1778,12 @@ def test_report_automation_with_nothing_new_to_report_queues_no_job( assert result.exit_code == 0, result.output assert "queued 0 reporting job(s)" in result.output, result.output assert len(app.queues["reporting"].jobs) == 0 + # A run with nothing to do is done. + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.job_intents == [] + assert run.execution_state == "succeeded" # Run on demand, it says why it queued nothing. result = runner.invoke(run_one_automation, ["--automation", str(automation.id)]) @@ -1836,9 +1850,19 @@ def test_report_automation_refuses_a_sensor_nobody_checked( mocker.patch( "flexmeasures.data.services.reporting.get_current_job", return_value=job ) + mocker.patch( + "flexmeasures.data.services.automations.get_current_job", return_value=job + ) with pytest.raises(ReportWritesUncheckedSensor, match=str(sensor1_id)): run_report_job(**job.kwargs) + # the refusal failed the run, saying why + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.execution_state == "failed" + assert run.last_error_type == "ReportWritesUncheckedSensor" + # a refused report covers nothing, so the next run still starts where the last successful one ended assert not app.redis_connection.get(f"automation-last-run:{automation.id}") diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index d479ad3c07..4813efecc8 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -5,6 +5,7 @@ import isodate import pytest from rq.job import Job +from sqlalchemy import select from sqlalchemy.exc import IntegrityError from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule @@ -223,6 +224,135 @@ def test_a_durable_run_schedules_what_it_was_planned_with( ) +def _planned_schedule_run(fresh_db, battery, sequential: bool): + """Set up a schedule automation on the battery, and the durable run a runner would have claimed for it.""" + message = message_for_trigger_schedule() + flex_model = message.pop("flex-model") + flex_model["sensor"] = battery.sensors[0].id + parameters = {**message, "flex-model": [flex_model], "sequential": sequential} + automation = build_schedule_automation( + battery, + name="Nightly schedules", + cronstr="0 0 * * *", + parameters=parameters, + ) + fresh_db.session.add(automation) + fresh_db.session.flush() + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=parameters, + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return automation, run + + +@pytest.mark.parametrize( + "sequential, expected_jobs", + [(False, ["schedule"]), (True, ["device-0", "wrap-up"])], +) +def test_a_schedule_run_records_how_its_jobs_ended( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + sequential, + expected_jobs, +): + """A schedule run records each job it queued, and succeeds once the workers finished all of them.""" + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + run = fresh_db.session.get(AutomationRun, run_id) + assert [intent.logical_job_key for intent in run.job_intents] == sorted( + expected_jobs + ) + assert {intent.status for intent in run.job_intents} == {"queued"} + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert {intent.status for intent in run.job_intents} == {"succeeded"} + assert run.execution_state == "succeeded" + assert run.execution_completed_at is not None + + +@pytest.mark.parametrize("sequential", [False, True]) +def test_a_schedule_run_records_a_failed_job( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, + sequential, +): + """A schedule run whose job fails is failed, with the job's error, and over once nothing of it runs any more. + + A failed device job of a sequential schedule leaves its wrap-up job waiting for good, which must not keep the run going. + """ + from flexmeasures.data.services import scheduling + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_to_db", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "failed" + assert run.last_error_type == "ValueError" + assert run.last_error_message == "The database is read-only today." + assert run.execution_completed_at is not None + + +def test_a_schedule_job_queued_outside_a_run_is_not_recorded( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, +): + """Running an automation on demand has no run to record its job on, and its job runs as it always did.""" + from flexmeasures.data.models.automations import AutomationRunJob + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, _ = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + returns = run_automation(automation) + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + job = Job.fetch(returns["job_id"], connection=app.queues["scheduling"].connection) + assert job.is_finished + assert fresh_db.session.scalars(select(AutomationRunJob)).all() == [] + + def test_run_day_ahead_schedule_automation( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 77ff20b5bb..c2ecd8ed54 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -417,3 +417,80 @@ def test_a_source_that_is_not_the_users_to_work_with_is_refused_for_a_plugin_typ assert "not yours to work with" in refusal.value.api_message finally: logout_user() + + +def _claimed_run(fresh_db, automation): + """Return the durable run a runner would have claimed for this automation.""" + from datetime import datetime, timezone + + from flexmeasures.data.models.automations import AutomationRun + + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=dict(automation.parameters), + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return run + + +def test_a_plugin_run_records_how_its_job_ended( + fresh_db, app, ingestion_plugin, ingestion_assets +): + """A plugin run names itself on its job, records the job, and succeeds once a worker finished it.""" + from flexmeasures.data.models.automations import AutomationRun + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + + result = run_automation(automation, automation_run=run) + job = Job.fetch(result["job_id"], connection=app.redis_connection) + assert job.meta["trigger"]["automation_run_id"] == run_id + work_on_rq(app.queues["ingestion"], job=job) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("job", "succeeded") + ] + assert run.execution_state == "succeeded" + + +def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A worker may finish a job before the run that queued it has recorded it, so the run takes the outcome from Redis.""" + from flexmeasures.data.models.automations import AutomationRun + from flexmeasures.data.services import automations as automation_services + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + record_automation_run_jobs = automation_services.record_automation_run_jobs + held_back = [] + monkeypatch.setattr( + automation_services, + "record_automation_run_jobs", + lambda *args: held_back.append(args), + ) + result = run_automation(automation, automation_run=run) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + record_automation_run_jobs(*held_back[0]) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "succeeded" From da14723a3c893c01f64788505f1981e6345d3a1b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 03/20] docs: every automation run records how its jobs ended Context: - Schedule, report and plugin runs now record how their jobs ended, as forecast runs did. Change: - Say which jobs each type of run records, that a run without jobs has succeeded, and that a schedule which fell back counts as failed. Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f8a6d3feb7..737fd4b2de 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -268,8 +268,11 @@ and the run's record says what went wrong on each attempt. Retrying a failed dispatch this way is what a forecast run does. A schedule run is recorded, claimed and reported in just the same way, but is left where it failed rather than dispatched again, because its jobs get a fresh ID on every dispatch, so a retry could not tell an already queued schedule from a missing one. -A forecast run also records each job it created, and how that job ended, which is what its *execution* state describes. -A schedule or report run records its dispatch in the same way, while its execution state stays ``pending``, because the jobs of those runs are not recorded individually yet. +A run also records each job it created, and how that job ended, which is what its *execution* state describes. +This holds for every type of automation: a schedule run records its scheduling job (or, for a sequential schedule, each device job and the job wrapping them up), +a report run its reporting job, and a plugin run the job its plugin queued. +A run which queued no jobs, such as a report run with nothing new to report on, has succeeded. +A job which a schedule queues to fall back on a simpler scheduler is not one of the run's jobs, so a run whose schedule fell back has failed, with the error that made it fall back. A run tracks two things separately: how far its *dispatch* got (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), and how its *execution* by the workers ended (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``). Each attempt to dispatch a run is recorded too, with the runner which made it, what it queued, and why it failed if it did. From 2441424fd5904df974475afaafc956ff1da69123 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:50:15 +0100 Subject: [PATCH 04/20] changelog: append PR #2670 to the durable automation runs entry Context: - Schedule, report and plugin runs now record how their jobs ended, not only forecast runs. Change: - Append PR #2670 to the #2457 entry, and drop "for its forecast jobs", which no longer holds. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4998df87c2..fcfa8d1415 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -112,7 +112,7 @@ Automations arrived over several pull requests. This is what each of them contri * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_] * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2299 `_] -* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and, for its forecast jobs, how they ended [see `PR #2457 `_] +* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and how its jobs ended [see `PR #2457 `_ and `PR #2670 `_] * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization [see `PR #2553 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] From 56e5f8300548dead50abb6cc039f2f2e2d4290d2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:41 +0100 Subject: [PATCH 05/20] data/models: record when an automation run was reported by monitoring Context: - #2519: a monitor command will email a digest of failed automation runs, and has to report each failure once, also when a monitoring run is missed. Change: - Add a nullable alerted_at column to AutomationRun, with a migration. - Add a partial index on the runs the monitor looks at (not reported yet, and failed, running or not fully dispatched), so that its hourly query does not scan the whole run history. Signed-off-by: Mohamed Belhsan Hmida --- ...1c362_add_alerted_at_to_automation_runs.py | 43 +++++++++++++++++++ flexmeasures/data/models/automations.py | 3 ++ 2 files changed, 46 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py new file mode 100644 index 0000000000..eac54bffce --- /dev/null +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -0,0 +1,43 @@ +"""Record when an automation run was reported by `flexmeasures monitor automations` + +Revision ID: d4b7e9a1c362 +Revises: c5e1a7b94d20 +Create Date: 2026-10-07 10:00:00.000000 + +The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. +Every run recorded before this upgrade has no mark, so the first monitoring run reports the failures already on record, collapsed per automation. + +The partial index holds only the runs the monitor has to look at: +those not reported yet which failed, are still running, or never finished their dispatch. +Every other run, which is nearly all of them, stays out of it. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "d4b7e9a1c362" +down_revision = "c5e1a7b94d20" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "automation_run", + sa.Column("alerted_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "automation_run_unalerted_idx", + "automation_run", + ["automation_id"], + postgresql_where=sa.text( + "alerted_at IS NULL" + " AND (execution_state = 'failed' OR execution_state = 'running' OR dispatch_completed_at IS NULL)" + ), + ) + + +def downgrade(): + op.drop_index("automation_run_unalerted_idx", table_name="automation_run") + op.drop_column("automation_run", "alerted_at") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index 78e275f533..2b419ee1fc 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -243,6 +243,8 @@ class AutomationRun(db.Model): execution_completed_at = db.Column(db.DateTime(timezone=True), nullable=True) last_error_type = db.Column(db.String(160), nullable=True) last_error_message = db.Column(db.Text, nullable=True) + # When `flexmeasures monitor automations` reported this run as failed or stuck, so that it is reported once. + alerted_at = db.Column(db.DateTime(timezone=True), nullable=True) parameters = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) plan = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) @@ -272,6 +274,7 @@ class AutomationRun(db.Model): "dispatch_completed_at", "execution_started_at", "execution_completed_at", + "alerted_at", ) def validate_datetime_is_aware( self, key: str, value: datetime | None From 39ca4a05d89b1c64612d21e4fd86a391b5d3ade1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:41 +0100 Subject: [PATCH 06/20] utils/config: add FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER Context: - #2519: a run whose worker died stays "running" for good, so monitoring needs a threshold after which it counts as stuck. Change: - Add the setting, defaulting to six hours, well above any job timeout. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/utils/config_defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index a163c09035..3f681a4a6f 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -125,6 +125,8 @@ class Config(object): [] ) # Deprecated. Use FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS instead. FLEXMEASURES_MONITORING_MAIL_RECIPIENTS: list[str] = [] + # How long an automation run may execute before `flexmeasures monitor automations` reports it as stuck. + FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER: timedelta = timedelta(hours=6) FLEXMEASURES_PLATFORM_NAME: str | list[str | tuple[str, list[str]]] = "FlexMeasures" FLEXMEASURES_MODE: str = "" From ddfbe7ade0b000843924f9609c90934661ce7ccc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 07/20] data/services: find the automation runs that failed and were not reported yet Context: - #2519: monitoring reports runs whose execution failed, which are stuck, or whose dispatch failed for good. Change: - Add get_unalerted_failed_automation_runs, with one criterion for every automation type: only a forecast run with dispatch attempts to spare is left out, as it will be dispatched again, and so is a run that a runner still holds a live claim on. - Add group_failed_runs_by_automation, collapsing failures per automation with their counts, latest run and latest error. Signed-off-by: Mohamed Belhsan Hmida --- .../data/services/automation_monitoring.py | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 flexmeasures/data/services/automation_monitoring.py diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py new file mode 100644 index 0000000000..0b82190887 --- /dev/null +++ b/flexmeasures/data/services/automation_monitoring.py @@ -0,0 +1,150 @@ +"""Find the automation runs that failed, so that monitoring can report them.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from sqlalchemy import and_, not_, or_, select +from sqlalchemy.orm import selectinload + +from flexmeasures.data import db +from flexmeasures.data.models.automations import Automation, AutomationRun +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.services.automations import AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS +from flexmeasures.utils.time_utils import server_now + +EXECUTION_FAILED = "execution failed" +STUCK = "stuck" +DISPATCH_FAILED = "dispatch failed" + + +@dataclass(frozen=True) +class FailedAutomationRun: + """An automation run that failed, with the way it failed.""" + + run: AutomationRun + reason: str + + +@dataclass +class AutomationFailures: + """The failed runs of one automation, collapsed into what a digest says about them.""" + + automation: Automation + failed_runs: list[FailedAutomationRun] = field(default_factory=list) + + @property + def reason_counts(self) -> Counter: + """How many runs failed in each way.""" + return Counter(failed_run.reason for failed_run in self.failed_runs) + + @property + def last_run(self) -> AutomationRun: + """The failed run that was due most recently.""" + return max( + (failed_run.run for failed_run in self.failed_runs), + key=lambda run: (run.scheduled_at, run.id), + ) + + @property + def last_error(self) -> str | None: + """The error of the most recent failed run that recorded one, if any did.""" + runs_with_an_error = [ + failed_run.run + for failed_run in self.failed_runs + if failed_run.run.last_error_type or failed_run.run.last_error_message + ] + if not runs_with_an_error: + return None + run = max(runs_with_an_error, key=lambda run: (run.scheduled_at, run.id)) + return ": ".join( + part for part in (run.last_error_type, run.last_error_message) if part + ) + + +def _dispatch_failed_for_good(now: datetime): + """Return the criterion for a run whose dispatch did not finish and will not be attempted again. + + No runner may hold a live claim on it, or it is still being dispatched. + Only a forecast run is dispatched again, until its attempts are used up (see `get_dispatchable_automation_runs`), + so a run of any other type has failed for good after its first attempt. + """ + return and_( + AutomationRun.dispatch_completed_at.is_(None), + or_( + AutomationRun.claim_owner.is_(None), + AutomationRun.claim_expires_at.is_(None), + AutomationRun.claim_expires_at <= now, + ), + not_( + and_( + AutomationRun.automation_type == "forecasting", + AutomationRun.attempt_count < AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + ), + ) + + +def get_unalerted_failed_automation_runs( + stuck_after: timedelta, + now: datetime | None = None, +) -> list[FailedAutomationRun]: + """Return the automation runs that failed and were not reported yet, oldest first. + + A run failed when its execution failed, when its execution has been running for longer than ``stuck_after``, + or when its dispatch failed and will not be attempted again. + This holds for every automation type, although only the types that record their execution can fail on it. + + :param stuck_after: How long a run may execute before it counts as stuck. + :param now: The moment to judge by, which defaults to the server's current time. + """ + if now is None: + now = server_now() + now = now.astimezone(timezone.utc) + execution_failed = AutomationRun.execution_state == "failed" + stuck = and_( + AutomationRun.execution_state == "running", + AutomationRun.execution_started_at <= now - stuck_after, + ) + runs = db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.alerted_at.is_(None), + or_(execution_failed, stuck, _dispatch_failed_for_good(now)), + ) + .options( + selectinload(AutomationRun.automation) + .joinedload(Automation.asset) + .joinedload(GenericAsset.owner) + ) + .order_by( + AutomationRun.automation_id, AutomationRun.scheduled_at, AutomationRun.id + ) + ).all() + failed_runs = [] + for run in runs: + if run.execution_state == "failed": + reason = EXECUTION_FAILED + elif ( + run.execution_state == "running" + and run.execution_started_at <= now - stuck_after + ): + reason = STUCK + else: + reason = DISPATCH_FAILED + failed_runs.append(FailedAutomationRun(run=run, reason=reason)) + return failed_runs + + +def group_failed_runs_by_automation( + failed_runs: list[FailedAutomationRun], +) -> list[AutomationFailures]: + """Collapse failed runs per automation, keeping the order in which the automations first appear.""" + failures: dict[int, AutomationFailures] = {} + for failed_run in failed_runs: + automation = failed_run.run.automation + failures.setdefault(automation.id, AutomationFailures(automation=automation)) + failures[automation.id].failed_runs.append(failed_run) + return list(failures.values()) From 3cf9ef6449e8c5fb5351a4bed89a529da66fcc0c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 08/20] cli: add flexmeasures monitor automations Context: - #2519: automations fail silently unless someone watches their asset's Automations page. Change: - Email one digest of the failed runs to the monitoring recipients (or --recipient), collapsed per automation, and mark the runs as reported once it is sent. - Send nothing when nothing failed, and refuse to run without recipients, rather than marking runs as reported that nobody heard of. - --stuck-after-minutes overrides FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER. - Kept out of flexmeasures jobs run-automations, so a mail problem never holds up dispatching. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/monitor.py | 122 ++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index 2c4edebbe0..c255a3725b 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -21,6 +21,11 @@ from flexmeasures.data import db from flexmeasures.data.models.task_runs import LatestTaskRun +from flexmeasures.data.services.automation_monitoring import ( + AutomationFailures, + get_unalerted_failed_automation_runs, + group_failed_runs_by_automation, +) from flexmeasures.data.models.user import Account, User from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.api.common.schemas.users import UserIdField @@ -505,4 +510,121 @@ def monitor_last_seen( db.session.commit() +# Error messages can carry a whole traceback, which does not help a digest that lists many automations. +MAX_ERROR_LENGTH_IN_DIGEST = 500 + + +def _format_utc(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def _format_duration(duration: timedelta) -> str: + """Say how long a threshold is, exactly, which a rounded duration such as humanize's would not.""" + minutes = int(duration.total_seconds() // 60) + if minutes % 60 == 0: + hours = minutes // 60 + return "1 hour" if hours == 1 else f"{hours} hours" + return "1 minute" if minutes == 1 else f"{minutes} minutes" + + +def _times(count: int) -> str: + return "once" if count == 1 else f"{count} times" + + +def describe_automation_failures(failures: AutomationFailures) -> str: + """Describe the failed runs of one automation in a few lines, for the monitoring digest.""" + automation = failures.automation + asset = automation.asset + if asset.owner is None: + where = f"public asset {asset.id} '{asset.name}'" + else: + where = f"asset {asset.id} '{asset.name}' of organisation '{asset.owner.name}'" + reasons = ", ".join( + f"{reason}: {count}" for reason, count in failures.reason_counts.items() + ) + last_run = failures.last_run + lines = [ + f"Automation {automation.id} '{automation.name}' ({automation.type}) on {where}:", + f" failed {_times(len(failures.failed_runs))} ({reasons})," + f" last at {_format_utc(last_run.scheduled_at)} (run {last_run.id})", + ] + last_error = failures.last_error + if last_error: + if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: + last_error = last_error[:MAX_ERROR_LENGTH_IN_DIGEST] + " ..." + lines.append(f" last error: {last_error}") + return "\n".join(lines) + + +@fm_monitor.command("automations") +@with_appcontext +@click.option( + "--stuck-after-minutes", + type=click.IntRange(min=1), + default=None, + help="Report a run as stuck once it has been executing for this many minutes. Defaults to the FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER setting (6 hours).", +) +@click.option( + "--recipient", + type=str, + multiple=True, + help="User ID or email address to send the digest to. Use multiple times if needed. If not used, the default monitoring mail recipients are informed.", +) +def monitor_automations( + stuck_after_minutes: int | None = None, + recipient: tuple[str, ...] = (), +): + """ + Email a digest of the automation runs that failed since the last check. + + A run is reported when its execution failed, when it has been executing for too long (see --stuck-after-minutes), + or when its dispatch failed and will not be attempted again. + Each run is reported once, so a monitoring run that was missed is made up for by the next one. + Repeated failures are collapsed per automation, and no email is sent when nothing failed. + + Meant to be run regularly, for instance hourly from cron. + """ + email_recipients = get_monitoring_email_recipients(recipient) + if not email_recipients: + # Marking runs as reported without telling anyone would hide them from the next monitoring run. + raise click.UsageError( + "Nobody to send the digest to. Set FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS or use --recipient." + ) + if stuck_after_minutes is not None: + stuck_after = timedelta(minutes=stuck_after_minutes) + else: + stuck_after = app.config["FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER"] + + now = server_now() + failed_runs = get_unalerted_failed_automation_runs(stuck_after, now=now) + if not failed_runs: + click.secho( + "All good ― no automation runs failed since the last check.", + **MsgStyle.SUCCESS, + ) + return + failures = group_failed_runs_by_automation(failed_runs) + + subject = f"Failed runs of {len(failures)} automation(s)" + body = ( + f"These automations have runs that failed, or that have been executing for longer than {_format_duration(stuck_after)}, which were not reported before:\n\n" + + "\n\n".join(describe_automation_failures(f) for f in failures) + + "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." + ) + capture_message_for_sentry(subject) + email = Message(subject=subject, bcc=email_recipients) + email.body = body + app.mail.send(email) + app.logger.error(body) + + # Only once the digest is out, so that a failure to send it leaves the runs to the next monitoring run. + for failed_run in failed_runs: + failed_run.run.alerted_at = now + db.session.commit() + click.secho( + f"Reported {len(failed_runs)} failed run(s) of {len(failures)} automation(s) to {', '.join(email_recipients)}.", + **MsgStyle.WARN, + ) + + app.cli.add_command(fm_monitor) From 47e200c827a8452568eb348903f8e89906f7a1e2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:42 +0100 Subject: [PATCH 09/20] cli/tests: test flexmeasures monitor automations Context: - #2519: cover the new monitor command. Change: - Drive forecast runs to a failed execution, a stuck execution and an exhausted dispatch through the real runner and worker, and check each is reported once, and not before it counts. - Check which runs count as failed for other types, the threshold's config and CLI override, collapsing per automation, and refusing to run without recipients. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_monitor.py | 3 +- .../test_monitor_automations_fresh_db.py | 353 ++++++++++++++++++ 2 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 flexmeasures/cli/tests/test_monitor_automations_fresh_db.py diff --git a/flexmeasures/cli/tests/test_monitor.py b/flexmeasures/cli/tests/test_monitor.py index caa18fdd6f..ebcfc83da0 100644 --- a/flexmeasures/cli/tests/test_monitor.py +++ b/flexmeasures/cli/tests/test_monitor.py @@ -10,6 +10,7 @@ def test_monitor_help(app): from flexmeasures.cli.monitor import ( fm_monitor, + monitor_automations, monitor_last_seen, monitor_latest_run, ) @@ -19,7 +20,7 @@ def test_monitor_help(app): result = runner.invoke(fm_monitor, ["--help"]) assert result.exit_code == 0 - for command in (monitor_last_seen, monitor_latest_run): + for command in (monitor_last_seen, monitor_latest_run, monitor_automations): result = runner.invoke(command, ["--help"]) assert result.exit_code == 0 diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py new file mode 100644 index 0000000000..e112231904 --- /dev/null +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -0,0 +1,353 @@ +"""Tests for `flexmeasures monitor automations`, which emails a digest of failed automation runs.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from flexmeasures.cli.tests.utils import to_flags +from flexmeasures.data.models.automations import Automation, AutomationRun + +RECIPIENT = "ops@example.test" +DUE_AT = datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc) + + +@pytest.fixture(scope="function") +def clean_redis(app): + app.redis_connection.flushdb() + yield + app.redis_connection.flushdb() + + +@pytest.fixture() +def forecast_automation( + app, fresh_db, clean_redis, setup_fresh_test_forecast_data, freeze_server_now +): + """Create a forecast automation which is due at the frozen minute.""" + from flexmeasures.cli.data_add import add_automation + + freeze_server_now(DUE_AT - timedelta(minutes=2)) + sensor = setup_fresh_test_forecast_data["solar-sensor"] + result = app.test_cli_runner().invoke( + add_automation, + to_flags( + { + "asset": sensor.generic_asset_id, + "name": "Monitored forecasts", + "cron": "0 1 * * *", + "timezone": "UTC", + "sensor": sensor.id, + "duration": "PT2H", + "forecast-frequency": "PT1H", + "max-forecast-horizon": "PT2H", + "retrain-frequency": "PT1H", + } + ), + ) + assert result.exit_code == 0, result.output + freeze_server_now(DUE_AT) + return fresh_db.session.scalars(select(Automation)).one() + + +def run_automations(app, exit_code: int = 0): + from flexmeasures.cli.jobs import run_automations as run_automations_command + + result = app.test_cli_runner().invoke(run_automations_command) + assert result.exit_code == exit_code, result.output + + +def monitor(app, *args: str): + """Run the monitor, returning its result and the emails it sent.""" + from flexmeasures.cli.monitor import monitor_automations + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", RECIPIENT, *args] + ) + return result, outbox + + +def perform_job(app, intent): + from rq import SimpleWorker + + queue = app.queues["forecasting"] + worker = SimpleWorker([queue], connection=queue.connection) + worker.perform_job(queue.fetch_job(intent.rq_job_id), queue) + + +def add_run( + db, automation: Automation, scheduled_at: datetime, **columns +) -> AutomationRun: + """Record a run of the automation in whatever state the test needs, dispatched by default.""" + columns = { + "automation_type": automation.type, + "dispatch_state": "queued", + "dispatch_completed_at": scheduled_at, + "execution_state": "pending", + "attempt_count": 1, + **columns, + } + run = AutomationRun( + automation=automation, + scheduled_at=scheduled_at, + schedule_revision=automation.schedule_revision, + generator_id=automation.generator_id, + parameters={}, + plan={}, + **columns, + ) + db.session.add(run) + db.session.commit() + return run + + +def test_a_failed_forecast_is_reported_once(app, fresh_db, forecast_automation): + """A forecast run whose job failed on the worker is emailed about, and only the first time the monitor runs.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + queue = app.queues["forecasting"] + wrap_up = next(i for i in run.job_intents if i.kind == "forecast-wrap-up") + cycle = next(i for i in run.job_intents if i.kind == "forecast-cycle") + # Make the wrap-up job fail by taking away one of the cycle jobs it reports on. + queue.fetch_job(cycle.rq_job_id).delete() + perform_job(app, wrap_up) + fresh_db.session.refresh(run) + assert run.execution_state == "failed" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert outbox[0].bcc == [RECIPIENT] + assert outbox[0].subject == "Failed runs of 1 automation(s)" + body = outbox[0].body + assert ( + f"Automation {forecast_automation.id} 'Monitored forecasts' (forecasting)" + in body + ) + assert "failed once (execution failed: 1), last at 2026-08-05 01:00 UTC" in body + assert f"last error: {run.last_error_type}" in body + fresh_db.session.refresh(run) + assert run.alerted_at == DUE_AT + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + assert "All good" in result.output + + +def test_a_running_forecast_is_reported_once_it_is_stuck( + app, fresh_db, forecast_automation, freeze_server_now +): + """A forecast run which started executing is reported once it has been at it for longer than the configured threshold.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + # The wrap-up job succeeds while the cycle jobs have not run, so the run keeps executing. + perform_job(app, next(i for i in run.job_intents if i.kind == "forecast-wrap-up")) + fresh_db.session.refresh(run) + assert run.execution_state == "running" + + freeze_server_now(DUE_AT + timedelta(hours=5, minutes=59)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + freeze_server_now(DUE_AT + timedelta(hours=6)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (stuck: 1)" in outbox[0].body + assert "last error" not in outbox[0].body + + +def test_the_stuck_threshold_comes_from_the_config_or_the_command_line( + app, fresh_db, forecast_automation, freeze_server_now, monkeypatch +): + """The command-line option overrides the configured threshold, which overrides the default of six hours.""" + stuck_since = DUE_AT - timedelta(hours=2) + add_run( + fresh_db, + forecast_automation, + stuck_since, + execution_state="running", + execution_started_at=stuck_since, + ) + + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER", timedelta(hours=1) + ) + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "longer than 1 hour" in outbox[0].body + + +def test_a_forecast_dispatch_is_reported_once_its_attempts_are_used_up( + app, fresh_db, forecast_automation, freeze_server_now, mocker +): + """A forecast run whose dispatch failed is retried, so it is only reported once it will not be tried again.""" + from flexmeasures.data.services.automations import ( + AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + + mocker.patch.object( + app.queues["forecasting"], + "enqueue_job", + side_effect=RuntimeError("redis unavailable"), + ) + run_automations(app, exit_code=1) + run = fresh_db.session.scalars(select(AutomationRun)).one() + assert run.dispatch_state == "failed" + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + # Each attempt waits longer than the last, so stepping an hour ahead each time takes them all. + for attempt_no in range(2, AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + 1): + freeze_server_now(DUE_AT + timedelta(hours=attempt_no)) + result, outbox = monitor(app) + assert outbox == [], f"reported before attempt {attempt_no}" + run_automations(app, exit_code=1) + fresh_db.session.refresh(run) + assert run.attempt_count == AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (dispatch failed: 1)" in outbox[0].body + assert "last error: RuntimeError: redis unavailable" in outbox[0].body + + +def test_which_runs_count_as_failed(fresh_db, forecast_automation): + """Failures are judged the same way for every automation type, except that only a forecast run is dispatched again.""" + from flexmeasures.data.services.automation_monitoring import ( + get_unalerted_failed_automation_runs, + ) + + def add(minute: int, **columns) -> AutomationRun: + return add_run( + fresh_db, forecast_automation, DUE_AT - timedelta(minutes=minute), **columns + ) + + expected = { + add(1, execution_state="failed").id: "execution failed", + add( + 2, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=7), + ).id: "stuck", + # A schedule run is never dispatched again, so its first failed attempt is its last. + add( + 3, + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + claim_expires_at=DUE_AT + timedelta(minutes=1), + ).id: "dispatch failed", + # A runner which died mid-dispatch leaves its claim behind, until its lease runs out. + add( + 4, + automation_type="reporting", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:dead", + claim_expires_at=DUE_AT - timedelta(minutes=1), + ).id: "dispatch failed", + } + not_failed = [ + add(5, execution_state="succeeded"), + add( + 6, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=5), + ), + # A forecast run with attempts to spare will be dispatched again. + add(7, dispatch_state="failed", dispatch_completed_at=None), + # A runner is still dispatching this one. + add( + 8, + automation_type="scheduling", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:alive", + claim_expires_at=DUE_AT + timedelta(minutes=5), + ), + add(9, execution_state="failed", alerted_at=DUE_AT - timedelta(hours=1)), + ] + + failed_runs = get_unalerted_failed_automation_runs(timedelta(hours=6), now=DUE_AT) + + assert {f.run.id: f.reason for f in failed_runs} == expected + assert not {run.id for run in not_failed} & {f.run.id for f in failed_runs} + + +def test_repeated_failures_are_collapsed_per_automation( + app, fresh_db, forecast_automation +): + """An automation which failed several times takes up one entry, naming its latest failure and error.""" + for hours_ago, error in ((3, "first"), (1, "latest"), (2, "middle")): + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=hours_ago), + execution_state="failed", + last_error_type="ValueError", + last_error_message=error, + ) + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(minutes=30), + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert body.count(f"Automation {forecast_automation.id} ") == 1 + assert ( + "failed 4 times (execution failed: 3, dispatch failed: 1), last at 2026-08-05 00:30 UTC" + in body + ) + # The dispatch failure recorded no error, so the latest error is that of the latest failed execution. + assert "last error: ValueError: latest" in body + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + +def test_without_recipients_nothing_is_marked_as_reported( + app, fresh_db, forecast_automation, monkeypatch +): + """Without anyone to email, the monitor refuses to run, rather than marking runs as reported that nobody heard of.""" + from flexmeasures.cli.monitor import monitor_automations + + monkeypatch.setitem( + app.config, "FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS", [] + ) + monkeypatch.setitem(app.config, "FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", []) + run = add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result = app.test_cli_runner().invoke(monitor_automations) + + assert result.exit_code == 2 + assert "Nobody to send the digest to" in result.output + fresh_db.session.refresh(run) + assert run.alerted_at is None From 379a8b810fb66f9485f6dbf35f26156837ef7f88 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:51:43 +0100 Subject: [PATCH 10/20] docs: document flexmeasures monitor automations Context: - #2519: new monitor command and setting. Change: - Describe what the command reports in the automations feature docs, and point to it from the host's error monitoring page. - List the command, document the stuck threshold setting, and add a CLI changelog entry. Signed-off-by: Mohamed Belhsan Hmida --- documentation/cli/change_log.rst | 1 + documentation/cli/commands.rst | 1 + documentation/configuration.rst | 13 ++++++++++++- documentation/features/automations.rst | 25 +++++++++++++++++++++++++ documentation/host/error-monitoring.rst | 11 +++++++++++ 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 2a8a3983a6..4a15790a58 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -7,6 +7,7 @@ FlexMeasures CLI Changelog since v1.1.0 | September XX, 2026 ================================= +* Add ``flexmeasures monitor automations`` to email a digest of the automation runs that failed since the last check: runs whose execution failed, which have been executing for too long (``--stuck-after-minutes``), or whose dispatch failed for good. Each run is reported once, and repeated failures are collapsed per automation. * Add ``flexmeasures show automations`` to list all automations, including inactive ones, with the IDs which the edit, delete and run commands expect, and ``flexmeasures show automations --id `` to show one automation in detail: its recurrence in words, timezone, cursor, data generator, parameters and the sensors it reads from and writes to. * Fix ``flexmeasures add schedule --dry-run`` saving a schedule when combined with ``--as-job``: the combination is now rejected, and a dry run reports the beliefs and events it would have saved. * Add ``flexmeasures add forecasts --dry-run`` to compute a forecast and see what it would record, without saving anything to the database. It cannot be combined with ``--as-job``. diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 370dbf6eb8..bcd4a13dc8 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -106,6 +106,7 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures monitor latest-run`` Check if the given task's last successful execution happened less than the allowed time ago. ``flexmeasures monitor last-seen`` Check if given users last contact (via a request) happened less than the allowed time ago. +``flexmeasures monitor automations`` Email a digest of the automation runs that failed since the last check. ================================================= ======================================= diff --git a/documentation/configuration.rst b/documentation/configuration.rst index e760f04622..bfeb155059 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -832,11 +832,22 @@ Default: ``None`` FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run`` and ``flexmeasures monitor last-seen`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. +E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run``, ``flexmeasures monitor last-seen`` and ``flexmeasures monitor automations`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. Default: ``[]`` +.. _monitor_automations_stuck_after: + +FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +How long an automation run may execute before ``flexmeasures monitor automations`` reports it as stuck, for instance because the worker running its job died. +Its ``--stuck-after-minutes`` option overrides this setting. + +Default: ``timedelta(hours=6)`` + + .. _redis-config: Redis diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f8a6d3feb7..1b3c4c68e3 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -278,6 +278,31 @@ This is what an operator needs to tell a run which failed before queueing anythi Editing an automation's cron string or timezone, or reactivating it, counts up its schedule revision. Runs of the old and the new schedule therefore stay distinct, even when they fall on the same scheduled UTC time. +.. _monitoring_automations: + +Monitoring automation runs +-------------------------- + +As a host, you can have FlexMeasures email you a digest of the automation runs that failed, by running ``flexmeasures monitor automations`` regularly, for instance every hour: + +.. code-block:: bash + + 0 * * * * flexmeasures monitor automations + +A run is reported when its execution failed, when it has been executing for longer than six hours, or when its dispatch failed and will not be attempted again. +A forecast run therefore counts once its five dispatch attempts are used up, and a run of any other type after its first failed attempt. +The six hours are set by :ref:`monitor_automations_stuck_after`, and ``--stuck-after-minutes`` overrides that setting. + +Each run is reported once, so if one monitoring run is missed, the next one still reports what failed in the meantime. +Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it last did and with which error. +No email is sent when nothing failed. + +The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). +The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. + +.. note:: Until schedule, report and plugin runs record how their jobs ended, only forecast runs can be reported for a failed or stuck execution. + A failed dispatch is reported for every type. + Running one automation on demand -------------------------------- diff --git a/documentation/host/error-monitoring.rst b/documentation/host/error-monitoring.rst index 1ff8d049cf..ac98a8ec1f 100644 --- a/documentation/host/error-monitoring.rst +++ b/documentation/host/error-monitoring.rst @@ -36,6 +36,17 @@ If you run distinct filters, such as separate checks per account, account group .. todo:: Adding roles and assigning them to accounts is not supported by the UI yet (user roles can be added in the UI). Account roles can be added with ``flexmeasures add account-role``. +Monitoring automations +----------------------- + +The CLI task ``flexmeasures monitor automations`` emails a digest of the automation runs that failed since its last check, with repeated failures collapsed per automation. +Run it from cron, for instance every hour; see :ref:`monitoring_automations` for what it reports. + +.. code-block:: bash + + $ flexmeasures monitor automations --recipient alerts@example.com + + Monitoring task runs --------------------- From f58aaa628992b355c3f7d114518de0fb6f43aa02 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:15:00 +0100 Subject: [PATCH 11/20] data/migrations: revise main's latest migration with the alerted_at migration Context: - Main gained migrations since this branch started, which left two Alembic heads. Change: - Make d4b7e9a1c362 revise f7c2e8b1a409, main's head, rather than c5e1a7b94d20. Signed-off-by: Mohamed Belhsan Hmida --- .../d4b7e9a1c362_add_alerted_at_to_automation_runs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py index eac54bffce..5eb4e48598 100644 --- a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -1,7 +1,7 @@ """Record when an automation run was reported by `flexmeasures monitor automations` Revision ID: d4b7e9a1c362 -Revises: c5e1a7b94d20 +Revises: f7c2e8b1a409 Create Date: 2026-10-07 10:00:00.000000 The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. @@ -17,7 +17,7 @@ # revision identifiers, used by Alembic. revision = "d4b7e9a1c362" -down_revision = "c5e1a7b94d20" +down_revision = "f7c2e8b1a409" branch_labels = None depends_on = None From d37502eab2f3de09963ac70f2e4f349787fbd8a5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:17:55 +0100 Subject: [PATCH 12/20] tests: report failed schedule, report and plugin runs in the monitoring digest Context: - #2670 has schedule, report and plugin runs record how their jobs ended, so their failed executions can now reach the host digest. Change: - Fail a schedule, a report and a plugin job on the worker, and check the digest names the automation, the failed execution and the job's error. - The schedule and plugin runs are marked dispatched, as a runner would, so that only their execution can make them count. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_automations.py | 52 +++++++++++++++++++ .../data/tests/test_automations_fresh_db.py | 44 ++++++++++++++++ .../tests/test_plugin_automations_fresh_db.py | 38 ++++++++++++++ 3 files changed, 134 insertions(+) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 8dd8172da3..4d62131138 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1869,6 +1869,58 @@ def test_report_automation_refuses_a_sensor_nobody_checked( assert not app.redis_connection.get(f"automation-last-run:{automation.id}") +def test_a_failed_report_run_is_reported_by_monitoring( + app, + fresh_db, + setup_dummy_data_fresh_db, + clean_redis, + tmp_path, + freeze_server_now, + mocker, +): + """A report run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.data_add import add_automation + from flexmeasures.cli.jobs import run_automations + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.models.reporting.pandas_reporter import PandasReporter + from flexmeasures.utils.job_utils import work_on_rq + + sensor1_id, sensor2_id, report_sensor_id, _ = setup_dummy_data_fresh_db + report_sensor = fresh_db.session.get(Sensor, report_sensor_id) + cli_input = _report_automation_cli_input( + tmp_path, + sensor1_id, + sensor2_id, + report_sensor_id, + parameters_extra={"start-offset": "DB"}, + asset_id=report_sensor.generic_asset_id, + ) + cli_input[cli_input.index("0 1 * * *")] = "* * * * *" # due every minute + cli_input += ["--timezone", "UTC"] + freeze_server_now(datetime(2023, 4, 10, 10, 0, 30, tzinfo=timezone.utc)) + runner = app.test_cli_runner() + result = runner.invoke(add_automation, cli_input) + assert "Successfully created" in result.output, result.output + automation = fresh_db.session.execute(select(Automation)).scalar_one() + result = runner.invoke(run_automations) + assert "queued 1 reporting job(s)" in result.output, result.output + + mocker.patch.object( + PandasReporter, "compute", side_effect=KeyError("df_agg is missing") + ) + work_on_rq(app.queues["reporting"]) + + with app.mail.record_messages() as outbox: + result = runner.invoke(monitor_automations, ["--recipient", "ops@example.test"]) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' (reporting)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: KeyError: 'df_agg is missing'" in body + + def _two_output_report_automation_cli_input( tmp_path, sensor1_id, diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index 9c18496dd5..6eee1ca831 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -330,6 +330,50 @@ def refuse_to_save(*args, **kwargs): assert run.execution_completed_at is not None +def test_a_failed_schedule_run_is_reported_by_monitoring( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, +): + """A schedule run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services import scheduling + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_generator_results", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} 'Nightly schedules' (scheduling)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ValueError: The database is read-only today." in body + + def test_a_schedule_job_queued_outside_a_run_is_not_recorded( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 22441406d1..4395b510ee 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -519,6 +519,44 @@ def test_a_plugin_run_records_how_its_job_ended( assert run.execution_state == "succeeded" +def test_a_failed_plugin_run_is_reported_by_monitoring( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A plugin run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + + def fail_to_ingest(generator, value): + raise ConnectionError("The meter portal is down.") + + monkeypatch.setattr(MockIngestionGenerator, "_compute", fail_to_ingest) + result = run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' ({automation.type})" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ConnectionError: The meter portal is down." in body + + def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch ): From 3734a4eceb01f349db2e2b96fd616cecee49a30d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:17:55 +0100 Subject: [PATCH 13/20] docs: drop the note that only forecast runs are monitored for their execution Context: - Since #2670, every automation type records how its run's jobs ended. Change: - Remove the note from the monitoring section of the automations docs. Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index 2a23c639be..efc3cc3249 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -303,9 +303,6 @@ No email is sent when nothing failed. The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. -.. note:: Until schedule, report and plugin runs record how their jobs ended, only forecast runs can be reported for a failed or stuck execution. - A failed dispatch is reported for every type. - Running one automation on demand -------------------------------- From d910da47dd35b964c2f451a5f88700182ba97087 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:30:56 +0100 Subject: [PATCH 14/20] changelog: add PR #2671 to the automations, in detail list Context: - PR #2671 adds flexmeasures monitor automations. Change: - Add one bullet for it to the per-PR automations list. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 05b1dd1c17..aa44969680 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -130,6 +130,7 @@ Automations arrived over several pull requests. This is what each of them contri * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization. A plugin job's result is kept for as long as results on its own queue are kept, and reports the number of beliefs it saved [see `PR #2553 `_ and `PR #2644 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] +* Hosts can be emailed a digest of the automation runs that failed, got stuck or could not be queued, with ``flexmeasures monitor automations``, run from cron; each run is reported once, and repeated failures are collapsed per automation [see `PR #2671 `_] v1.0.1 | September 9, 2026 From 5f371081e3106f63536cedcdb07715acc132ffae Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:53:10 +0100 Subject: [PATCH 15/20] docs/api: run-stats records execution state for every automation type Context: - The v3.0-33 entry on run-stats said only forecast runs record their execution state, which PR #2670 makes untrue. Change: - Say execution state and job records are kept for every type of automation. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index af8f7d2f7b..b00201c2fc 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -51,7 +51,7 @@ v3.0-33 | September 1, 2026 - Added ``POST /api/v3_0/assets//reports/trigger`` to queue a one-off report as a background job. It returns ``202 Accepted`` with the canonical ``job`` and ``job-url`` fields, and shares the trigger rate limit with forecast and schedule endpoints. - Added ``GET /api/v3_0/assets//automations`` and ``GET /api/v3_0/assets//automations/`` for listing and inspecting forecast automations, including the sensors an automation reads from and writes to. Each automation shows the IANA ``timezone`` in which its cron expression is interpreted, and a ``cursor``: the offset-aware UTC time of the most recent run it committed to. The cursor advances just before queueing, so it does not indicate that queueing or the forecast itself succeeded. Asset job entries now include ``created_via`` provenance; automation identity is included only when the caller may read that automation. - Added ``GET /api/v3_0/sources/`` to show the full record of one data source, including the attributes in which data generators store their configuration. -- Automation responses now also include ``schedule-revision``, which counts the execution-affecting edits made to the automation's schedule, and automation detail responses gained a ``run-stats`` object. It summarizes the durable runs of that automation and describes the most recent ones: their scheduled time, dispatch state (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), execution state (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``), attempt count, intended and queued job counts, timestamps, last error, latest attempt, and the individual jobs they created. Execution state, and the jobs a run created, are recorded for forecast automations; for the other types a run reports its dispatch, while its execution state stays ``pending``. Both additions are backward compatible: no existing field changed. +- Automation responses now also include ``schedule-revision``, which counts the execution-affecting edits made to the automation's schedule, and automation detail responses gained a ``run-stats`` object. It summarizes the durable runs of that automation and describes the most recent ones: their scheduled time, dispatch state (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), execution state (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``), attempt count, intended and queued job counts, timestamps, last error, latest attempt, and the individual jobs they created. Execution state, and the jobs a run created, are recorded for every type of automation. Both additions are backward compatible: no existing field changed. - ``GET /api/v3_0/sensors//stats`` now reports an ``All sources`` entry summarising every data source, whenever more than one recorded. Its mean divides by the values that were summed, not by ``Number of values``, which also counts rows holding NaN. v3.0-32 | August 11, 2026 From cc5295af9de3bf9744df1dbf93541ee3106551ac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 20 Oct 2026 12:01:24 +0200 Subject: [PATCH 16/20] Record the error of a job that failed before its run recorded it by its stored summary Every queue's exception handler now stores the type and message of a job's error in its meta (#2698), so a run catching up on a job that already failed records that type, rather than a stand-in type around the last traceback line. The meta is read afresh, as the run holds the job as it was when queued. A job without such a summary is still described by its traceback, without a type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0129WrXeJ5gia2pctFH93BqC Signed-off-by: F.N. Claessen --- flexmeasures/data/services/automations.py | 48 ++++++++++++------- .../tests/test_plugin_automations_fresh_db.py | 44 +++++++++++++++++ 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index b3590f6520..d5d88a9056 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -46,6 +46,7 @@ asset_is_in_subtree, descendants_cte, ) +from flexmeasures.data.services.utils import failed_job_reason from flexmeasures.data.services.data_generators import ( check_sensor_access, resolve_data_generator_sensors, @@ -561,7 +562,19 @@ def record_automation_job_failed( logical_job_key: str | None, error: BaseException, ) -> None: - """Record that a worker failed an automation-created job. + """Record that a worker failed an automation-created job, with the error it failed on.""" + _record_automation_job_failure( + run_id, logical_job_key, error.__class__.__name__, str(error) + ) + + +def _record_automation_job_failure( + run_id: int | None, + logical_job_key: str | None, + error_type: str | None, + error_message: str, +) -> None: + """Record that a worker failed an automation-created job, by the type and message of the error it failed on. The job may well have failed on the database itself, which leaves the session in an aborted transaction where every further statement is refused. @@ -587,12 +600,12 @@ def record_automation_job_failed( return intent.status = "failed" intent.finished_at = now - intent.last_error_type = error.__class__.__name__ - intent.last_error_message = str(error) + intent.last_error_type = error_type + intent.last_error_message = error_message run = intent.run _refresh_run_execution_state(run, now) - run.last_error_type = error.__class__.__name__ - run.last_error_message = str(error) + run.last_error_type = error_type + run.last_error_message = error_message db.session.commit() @@ -615,10 +628,6 @@ def reconcile_automation_job_intent(intent: AutomationRunJob) -> bool: return False -class AutomationJobFailed(Exception): - """Stands in for the error of a job that failed before its run had a record of it, as only its traceback is left.""" - - def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> None: """Record the jobs that a report or plugin run queued, so that the run follows what becomes of them. @@ -665,14 +674,19 @@ def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> if status == JobStatus.FINISHED: record_automation_job_succeeded(run_id, key) elif status == JobStatus.FAILED: - traceback_lines = (job.exc_info or "").strip().splitlines() - record_automation_job_failed( - run_id, - key, - AutomationJobFailed( - traceback_lines[-1] if traceback_lines else "unknown error" - ), - ) + _record_automation_job_failure(run_id, key, *_error_of_failed_job(job)) + + +def _error_of_failed_job(job: Job) -> tuple[str | None, str]: + """Return the type and message of the error a job already failed on, which its queue's exception handler stored (see `store_job_exception`). + + A job that failed without such a summary is described as well as its traceback allows, without a type. + """ + # The job was handed over when it was queued, before the worker stored anything on it. + exception = job.get_meta(refresh=True).get("exception") + if isinstance(exception, dict) and exception.get("type"): + return exception["type"], exception.get("message", "") + return None, failed_job_reason(job) or "unknown error" def record_outcome_on_automation_run(job_function): diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 22441406d1..4b9463f56d 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -548,3 +548,47 @@ def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( fresh_db.session.expire_all() run = fresh_db.session.get(AutomationRun, run_id) assert run.execution_state == "succeeded" + + +def test_a_job_that_failed_before_its_run_recorded_it_is_caught_up_on_with_its_error( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A worker may fail a job before the run that queued it has recorded it, so the run takes the error from Redis. + + That is the summary the queue's exception handler stored, so the run names the error's own type, rather than a stand-in. + """ + from flexmeasures.cli.jobs import handle_worker_exception + from flexmeasures.data.models.automations import AutomationRun + from flexmeasures.data.services import automations as automation_services + + def fail(self, value): + raise ValueError("no readings to ingest") + + monkeypatch.setattr(MockIngestionGenerator, "_compute", fail) + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + record_automation_run_jobs = automation_services.record_automation_run_jobs + held_back = [] + monkeypatch.setattr( + automation_services, + "record_automation_run_jobs", + lambda *args: held_back.append(args), + ) + result = run_automation(automation, automation_run=run) + work_on_rq( + app.queues["ingestion"], + exc_handler=handle_worker_exception, + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + record_automation_run_jobs(*held_back[0]) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "failed" + assert (run.last_error_type, run.last_error_message) == ( + "ValueError", + "no readings to ingest", + ) From e83eb290e48e3cfc79f3d826dadb4aff8382b7eb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 29 Oct 2026 09:10:02 +0100 Subject: [PATCH 17/20] data/migrations: revise main's latest migration with the alerted_at migration Context: - Main gained a merge migration since the last sync, which left two Alembic heads. Change: - Make d4b7e9a1c362 revise bf22b21d0990, main's head, rather than f7c2e8b1a409. Signed-off-by: Mohamed Belhsan Hmida --- .../d4b7e9a1c362_add_alerted_at_to_automation_runs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py index 5eb4e48598..16f35b0cb2 100644 --- a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -1,7 +1,7 @@ """Record when an automation run was reported by `flexmeasures monitor automations` Revision ID: d4b7e9a1c362 -Revises: f7c2e8b1a409 +Revises: bf22b21d0990 Create Date: 2026-10-07 10:00:00.000000 The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. @@ -17,7 +17,7 @@ # revision identifiers, used by Alembic. revision = "d4b7e9a1c362" -down_revision = "f7c2e8b1a409" +down_revision = "bf22b21d0990" branch_labels = None depends_on = None From b57c036cfde5b7cb8b9f298cdb5ab7c80d7c535b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 15 Nov 2026 23:56:32 +0100 Subject: [PATCH 18/20] utils/time_utils: move the exact duration formatting out of the monitor CLI Requested in review, so other code can reuse it. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/monitor.py | 13 ++----------- flexmeasures/utils/tests/test_time_utils.py | 15 +++++++++++++++ flexmeasures/utils/time_utils.py | 13 +++++++++++++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index c255a3725b..e29bd71f0c 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -29,7 +29,7 @@ from flexmeasures.data.models.user import Account, User from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.api.common.schemas.users import UserIdField -from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.time_utils import exact_duration_str, server_now from flexmeasures.cli.utils import LoggedClickExceptionGroup, MsgStyle @@ -518,15 +518,6 @@ def _format_utc(moment: datetime) -> str: return moment.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") -def _format_duration(duration: timedelta) -> str: - """Say how long a threshold is, exactly, which a rounded duration such as humanize's would not.""" - minutes = int(duration.total_seconds() // 60) - if minutes % 60 == 0: - hours = minutes // 60 - return "1 hour" if hours == 1 else f"{hours} hours" - return "1 minute" if minutes == 1 else f"{minutes} minutes" - - def _times(count: int) -> str: return "once" if count == 1 else f"{count} times" @@ -607,7 +598,7 @@ def monitor_automations( subject = f"Failed runs of {len(failures)} automation(s)" body = ( - f"These automations have runs that failed, or that have been executing for longer than {_format_duration(stuck_after)}, which were not reported before:\n\n" + f"These automations have runs that failed, or that have been executing for longer than {exact_duration_str(stuck_after)}, which were not reported before:\n\n" + "\n\n".join(describe_automation_failures(f) for f in failures) + "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." ) diff --git a/flexmeasures/utils/tests/test_time_utils.py b/flexmeasures/utils/tests/test_time_utils.py index 9c637acdf7..ac75c877c5 100644 --- a/flexmeasures/utils/tests/test_time_utils.py +++ b/flexmeasures/utils/tests/test_time_utils.py @@ -10,6 +10,7 @@ from flexmeasures.utils.time_utils import ( as_utc_isoformat, duration_isoformat, + exact_duration_str, server_now, naturalized_datetime_str, get_most_recent_clocktime_window, @@ -309,3 +310,17 @@ def test_to_utc_timestamp(input_value, expected_output): assert result is None else: assert result == pytest.approx(expected_output) + + +@pytest.mark.parametrize( + "duration, expected", + [ + (timedelta(hours=6), "6 hours"), + (timedelta(hours=1), "1 hour"), + (timedelta(minutes=90), "90 minutes"), + (timedelta(minutes=1), "1 minute"), + (timedelta(days=1), "24 hours"), + ], +) +def test_exact_duration_str(duration: timedelta, expected: str): + assert exact_duration_str(duration) == expected diff --git a/flexmeasures/utils/time_utils.py b/flexmeasures/utils/time_utils.py index 16f8a52f23..42822d2d1c 100644 --- a/flexmeasures/utils/time_utils.py +++ b/flexmeasures/utils/time_utils.py @@ -305,6 +305,19 @@ def duration_isoformat(duration: timedelta): return re.sub("%P", repl, "P%P") +def exact_duration_str(duration: timedelta) -> str: + """Say how long a duration is in whole hours or minutes, exactly, which a rounded duration such as humanize's would not. + + For example, 6 hours is "6 hours" and 90 minutes is "90 minutes". + Seconds are left out. + """ + minutes = int(duration.total_seconds() // 60) + if minutes % 60 == 0: + hours = minutes // 60 + return "1 hour" if hours == 1 else f"{hours} hours" + return "1 minute" if minutes == 1 else f"{minutes} minutes" + + def determine_minimum_resampling_resolution( event_resolutions: list[timedelta], fallback_resolution: timedelta = timedelta(0), From ff3ec20dc27e94e453a1f03a7532f18bb461d077 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 15 Nov 2026 23:58:49 +0100 Subject: [PATCH 19/20] data/migrations: count runs that failed before the upgrade as reported Otherwise a host's first digest lists every run that ever failed. The upgrade marks the runs that the monitor would report now, by its rules at this revision, and leaves out runs that may still be retried or finish. Signed-off-by: Mohamed Belhsan Hmida --- ...e9a1c362_add_alerted_at_to_automation_runs.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py index 16f35b0cb2..74230d5642 100644 --- a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -5,7 +5,9 @@ Create Date: 2026-10-07 10:00:00.000000 The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. -Every run recorded before this upgrade has no mark, so the first monitoring run reports the failures already on record, collapsed per automation. +The upgrade marks the runs that had already failed for good as reported, so that the first monitoring run does not report every failure on record. +It judges them by the monitor's rules as they stand at this revision, with the default stuck threshold of 6 hours, +and leaves out any run that may still be retried or finish. The partial index holds only the runs the monitor has to look at: those not reported yet which failed, are still running, or never finished their dispatch. @@ -36,6 +38,18 @@ def upgrade(): " AND (execution_state = 'failed' OR execution_state = 'running' OR dispatch_completed_at IS NULL)" ), ) + # Copied from the monitor's query rather than imported, so this migration keeps doing what it did when the query changes. + op.execute(""" + UPDATE automation_run + SET alerted_at = now() + WHERE execution_state = 'failed' + OR (execution_state = 'running' AND execution_started_at <= now() - interval '6 hours') + OR ( + dispatch_completed_at IS NULL + AND (claim_owner IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= now()) + AND NOT (automation_type = 'forecasting' AND attempt_count < 5) + ) + """) def downgrade(): From 644b6042fb45546bcb37e312f1e01f50833b5085 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 16 Nov 2026 00:00:44 +0100 Subject: [PATCH 20/20] cli/monitor: link each automation in the digest to its info on the Automations page The digest is sent from the CLI, where FlexMeasures does not know its own address, so the new FLEXMEASURES_BASE_URL setting gives it. Without the setting, the digest carries no links, as before. The Automations page opens the tab and info panel of the automation named in ?automation=, which show its latest run, its error and its jobs. Signed-off-by: Mohamed Belhsan Hmida --- documentation/configuration.rst | 12 ++++++++ flexmeasures/cli/monitor.py | 17 ++++++++++- .../test_monitor_automations_fresh_db.py | 28 +++++++++++++++++++ .../templates/assets/asset_automations.html | 21 ++++++++++++++ flexmeasures/utils/config_defaults.py | 2 ++ 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 2e118a7d72..f88d48791f 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -879,6 +879,18 @@ Its ``--stuck-after-minutes`` option overrides this setting. Default: ``timedelta(hours=6)`` +.. _base_url: + +FLEXMEASURES_BASE_URL +^^^^^^^^^^^^^^^^^^^^^ + +Where this server is reached on the web, for instance ``"https://ems.example.com"``. +Emails sent from the CLI, outside of any request, use it to link to the web pages they are about. +For instance, the digest of ``flexmeasures monitor automations`` links each failing automation to the Automations page of its asset. +Without it, these emails carry no links. + +Default: ``None`` + .. _redis-config: diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index e29bd71f0c..f8fdaafe85 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -522,6 +522,14 @@ def _times(count: int) -> str: return "once" if count == 1 else f"{count} times" +def automation_page_url(automation) -> str | None: + """Link to the Automations page of the automation's asset, opening that automation's info, if the server knows its own address.""" + base_url = app.config.get("FLEXMEASURES_BASE_URL") + if not base_url: + return None + return f"{base_url.rstrip('/')}/assets/{automation.asset_id}/automations?automation={automation.id}" + + def describe_automation_failures(failures: AutomationFailures) -> str: """Describe the failed runs of one automation in a few lines, for the monitoring digest.""" automation = failures.automation @@ -544,6 +552,9 @@ def describe_automation_failures(failures: AutomationFailures) -> str: if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: last_error = last_error[:MAX_ERROR_LENGTH_IN_DIGEST] + " ..." lines.append(f" last error: {last_error}") + url = automation_page_url(automation) + if url: + lines.append(f" runs and jobs: {url}") return "\n".join(lines) @@ -600,7 +611,11 @@ def monitor_automations( body = ( f"These automations have runs that failed, or that have been executing for longer than {exact_duration_str(stuck_after)}, which were not reported before:\n\n" + "\n\n".join(describe_automation_failures(f) for f in failures) - + "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." + + ( + "\n\nWe suggest to check the logs." + if app.config.get("FLEXMEASURES_BASE_URL") + else "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." + ) ) capture_message_for_sentry(subject) email = Message(subject=subject, bcc=email_recipients) diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py index e112231904..7859b8d022 100644 --- a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -351,3 +351,31 @@ def test_without_recipients_nothing_is_marked_as_reported( assert "Nobody to send the digest to" in result.output fresh_db.session.refresh(run) assert run.alerted_at is None + + +def test_the_digest_links_to_the_automation_once_the_base_url_is_set( + app, fresh_db, forecast_automation, monkeypatch +): + """With the server's own address configured, each automation in the digest links to its info on the Automations page.""" + add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + link = f"/assets/{forecast_automation.asset_id}/automations?automation={forecast_automation.id}" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert link not in outbox[0].body + + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=1), + execution_state="failed", + ) + monkeypatch.setitem( + app.config, "FLEXMEASURES_BASE_URL", "https://ems.example.test/" + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert f" runs and jobs: https://ems.example.test{link}" in outbox[0].body diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 3ddbb9285e..e70b7f2a2c 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -824,6 +824,26 @@
Recently created jobs // The types the page keeps a tab and a table for. const AUTOMATION_TYPES = Object.keys(automationTypes); + /* A link can point at one automation (?automation=), as the monitoring digest's links do. + Once the listing is in, its tab and its info panel are opened, which show its latest run, its error and its jobs. */ + let automationToOpen = Number(new URLSearchParams(window.location.search).get("automation")) || null; + + function openLinkedAutomation() { + const automation = automationToOpen === null ? undefined : automationsById.get(automationToOpen); + automationToOpen = null; // only on the first load, not on every refresh + if (automation === undefined) { + return; + } + const tab = document.getElementById(`${automation.type}-tab`); + if (tab) { + bootstrap.Tab.getOrCreateInstance(tab).show(); + } + const modal = document.getElementById(`AutomationInfoModal-${automation.id}`); + if (modal) { + bootstrap.Modal.getOrCreateInstance(modal).show(); + } + } + function loadAutomations({ rebuild = false } = {}) { // One refresh at a time: a slow response must not pile up behind the next tick. if (loadInFlight) { @@ -846,6 +866,7 @@
Recently created jobs } } markAutomationsRefreshed(); + openLinkedAutomation(); // The listing counts the jobs itself, so it is also where it says when Redis could not be reached. if (res["redis-connection-err"]) { showAutomationsError(res["redis-connection-err"]); diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 967611d7dd..27c7c43865 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -127,6 +127,8 @@ class Config(object): FLEXMEASURES_MONITORING_MAIL_RECIPIENTS: list[str] = [] # How long an automation run may execute before `flexmeasures monitor automations` reports it as stuck. FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER: timedelta = timedelta(hours=6) + # Where this server is reached on the web, e.g. "https://ems.example.com", for links in emails sent outside of a request, such as monitoring digests. + FLEXMEASURES_BASE_URL: str | None = None FLEXMEASURES_PLATFORM_NAME: str | list[str | tuple[str, list[str]]] = "FlexMeasures" FLEXMEASURES_MODE: str = ""