From 6c7798aef065afddc1adb14aa6942018e890be23 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:22 +0100 Subject: [PATCH 01/47] data/services/automations: record how schedule, report and plugin runs end Context: - Only forecast runs followed their jobs, so schedule, report and plugin runs stayed at execution state 'pending' whatever became of their jobs. Change: - record_automation_run_jobs records the jobs such a run queued, then takes the outcome of any job that already finished or failed from Redis; a run with no jobs has succeeded. - The record_outcome_on_automation_run decorator records a job's start, success or failure on its run, found through the job's RQ ID, and wraps make_schedule, the sequential wrap-up job, run_report_job and execute_automation_job. - Plugin jobs carry automation_run_id in their trigger. - A schedule that fell back counts as a failed run, as the fallback job is not one of the run's recorded jobs. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/automations.py | 17 ++- flexmeasures/data/services/automations.py | 139 +++++++++++++++++++++- flexmeasures/data/services/reporting.py | 2 + flexmeasures/data/services/scheduling.py | 3 + 4 files changed, 156 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/automations.py b/flexmeasures/data/automations.py index dc97a37cc6..e6ea8a3909 100644 --- a/flexmeasures/data/automations.py +++ b/flexmeasures/data/automations.py @@ -15,6 +15,7 @@ from flexmeasures.data import db from flexmeasures.data.models.data_sources import DataGenerator +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.utils.job_utils import KNOWN_JOB_QUEUES @@ -180,8 +181,8 @@ def run(self, automation, automation_run=None, scheduled_at=None): A durable run (see `dispatch_automation_run`) carries the parameters and the jobs its run was planned with, which the built-in types read to pick up where an earlier attempt at the same run stopped. - A plugin type's run is dispatched with the parameters its run was planned with, but records no jobs of its own, - so such a run reports how far its dispatch got and leaves its execution state at 'pending'. + A plugin type's run is dispatched with the parameters its run was planned with, + and records its one job once queued, so that the run follows how the job ends. """ if self.generator_class is None: from flexmeasures.data.services.automations import ( @@ -212,6 +213,9 @@ def run(self, automation, automation_run=None, scheduled_at=None): parameters = planned_parameters db.session.commit() queue = current_app.queues[self.queue] + trigger = {"origin": "automation", "automation_id": automation.id} + if automation_run is not None: + trigger["automation_run_id"] = automation_run.id job = Job.create( execute_automation_job, kwargs={ @@ -232,7 +236,7 @@ def run(self, automation, automation_run=None, scheduled_at=None): ).total_seconds() ), meta={ - "trigger": {"origin": "automation", "automation_id": automation.id}, + "trigger": trigger, "data_source_info": {"id": source_id}, "asset_id": automation.asset_id, "asset_or_sensor": {"class": "Asset", "id": automation.asset_id}, @@ -245,6 +249,12 @@ def run(self, automation, automation_run=None, scheduled_at=None): queue=self.queue, asset_or_sensor_type="asset", ) + if automation_run is not None: + from flexmeasures.data.services.automations import ( + record_automation_run_jobs, + ) + + record_automation_run_jobs(automation_run.id, [("job", self.type_id, job)]) return {"job_id": job.id, "n_jobs": 1} @@ -395,6 +405,7 @@ def check_execution_access(automation, sensors): ) +@record_outcome_on_automation_run def execute_automation_job(automation_id: int, data_source_id: int, parameters: dict): """Compute and persist declared results with provenance in a plugin-enabled worker.""" from flexmeasures.data.models.automations import Automation diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 838dd4e649..4ccd0a07a6 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -5,6 +5,7 @@ from __future__ import annotations from contextlib import contextmanager +import functools from copy import copy, deepcopy from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -21,7 +22,8 @@ from isodate.isoerror import ISO8601Error from flask import current_app from marshmallow import ValidationError -from rq.job import Job +from rq import get_current_job +from rq.job import Job, JobStatus from sqlalchemy import func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import selectinload @@ -574,6 +576,116 @@ def reconcile_automation_job_intent(intent: AutomationRunJob) -> bool: return False +class AutomationJobFailed(Exception): + """Stands in for the error of a job that failed before its run had a record of it, as only its traceback is left.""" + + +def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> None: + """Record the jobs that a schedule, report or plugin run queued, so that the run follows what becomes of them. + + A forecast run records its jobs before queueing them, under IDs it derives from the run (see `ensure_automation_run_job_intents`). + The other types get their job IDs from RQ, and a sequential schedule queues each device job as it creates it, + so their jobs are recorded once they are queued. + A worker may have finished one of them in the meantime, when there was no record yet to report to, + so whatever Redis already knows to be over is taken from there. + A run which queued no jobs, such as a report with nothing new to report on, has nothing left to do, so it succeeded. + + :param jobs: the logical job key, the kind and the RQ job of each job, in the order they were queued. + """ + run = db.session.get(AutomationRun, run_id) + if run is None: + raise ValueError(f"Automation run {run_id} does not exist.") + now = _now_utc() + if not jobs: + run.execution_state = "succeeded" + run.execution_completed_at = now + db.session.commit() + return + run.plan = { + **dict(run.plan or {}), + "jobs": [ + {"logical_job_key": key, "kind": kind, "rq_job_id": job.id} + for key, kind, job in jobs + ], + } + for key, kind, job in jobs: + db.session.add( + AutomationRunJob( + run=run, + logical_job_key=key, + rq_job_id=job.id, + queue=job.origin, + kind=kind, + status="queued", + enqueued_at=now, + ) + ) + run.first_enqueued_at = run.first_enqueued_at or now + db.session.commit() + for key, _, job in jobs: + status = job.get_status(refresh=True) + if status == JobStatus.FINISHED: + record_automation_job_succeeded(run_id, key) + elif status == JobStatus.FAILED: + traceback_lines = (job.exc_info or "").strip().splitlines() + record_automation_job_failed( + run_id, + key, + AutomationJobFailed( + traceback_lines[-1] if traceback_lines else "unknown error" + ), + ) + + +def record_outcome_on_automation_run(job_function): + """Record a job's start and outcome on the automation run that queued it, if one did. + + Decorates the function a schedule, report or plugin job runs. + The job is looked up by its RQ ID among the jobs its run recorded (see `record_automation_run_jobs`), + so a job queued outside a run, or on behalf of a run that never recorded it (such as a fallback schedule job), is left alone. + A forecast job reports on its own, as its run hands it the key it was recorded under. + """ + + @functools.wraps(job_function) + def wrapper(*args, **kwargs): + rq_job = get_current_job() + run_id = ( + (rq_job.meta.get("trigger") or {}).get("automation_run_id") + if rq_job is not None + else None + ) + if run_id is None: + return job_function(*args, **kwargs) + # The run records its jobs once they are queued, so a job which starts right away may not find itself yet, + # in which case it looks again once it is done (or the run catches up from Redis, see `record_automation_run_jobs`). + logical_job_key = _logical_job_key(run_id, rq_job.id) + record_automation_job_started(run_id, logical_job_key) + try: + result = job_function(*args, **kwargs) + except Exception as exc: + # The job may have failed on the database itself, so get a usable session before looking anything up. + db.session.rollback() + record_automation_job_failed( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id), exc + ) + raise + record_automation_job_succeeded( + run_id, logical_job_key or _logical_job_key(run_id, rq_job.id) + ) + return result + + return wrapper + + +def _logical_job_key(run_id: int, rq_job_id: str) -> str | None: + """Return the key under which a run recorded the job with this RQ ID, if it did.""" + return db.session.scalar( + select(AutomationRunJob.logical_job_key).filter_by( + run_id=run_id, rq_job_id=rq_job_id + ) + ) + + # Fields naming a sensor that a scheduler records its results on, rather than reads from. # A scheduler hands its results to `make_schedule` as (sensor, data) pairs, and these are the fields that decide which sensors those are: # besides the power sensor of each device in the flex-model, its state of charge and its consumption and production sensors, @@ -2452,6 +2564,8 @@ def _run_report_automation( current_app.logger.info( f"Report automation {automation.id} has nothing new to report on, up to {parameters['end']}, so it queues no job." ) + if automation_run is not None: + record_automation_run_jobs(automation_run.id, []) return { "job_id": None, "n_jobs": 0, @@ -2472,7 +2586,13 @@ def _run_report_automation( automation_id=automation.id, automation_run_id=automation_run.id if automation_run is not None else None, ) - return reporter.compute(as_job=True, parameters=parameters) + returns = reporter.compute(as_job=True, parameters=parameters) + if automation_run is not None: + job = Job.fetch( + returns["job_id"], connection=current_app.queues["reporting"].connection + ) + record_automation_run_jobs(automation_run.id, [("report", "report", job)]) + return returns def _run_schedule_automation( @@ -2529,5 +2649,20 @@ def _run_schedule_automation( trigger=trigger, **scheduler_kwargs, ) + if automation_run is not None: + if trigger_data["sequential"]: + # The returned job is the wrap-up job, which names the device jobs it waits for. + connection = current_app.queues["scheduling"].connection + run_jobs = [ + ( + f"device-{index}", + "schedule-device", + Job.fetch(job_id, connection=connection), + ) + for index, job_id in enumerate(job.args[0]) + ] + [("wrap-up", "schedule-wrap-up", job)] + else: + run_jobs = [("schedule", "schedule", job)] + record_automation_run_jobs(automation_run.id, run_jobs) n_jobs = len(job.args[0]) + 1 if trigger_data["sequential"] else 1 return {"job_id": job.id, "n_jobs": n_jobs} diff --git a/flexmeasures/data/services/reporting.py b/flexmeasures/data/services/reporting.py index e22230f99c..5fc2b54ed9 100644 --- a/flexmeasures/data/services/reporting.py +++ b/flexmeasures/data/services/reporting.py @@ -11,6 +11,7 @@ from flexmeasures.data import db from flexmeasures.data.schemas.reporting import ReporterParametersSchema +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.utils import save_to_db if TYPE_CHECKING: @@ -88,6 +89,7 @@ class ReportWritesUncheckedSensor(PermissionError): """Raised when a reporter returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def run_report_job(data_source_id: int, parameters: dict) -> list[dict]: """Compute and store a report in a reporting worker. diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index eaaee0a274..488e2cd94e 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -36,6 +36,7 @@ from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException from flexmeasures.data.models.planning.process import ProcessScheduler from flexmeasures.data.services.scheduling_result import SchedulingJobResult +from flexmeasures.data.services.automations import record_outcome_on_automation_run from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.generic_assets import GenericAsset as Asset from flexmeasures.data.models.data_sources import DataSource @@ -347,6 +348,7 @@ def create_scheduling_job( return job +@record_outcome_on_automation_run def cb_done_sequential_scheduling_job(jobs_ids: list[str]): """ TODO: maybe check if any of the subjobs used a fallback scheduler or accrued a relaxation penalty. @@ -912,6 +914,7 @@ class ScheduleWritesUncheckedSensor(PermissionError): """Raised when a scheduler returns results for a sensor that nobody's permissions were checked against.""" +@record_outcome_on_automation_run def make_schedule( # noqa: C901 sensor_id: int | None = None, start: datetime | None = None, From 196ccc15fc2a95561c0052ea55982eaf8fe4f904 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 02/47] data/tests: cover how schedule, report and plugin runs end Context: - Schedule, report and plugin runs now record how their jobs ended. Change: - Schedule runs, single and sequential, succeed once their jobs did and fail with the job's error; a job queued outside a run is not recorded. - A plugin run records its job's outcome, also when the job finished before the run recorded it. - Report run tests check the run's outcome: succeeded, nothing to do, and refused. Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/tests/test_automations.py | 24 ++++ .../data/tests/test_automations_fresh_db.py | 130 ++++++++++++++++++ .../tests/test_plugin_automations_fresh_db.py | 77 +++++++++++ 3 files changed, 231 insertions(+) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index fdc0a35ac9..2f782b8fe6 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1724,6 +1724,14 @@ def test_run_report_automation( ) assert (stored_report.values.T == [1, 2 + 3, 4 + 5, 6 + 7, 8 + 9]).all() + # the run followed its job to the end + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run.id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("report", "succeeded") + ] + assert run.execution_state == "succeeded" + # the successful job recorded the end of the report window as covered import pandas as pd @@ -1770,6 +1778,12 @@ def test_report_automation_with_nothing_new_to_report_queues_no_job( assert result.exit_code == 0, result.output assert "queued 0 reporting job(s)" in result.output, result.output assert len(app.queues["reporting"].jobs) == 0 + # A run with nothing to do is done. + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.job_intents == [] + assert run.execution_state == "succeeded" # Run on demand, it says why it queued nothing. result = runner.invoke(run_one_automation, ["--automation", str(automation.id)]) @@ -1836,9 +1850,19 @@ def test_report_automation_refuses_a_sensor_nobody_checked( mocker.patch( "flexmeasures.data.services.reporting.get_current_job", return_value=job ) + mocker.patch( + "flexmeasures.data.services.automations.get_current_job", return_value=job + ) with pytest.raises(ReportWritesUncheckedSensor, match=str(sensor1_id)): run_report_job(**job.kwargs) + # the refusal failed the run, saying why + from flexmeasures.data.models.automations import AutomationRun + + run = fresh_db.session.execute(select(AutomationRun)).scalar_one() + assert run.execution_state == "failed" + assert run.last_error_type == "ReportWritesUncheckedSensor" + # a refused report covers nothing, so the next run still starts where the last successful one ended assert not app.redis_connection.get(f"automation-last-run:{automation.id}") diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index d479ad3c07..4813efecc8 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -5,6 +5,7 @@ import isodate import pytest from rq.job import Job +from sqlalchemy import select from sqlalchemy.exc import IntegrityError from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule @@ -223,6 +224,135 @@ def test_a_durable_run_schedules_what_it_was_planned_with( ) +def _planned_schedule_run(fresh_db, battery, sequential: bool): + """Set up a schedule automation on the battery, and the durable run a runner would have claimed for it.""" + message = message_for_trigger_schedule() + flex_model = message.pop("flex-model") + flex_model["sensor"] = battery.sensors[0].id + parameters = {**message, "flex-model": [flex_model], "sequential": sequential} + automation = build_schedule_automation( + battery, + name="Nightly schedules", + cronstr="0 0 * * *", + parameters=parameters, + ) + fresh_db.session.add(automation) + fresh_db.session.flush() + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=parameters, + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return automation, run + + +@pytest.mark.parametrize( + "sequential, expected_jobs", + [(False, ["schedule"]), (True, ["device-0", "wrap-up"])], +) +def test_a_schedule_run_records_how_its_jobs_ended( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + sequential, + expected_jobs, +): + """A schedule run records each job it queued, and succeeds once the workers finished all of them.""" + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + run = fresh_db.session.get(AutomationRun, run_id) + assert [intent.logical_job_key for intent in run.job_intents] == sorted( + expected_jobs + ) + assert {intent.status for intent in run.job_intents} == {"queued"} + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert {intent.status for intent in run.job_intents} == {"succeeded"} + assert run.execution_state == "succeeded" + assert run.execution_completed_at is not None + + +@pytest.mark.parametrize("sequential", [False, True]) +def test_a_schedule_run_records_a_failed_job( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, + sequential, +): + """A schedule run whose job fails is failed, with the job's error, and over once nothing of it runs any more. + + A failed device job of a sequential schedule leaves its wrap-up job waiting for good, which must not keep the run going. + """ + from flexmeasures.data.services import scheduling + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential + ) + run_id = run.id + run_automation(automation, automation_run=run) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_to_db", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "failed" + assert run.last_error_type == "ValueError" + assert run.last_error_message == "The database is read-only today." + assert run.execution_completed_at is not None + + +def test_a_schedule_job_queued_outside_a_run_is_not_recorded( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, +): + """Running an automation on demand has no run to record its job on, and its job runs as it always did.""" + from flexmeasures.data.models.automations import AutomationRunJob + from flexmeasures.data.services.scheduling import handle_scheduling_exception + from flexmeasures.utils.job_utils import work_on_rq + + automation, _ = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + returns = run_automation(automation) + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + job = Job.fetch(returns["job_id"], connection=app.queues["scheduling"].connection) + assert job.is_finished + assert fresh_db.session.scalars(select(AutomationRunJob)).all() == [] + + def test_run_day_ahead_schedule_automation( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 77ff20b5bb..c2ecd8ed54 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -417,3 +417,80 @@ def test_a_source_that_is_not_the_users_to_work_with_is_refused_for_a_plugin_typ assert "not yours to work with" in refusal.value.api_message finally: logout_user() + + +def _claimed_run(fresh_db, automation): + """Return the durable run a runner would have claimed for this automation.""" + from datetime import datetime, timezone + + from flexmeasures.data.models.automations import AutomationRun + + run = AutomationRun( + automation=automation, + scheduled_at=datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc), + schedule_revision=automation.schedule_revision, + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state="claimed", + execution_state="pending", + parameters=dict(automation.parameters), + plan={}, + ) + fresh_db.session.add(run) + fresh_db.session.commit() + return run + + +def test_a_plugin_run_records_how_its_job_ended( + fresh_db, app, ingestion_plugin, ingestion_assets +): + """A plugin run names itself on its job, records the job, and succeeds once a worker finished it.""" + from flexmeasures.data.models.automations import AutomationRun + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + + result = run_automation(automation, automation_run=run) + job = Job.fetch(result["job_id"], connection=app.redis_connection) + assert job.meta["trigger"]["automation_run_id"] == run_id + work_on_rq(app.queues["ingestion"], job=job) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert [(intent.logical_job_key, intent.status) for intent in run.job_intents] == [ + ("job", "succeeded") + ] + assert run.execution_state == "succeeded" + + +def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A worker may finish a job before the run that queued it has recorded it, so the run takes the outcome from Redis.""" + from flexmeasures.data.models.automations import AutomationRun + from flexmeasures.data.services import automations as automation_services + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + run_id = run.id + record_automation_run_jobs = automation_services.record_automation_run_jobs + held_back = [] + monkeypatch.setattr( + automation_services, + "record_automation_run_jobs", + lambda *args: held_back.append(args), + ) + result = run_automation(automation, automation_run=run) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + record_automation_run_jobs(*held_back[0]) + + fresh_db.session.expire_all() + run = fresh_db.session.get(AutomationRun, run_id) + assert run.execution_state == "succeeded" From da14723a3c893c01f64788505f1981e6345d3a1b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:42:28 +0100 Subject: [PATCH 03/47] docs: every automation run records how its jobs ended Context: - Schedule, report and plugin runs now record how their jobs ended, as forecast runs did. Change: - Say which jobs each type of run records, that a run without jobs has succeeded, and that a schedule which fell back counts as failed. Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index f8a6d3feb7..737fd4b2de 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -268,8 +268,11 @@ and the run's record says what went wrong on each attempt. Retrying a failed dispatch this way is what a forecast run does. A schedule run is recorded, claimed and reported in just the same way, but is left where it failed rather than dispatched again, because its jobs get a fresh ID on every dispatch, so a retry could not tell an already queued schedule from a missing one. -A forecast run also records each job it created, and how that job ended, which is what its *execution* state describes. -A schedule or report run records its dispatch in the same way, while its execution state stays ``pending``, because the jobs of those runs are not recorded individually yet. +A run also records each job it created, and how that job ended, which is what its *execution* state describes. +This holds for every type of automation: a schedule run records its scheduling job (or, for a sequential schedule, each device job and the job wrapping them up), +a report run its reporting job, and a plugin run the job its plugin queued. +A run which queued no jobs, such as a report run with nothing new to report on, has succeeded. +A job which a schedule queues to fall back on a simpler scheduler is not one of the run's jobs, so a run whose schedule fell back has failed, with the error that made it fall back. A run tracks two things separately: how far its *dispatch* got (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), and how its *execution* by the workers ended (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``). Each attempt to dispatch a run is recorded too, with the runner which made it, what it queued, and why it failed if it did. From 2441424fd5904df974475afaafc956ff1da69123 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 7 Oct 2026 01:50:15 +0100 Subject: [PATCH 04/47] changelog: append PR #2670 to the durable automation runs entry Context: - Schedule, report and plugin runs now record how their jobs ended, not only forecast runs. Change: - Append PR #2670 to the #2457 entry, and drop "for its forecast jobs", which no longer holds. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4998df87c2..fcfa8d1415 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -112,7 +112,7 @@ Automations arrived over several pull requests. This is what each of them contri * Every automation times its runs the same way: a fixed ``start``, ``end`` or ``prior`` in its parameters is refused, as every run would share that moment, and two of ``start-offset``, ``end-offset`` and ``duration`` describe the period each run covers instead, with the offsets applied to the time the run was due on the automation's own clock, so that, for instance, a schedule automation can plan the whole of the next day [see `PR #2551 `_] * Look up automations from the command line with ``flexmeasures show automations``, which lists them all (inactive ones included) with the IDs that the edit, delete and run commands expect, and, with ``--id``, shows a single automation's recurrence, cursor, parameters and the sensors it reads from and writes to [see `PR #2533 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2299 `_] -* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and, for its forecast jobs, how they ended [see `PR #2457 `_] +* Automations now keep a durable record of every scheduled run, so a forecast run which failed before queueing any work is simply picked up again, while one which failed halfway only queues the jobs it still owes; an automation's details show, per run, what it queued, how many attempts that took, and how its jobs ended [see `PR #2457 `_ and `PR #2670 `_] * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization [see `PR #2553 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] From 5f371081e3106f63536cedcdb07715acc132ffae Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 12 Oct 2026 02:53:10 +0100 Subject: [PATCH 05/47] docs/api: run-stats records execution state for every automation type Context: - The v3.0-33 entry on run-stats said only forecast runs record their execution state, which PR #2670 makes untrue. Change: - Say execution state and job records are kept for every type of automation. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index af8f7d2f7b..b00201c2fc 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -51,7 +51,7 @@ v3.0-33 | September 1, 2026 - Added ``POST /api/v3_0/assets//reports/trigger`` to queue a one-off report as a background job. It returns ``202 Accepted`` with the canonical ``job`` and ``job-url`` fields, and shares the trigger rate limit with forecast and schedule endpoints. - Added ``GET /api/v3_0/assets//automations`` and ``GET /api/v3_0/assets//automations/`` for listing and inspecting forecast automations, including the sensors an automation reads from and writes to. Each automation shows the IANA ``timezone`` in which its cron expression is interpreted, and a ``cursor``: the offset-aware UTC time of the most recent run it committed to. The cursor advances just before queueing, so it does not indicate that queueing or the forecast itself succeeded. Asset job entries now include ``created_via`` provenance; automation identity is included only when the caller may read that automation. - Added ``GET /api/v3_0/sources/`` to show the full record of one data source, including the attributes in which data generators store their configuration. -- Automation responses now also include ``schedule-revision``, which counts the execution-affecting edits made to the automation's schedule, and automation detail responses gained a ``run-stats`` object. It summarizes the durable runs of that automation and describes the most recent ones: their scheduled time, dispatch state (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), execution state (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``), attempt count, intended and queued job counts, timestamps, last error, latest attempt, and the individual jobs they created. Execution state, and the jobs a run created, are recorded for forecast automations; for the other types a run reports its dispatch, while its execution state stays ``pending``. Both additions are backward compatible: no existing field changed. +- Automation responses now also include ``schedule-revision``, which counts the execution-affecting edits made to the automation's schedule, and automation detail responses gained a ``run-stats`` object. It summarizes the durable runs of that automation and describes the most recent ones: their scheduled time, dispatch state (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), execution state (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``), attempt count, intended and queued job counts, timestamps, last error, latest attempt, and the individual jobs they created. Execution state, and the jobs a run created, are recorded for every type of automation. Both additions are backward compatible: no existing field changed. - ``GET /api/v3_0/sensors//stats`` now reports an ``All sources`` entry summarising every data source, whenever more than one recorded. Its mean divides by the values that were summed, not by ``Number of values``, which also counts rows holding NaN. v3.0-32 | August 11, 2026 From c9086ee2a44a815c305dfbd9c8996e0c6edff74d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:46 +0100 Subject: [PATCH 06/47] data: let an automation's runs wait for the runs of other automations Context: - Issue #2586: an automation consuming another's results could only be ordered by staggering cron strings Change: - New automation_dependency table, and 'waiting' and 'skipped' run states - A run waits for the upstream runs scheduled for the same time, and is dispatched once they ended, or skipped if one failed (unless failure is allowed) - Waiting runs are released when the upstream run ends, and by the runner's tick as a backstop - Dependencies are validated: same schedule, no cycles, readable upstream automations Signed-off-by: Mohamed Belhsan Hmida --- ...8e4c2d7f913_add_automation_dependencies.py | 98 ++++ flexmeasures/data/models/automations.py | 52 +- flexmeasures/data/services/automations.py | 525 +++++++++++++++++- 3 files changed, 662 insertions(+), 13 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py diff --git a/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py new file mode 100644 index 0000000000..4f70e17ae4 --- /dev/null +++ b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py @@ -0,0 +1,98 @@ +"""add automation dependencies + +Let an automation's runs wait for the run of another automation at the same scheduled time. +A run that waits is 'waiting' to be dispatched, and a run whose upstream run failed is 'skipped'. + +Revision ID: b8e4c2d7f913 +Revises: f7c2e8b1a409 +Create Date: 2026-10-14 10:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "b8e4c2d7f913" +down_revision = "f7c2e8b1a409" +branch_labels = None +depends_on = None + +DISPATCH_STATE_CK = "automation_run_automation_run_dispatch_state_ck" +EXECUTION_STATE_CK = "automation_run_automation_run_execution_state_ck" + + +def upgrade(): + op.create_table( + "automation_dependency", + sa.Column("automation_id", sa.Integer(), nullable=False), + sa.Column("upstream_id", sa.Integer(), nullable=False), + sa.Column( + "allow_failure", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.ForeignKeyConstraint( + ["automation_id"], + ["automation.id"], + name=op.f("automation_dependency_automation_id_automation_fkey"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["upstream_id"], + ["automation.id"], + name=op.f("automation_dependency_upstream_id_automation_fkey"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "automation_id", "upstream_id", name=op.f("automation_dependency_pkey") + ), + ) + op.alter_column("automation_dependency", "allow_failure", server_default=None) + op.create_index( + op.f("automation_dependency_upstream_id_idx"), + "automation_dependency", + ["upstream_id"], + ) + + op.drop_constraint(op.f(DISPATCH_STATE_CK), "automation_run", type_="check") + op.create_check_constraint( + op.f(DISPATCH_STATE_CK), + "automation_run", + "dispatch_state IN ('pending', 'claimed', 'waiting', 'partially_queued', 'queued', 'failed')", + ) + op.drop_constraint(op.f(EXECUTION_STATE_CK), "automation_run", type_="check") + op.create_check_constraint( + op.f(EXECUTION_STATE_CK), + "automation_run", + "execution_state IN ('pending', 'running', 'succeeded', 'failed', 'canceled', 'skipped')", + ) + + +def downgrade(): + # The narrower constraints below would refuse these runs, so give them the states they had before a run could wait. + # A waiting run was claimed and never queued anything, and a skipped run never ran. + op.execute( + "UPDATE automation_run SET dispatch_state = 'failed' WHERE dispatch_state = 'waiting'" + ) + op.execute( + "UPDATE automation_run SET execution_state = 'canceled' WHERE execution_state = 'skipped'" + ) + op.drop_constraint(op.f(EXECUTION_STATE_CK), "automation_run", type_="check") + op.create_check_constraint( + op.f(EXECUTION_STATE_CK), + "automation_run", + "execution_state IN ('pending', 'running', 'succeeded', 'failed', 'canceled')", + ) + op.drop_constraint(op.f(DISPATCH_STATE_CK), "automation_run", type_="check") + op.create_check_constraint( + op.f(DISPATCH_STATE_CK), + "automation_run", + "dispatch_state IN ('pending', 'claimed', 'partially_queued', 'queued', 'failed')", + ) + op.drop_index( + op.f("automation_dependency_upstream_id_idx"), + table_name="automation_dependency", + ) + op.drop_table("automation_dependency") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index cbba34f811..ad5ef6a5eb 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -123,6 +123,16 @@ class Automation(db.Model, AuthModelMixin): passive_deletes=True, order_by="desc(AutomationRun.scheduled_at)", ) + # The automations whose run for the same scheduled time each run of this one waits for (see `AutomationDependency`). + dependencies = db.relationship( + "AutomationDependency", + foreign_keys="AutomationDependency.automation_id", + back_populates="automation", + lazy=True, + cascade="all, delete-orphan", + passive_deletes=True, + order_by="AutomationDependency.upstream_id", + ) @validates("timezone") def validate_timezone(self, key: str, timezone: str) -> str: @@ -183,6 +193,42 @@ def output_sensors(self) -> list: return get_automation_sensors(self)["output_sensors"] +class AutomationDependency(db.Model): + """One automation that another automation's runs wait for. + + A run of the dependent automation is dispatched only once the upstream automation's run for the same scheduled time has ended, + and is skipped if that run failed, unless the dependency allows for failure. + """ + + __tablename__ = "automation_dependency" + + automation_id = db.Column( + db.Integer, + db.ForeignKey("automation.id", ondelete="CASCADE"), + primary_key=True, + ) + upstream_id = db.Column( + db.Integer, + db.ForeignKey("automation.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + # Whether the dependent run goes ahead when the upstream run failed, rather than being skipped. + allow_failure = db.Column(db.Boolean, nullable=False, default=False) + + automation = db.relationship( + "Automation", foreign_keys=[automation_id], back_populates="dependencies" + ) + upstream = db.relationship("Automation", foreign_keys=[upstream_id]) + + def __repr__(self): + return "" % ( + self.automation_id, + self.upstream_id, + ", also when it fails" if self.allow_failure else "", + ) + + # A job intent counts as dispatched from this status onwards: it is in Redis, whatever became of it since. AUTOMATION_RUN_JOB_QUEUED_OR_LATER = ( "queued", @@ -205,11 +251,11 @@ class AutomationRun(db.Model): name="automation_run_occurrence_uq", ), db.CheckConstraint( - "dispatch_state IN ('pending', 'claimed', 'partially_queued', 'queued', 'failed')", + "dispatch_state IN ('pending', 'claimed', 'waiting', 'partially_queued', 'queued', 'failed')", name="automation_run_dispatch_state_ck", ), db.CheckConstraint( - "execution_state IN ('pending', 'running', 'succeeded', 'failed', 'canceled')", + "execution_state IN ('pending', 'running', 'succeeded', 'failed', 'canceled', 'skipped')", name="automation_run_execution_state_ck", ), ) @@ -233,7 +279,9 @@ class AutomationRun(db.Model): schedule_revision = db.Column(db.Integer, nullable=False) automation_type = db.Column(db.String(80), nullable=False) generator_id = db.Column(db.Integer, nullable=True) + # A run that is 'waiting' holds back its jobs until the upstream runs it depends on have ended (see `AutomationDependency`). dispatch_state = db.Column(db.String(32), nullable=False, default="pending") + # A run that is 'skipped' queued no jobs, because an upstream run it depends on did not succeed. execution_state = db.Column(db.String(32), nullable=False, default="pending") claim_owner = db.Column(db.String(128), nullable=True) claimed_at = db.Column(db.DateTime(timezone=True), nullable=True) diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 87a6c95468..5fa66a9745 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -25,13 +25,14 @@ from rq import get_current_job from rq.job import Job, JobStatus from sqlalchemy import func, or_, select, update -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, SQLAlchemyError from sqlalchemy.orm import selectinload from flexmeasures import Forecaster, Reporter from flexmeasures.data import db from flexmeasures.data.models.automations import ( Automation, + AutomationDependency, AutomationRun, AutomationRunAttempt, AutomationRunJob, @@ -72,6 +73,11 @@ "queued", "failed", ) +# The automation types whose runs are dispatched a second time, after an attempt that failed. +# A forecast run's jobs carry IDs derived from the run, so a retry recognizes the ones it already queued. +# A schedule run's jobs get a fresh ID on every dispatch, so retrying one would duplicate its schedules, +# which is why such a run is recorded and reported, but left where it failed. +RETRIED_AUTOMATION_TYPES = ("forecasting",) @dataclass(frozen=True) @@ -258,12 +264,16 @@ def claim_existing_automation_run( run: AutomationRun, owner: str | None = None, lease: timedelta = AUTOMATION_RUN_CLAIM_LEASE, + dispatch_states: tuple[str, ...] = AUTOMATION_RUN_RESUMABLE_DISPATCH_STATES, ) -> ClaimedAutomationRun | None: """Claim a durable automation run whose dispatch is unfinished and unclaimed. A run is only up for grabs once no other runner holds a live claim on it, because the dispatch state turns to 'partially_queued' while the owning runner is still queueing the rest of its jobs. A runner which fails releases its own claim, so its run is immediately retryable. + + :param dispatch_states: the dispatch states the run may be in to be claimed; + a run that is 'waiting' for its upstream runs is claimed only once they ended (see `release_automation_runs_waiting_for`). """ owner = owner or _runner_owner() now = _now_utc() @@ -271,7 +281,7 @@ def claim_existing_automation_run( update(AutomationRun) .where( AutomationRun.id == run.id, - AutomationRun.dispatch_state.in_(AUTOMATION_RUN_RESUMABLE_DISPATCH_STATES), + AutomationRun.dispatch_state.in_(dispatch_states), AutomationRun.dispatch_completed_at.is_(None), AutomationRun.attempt_count < AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, _claim_is_available(now), @@ -301,7 +311,12 @@ def get_dispatchable_automation_runs( now: datetime | None = None, owner: str | None = None, ) -> list[ClaimedAutomationRun]: - """Claim new due occurrences and resumable durable runs for dispatch.""" + """Claim new due occurrences, resumable durable runs, and waiting runs whose upstream runs ended, for dispatch. + + A waiting run is normally dispatched as soon as the last upstream run it waits for ends (see `release_automation_runs_waiting_for`). + Looking at the waiting runs here, too, catches the ones whose release was missed, + for instance because the worker that ended the upstream run went down before it got to them. + """ if now is None: now = _now_utc() now = floor_to_minute(now) @@ -316,11 +331,7 @@ def get_dispatchable_automation_runs( .join(Automation) .where( Automation.active.is_(True), - # Only a forecast run can be dispatched a second time safely. - # Its jobs carry IDs derived from the run, so a retry recognizes the ones it already queued. - # A schedule run's jobs get a fresh ID on every dispatch, so retrying one would duplicate its schedules, - # which is why such a run is recorded and reported, but left where it failed. - AutomationRun.automation_type == "forecasting", + AutomationRun.automation_type.in_(RETRIED_AUTOMATION_TYPES), AutomationRun.dispatch_state.in_(AUTOMATION_RUN_RESUMABLE_DISPATCH_STATES), AutomationRun.dispatch_completed_at.is_(None), # A run which used up its attempts is left as it is, rather than dispatched again every minute. @@ -336,9 +347,256 @@ def get_dispatchable_automation_runs( claimed = claim_existing_automation_run(run, owner=owner) if claimed is not None: claimed_runs.append(claimed) + + waiting_runs = db.session.scalars( + select(AutomationRun) + .join(Automation) + .where( + Automation.active.is_(True), + AutomationRun.dispatch_state == "waiting", + _claim_is_available(now), + ) + .order_by(AutomationRun.scheduled_at, AutomationRun.id) + ).all() + for run in waiting_runs: + if check_upstream_runs(run)[0] == "wait": + continue + claimed = claim_existing_automation_run( + run, owner=owner, dispatch_states=("waiting",) + ) + if claimed is not None: + claimed_runs.append(claimed) return claimed_runs +def _dispatch_was_given_up(run: AutomationRun) -> bool: + """Whether a run's dispatch failed for good, so that it will never queue (the rest of) its jobs.""" + return ( + run.dispatch_completed_at is None + and run.dispatch_state in ("failed", "partially_queued") + # A runner that is still queueing the jobs of a run holds a claim on it, and may yet finish. + and run.claim_owner is None + and ( + run.automation_type not in RETRIED_AUTOMATION_TYPES + or run.attempt_count >= AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + ) + ) + + +def _upstream_run_state(run: AutomationRun) -> str: + """Say whether an upstream run 'succeeded', 'failed', or is still 'pending', as far as the runs waiting for it are concerned. + + A run that was skipped, or canceled, did not deliver what the runs waiting for it need, so it counts as failed. + So does a run whose dispatch was given up, as it will never end otherwise. + """ + if run.execution_completed_at is not None: + return "succeeded" if run.execution_state == "succeeded" else "failed" + if _dispatch_was_given_up(run): + return "failed" + return "pending" + + +def _state_without_upstream_run(upstream: Automation, run: AutomationRun) -> str: + """Say what it means that an upstream automation has no run scheduled for the same time as this run. + + - 'pending': the upstream automation is due at that time, but no runner claimed its run yet. + - 'passed-over': the upstream automation was due at that time, but moved past it without a run, + as missed runs are coalesced or left out (see `get_due_automations`). + - 'not-scheduled': the upstream automation was never going to run at that time, + as it is inactive, recurs on another schedule, or did not exist yet. + """ + automation = run.automation + if ( + not upstream.active + or (upstream.cronstr, upstream.timezone) + != (automation.cronstr, automation.timezone) + # An automation's first run can be the one due in the minute it was created (see `get_initial_cursor`). + or floor_to_minute(upstream.created_at) > run.scheduled_at + ): + return "not-scheduled" + if upstream.cursor is None or upstream.cursor < run.scheduled_at: + return "pending" + return "passed-over" + + +def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: + """Decide whether a run can be dispatched yet, going by the runs of the automations it waits for. + + Each upstream automation's run for the same scheduled time counts. + The run is skipped as soon as one of those failed or will not happen, unless its dependency allows for that, + as its jobs would compute with input that was never produced. + Otherwise, it waits for as long as one of them has yet to end. + An upstream automation that was never going to run at that time is left out, and the run goes ahead without it. + + :returns: 'wait', 'skip' or 'go', and for each dependency, what was found: + the upstream automation, whether its failure is allowed for, its run (if any) and the state of that run. + """ + found = [] + for dependency in run.automation.dependencies: + upstream_run = db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.automation_id == dependency.upstream_id, + AutomationRun.scheduled_at == run.scheduled_at, + ) + .order_by(AutomationRun.schedule_revision.desc()) + .limit(1) + ).first() + found.append( + { + "automation": dependency.upstream_id, + "allow-failure": dependency.allow_failure, + "run": upstream_run.id if upstream_run is not None else None, + "state": ( + _upstream_run_state(upstream_run) + if upstream_run is not None + else _state_without_upstream_run(dependency.upstream, run) + ), + } + ) + if any( + upstream["state"] in ("failed", "passed-over") and not upstream["allow-failure"] + for upstream in found + ): + return "skip", found + if any(upstream["state"] == "pending" for upstream in found): + return "wait", found + return "go", found + + +def _describe_upstream_runs( + found: list[dict[str, Any]], states: tuple[str, ...] +) -> str: + """Name the upstream automations whose runs are in one of these states, for a message.""" + return ", ".join( + f"{upstream['automation']}" + + (f" (run {upstream['run']})" if upstream["run"] is not None else "") + for upstream in found + if upstream["state"] in states + ) + + +def _hold_automation_run( + claimed_run: ClaimedAutomationRun, found: list[dict[str, Any]] +) -> dict[str, Any]: + """Have a run wait for its upstream runs, releasing the claim on it, as no runner has to hold on to it meanwhile.""" + run = claimed_run.run + run.dispatch_state = "waiting" + run.claim_owner = None + run.claim_expires_at = None + _finish_attempt(claimed_run.attempt, "waiting", 0) + db.session.commit() + message = f"It waits for the run(s) of automation(s) {_describe_upstream_runs(found, ('pending',))} to end." + # An upstream run may have ended between deciding to wait and saying so, + # in which case nothing else would come back to this run before the next runner tick. + released = release_waiting_automation_run(run) + if released is not None: + return released + return { + "run_id": run.id, + "job_id": None, + "n_jobs": 0, + "dispatch_state": "waiting", + "message": message, + } + + +def _skip_automation_run( + claimed_run: ClaimedAutomationRun, found: list[dict[str, Any]] +) -> dict[str, Any]: + """End a run without queueing any jobs, as an upstream run it waits for did not succeed.""" + run = claimed_run.run + now = _now_utc() + reason = f"Skipped, as the run(s) of automation(s) {_describe_upstream_runs(found, ('failed', 'passed-over'))} that this run waits for did not succeed or did not happen." + run.dispatch_state = "queued" + run.dispatch_completed_at = now + run.execution_state = "skipped" + run.execution_completed_at = now + run.last_error_type = "UpstreamRunFailed" + run.last_error_message = reason + run.claim_owner = None + run.claim_expires_at = None + _finish_attempt(claimed_run.attempt, "skipped", 0) + db.session.commit() + # Whatever waits for this run cannot have its input either. + release_automation_runs_waiting_for(run) + return { + "run_id": run.id, + "job_id": None, + "n_jobs": 0, + "dispatch_state": "skipped", + "message": reason, + } + + +def release_waiting_automation_run(run: AutomationRun) -> dict[str, Any] | None: + """Dispatch a waiting run, or skip it, if the upstream runs it waits for ended. + + Another worker or runner may be releasing the same run at the same time, so it is claimed first, and only one of them gets it. + A failure to dispatch is recorded on the run (see `dispatch_automation_run`), rather than handed to whoever ended the upstream run, + whose own job succeeded or failed regardless. + + :returns: what the dispatch returned, or None if the run was left waiting. + """ + db.session.refresh(run) + if run.dispatch_state != "waiting" or not run.automation.active: + return None + if check_upstream_runs(run)[0] == "wait": + return None + claimed = claim_existing_automation_run(run, dispatch_states=("waiting",)) + if claimed is None: + return None + try: + return dispatch_automation_run(claimed) + except Exception as exc: + # Any error, whatever its kind, was recorded on the run by the dispatch, which is what the run's owner gets to see. + # The upstream run ended either way, so its job does not fail on account of the run that waited for it. + current_app.logger.warning( + f"Automation run {run.id} failed to dispatch once its upstream runs ended: {exc}" + ) + return None + + +def release_automation_runs_waiting_for(upstream_run: AutomationRun) -> None: + """Dispatch, or skip, the runs that wait for this upstream run, now that it ended. + + This is called wherever a run ends: when its last job ends, when it queued no jobs, when its dispatch failed for good, and when it was skipped. + A runner also looks at the waiting runs on every tick (see `get_dispatchable_automation_runs`), in case a release here was missed. + """ + waiting_runs = db.session.scalars( + select(AutomationRun) + .join( + AutomationDependency, + AutomationDependency.automation_id == AutomationRun.automation_id, + ) + .where( + AutomationDependency.upstream_id == upstream_run.automation_id, + AutomationRun.scheduled_at == upstream_run.scheduled_at, + AutomationRun.dispatch_state == "waiting", + ) + .order_by(AutomationRun.id) + ).all() + for run in waiting_runs: + release_waiting_automation_run(run) + + +def _release_runs_waiting_for_ended_run(run: AutomationRun) -> None: + """Release the runs waiting for this run, if it ended, without letting a database error there reach whoever ended it. + + The callers record how a job or a dispatch ended, which stands whatever becomes of the runs waiting for it; + a runner picks those up on its next tick instead (see `get_dispatchable_automation_runs`). + """ + if run.execution_completed_at is None and not _dispatch_was_given_up(run): + return + try: + release_automation_runs_waiting_for(run) + except SQLAlchemyError as exc: + db.session.rollback() + current_app.logger.warning( + f"Could not release the automation runs waiting for run {run.id}, so a runner will on its next tick: {exc}" + ) + + def ensure_automation_run_job_intents( run_id: int, job_specs: list[dict[str, Any]] ) -> list[AutomationRunJob]: @@ -448,6 +706,7 @@ def mark_automation_run_dispatch_failed( run.claim_expires_at = _now_utc() + _retry_delay(run.attempt_count) _finish_attempt(attempt, run.dispatch_state, queued_count, error) db.session.commit() + _release_runs_waiting_for_ended_run(run) def record_automation_job_started( @@ -515,6 +774,7 @@ def record_automation_job_succeeded( intent.finished_at = now _refresh_run_execution_state(intent.run, now) db.session.commit() + _release_runs_waiting_for_ended_run(intent.run) def record_automation_job_failed( @@ -555,6 +815,7 @@ def record_automation_job_failed( run.last_error_type = error.__class__.__name__ run.last_error_message = str(error) db.session.commit() + _release_runs_waiting_for_ended_run(run) def reconcile_automation_job_intent(intent: AutomationRunJob) -> bool: @@ -600,6 +861,7 @@ def record_automation_run_jobs(run_id: int, jobs: list[tuple[str, str, Job]]) -> run.execution_state = "succeeded" run.execution_completed_at = now db.session.commit() + _release_runs_waiting_for_ended_run(run) return run.plan = { **dict(run.plan or {}), @@ -1969,8 +2231,12 @@ def create_automation( source: DataSource | None = None, origin: str = "API", check_permissions: bool = False, + dependencies: list[dict[str, Any]] | None = None, ) -> tuple[Automation, list[str]]: - """Create an automation through its registered handler, without committing.""" + """Create an automation through its registered handler, without committing. + + :param dependencies: the automations whose runs this automation's runs wait for (see `set_automation_dependencies`). + """ from flask_security import current_user from flexmeasures.data.automations import ( get_automation_handler, @@ -1978,6 +2244,15 @@ def create_automation( ) validate_automation_type(automation_type) + if dependencies: + # Check these first, so that nothing is set up for an automation that is refused. + validate_automation_dependencies( + None, + dependencies, + cronstr=cronstr, + timezone=timezone or get_default_automation_timezone(asset), + check_permissions=check_permissions, + ) automation, warnings = get_automation_handler(automation_type).create( asset=asset, name=name, @@ -1998,6 +2273,8 @@ def create_automation( raise Unauthorized() automation.execution_user_id = current_user.id + if dependencies: + set_automation_dependencies(automation, dependencies, origin=origin) return automation, warnings @@ -2181,18 +2458,50 @@ def update_automation( timezone: str | None = None, active: bool | None = None, origin: str = "API", + dependencies: list[dict[str, Any]] | None = None, + check_permissions: bool = False, ) -> list[str]: - """Update an automation's name, cron string, timezone and/or activation status (not committed yet). + """Update an automation's name, cron string, timezone, activation status and/or dependencies (not committed yet). Anything that changes which runs are due, namely the recurrence, the timezone and reactivation, also resets the cursor to just before the minute of the change, so that runs from before it are not caught up on, while a run due in that very minute still is. + A run waits for the upstream runs scheduled for the same time, + so the recurrence and the timezone can only change in step with the automations it waits for, + and not at all while other automations wait for it. An audit log record is added to the asset. + :param dependencies: the automations whose runs this automation's runs wait for, replacing the ones it had; None leaves them as they are. + :param check_permissions: whether to require that the current user may read the upstream automations named in `dependencies`. + :raises marshmallow.ValidationError: if the dependencies, or the new schedule, are not possible. :returns: a list of (human-readable) changes; empty if nothing changed. """ from flexmeasures.data.models.audit_log import AssetAuditLog + # Check everything before changing anything, so that a refused update leaves the automation as it was. + validate_dependents_keep_schedule(automation, cronstr, timezone) + schedule_changes = (cronstr is not None and cronstr != automation.cronstr) or ( + timezone is not None and timezone != automation.timezone + ) + if dependencies is not None or schedule_changes: + validate_automation_dependencies( + automation, + ( + dependencies + if dependencies is not None + else [ + { + "upstream": dependency.upstream, + "allow_failure": dependency.allow_failure, + } + for dependency in automation.dependencies + ] + ), + cronstr=cronstr, + timezone=timezone, + check_permissions=check_permissions and dependencies is not None, + ) + changes = [] reset_cursor = False if name is not None and name != automation.name: @@ -2216,12 +2525,195 @@ def update_automation( # even where they fall on the same scheduled UTC time. automation.schedule_revision += 1 automation.cursor = get_initial_cursor() + if dependencies is not None: + # This records its own audit log line, as creating an automation with dependencies does. + dependency_changes = set_automation_dependencies( + automation, dependencies, origin=origin + ) + else: + dependency_changes = [] if changes: AssetAuditLog.add_record( automation.asset, f"Updated automation '{automation.name}' ({automation.id}): {'; '.join(changes)}. Via {origin}.", ) - return changes + return changes + dependency_changes + + +def _describe_dependency(upstream: Automation, allow_failure: bool) -> str: + """Name an upstream automation in a message, and say whether its failure is allowed for.""" + return f"{upstream.id}{' (also when it fails)' if allow_failure else ''}" + + +def _upstream_ids_reachable_from(automation_ids: set[int]) -> set[int]: + """Return every automation that the given automations wait for, directly or through others.""" + reached: set[int] = set() + frontier = set(automation_ids) + while frontier: + upstream_ids = set( + db.session.scalars( + select(AutomationDependency.upstream_id).where( + AutomationDependency.automation_id.in_(frontier) + ) + ).all() + ) + frontier = upstream_ids - reached + reached |= upstream_ids + return reached + + +def validate_automation_dependencies( + automation: Automation | None, + dependencies: list[dict[str, Any]], + cronstr: str | None = None, + timezone: str | None = None, + check_permissions: bool = False, +) -> None: + """Check that an automation may wait for these upstream automations. + + A run waits for the upstream run scheduled for the very same time, so an upstream automation has to recur on the same schedule: + the same cron string, read in the same timezone. + An automation waiting for itself, directly or through others, would never run, so such a cycle is refused. + + :param automation: the automation that is to wait, or None for one that is about to be created. + :param dependencies: dicts with the upstream automation under "upstream", and whether its failure is allowed for under "allow_failure". + :param cronstr: the cron string the automation is about to have, if not the one it has. + :param timezone: the timezone the automation is about to have, if not the one it has. + :param check_permissions: whether to require that the current user may read each upstream automation (and so its asset). + :raises marshmallow.ValidationError: if a dependency is not possible, under the "depends-on" field. + :raises werkzeug.exceptions.Forbidden: if the current user may not read an upstream automation. + """ + from flexmeasures.auth.policy import check_access + + if automation is not None: + cronstr = cronstr or automation.cronstr + timezone = timezone or automation.timezone + automation_id = automation.id if automation is not None else None + errors = [] + upstream_ids = [dependency["upstream"].id for dependency in dependencies] + duplicates = sorted( + { + upstream_id + for upstream_id in upstream_ids + if upstream_ids.count(upstream_id) > 1 + } + ) + if duplicates: + errors.append( + f"Automation(s) {', '.join(map(str, duplicates))} are named more than once." + ) + for dependency in dependencies: + upstream = dependency["upstream"] + if check_permissions: + check_access(upstream, "read") + if automation_id is not None and upstream.id == automation_id: + errors.append(f"Automation {upstream.id} cannot wait for itself.") + continue + if upstream.cronstr != cronstr or upstream.timezone != timezone: + errors.append( + f"Automation {upstream.id} recurs per cron string '{upstream.cronstr}' in timezone '{upstream.timezone}'," + f" while this automation recurs per '{cronstr}' in '{timezone}'." + " A run waits for the upstream run scheduled for the same time, so both have to recur on the same schedule." + ) + # An automation that is about to be created has nothing waiting for it yet, so it cannot close a cycle. + if automation_id is not None and automation_id in _upstream_ids_reachable_from( + {upstream_id for upstream_id in upstream_ids if upstream_id != automation_id} + ): + errors.append( + f"Automation {automation_id} would end up waiting for itself, as one of the automations it waits for already waits for it, directly or through others." + ) + if errors: + raise ValidationError({"depends-on": errors}) + + +def validate_dependents_keep_schedule( + automation: Automation, cronstr: str | None, timezone: str | None +) -> None: + """Refuse a new schedule for an automation that other automations wait for, as their runs would no longer line up with its runs. + + :raises marshmallow.ValidationError: under the "cron" field, naming the automations that wait for this one. + """ + new_cronstr = cronstr or automation.cronstr + new_timezone = timezone or automation.timezone + if (new_cronstr, new_timezone) == (automation.cronstr, automation.timezone): + return + dependent_ids = db.session.scalars( + select(AutomationDependency.automation_id) + .where(AutomationDependency.upstream_id == automation.id) + .order_by(AutomationDependency.automation_id) + ).all() + if dependent_ids: + raise ValidationError( + { + "cron": [ + f"Automation(s) {', '.join(map(str, dependent_ids))} wait for this automation's runs, so it has to keep recurring per '{automation.cronstr}' in '{automation.timezone}'." + " Remove those dependencies first, or change the schedule of the automations that wait for it, too, once they no longer do." + ] + } + ) + + +def set_automation_dependencies( + automation: Automation, + dependencies: list[dict[str, Any]], + origin: str = "API", +) -> list[str]: + """Replace the automations that an automation's runs wait for (not committed yet). + + Validate them first (see `validate_automation_dependencies`). + An audit log record is added to the asset if anything changed. + + :param dependencies: dicts with the upstream automation under "upstream", and whether its failure is allowed for under "allow_failure". + :returns: a list of (human-readable) changes; empty if nothing changed. + """ + from flexmeasures.data.models.audit_log import AssetAuditLog + + current = { + dependency.upstream_id: dependency.allow_failure + for dependency in automation.dependencies + } + wanted = { + dependency["upstream"].id: bool(dependency.get("allow_failure", False)) + for dependency in dependencies + } + if current == wanted: + return [] + upstreams = { + dependency["upstream"].id: dependency["upstream"] for dependency in dependencies + } + automation.dependencies = [ + AutomationDependency( + automation=automation, + upstream=upstreams[upstream_id], + allow_failure=allow_failure, + ) + for upstream_id, allow_failure in sorted(wanted.items()) + ] + described = ( + ", ".join( + _describe_dependency(upstreams[upstream_id], allow_failure) + for upstream_id, allow_failure in sorted(wanted.items()) + ) + or "none" + ) + change = f"depends on: {described}" + if automation.id is not None: + AssetAuditLog.add_record( + automation.asset, + f"Updated automation '{automation.name}' ({automation.id}): {change}. Via {origin}.", + ) + return [change] + + +def serialize_automation_dependencies(automation: Automation) -> list[dict[str, Any]]: + """Render the automations an automation waits for, as the API names them.""" + return [ + { + "automation": dependency.upstream_id, + "allow-failure": dependency.allow_failure, + } + for dependency in automation.dependencies + ] def delete_automation(automation: Automation, origin: str = "API"): @@ -2258,6 +2750,8 @@ def serialize_automation_run(run: AutomationRun) -> dict[str, Any]: if run.execution_completed_at else None ), + # What the run found of the upstream runs it waits for, when it last looked (see `check_upstream_runs`). + "upstream-runs": list((run.plan or {}).get("upstream-runs", [])), "claim-owner": run.claim_owner, "claim-expires-at": ( run.claim_expires_at.isoformat() if run.claim_expires_at else None @@ -2447,6 +2941,15 @@ def dispatch_automation_run( raise AutomationRunClaimLost( f"Automation run {run.id} is no longer claimed by {owner}, so another runner is dispatching it." ) + if run.automation.dependencies and not run.job_intents: + # A run that queued jobs before was past its upstream runs already, so a retry does not wait for them again. + verdict, found = check_upstream_runs(run) + run.plan = {**dict(run.plan or {}), "upstream-runs": found} + if verdict == "wait": + return _hold_automation_run(claimed_run, found) + if verdict == "skip": + return _skip_automation_run(claimed_run, found) + db.session.commit() try: returns = run_automation(run.automation, automation_run=run) except Exception as exc: From 57686c2e2abd7a40f31dc921556efb174a6548d5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:47 +0100 Subject: [PATCH 07/47] data/services: copy an automation's dependencies along with an asset copy Context: - Issue #2586 asks what a copy does with a dependency Change: - A copy waits for the copied upstream automation where that was copied too, and for the original one otherwise Signed-off-by: Mohamed Belhsan Hmida --- .../data/services/automation_copies.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/services/automation_copies.py b/flexmeasures/data/services/automation_copies.py index 2d0eba027a..d377852a0f 100644 --- a/flexmeasures/data/services/automation_copies.py +++ b/flexmeasures/data/services/automation_copies.py @@ -29,6 +29,7 @@ from flexmeasures.data.services.automations import ( WINDOW_FIELDS, get_forecast_output_sensor, + set_automation_dependencies, validate_automation_output_scope, ) from flexmeasures.data.services.data_sources import get_or_create_source @@ -92,6 +93,8 @@ def copy_automations( with its references to sensors inside the subtree pointed at the new sensors. A copy keeps its name, type, cron expression and timezone, but starts inactive and with a fresh cursor, so that it inherits neither the original's run history nor its queued jobs. + A copy waits for the copies of the automations its original waits for, where those were copied, too, + and for the very automations its original waits for otherwise. An automation that cannot be copied safely is skipped rather than failing the asset copy: the copied asset, its sensors and the other automations are kept. @@ -120,10 +123,11 @@ def copy_automations( ) skipped: list[SkippedAutomation] = [] + copies: dict[int, Automation] = {} for automation in source_automations: try: with db.session.begin_nested(): - _copy_automation( + copies[automation.id] = _copy_automation( automation=automation, asset_id_map=asset_id_map, sensor_id_map=sensor_id_map, @@ -148,9 +152,37 @@ def copy_automations( reason=str(e), ) ) + _copy_dependencies(source_automations, copies) return skipped +def _copy_dependencies( + source_automations: list[Automation], copies: dict[int, Automation] +) -> None: + """Have each copy wait for what its original waits for, pointing at the copies where there are any. + + This happens once all automations were copied, as an automation may wait for one that was copied after it. + The copies recur on the schedules of their originals, and a copied set of dependencies cannot form a cycle the originals did not, + so the dependencies need no validation beyond what the originals had. + """ + for automation in source_automations: + copied_automation = copies.get(automation.id) + if copied_automation is None or not automation.dependencies: + continue + set_automation_dependencies( + copied_automation, + [ + { + "upstream": copies.get(dependency.upstream_id, dependency.upstream), + "allow_failure": dependency.allow_failure, + } + for dependency in automation.dependencies + ], + origin="asset copy", + ) + db.session.flush() + + def _copy_automation( automation: Automation, asset_id_map: dict[int, int], From 06dd24dd3785afdea68db92b315a8d9c63fcc48e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:47 +0100 Subject: [PATCH 08/47] api/v3_0: name an automation's dependencies with depends-on Context: - Issue #2586 Change: - depends-on on creating, updating and reading automations, and upstream-runs on run stats - PATCH checks the user may read the upstream automations, and refuses a schedule that breaks a dependency Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/assets.py | 28 ++++++++-- flexmeasures/data/schemas/automations.py | 67 ++++++++++++++++++++++- flexmeasures/ui/static/openapi-specs.json | 63 ++++++++++++++++++++- 3 files changed, 150 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 10736f20ec..da3fc54cf0 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1781,6 +1781,11 @@ def post_automation(self, automation_data: dict, id: int, asset: GenericAsset): The automation can only involve sensors that you have access to yourself: read access to the sensors it reads data from, and permission to record data on the sensors it writes to. + + An automation can wait for other automations that it consumes the results of, listed in `depends-on`. + Each of its runs then waits for their runs scheduled for the same time, and queues its jobs once those ended, + or is skipped if one of them failed, unless that dependency has `allow-failure` set. + The automations it waits for can be on any asset you can read, and have to recur on the same schedule. security: - ApiKeyAuth: [] parameters: @@ -1878,15 +1883,17 @@ def patch_automation( self, automation_data: dict, id: int, automation_id: int, asset: GenericAsset ): """ - .. :quickref: Assets; Update an automation's name, cron string, timezone or activation status. + .. :quickref: Assets; Update an automation's name, cron string, timezone, activation status or dependencies. --- patch: - summary: Update an automation's name, cron string, timezone or activation status. + summary: Update an automation's name, cron string, timezone, activation status or dependencies. description: | - Any subset of the fields `name`, `cronstr`, `timezone` and `active` can be sent. + Any subset of the fields `name`, `cron`, `timezone`, `active` and `depends-on` can be sent. Changing the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change, so runs from before it are not caught up on, while a run due in that very minute still is. + An automation's runs wait for the runs of the automations in `depends-on` that are scheduled for the same time, + so the recurrence and the timezone can only change along with `depends-on`, and not while other automations depend on this one. Other automation fields cannot be updated; instead, create a new automation. Requires permission to add data under the asset. security: @@ -1913,6 +1920,13 @@ def patch_automation( summary: Deactivate the automation value: active: false + wait_for_another_automation: + summary: Wait for the runs of automation 4 + description: >- + Each run waits for the run of automation 4 scheduled for the same time, and is skipped if that one failed. + value: + depends-on: + - automation: 4 responses: 200: description: PROCESSED @@ -1932,7 +1946,13 @@ def patch_automation( return { "message": f"Asset {asset.id} has no automation with id {automation_id}." }, 404 - update_automation(automation, origin="API", **automation_data) + try: + update_automation( + automation, origin="API", check_permissions=True, **automation_data + ) + except ValidationError as e: + db.session.rollback() + return unprocessable_entity(e.messages) db.session.commit() response = automation_schema.dump(automation) response["recurrence-description"] = describe_cronstr(automation.cronstr) diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py index 458e617413..225965e9c9 100644 --- a/flexmeasures/data/schemas/automations.py +++ b/flexmeasures/data/schemas/automations.py @@ -77,6 +77,36 @@ def _serialize(self, automation, attr, data, **kwargs): return automation.id +class AutomationDependencySchema(Schema): + """One automation whose runs another automation's runs wait for.""" + + # The loaded names are the ones `set_automation_dependencies` takes. + upstream = AutomationIdField( + required=True, + data_key="automation", + metadata={ + "description": "ID of the automation to wait for. It has to recur on the same schedule (the same cron string, in the same timezone).", + "example": 4, + }, + ) + allow_failure = fields.Bool( + load_default=False, + data_key="allow-failure", + metadata={ + "description": "Whether a run goes ahead when the upstream run failed or did not happen, rather than being skipped.", + "example": False, + }, + ) + + +DEPENDS_ON_DESCRIPTION = ( + "The automations whose runs each run of this automation waits for: the runs scheduled for the same time." + " A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure." + " Upstream automations can be on any asset you can read, and have to recur on the same schedule." + " A run on demand does not wait." +) + + class AutomationCreationSchema(Schema): """Request schema for creating an automation (the asset comes from the URL path). @@ -132,6 +162,15 @@ class AutomationCreationSchema(Schema): "example": 6, }, ) + dependencies = fields.List( + fields.Nested(AutomationDependencySchema), + data_key="depends-on", + load_default=list, + metadata={ + "description": DEPENDS_ON_DESCRIPTION, + "example": [{"automation": 4, "allow-failure": False}], + }, + ) @validates_schema def validate_generator_is_named_once(self, data: dict, **kwargs): @@ -161,7 +200,7 @@ def validate_generator_is_named_once(self, data: dict, **kwargs): class AutomationUpdateSchema(Schema): - """Request schema for updating an automation's name, recurrence, timezone and/or activation status. + """Request schema for updating an automation's name, recurrence, timezone, activation status and/or dependencies. The parameters cannot be updated, so the sensors an automation involves stay the ones its creator was checked against. """ @@ -175,6 +214,15 @@ class AutomationUpdateSchema(Schema): } ) active = fields.Bool() + dependencies = fields.List( + fields.Nested(AutomationDependencySchema), + data_key="depends-on", + metadata={ + "description": DEPENDS_ON_DESCRIPTION + + " Replaces the automations it depended on; send an empty list to have its runs wait for none.", + "example": [{"automation": 4, "allow-failure": True}], + }, + ) class AutomationSchema(ma.SQLAlchemySchema): @@ -221,6 +269,15 @@ class Meta: }, ) active = ma.auto_field() + depends_on = fields.Method( + serialize="dump_depends_on", + data_key="depends-on", + dump_only=True, + metadata={ + "description": DEPENDS_ON_DESCRIPTION, + "example": [{"automation": 4, "allow-failure": False}], + }, + ) @staticmethod def _in_automation_timezone( @@ -247,6 +304,14 @@ def dump_cursor(self, automation: Automation) -> str | None: """Render the cursor in the automation's own timezone.""" return self._in_automation_timezone(automation, automation.cursor) + def dump_depends_on(self, automation: Automation) -> list[dict]: + """Render the automations this automation's runs wait for.""" + from flexmeasures.data.services.automations import ( + serialize_automation_dependencies, + ) + + return serialize_automation_dependencies(automation) + @validates("type") def validate_type(self, type: str, **kwargs): validate_automation_type(type) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 8c51b957f0..5f459ec427 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3636,8 +3636,8 @@ ] }, "patch": { - "summary": "Update an automation's name, cron string, timezone or activation status.", - "description": "Any subset of the fields `name`, `cronstr`, `timezone` and `active` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", + "summary": "Update an automation's name, cron string, timezone, activation status or dependencies.", + "description": "Any subset of the fields `name`, `cron`, `timezone`, `active` and `depends-on` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nAn automation's runs wait for the runs of the automations in `depends-on` that are scheduled for the same time,\nso the recurrence and the timezone can only change along with `depends-on`, and not while other automations depend on this one.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", "security": [ { "ApiKeyAuth": [] @@ -3675,6 +3675,17 @@ "value": { "active": false } + }, + "wait_for_another_automation": { + "summary": "Wait for the runs of automation 4", + "description": "Each run waits for the run of automation 4 scheduled for the same time, and is skipped if that one failed.", + "value": { + "depends-on": [ + { + "automation": 4 + } + ] + } } } } @@ -3789,7 +3800,7 @@ }, "post": { "summary": "Create an automation on an asset.", - "description": "Create a recurring forecasting, scheduling, reporting or plugin-defined task on the asset.\nThe parameters are validated by the schema matching the automation type:\nforecast parameters for type `forecasting`,\na schedule trigger message (without the asset id) for type `scheduling`,\nreport parameters for type `reporting`, or the registered plugin schema.\nRequires permission to add data under the asset.\n\nAn automation runs again and again, so its parameters cannot fix a moment in time:\na `start`, `end` or `prior` among them is refused.\nSay instead how the period each run covers relates to that run,\nwith two of `start-offset`, `end-offset` and `duration`.\nThe offsets are chains of comma-separated Pandas offsets, plus `DB` (day begin) and `HB` (hour begin),\napplied to the time the run was due, on the automation's own clock.\nLeave the timing out to start at the time of each run,\nor, for a report, to cover the period since the last successful report.\n\nThe automation can only involve sensors that you have access to yourself:\nread access to the sensors it reads data from,\nand permission to record data on the sensors it writes to.\n", + "description": "Create a recurring forecasting, scheduling, reporting or plugin-defined task on the asset.\nThe parameters are validated by the schema matching the automation type:\nforecast parameters for type `forecasting`,\na schedule trigger message (without the asset id) for type `scheduling`,\nreport parameters for type `reporting`, or the registered plugin schema.\nRequires permission to add data under the asset.\n\nAn automation runs again and again, so its parameters cannot fix a moment in time:\na `start`, `end` or `prior` among them is refused.\nSay instead how the period each run covers relates to that run,\nwith two of `start-offset`, `end-offset` and `duration`.\nThe offsets are chains of comma-separated Pandas offsets, plus `DB` (day begin) and `HB` (hour begin),\napplied to the time the run was due, on the automation's own clock.\nLeave the timing out to start at the time of each run,\nor, for a report, to cover the period since the last successful report.\n\nThe automation can only involve sensors that you have access to yourself:\nread access to the sensors it reads data from,\nand permission to record data on the sensors it writes to.\n\nAn automation can wait for other automations that it consumes the results of, listed in `depends-on`.\nEach of its runs then waits for their runs scheduled for the same time, and queues its jobs once those ended,\nor is skipped if one of them failed, unless that dependency has `allow-failure` set.\nThe automations it waits for can be on any asset you can read, and have to recur on the same schedule.\n", "security": [ { "ApiKeyAuth": [] @@ -6783,6 +6794,26 @@ ], "additionalProperties": false }, + "AutomationDependency": { + "type": "object", + "properties": { + "automation": { + "type": "integer", + "description": "ID of the automation to wait for. It has to recur on the same schedule (the same cron string, in the same timezone).", + "example": 4 + }, + "allow-failure": { + "type": "boolean", + "default": false, + "description": "Whether a run goes ahead when the upstream run failed or did not happen, rather than being skipped.", + "example": false + } + }, + "required": [ + "automation" + ], + "additionalProperties": false + }, "AutomationCreationSchema": { "type": "object", "properties": { @@ -6838,6 +6869,19 @@ ], "description": "ID of an existing data source to reuse, instead of naming a `data-generator` and its `config`.", "example": 6 + }, + "depends-on": { + "type": "array", + "description": "The automations whose runs each run of this automation waits for: the runs scheduled for the same time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and have to recur on the same schedule. A run on demand does not wait.", + "example": [ + { + "automation": 4, + "allow-failure": false + } + ], + "items": { + "$ref": "#/components/schemas/AutomationDependency" + } } }, "required": [ @@ -6864,6 +6908,19 @@ }, "active": { "type": "boolean" + }, + "depends-on": { + "type": "array", + "description": "The automations whose runs each run of this automation waits for: the runs scheduled for the same time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and have to recur on the same schedule. A run on demand does not wait. Replaces the automations it depended on; send an empty list to have its runs wait for none.", + "example": [ + { + "automation": 4, + "allow-failure": true + } + ], + "items": { + "$ref": "#/components/schemas/AutomationDependency" + } } }, "additionalProperties": false From a6b40b0f8f5d9d7223e7a459d223091d3b67c47f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:47 +0100 Subject: [PATCH 09/47] cli: --depends-on and --allow-failure-of for add and edit automation Context: - Issue #2586 Change: - add/edit automation take --depends-on and --allow-failure-of, edit also --no-dependencies - show automation lists dependencies, run-automations reports waiting and skipped runs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/cli/data_add.py | 33 ++++++++++++++++- flexmeasures/cli/data_edit.py | 68 ++++++++++++++++++++++++++++++----- flexmeasures/cli/data_show.py | 9 +++++ flexmeasures/cli/jobs.py | 16 +++++++++ flexmeasures/cli/utils.py | 20 +++++++++++ 5 files changed, 136 insertions(+), 10 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 3dc776225d..e962d00731 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -45,10 +45,12 @@ DeprecatedOptionsCommand, LoggedClickExceptionGroup, add_cli_options_from_schema, + assemble_automation_dependencies, split_commas, ) from flexmeasures.data import db from flexmeasures.data.automations import get_automation_handler, get_automation_types +from flexmeasures.data.models.automations import Automation from flexmeasures.data.scripts.data_gen import ( add_transmission_zone_asset, populate_initial_structure, @@ -86,7 +88,11 @@ ) from flexmeasures.data.models.data_sources import DataSource, DEFAULT_DATASOURCE_TYPES from flexmeasures.data.models.annotations import Annotation, get_or_create_annotation -from flexmeasures.data.schemas.automations import CronField, TimezoneField +from flexmeasures.data.schemas.automations import ( + AutomationIdField, + CronField, + TimezoneField, +) from flexmeasures.data.schemas import ( AccountIdField, AwareDateTimeField, @@ -1871,6 +1877,24 @@ def add_forecast( # noqa: C901 help="How long the period each run covers lasts, as an ISO 8601 duration, such as P1D." " For a forecast or schedule automation, the period then starts at the run time, unless --start-offset or --end-offset says otherwise.", ) +@click.option( + "--depends-on", + "depends_on", + multiple=True, + type=AutomationIdField(), + help="ID of an automation whose runs each run of this one waits for: the run scheduled for the same time." + " A run queues its jobs once the runs it waits for ended, and is skipped if one of them failed (but see --allow-failure-of)." + " The automation waited for has to recur on the same schedule, so --cron and --timezone have to match it." + " Repeat to wait for several automations.", +) +@click.option( + "--allow-failure-of", + "allow_failure_of", + multiple=True, + type=AutomationIdField(), + help="ID of an automation given with --depends-on whose failed (or missed) run should not stop this automation's run." + " Repeat for several.", +) @add_cli_options_from_schema( ForecasterParametersSchema(), hidden=True, @@ -1892,6 +1916,8 @@ def add_automation( # noqa: C901 source: DataSource | None = None, config_file: TextIOBase | None = None, parameters_file: TextIOBase | None = None, + depends_on: tuple[Automation, ...] = (), + allow_failure_of: tuple[Automation, ...] = (), **kwargs, ): """ @@ -1920,6 +1946,9 @@ def add_automation( # noqa: C901 Alternatively, pass an existing data source (--source) to reuse the forecaster and configuration stored on it. + An automation that consumes what another automation produces can wait for it (--depends-on), + so that each of its runs queues its jobs only once the other automation's run for the same time ended. + Every forecaster and pipeline option that `flexmeasures add forecast` accepts is accepted here, too, but is left out of the help text above to keep it focused on the automation itself; run `flexmeasures add forecast --help` to see them. @@ -1932,6 +1961,7 @@ def add_automation( # noqa: C901 except (ValueError, NotImplementedError) as e: available = ", ".join(get_automation_types()) raise click.UsageError(f"{e} Available types: {available}.") + dependencies = assemble_automation_dependencies(depends_on, allow_failure_of) # Only a forecast automation has a default generator: # a report automation has to name its reporter, and the service says so, while a schedule automation resolves its own. @@ -1991,6 +2021,7 @@ def add_automation( # noqa: C901 config=config, source=source, origin="CLI", + dependencies=dependencies, ) except ValidationError as e: # The messages name the part of the request at fault, which is not always the parameters. diff --git a/flexmeasures/cli/data_edit.py b/flexmeasures/cli/data_edit.py index 4db8c41af6..6ceea925dd 100644 --- a/flexmeasures/cli/data_edit.py +++ b/flexmeasures/cli/data_edit.py @@ -8,6 +8,7 @@ import pandas as pd from flask import current_app as app from flask.cli import with_appcontext +from marshmallow import ValidationError import json from flexmeasures.data.models.user import Account, Plan from flexmeasures.data.schemas.account import AccountIdField @@ -35,6 +36,7 @@ DeprecatedOption, DeprecatedOptionsCommand, abort, + assemble_automation_dependencies, validate_rate_limit_cli, ) from flexmeasures.utils.flexmeasures_inflection import pluralize @@ -98,22 +100,70 @@ def fm_edit_data(): default=None, help="Activate or deactivate the automation.", ) +@click.option( + "--depends-on", + "depends_on", + multiple=True, + type=AutomationIdField(), + help="ID of an automation whose runs each run of this one waits for: the run scheduled for the same time." + " Replaces the automations it waited for. Repeat to wait for several automations.", +) +@click.option( + "--allow-failure-of", + "allow_failure_of", + multiple=True, + type=AutomationIdField(), + help="ID of an automation given with --depends-on whose failed (or missed) run should not stop this automation's run." + " Repeat for several.", +) +@click.option( + "--no-dependencies", + "no_dependencies", + is_flag=True, + help="Stop waiting for other automations.", +) def edit_automation( automation: Automation, name: str | None = None, cronstr: str | None = None, timezone: str | None = None, active: bool | None = None, + depends_on: tuple[Automation, ...] = (), + allow_failure_of: tuple[Automation, ...] = (), + no_dependencies: bool = False, ): - """Edit the name, recurrence, timezone or activation status of an automation.""" - changes = update_automation( - automation, - name=name, - cronstr=cronstr, - timezone=timezone, - active=active, - origin="CLI", - ) + """Edit the name, recurrence, timezone, activation status or dependencies of an automation. + + The recurrence and timezone of an automation that waits for other automations have to stay those of the automations it waits for, + so pass --depends-on (or --no-dependencies) along with --cron or --timezone to change both. + """ + if no_dependencies and (depends_on or allow_failure_of): + raise click.UsageError( + "--no-dependencies cannot be combined with --depends-on or --allow-failure-of." + ) + if allow_failure_of and not depends_on: + raise click.UsageError( + "--allow-failure-of needs --depends-on, as it replaces all the automations this one waits for." + ) + if no_dependencies: + dependencies: list[dict] | None = [] + elif depends_on: + dependencies = assemble_automation_dependencies(depends_on, allow_failure_of) + else: + dependencies = None + try: + changes = update_automation( + automation, + name=name, + cronstr=cronstr, + timezone=timezone, + active=active, + origin="CLI", + dependencies=dependencies, + ) + except ValidationError as e: + db.session.rollback() + abort(f"Cannot update automation {automation.id}: {e.messages}") if not changes: click.secho("Nothing to change.", **MsgStyle.WARN) return diff --git a/flexmeasures/cli/data_show.py b/flexmeasures/cli/data_show.py index b2735c1c83..73091a8a1e 100644 --- a/flexmeasures/cli/data_show.py +++ b/flexmeasures/cli/data_show.py @@ -579,6 +579,15 @@ def _show_automation(automation: Automation): ), ), ("Parameters", json.dumps(automation.parameters, indent=4)), + ( + "Depends on", + "\n".join( + f"{dependency.upstream.name} (ID: {dependency.upstream_id})" + + (", also when it fails" if dependency.allow_failure else "") + for dependency in automation.dependencies + ) + or "none", + ), ], headers=["Field", "Value"], ) diff --git a/flexmeasures/cli/jobs.py b/flexmeasures/cli/jobs.py index 8637f2cf61..2b4869419a 100644 --- a/flexmeasures/cli/jobs.py +++ b/flexmeasures/cli/jobs.py @@ -94,6 +94,10 @@ def run_automations(): Durable automation run records claim each scheduled run and make queueing resumable. Failed dispatch attempts are retried safely by reusing the original run plan and deterministic job IDs. + + A run of an automation that depends on other automations waits for their runs scheduled for the same time, + and queues its jobs as soon as the last of them ended, or is skipped if one of them failed. + This command also dispatches the waiting runs whose upstream runs ended without them being released. """ now = floor_to_minute(server_now()) claimed_runs = get_dispatchable_automation_runs(now) @@ -108,6 +112,17 @@ def run_automations(): try: returns = dispatch_automation_run(claimed_run) n_jobs = returns["n_jobs"] + if returns.get("dispatch_state") in ("waiting", "skipped"): + click.secho( + f"Automation {automation.id} ('{automation.name}') run {claimed_run.run.id}, scheduled for {claimed_run.run.scheduled_at}, queued no jobs. {returns['message']}", + **( + MsgStyle.WARN + if returns["dispatch_state"] == "skipped" + else MsgStyle.SUCCESS + ), + ) + n_run += 1 + continue click.secho( f"Automation {automation.id} ('{automation.name}') run {claimed_run.run.id} queued {n_jobs} {automation.type} job(s) for asset {automation.asset_id}, scheduled for {claimed_run.run.scheduled_at}.", **MsgStyle.SUCCESS, @@ -153,6 +168,7 @@ def run_one_automation(automation: Automation): This does not affect the automation's recurrence: its cursor stays where it was, so the next recurring run still happens as scheduled (see `flexmeasures jobs run-automations`). Inactive automations can be run this way, too, which is how you can try one out before activating it. + A run on demand does not wait for the automations this one depends on, as it is meant to run now. """ try: returns = run_automation(automation) diff --git a/flexmeasures/cli/utils.py b/flexmeasures/cli/utils.py index b6e6f17d3b..070ab7362d 100644 --- a/flexmeasures/cli/utils.py +++ b/flexmeasures/cli/utils.py @@ -285,6 +285,26 @@ def validate_unique(ctx, param, value): return value +def assemble_automation_dependencies( + depends_on: tuple, allow_failure_of: tuple +) -> list[dict[str, Any]]: + """Combine the automations given with --depends-on and --allow-failure-of into the dependencies an automation is given. + + :raises click.UsageError: if --allow-failure-of names an automation that --depends-on does not. + """ + upstream_ids = {upstream.id for upstream in depends_on} + stray_ids = sorted({upstream.id for upstream in allow_failure_of} - upstream_ids) + if stray_ids: + raise click.UsageError( + f"--allow-failure-of names automation(s) {', '.join(map(str, stray_ids))}, which --depends-on does not." + ) + allowed_ids = {upstream.id for upstream in allow_failure_of} + return [ + {"upstream": upstream, "allow_failure": upstream.id in allowed_ids} + for upstream in depends_on + ] + + def abort(message: str): click.secho(message, **MsgStyle.ERROR) raise click.Abort() From 2dd69746108e81269f4ee9b403843d9022422c91 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:48 +0100 Subject: [PATCH 10/47] ui: pick the automations an automation waits for on the automations page Context: - Issue #2586 Change: - A 'Waits for' picker on the create and edit forms, copying the picked automation's schedule into the create form - The details panel lists the dependencies Signed-off-by: Mohamed Belhsan Hmida --- .../templates/assets/asset_automations.html | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 3ddbb9285e..407dda3d2e 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -192,6 +192,18 @@ +
+ + +
Lists the automations on this page. Use the API or the CLI to wait for an automation on another asset.
+
+ + +
+
+
+ + +
Lists the automations on this page. Use the API or the CLI to wait for an automation on another asset.
+
+ + +
+
@@ -492,8 +516,55 @@ /* Empty the creation form, so that the next automation starts from the defaults the form was rendered with. The form is shared with the Copy action, which leaves the previous automation's settings in it, so opening it blank has to clear them rather than trust that they were never there. */ + /* Fill a "Waits for" picker with the automations on this page, leaving out the one being edited, as it cannot wait for itself. + An automation it already waits for that is not on this page, such as one on another asset, is kept as an option of its own, + so that saving the form does not drop it. */ + function fillDependsOnPicker(selector, excludedId = null, selected = []) { + const picker = $(selector).empty(); + const selectedIds = new Set(selected.map((dependency) => dependency.automation)); + const onPage = [...automationsById.values()].filter((automation) => automation.id !== excludedId); + for (const automation of onPage) { + picker.append(new Option( + `${automation.id}: ${automation.name} (${automation["cron"]}, ${automation.timezone})`, + automation.id, + false, + selectedIds.has(automation.id), + )); + } + for (const dependency of selected) { + if (!automationsById.has(dependency.automation)) { + picker.append(new Option(`${dependency.automation} (not on this page)`, dependency.automation, false, true)); + } + } + } + + // Whether to allow for failure, as the checkbox says it, or for each dependency as it was, while the checkbox still says "some of them". + function dependsOnValues(selector, allowFailureSelector, previous = []) { + const allowFailureBox = document.querySelector(allowFailureSelector); + const previouslyAllowed = new Map(previous.map((dependency) => [dependency.automation, dependency["allow-failure"]])); + return ($(selector).val() || []).map((id) => { + const automationId = Number(id); + return { + automation: automationId, + "allow-failure": allowFailureBox.indeterminate + ? Boolean(previouslyAllowed.get(automationId)) + : allowFailureBox.checked, + }; + }); + } + + // One checkbox stands for every dependency, so where they differ, it says neither, until it is clicked. + function showAllowFailure(allowFailureSelector, dependencies) { + const allowFailureBox = document.querySelector(allowFailureSelector); + const allowed = dependencies.filter((dependency) => dependency["allow-failure"]).length; + allowFailureBox.checked = dependencies.length > 0 && allowed === dependencies.length; + allowFailureBox.indeterminate = allowed > 0 && allowed < dependencies.length; + } + function resetNewAutomationForm() { document.getElementById("newAutomationForm").reset(); + fillDependsOnPicker("#automationDependsOn"); + showAllowFailure("#automationAllowFailure", []); // Resetting restores the type the form was rendered with, and the generator fields follow whichever type that is. clearSelectedSource(); showGeneratorFields(); @@ -510,6 +581,8 @@ $("#automationTimezone").val(values.timezone); $("#automationParameters").val(values.parameters); $("#automationActive").prop("checked", values.active); + fillDependsOnPicker("#automationDependsOn", null, values.dependsOn); + showAllowFailure("#automationAllowFailure", values.dependsOn); } // The name column allows 80 characters, and so does the API, so a long name loses its tail rather than the suffix that marks the copy. @@ -537,6 +610,8 @@ parameters: JSON.stringify(details?.parameters ?? {}, null, 4), active: false, sourceId: SOURCE_TYPE_PER_AUTOMATION_TYPE[automation.type] && sourceId ? sourceId : null, + // A copy computes from the same input as its original, so it waits for the same automations. + dependsOn: automation["depends-on"] || [], }; } @@ -689,6 +764,17 @@
Writes to

${sensorLinks(res["output-sensors"], res.source)}

Parameters
${esc(JSON.stringify(res.parameters, null, 4))}
+
Waits for
+

${dependsOnText(res["depends-on"])}

Durable runs
${esc(JSON.stringify({ total: runStats.total || 0, dispatch: runStats.dispatch || {}, execution: runStats.execution || {}, "latest-run": latestRunSummary }, null, 4))}
Recently created jobs @@ -972,6 +1061,8 @@
Recently created jobs $("#editAutomationCron").val(automation["cron"]); $("#editAutomationTimezone").val(automation.timezone); $("#editAutomationActive").prop("checked", automation.active); + fillDependsOnPicker("#editAutomationDependsOn", automation.id, automation["depends-on"] || []); + showAllowFailure("#editAutomationAllowFailure", automation["depends-on"] || []); $("#editAutomationErr").addClass("d-none").text(""); bootstrap.Modal.getOrCreateInstance(document.getElementById("editAutomationModal")).show(); }); @@ -1029,6 +1120,16 @@
Recently created jobs }); }); + /* A run waits for the upstream runs scheduled for the very same time, so an automation has to recur on the schedule of those it waits for. + Picking one therefore copies its schedule into the form, rather than leaving a mismatch for the server to refuse. */ + $("#automationDependsOn").on("change", function () { + const picked = ($(this).val() || []).map((id) => automationsById.get(Number(id))).filter(Boolean); + if (picked.length) { + $("#automationCron").val(picked[0]["cron"]); + $("#automationTimezone").val(picked[0].timezone); + } + }); + // Opening the form from the New automation button starts from the defaults, not from whatever a copy left in it. $("#newAutomationButton").on("click", () => resetNewAutomationForm()); @@ -1044,6 +1145,11 @@
Recently created jobs cron: $("#editAutomationCron").val(), timezone: $("#editAutomationTimezone").val(), active: $("#editAutomationActive").is(":checked"), + "depends-on": dependsOnValues( + "#editAutomationDependsOn", + "#editAutomationAllowFailure", + automationsById.get(Number(automationId))?.["depends-on"] || [], + ), }), success: () => location.reload(), error: function (xhr) { @@ -1186,6 +1292,7 @@
Recently created jobs timezone: $("#automationTimezone").val(), active: $("#automationActive").is(":checked"), parameters: parameters, + "depends-on": dependsOnValues("#automationDependsOn", "#automationAllowFailure"), }; if (reusesSource) { automationData.source = selectedAutomationSource.id; From 0e9eb6883870012e0bfe5c10af490f3b7432dcf7 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:36:48 +0100 Subject: [PATCH 11/47] tests: automation dependencies in the runner, API, CLI and asset copies Context: - Issue #2586 Change: - Runner tests with a real worker: wait then release, skip, allow-failure, cascade, missed release, passed-over and inactive upstreams - Validation, API permission and schedule checks, CLI options, and copied dependencies Signed-off-by: Mohamed Belhsan Hmida --- .../test_asset_copy_automations_fresh_db.py | 50 +++ ...st_automation_dependencies_api_fresh_db.py | 191 +++++++++ flexmeasures/cli/tests/test_automations.py | 96 +++++ .../test_automation_dependencies_fresh_db.py | 374 ++++++++++++++++++ 4 files changed, 711 insertions(+) create mode 100644 flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py create mode 100644 flexmeasures/data/tests/test_automation_dependencies_fresh_db.py diff --git a/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py index 4fba308614..49b4021273 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_copy_automations_fresh_db.py @@ -756,3 +756,53 @@ def test_a_copy_keeps_no_reference_its_maker_cannot_read( ] assert len(skipped) == 1 assert str(client_power.id) in skipped[0].reason + + +def test_a_copy_waits_for_the_copies_of_what_its_original_waits_for( + fresh_db, automated_site +): + """Within the copied subtree, a dependency points at the copied automation; outside it, at the very automation the original waits for.""" + from flexmeasures.data.models.automations import AutomationDependency + + neighbouring_automation = _add_automation( + fresh_db, + asset=automated_site["neighbour"], + name="Neighbouring power forecasts", + parameters={"sensor": automated_site["neighbouring_power"].id}, + ) + site_automation = automated_site["site_automation"] + fresh_db.session.add_all( + [ + AutomationDependency( + automation=site_automation, + upstream=automated_site["meter_automation"], + ), + AutomationDependency( + automation=site_automation, + upstream=neighbouring_automation, + allow_failure=True, + ), + ] + ) + fresh_db.session.commit() + + site_copy = copy_asset(automated_site["site"]).asset + + copied_site_automation = _automations_of(fresh_db, site_copy)[0] + copied_meter_automation = _automations_of(fresh_db, _child_of(fresh_db, site_copy))[ + 0 + ] + assert sorted( + (dependency.upstream_id, dependency.allow_failure) + for dependency in copied_site_automation.dependencies + ) == sorted( + [ + (copied_meter_automation.id, False), + (neighbouring_automation.id, True), + ] + ) + # The original keeps waiting for what it waited for. + assert {dependency.upstream_id for dependency in site_automation.dependencies} == { + automated_site["meter_automation"].id, + neighbouring_automation.id, + } diff --git a/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py new file mode 100644 index 0000000000..c5160d741a --- /dev/null +++ b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py @@ -0,0 +1,191 @@ +"""Naming the automations an automation waits for, through the API.""" + +from datetime import timedelta + +import pytest +from flask import url_for + +from flexmeasures.data.models.automations import Automation +from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.time_series import Sensor + + +@pytest.fixture +def upstream_automations( + fresh_db, setup_roles_users_fresh_db, setup_generic_assets_fresh_db +): + """An automation on the prosumer's own battery, one on a public asset, and one on the supplier's wind turbine, all recurring at 06:00 UTC.""" + assets = setup_generic_assets_fresh_db + battery = assets["test_battery"] + sensor = Sensor( + name="battery power", + unit="MW", + event_resolution=timedelta(minutes=15), + generic_asset=battery, + ) + generator = DataSource( + name="forecaster", type="forecaster", model="TrainPredictPipeline" + ) + fresh_db.session.add_all([sensor, generator]) + fresh_db.session.flush() + automations = { + key: Automation( + asset=assets[asset_key], + generator=generator, + type="forecasting", + name=f"Upstream on {asset_key}", + cronstr="0 6 * * *", + timezone="UTC", + parameters={}, + ) + for key, asset_key in ( + ("own", "test_battery"), + ("public", "troposphere"), + ("foreign", "test_wind_turbine"), + ) + } + fresh_db.session.add_all(automations.values()) + fresh_db.session.commit() + return battery, sensor, automations + + +def _post(client, battery, sensor, depends_on, cron="0 6 * * *"): + return client.post( + url_for("AssetAPI:post_automation", id=battery.id), + json={ + "name": "Downstream forecasts", + "cron": cron, + "timezone": "UTC", + "type": "forecasting", + "parameters": {"sensor": sensor.id}, + "depends-on": depends_on, + }, + ) + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_waiting_for_readable_automations( + app, fresh_db, upstream_automations, requesting_user +): + """Upstream automations can be on the user's own assets and on public ones, and the response names them.""" + battery, sensor, upstream = upstream_automations + with app.test_client() as client: + response = _post( + client, + battery, + sensor, + [ + {"automation": upstream["own"].id}, + {"automation": upstream["public"].id, "allow-failure": True}, + ], + ) + assert response.status_code == 201, response.json + assert response.json["depends-on"] == [ + {"automation": upstream["own"].id, "allow-failure": False}, + {"automation": upstream["public"].id, "allow-failure": True}, + ] + listing = client.get(url_for("AssetAPI:get_automations", id=battery.id)) + listed = { + automation["id"]: automation for automation in listing.json["automations"] + } + assert listed[response.json["id"]]["depends-on"] == response.json["depends-on"] + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_refuses_to_wait_for_an_unreadable_automation( + app, fresh_db, upstream_automations, requesting_user +): + battery, sensor, upstream = upstream_automations + with app.test_client() as client: + response = _post( + client, battery, sensor, [{"automation": upstream["foreign"].id}] + ) + assert response.status_code == 403, response.json + assert ( + fresh_db.session.query(Automation) + .filter_by(name="Downstream forecasts") + .count() + == 0 + ) + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_post_automation_refuses_another_schedule( + app, fresh_db, upstream_automations, requesting_user +): + battery, sensor, upstream = upstream_automations + with app.test_client() as client: + response = _post( + client, + battery, + sensor, + [{"automation": upstream["own"].id}], + cron="0 7 * * *", + ) + assert response.status_code == 422, response.json + assert "same schedule" in str(response.json["message"]["json"]["depends-on"]) + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_patch_automation_dependencies( + app, fresh_db, upstream_automations, requesting_user +): + """Dependencies can be replaced and cleared, and a schedule change that would break them is refused.""" + battery, sensor, upstream = upstream_automations + with app.test_client() as client: + created = _post(client, battery, sensor, [{"automation": upstream["own"].id}]) + assert created.status_code == 201, created.json + url = url_for( + "AssetAPI:patch_automation", id=battery.id, automation_id=created.json["id"] + ) + + refused = client.patch(url, json={"cron": "0 7 * * *"}) + assert refused.status_code == 422, refused.json + assert "depends-on" in refused.json["message"]["json"] + + replaced = client.patch( + url, + json={ + "depends-on": [ + {"automation": upstream["public"].id, "allow-failure": True} + ] + }, + ) + assert replaced.status_code == 200, replaced.json + assert replaced.json["depends-on"] == [ + {"automation": upstream["public"].id, "allow-failure": True} + ] + + cleared = client.patch(url, json={"cron": "0 7 * * *", "depends-on": []}) + assert cleared.status_code == 200, cleared.json + assert cleared.json["depends-on"] == [] + assert cleared.json["cron"] == "0 7 * * *" + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_patch_automation_refuses_to_wait_for_an_unreadable_automation( + app, fresh_db, upstream_automations, requesting_user +): + battery, sensor, upstream = upstream_automations + with app.test_client() as client: + response = client.patch( + url_for( + "AssetAPI:patch_automation", + id=battery.id, + automation_id=upstream["own"].id, + ), + json={"depends-on": [{"automation": upstream["foreign"].id}]}, + ) + assert response.status_code == 403, response.json + fresh_db.session.expire_all() + assert fresh_db.session.get(Automation, upstream["own"].id).dependencies == [] diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 8dd8172da3..4f24a46221 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -2970,3 +2970,99 @@ def test_show_automations_shows_a_schedule_automation( assert result.exit_code == 0, result.output assert re.search(r"^Type\s+scheduling$", result.output, re.MULTILINE) assert automation.parameters["duration"] in result.output + + +def _add_hourly_automation(runner, sensor_id: int, name: str, *extra_args): + from flexmeasures.cli.data_add import add_automation + + return runner.invoke( + add_automation, + [ + "--asset", + "1", + "--name", + name, + "--cron", + "0 * * * *", + "--timezone", + "UTC", + "--sensor", + str(sensor_id), + *extra_args, + ], + ) + + +def test_add_edit_and_show_automation_dependencies( + app, fresh_db, setup_dummy_data_fresh_db +): + """An automation can be told to wait for another with --depends-on, which edit replaces or clears, and show lists.""" + from flexmeasures.cli.data_edit import edit_automation + from flexmeasures.cli.data_show import list_automations + + runner = app.test_cli_runner() + sensor_id = setup_dummy_data_fresh_db[0] + assert _add_hourly_automation(runner, sensor_id, "Upstream").exit_code == 0 + upstream = fresh_db.session.scalars( + select(Automation).filter_by(name="Upstream") + ).one() + + result = _add_hourly_automation( + runner, + sensor_id, + "Downstream", + "--depends-on", + str(upstream.id), + "--allow-failure-of", + str(upstream.id), + ) + assert result.exit_code == 0, result.output + downstream = fresh_db.session.scalars( + select(Automation).filter_by(name="Downstream") + ).one() + assert [ + (dependency.upstream_id, dependency.allow_failure) + for dependency in downstream.dependencies + ] == [(upstream.id, True)] + + shown = runner.invoke(list_automations, ["--id", downstream.id]) + assert shown.exit_code == 0, shown.output + assert f"Upstream (ID: {upstream.id}), also when it fails" in shown.output + + # The upstream automation keeps its schedule while another one waits for it. + refused = runner.invoke( + edit_automation, ["--id", str(upstream.id), "--cron", "5 * * * *"] + ) + assert refused.exit_code != 0 + assert str(downstream.id) in refused.output + + cleared = runner.invoke( + edit_automation, ["--id", str(downstream.id), "--no-dependencies"] + ) + assert cleared.exit_code == 0, cleared.output + fresh_db.session.expire_all() + assert fresh_db.session.get(Automation, downstream.id).dependencies == [] + + +def test_add_automation_allow_failure_of_needs_depends_on( + app, fresh_db, setup_dummy_data_fresh_db +): + runner = app.test_cli_runner() + sensor_id = setup_dummy_data_fresh_db[0] + assert _add_hourly_automation(runner, sensor_id, "Upstream").exit_code == 0 + upstream = fresh_db.session.scalars( + select(Automation).filter_by(name="Upstream") + ).one() + + result = _add_hourly_automation( + runner, sensor_id, "Downstream", "--allow-failure-of", str(upstream.id) + ) + + assert result.exit_code == 2, result.output + assert "which --depends-on does not" in result.output + assert ( + fresh_db.session.scalars( + select(Automation).filter_by(name="Downstream") + ).one_or_none() + is None + ) diff --git a/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py new file mode 100644 index 0000000000..822f268a5f --- /dev/null +++ b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py @@ -0,0 +1,374 @@ +"""Automations that wait for the runs of other automations: validation, and how the runner holds, releases and skips their runs.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from marshmallow import ValidationError +from rq.job import Job +from sqlalchemy import select + +from flexmeasures.data.automations import register_automation_handler +from flexmeasures.data.models.automations import Automation, AutomationRun +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.services import automations as automation_services +from flexmeasures.data.services.automations import ( + create_automation, + dispatch_automation_run, + get_dispatchable_automation_runs, + run_automation, + update_automation, +) +from flexmeasures.data.tests.plugins.mock_ingestion import ( + MockIngestionGenerator, + __automation_types__, +) +from flexmeasures.utils.job_utils import work_on_rq + +# Every automation here recurs hourly and was created well before the run at 01:00, which a runner ticking at 01:00 dispatches. +CREATED_AT = datetime(2026, 8, 5, 0, 0, tzinfo=timezone.utc) +SCHEDULED_AT = datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc) +TICK = datetime(2026, 8, 5, 1, 0, 30, tzinfo=timezone.utc) + + +@pytest.fixture +def ingestion_plugin(app, monkeypatch): + monkeypatch.setattr(app, "automation_handlers", dict(app.automation_handlers)) + monkeypatch.setattr( + app, + "data_generators", + {key: dict(value) for key, value in app.data_generators.items()}, + ) + register_automation_handler(app, __automation_types__[0]) + app.redis_connection.flushdb() + yield + app.redis_connection.flushdb() + + +@pytest.fixture +def failing_value(monkeypatch): + """Have an ingestion job fail when its automation's parameters ask for a negative value.""" + compute = MockIngestionGenerator._compute + + def _compute(self, value): + if value < 0: + raise ValueError("The external system was unreachable.") + return compute(self, value) + + monkeypatch.setattr(MockIngestionGenerator, "_compute", _compute) + + +@pytest.fixture +def make_automation(fresh_db, ingestion_plugin): + """Create hourly ingestion automations on one asset, each due at 01:00, optionally waiting for others.""" + asset = GenericAsset( + name="Dependency site", + generic_asset_type=GenericAssetType(name="Dependency asset type"), + ) + sensor = Sensor(name="measurement", generic_asset=asset, unit="kWh") + fresh_db.session.add(sensor) + fresh_db.session.commit() + + def _make_automation( + name: str, + depends_on: list[Automation] = (), + allow_failure: bool = False, + value: float = 1.0, + cronstr: str = "0 * * * *", + tz: str = "UTC", + ) -> Automation: + automation, _ = create_automation( + asset=asset, + name=name, + cronstr=cronstr, + timezone=tz, + automation_type="mock-ingestion", + config={"sensor": sensor.id}, + parameters={"value": value}, + dependencies=[ + {"upstream": upstream, "allow_failure": allow_failure} + for upstream in depends_on + ], + ) + automation.created_at = CREATED_AT + automation.cursor = CREATED_AT + fresh_db.session.commit() + return automation + + return _make_automation + + +def _tick(now: datetime = TICK) -> dict[int, dict]: + """Claim and dispatch what a runner would at this moment, dependents first, which is the order that makes them wait.""" + claimed_runs = get_dispatchable_automation_runs(now) + claimed_runs.sort(key=lambda claimed: -claimed.run.automation_id) + return { + claimed.run.automation_id: dispatch_automation_run(claimed) + for claimed in claimed_runs + } + + +def _run_of(db, automation: Automation) -> AutomationRun: + db.session.expire_all() + return db.session.scalars( + select(AutomationRun).where(AutomationRun.automation_id == automation.id) + ).one() + + +def _work(app, run: AutomationRun) -> None: + """Have a worker run the jobs a run recorded.""" + for intent in run.job_intents: + work_on_rq( + app.queues[intent.queue], + job=Job.fetch(intent.rq_job_id, connection=app.redis_connection), + ) + + +def test_a_dependent_run_waits_and_then_runs_once_its_upstream_run_succeeded( + fresh_db, app, make_automation +): + """The dependent run queues nothing while the upstream job runs, and its job is queued as soon as that job succeeded.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + returns = _tick() + + assert returns[flows.id]["dispatch_state"] == "waiting" + flows_run = _run_of(fresh_db, flows) + assert flows_run.dispatch_state == "waiting" + assert flows_run.job_intents == [] + assert flows_run.claim_owner is None + snapper_run = _run_of(fresh_db, snapper) + assert flows_run.plan["upstream-runs"] == [ + { + "automation": snapper.id, + "allow-failure": False, + "run": snapper_run.id, + "state": "pending", + } + ] + + _work(app, snapper_run) + + flows_run = _run_of(fresh_db, flows) + assert flows_run.dispatch_state == "queued" + assert [intent.status for intent in flows_run.job_intents] == ["queued"] + assert [attempt.outcome for attempt in flows_run.attempts] == ["waiting", "queued"] + _work(app, flows_run) + assert _run_of(fresh_db, flows).execution_state == "succeeded" + + +def test_a_dependent_run_is_skipped_when_its_upstream_run_failed( + fresh_db, app, make_automation, failing_value +): + """Without allowing for failure, a failed upstream run ends the dependent run without queueing anything, and says why.""" + snapper = make_automation("Snap meter readings", value=-1) + flows = make_automation("Compute flows", depends_on=[snapper]) + _tick() + + _work(app, _run_of(fresh_db, snapper)) + + flows_run = _run_of(fresh_db, flows) + assert flows_run.execution_state == "skipped" + assert flows_run.execution_completed_at is not None + assert flows_run.dispatch_completed_at is not None + assert flows_run.job_intents == [] + assert flows_run.last_error_type == "UpstreamRunFailed" + assert f"automation(s) {snapper.id}" in flows_run.last_error_message + queued_for = { + Job.fetch(job_id, connection=app.redis_connection).meta["trigger"][ + "automation_id" + ] + for job_id in app.queues["ingestion"].job_ids + } + assert flows.id not in queued_for + + +def test_a_dependent_run_that_allows_for_failure_runs_anyway( + fresh_db, app, make_automation, failing_value +): + """Allowing for failure, the dependent run still waits for the upstream run, and then goes ahead although it failed.""" + snapper = make_automation("Snap meter readings", value=-1) + flows = make_automation("Compute flows", depends_on=[snapper], allow_failure=True) + assert _tick()[flows.id]["dispatch_state"] == "waiting" + + _work(app, _run_of(fresh_db, snapper)) + + flows_run = _run_of(fresh_db, flows) + assert flows_run.dispatch_state == "queued" + assert len(flows_run.job_intents) == 1 + + +def test_a_skip_cascades_down_a_chain(fresh_db, app, make_automation, failing_value): + """A run skipped for a failed upstream run did not deliver its own results, so what waits for it is skipped, too.""" + snapper = make_automation("Snap meter readings", value=-1) + flows = make_automation("Compute flows", depends_on=[snapper]) + totals = make_automation("Sum flows", depends_on=[flows]) + returns = _tick() + assert returns[totals.id]["dispatch_state"] == "waiting" + + _work(app, _run_of(fresh_db, snapper)) + + assert _run_of(fresh_db, flows).execution_state == "skipped" + assert _run_of(fresh_db, totals).execution_state == "skipped" + + +def test_a_waiting_run_whose_release_was_missed_is_picked_up_by_the_next_tick( + fresh_db, app, make_automation, monkeypatch +): + """Should the upstream run end without releasing its dependents, the runner dispatches them on its next tick.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + _tick() + with monkeypatch.context() as patch: + patch.setattr( + automation_services, + "release_automation_runs_waiting_for", + lambda run: None, + ) + _work(app, _run_of(fresh_db, snapper)) + assert _run_of(fresh_db, flows).dispatch_state == "waiting" + + returns = _tick(datetime(2026, 8, 5, 1, 1, 30, tzinfo=timezone.utc)) + + assert returns[flows.id]["dispatch_state"] == "queued" + assert _run_of(fresh_db, flows).dispatch_state == "queued" + + +def test_a_waiting_run_stays_waiting_on_a_tick_while_its_upstream_run_is_pending( + fresh_db, app, make_automation +): + """The runner leaves a waiting run alone for as long as the run it waits for has not ended.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + _tick() + + assert _tick(datetime(2026, 8, 5, 1, 1, 30, tzinfo=timezone.utc)) == {} + assert _run_of(fresh_db, flows).dispatch_state == "waiting" + + +def test_a_run_whose_upstream_automation_passed_over_that_time_is_skipped( + fresh_db, make_automation +): + """An upstream automation that moved past the scheduled time without a run there will never produce the input, so the run is skipped.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + # As if the upstream automation's missed run at 01:00 were left out by a bounded catch-up. + snapper.cursor = SCHEDULED_AT + fresh_db.session.commit() + + returns = _tick() + + assert returns[flows.id]["dispatch_state"] == "skipped" + assert _run_of(fresh_db, flows).plan["upstream-runs"][0]["state"] == "passed-over" + + +def test_a_run_goes_ahead_without_an_inactive_upstream_automation( + fresh_db, make_automation +): + """An upstream automation that was never going to run at that time is left out, and the record says so.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + snapper.active = False + fresh_db.session.commit() + + returns = _tick() + + assert returns[flows.id]["dispatch_state"] == "queued" + assert _run_of(fresh_db, flows).plan["upstream-runs"] == [ + { + "automation": snapper.id, + "allow-failure": False, + "run": None, + "state": "not-scheduled", + } + ] + + +def test_a_run_on_demand_does_not_wait(fresh_db, app, make_automation): + """Running an automation on demand means running it now, so it does not wait for the automations it depends on.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + returns = run_automation(flows) + + assert returns["n_jobs"] == 1 + assert app.queues["ingestion"].count == 1 + + +def test_dependencies_must_share_the_schedule(fresh_db, make_automation): + """A run waits for the upstream run scheduled for the same time, so a different cron string or timezone is refused.""" + snapper = make_automation("Snap meter readings") + with pytest.raises(ValidationError) as e: + make_automation("Compute flows", depends_on=[snapper], cronstr="5 * * * *") + assert "same schedule" in str(e.value.messages["depends-on"]) + fresh_db.session.rollback() + with pytest.raises(ValidationError): + make_automation("Compute flows", depends_on=[snapper], tz="Europe/Amsterdam") + + +def test_cycles_and_self_dependencies_are_refused(fresh_db, make_automation): + """An automation that ends up waiting for itself would never run.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + totals = make_automation("Sum flows", depends_on=[flows]) + + with pytest.raises(ValidationError) as e: + update_automation(snapper, dependencies=[{"upstream": totals}]) + assert "waiting for itself" in str(e.value.messages["depends-on"]) + with pytest.raises(ValidationError): + update_automation(snapper, dependencies=[{"upstream": snapper}]) + fresh_db.session.rollback() + assert _dependency_ids(fresh_db, snapper) == [] + + +def test_an_automation_others_wait_for_keeps_its_schedule(fresh_db, make_automation): + """Changing the schedule of an automation others wait for would leave their runs waiting for runs that never come.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + with pytest.raises(ValidationError) as e: + update_automation(snapper, cronstr="5 * * * *") + assert str(flows.id) in str(e.value.messages["cron"]) + assert snapper.cronstr == "0 * * * *" + + # The dependent automation can move along with dropping its dependency, after which the upstream one is free to move, too. + update_automation(flows, cronstr="5 * * * *", dependencies=[]) + fresh_db.session.commit() + update_automation(snapper, cronstr="5 * * * *") + fresh_db.session.commit() + assert snapper.cronstr == "5 * * * *" + + +def test_a_dependent_automation_cannot_move_away_from_its_upstream_schedule( + fresh_db, make_automation +): + """Changing only the schedule of an automation that waits for others is refused, as its runs would no longer line up.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + with pytest.raises(ValidationError): + update_automation(flows, cronstr="5 * * * *") + assert flows.cronstr == "0 * * * *" + + +def test_deleting_an_upstream_automation_drops_the_dependency( + fresh_db, make_automation +): + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + fresh_db.session.delete(snapper) + fresh_db.session.commit() + + assert _dependency_ids(fresh_db, flows) == [] + + +def _dependency_ids(db, automation: Automation) -> list[int]: + db.session.expire_all() + return [ + dependency.upstream_id + for dependency in db.session.get(Automation, automation.id).dependencies + ] From a29b23599bfc3986fd6323fa0155b163283c18f2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 03:37:25 +0100 Subject: [PATCH 12/47] docs: how an automation waits for other automations Context: - Issue #2586 Change: - New section on dependencies, the waiting and skipped run states, and that a run on demand does not wait Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/automations.rst | 39 +++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index 737fd4b2de..b73a5f77b2 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -223,6 +223,41 @@ It runs every night at 1 AM, an hour after midnight, so that the day's last read .. _running_automations: +.. _automation_dependencies: + +Waiting for other automations +----------------------------- + +Some automations use what another automation produces. +For example, a report that computes flows from a stock is only correct once another report has snapped the raw meter readings of that stock to a regular grid. +Rather than staggering their cron strings and hoping the first one finishes in time, let the second automation wait for the first: + +.. code-block:: bash + + flexmeasures add automation --asset 3 --name "Flows" --cron "*/5 * * * *" --timezone UTC \ + --type reporting --reporter PandasReporter --config flows-config.yml --parameters flows-parameters.yml \ + --depends-on 7 + +Each run of the waiting automation then waits for the run of automation 7 that is scheduled for the same time. +It queues its jobs as soon as that run has ended successfully, and is *skipped* if it failed, so that it never computes on input that was never produced. +A skipped run says which upstream run it was skipped for, and anything waiting for a skipped run is skipped, too. +To have a run go ahead even when the run it waits for failed, also pass ``--allow-failure-of 7``. + +An automation can wait for several others (repeat ``--depends-on``), and these can be on any asset you can read. +They have to recur on the same schedule, that is, with the same cron string in the same timezone, as a run waits for the upstream run scheduled for the very same time. +That is also why an automation that others wait for cannot change its schedule, and an automation that waits for others can only change its schedule together with its dependencies (see ``flexmeasures edit automation --help``). +A dependency that would have an automation end up waiting for itself is refused. + +Some cases where the upstream run does not happen: + +- If the upstream automation was due at that time but moved past it without a run, for instance because missed runs were coalesced while the runner was down, the waiting run is skipped (unless failure is allowed for). +- If the upstream automation was never going to run at that time, because it is inactive or did not exist yet, the waiting run goes ahead without it. Its run record says so. + +The API takes the same as ``depends-on``, a list like ``[{"automation": 7, "allow-failure": false}]``, and on the asset's *Automations* page the creation and edit forms let you pick the automations to wait for among those listed there. +Picking one copies its schedule into the creation form. + +When assets are copied along with their automations, a copy waits for the copy of the automation its original waited for, if that was copied too, and for the original one otherwise. + Running automations ------------------- @@ -274,7 +309,8 @@ a report run its reporting job, and a plugin run the job its plugin queued. A run which queued no jobs, such as a report run with nothing new to report on, has succeeded. A job which a schedule queues to fall back on a simpler scheduler is not one of the run's jobs, so a run whose schedule fell back has failed, with the error that made it fall back. -A run tracks two things separately: how far its *dispatch* got (``pending``, ``claimed``, ``partially_queued``, ``queued`` or ``failed``), and how its *execution* by the workers ended (``pending``, ``running``, ``succeeded``, ``failed`` or ``canceled``). +A run tracks two things separately: how far its *dispatch* got (``pending``, ``claimed``, ``waiting``, ``partially_queued``, ``queued`` or ``failed``), and how its *execution* by the workers ended (``pending``, ``running``, ``succeeded``, ``failed``, ``canceled`` or ``skipped``). +A run is ``waiting`` while the runs it depends on have not ended yet, and ``skipped`` if one of them failed (see :ref:`automation_dependencies`). Each attempt to dispatch a run is recorded too, with the runner which made it, what it queued, and why it failed if it did. This is what an operator needs to tell a run which failed before queueing anything, one which queued half its work, and one which queued everything but then failed while computing, apart from each other. @@ -303,6 +339,7 @@ That menu also lets you edit, copy, deactivate (or activate) and delete the auto The automation runs with the parameters it was created with, and the jobs it queues are recorded as its jobs, just like the jobs of a recurring run. An on-demand run does not affect the automation's recurrence: its cursor (see :ref:`automation_cursor`) stays where it was, so the next recurring run still happens as scheduled, and a run missed while the runner was down is still caught up. Inactive automations can be run this way, too, which is how you can try one out before activating it. +An on-demand run does not wait for the automations this one depends on, as it is meant to run now. Unlike a recurring run, an on-demand run is not protected against being started twice: asking for two runs in a row queues two runs. From 7d936276a73d30e8a80dacf372c4a11deb73ce1d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 14 Oct 2026 04:11:16 +0100 Subject: [PATCH 13/47] changelog: automation dependencies Context: - PR #2706 Change: - Entry in 'Automations, in detail' and in the CLI changelog Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + documentation/cli/change_log.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 44cb641e00..b4f75675f7 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -133,6 +133,7 @@ Automations arrived over several pull requests. This is what each of them contri * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization. A plugin job's result is kept for as long as results on its own queue are kept, and reports the number of beliefs it saved [see `PR #2553 `_ and `PR #2644 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] +* An automation can wait for the automations whose results it uses: each of its runs waits for their runs scheduled for the same time, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page [see `PR #2706 `_] v1.0.1 | September 9, 2026 diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 4553861db0..3268da093d 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -20,6 +20,7 @@ since v1.1.0 | September XX, 2026 * Add ``flexmeasures jobs run-automation --automation `` to queue the jobs for a single run of one automation, now, on top of its recurring runs. This leaves the automation's cursor alone, so its next recurring run still happens as scheduled, and inactive automations can be run this way, too. * ``flexmeasures delete sensor`` now warns which automations read from or write to a sensor before it is deleted, as an automation refers to its sensors by ID and would fail on its next run. * ``flexmeasures jobs run-automations`` now records a durable run for each scheduled run it claims, and retries the forecast runs whose queueing did not finish. A run which failed before queueing anything is dispatched again in full, and one which queued only part of its jobs resumes from its stored plan, reusing the job IDs it already queued. Each attempt is recorded with its owner, outcome and error, and the command reports the run and attempt it is working on. A run is only picked up by another runner once the claim lease of the runner holding it has expired, which is how a runner that died mid-queueing hands its work over. +* Add ``--depends-on`` and ``--allow-failure-of`` to ``flexmeasures add automation`` and ``flexmeasures edit automation`` (plus ``--no-dependencies`` on edit), to have an automation's runs wait for the runs of other automations scheduled for the same time. * ``flexmeasures edit automation`` now counts up the automation's schedule revision whenever it rebases the cursor (on a changed cron string or timezone, or on reactivation), which keeps the durable runs of the old and the new schedule apart, even at the same scheduled UTC time. * ``flexmeasures add report`` saves all of a report's outputs or none of them, where each output used to be committed on its own, and says for each output how many beliefs it saved. ``flexmeasures add forecasts`` saves the forecasts of a forecaster that a plugin registers, which used to be dropped while the command reported them as created. From 31632d7e00b67ade7db21fdd1f0a3f0bc7d4e61b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 30 Oct 2026 02:42:12 +0100 Subject: [PATCH 14/47] data/migrations: put automation dependencies after main's merge migration bf22b21d0990 Signed-off-by: Mohamed Belhsan Hmida --- .../versions/b8e4c2d7f913_add_automation_dependencies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py index 4f70e17ae4..010156e195 100644 --- a/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py +++ b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py @@ -4,7 +4,7 @@ A run that waits is 'waiting' to be dispatched, and a run whose upstream run failed is 'skipped'. Revision ID: b8e4c2d7f913 -Revises: f7c2e8b1a409 +Revises: bf22b21d0990 Create Date: 2026-10-14 10:00:00.000000 """ @@ -14,7 +14,7 @@ # revision identifiers, used by Alembic. revision = "b8e4c2d7f913" -down_revision = "f7c2e8b1a409" +down_revision = "bf22b21d0990" branch_labels = None depends_on = None From 2677b4fa418bf15a55ce1ca22f81e615c0cf588c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 30 Oct 2026 03:00:01 +0100 Subject: [PATCH 15/47] Let an automation wait for automations on another schedule A run now waits for each upstream automation's latest run due at or before its own scheduled time, by the upstream automation's own cron string and timezone, so for instance an hourly forecast can wait for meter data pulled every 15 minutes. The same-schedule check on creation and edit is dropped, and so is the UI copying the upstream schedule into the form. The run record names the time the upstream run was due at (due-at). Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- documentation/cli/change_log.rst | 2 +- documentation/features/automations.rst | 8 +- flexmeasures/api/v3_0/assets.py | 8 +- ...st_automation_dependencies_api_fresh_db.py | 21 ++- flexmeasures/cli/data_add.py | 6 +- flexmeasures/cli/data_edit.py | 8 +- flexmeasures/cli/jobs.py | 2 +- flexmeasures/cli/tests/test_automations.py | 9 +- ...8e4c2d7f913_add_automation_dependencies.py | 2 +- flexmeasures/data/models/automations.py | 4 +- flexmeasures/data/schemas/automations.py | 6 +- flexmeasures/data/services/automations.py | 150 +++++++----------- .../test_automation_dependencies_fresh_db.py | 120 +++++++++----- flexmeasures/ui/static/openapi-specs.json | 12 +- .../templates/assets/asset_automations.html | 16 +- 16 files changed, 185 insertions(+), 191 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9971d9c41e..83bee5515e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -179,7 +179,7 @@ Automations arrived over several pull requests. This is what each of them contri * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization. A plugin job's result is kept for as long as results on its own queue are kept, and reports the number of beliefs it saved [see `PR #2553 `_ and `PR #2644 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] -* An automation can wait for the automations whose results it uses: each of its runs waits for their runs scheduled for the same time, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page [see `PR #2706 `_] +* An automation can wait for the automations whose results it uses: each of its runs waits for their latest runs due at or before its own scheduled time, also on another schedule, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page [see `PR #2706 `_] * Update Readme and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 548a242d0a..f0f250f42f 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -12,7 +12,7 @@ since v1.1.0 | September XX, 2026 * Add ``flexmeasures jobs run-automations`` to queue jobs for all automations that are due to run this minute from standard five-field cron expressions. Run this command once per minute. Each scheduled run is claimed durably, so its jobs are queued exactly once even when several runners overlap. Runs missed while the runner was down are caught up once, with several missed forecast runs coalesced into the latest useful forecast, and a run at a skipped or repeated daylight-saving-time hour happens exactly once. * Add ``flexmeasures jobs run-automation --automation `` to queue the jobs for a single run of one automation, now, on top of its recurring runs. This leaves the automation's cursor alone, so its next recurring run still happens as scheduled, and inactive automations can be run this way, too. * ``flexmeasures jobs run-automations`` now records a durable run for each scheduled run it claims, and retries the forecast and schedule runs whose queueing did not finish. A run which failed before queueing anything is dispatched again in full, and one which queued only part of its jobs resumes from its stored plan, reusing the job IDs it already queued. Each attempt is recorded with its owner, outcome and error, and the command reports the run and attempt it is working on. A run is only picked up by another runner once the claim lease of the runner holding it has expired, which is how a runner that died mid-queueing hands its work over. -* Add ``--depends-on`` and ``--allow-failure-of`` to ``flexmeasures add automation`` and ``flexmeasures edit automation`` (plus ``--no-dependencies`` on edit), to have an automation's runs wait for the runs of other automations scheduled for the same time. +* Add ``--depends-on`` and ``--allow-failure-of`` to ``flexmeasures add automation`` and ``flexmeasures edit automation`` (plus ``--no-dependencies`` on edit), to have an automation's runs wait for the latest runs of other automations due at or before their own scheduled time. * ``flexmeasures edit automation`` now counts up the automation's schedule revision whenever it rebases the cursor (on a changed cron string or timezone, or on reactivation), which keeps the durable runs of the old and the new schedule apart, even at the same scheduled UTC time. * ``flexmeasures add automation --type scheduling`` refuses a flex config field which fixes a moment in time, such as ``soc-at-start`` or a ``soc-targets`` entry with a ``datetime``, naming the field: a recurring schedule automation computes a fresh schedule on every run, so such a value would be stale on the next one. * ``flexmeasures add automation`` refuses a fixed ``start``, ``end`` or ``prior`` in the parameters of any automation, as every run would share that moment. The parameters take ``start-offset`` and ``end-offset`` instead, or either one with a ``duration``, applied to the time each run was due, on the automation's own clock, and so do the new ``--start-offset`` and ``--end-offset`` options, next to ``--duration``. diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index 2ce5ef2bb8..dc28bc2c2c 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -239,14 +239,15 @@ Rather than staggering their cron strings and hoping the first one finishes in t --type reporting --reporter PandasReporter --config flows-config.yml --parameters flows-parameters.yml \ --depends-on 7 -Each run of the waiting automation then waits for the run of automation 7 that is scheduled for the same time. +Each run of the waiting automation then waits for the latest run of automation 7 that is due at or before its own scheduled time. It queues its jobs as soon as that run has ended successfully, and is *skipped* if it failed, so that it never computes on input that was never produced. A skipped run says which upstream run it was skipped for, and anything waiting for a skipped run is skipped, too. To have a run go ahead even when the run it waits for failed, also pass ``--allow-failure-of 7``. An automation can wait for several others (repeat ``--depends-on``), and these can be on any asset you can read. -They have to recur on the same schedule, that is, with the same cron string in the same timezone, as a run waits for the upstream run scheduled for the very same time. -That is also why an automation that others wait for cannot change its schedule, and an automation that waits for others can only change its schedule together with its dependencies (see ``flexmeasures edit automation --help``). +They may recur on another schedule, as each run waits for the latest upstream run due at or before its own scheduled time, by the upstream automation's own cron string and timezone. +For instance, an hourly forecast that waits for meter data pulled every 15 minutes runs at 10:00 once the pull due at 10:00 ended. +The other way around, a forecast every 15 minutes that waits for an hourly pull runs at 10:15, 10:30 and 10:45 on the pull due at 10:00, so a failure of that one pull has all three of them skipped. A dependency that would have an automation end up waiting for itself is refused. Some cases where the upstream run does not happen: @@ -255,7 +256,6 @@ Some cases where the upstream run does not happen: - If the upstream automation was never going to run at that time, because it is inactive or did not exist yet, the waiting run goes ahead without it. Its run record says so. The API takes the same as ``depends-on``, a list like ``[{"automation": 7, "allow-failure": false}]``, and on the asset's *Automations* page the creation and edit forms let you pick the automations to wait for among those listed there. -Picking one copies its schedule into the creation form. When assets are copied along with their automations, a copy waits for the copy of the automation its original waited for, if that was copied too, and for the original one otherwise. diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index da3fc54cf0..b7de43c7e2 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1783,9 +1783,9 @@ def post_automation(self, automation_data: dict, id: int, asset: GenericAsset): and permission to record data on the sensors it writes to. An automation can wait for other automations that it consumes the results of, listed in `depends-on`. - Each of its runs then waits for their runs scheduled for the same time, and queues its jobs once those ended, + Each of its runs then waits for their latest runs due at or before its own scheduled time, and queues its jobs once those ended, or is skipped if one of them failed, unless that dependency has `allow-failure` set. - The automations it waits for can be on any asset you can read, and have to recur on the same schedule. + The automations it waits for can be on any asset you can read, and may recur on another schedule. security: - ApiKeyAuth: [] parameters: @@ -1892,7 +1892,7 @@ def patch_automation( Any subset of the fields `name`, `cron`, `timezone`, `active` and `depends-on` can be sent. Changing the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change, so runs from before it are not caught up on, while a run due in that very minute still is. - An automation's runs wait for the runs of the automations in `depends-on` that are scheduled for the same time, + An automation's runs wait for the latest runs of the automations in `depends-on` that are due at or before their own scheduled time, so the recurrence and the timezone can only change along with `depends-on`, and not while other automations depend on this one. Other automation fields cannot be updated; instead, create a new automation. Requires permission to add data under the asset. @@ -1923,7 +1923,7 @@ def patch_automation( wait_for_another_automation: summary: Wait for the runs of automation 4 description: >- - Each run waits for the run of automation 4 scheduled for the same time, and is skipped if that one failed. + Each run waits for the latest run of automation 4 due at or before its own time, and is skipped if that one failed. value: depends-on: - automation: 4 diff --git a/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py index c5160d741a..8f9283055d 100644 --- a/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py @@ -116,9 +116,10 @@ def test_post_automation_refuses_to_wait_for_an_unreadable_automation( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_post_automation_refuses_another_schedule( +def test_post_automation_waiting_for_one_on_another_schedule( app, fresh_db, upstream_automations, requesting_user ): + """A run waits for the latest upstream run due at or before its own time, so the schedules need not match.""" battery, sensor, upstream = upstream_automations with app.test_client() as client: response = _post( @@ -128,8 +129,10 @@ def test_post_automation_refuses_another_schedule( [{"automation": upstream["own"].id}], cron="0 7 * * *", ) - assert response.status_code == 422, response.json - assert "same schedule" in str(response.json["message"]["json"]["depends-on"]) + assert response.status_code == 201, response.json + assert response.json["depends-on"] == [ + {"automation": upstream["own"].id, "allow-failure": False} + ] @pytest.mark.parametrize( @@ -138,7 +141,7 @@ def test_post_automation_refuses_another_schedule( def test_patch_automation_dependencies( app, fresh_db, upstream_automations, requesting_user ): - """Dependencies can be replaced and cleared, and a schedule change that would break them is refused.""" + """Dependencies can be replaced and cleared, and the schedule can change while they stay.""" battery, sensor, upstream = upstream_automations with app.test_client() as client: created = _post(client, battery, sensor, [{"automation": upstream["own"].id}]) @@ -147,9 +150,11 @@ def test_patch_automation_dependencies( "AssetAPI:patch_automation", id=battery.id, automation_id=created.json["id"] ) - refused = client.patch(url, json={"cron": "0 7 * * *"}) - assert refused.status_code == 422, refused.json - assert "depends-on" in refused.json["message"]["json"] + moved = client.patch(url, json={"cron": "0 7 * * *"}) + assert moved.status_code == 200, moved.json + assert moved.json["depends-on"] == [ + {"automation": upstream["own"].id, "allow-failure": False} + ] replaced = client.patch( url, @@ -164,7 +169,7 @@ def test_patch_automation_dependencies( {"automation": upstream["public"].id, "allow-failure": True} ] - cleared = client.patch(url, json={"cron": "0 7 * * *", "depends-on": []}) + cleared = client.patch(url, json={"depends-on": []}) assert cleared.status_code == 200, cleared.json assert cleared.json["depends-on"] == [] assert cleared.json["cron"] == "0 7 * * *" diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 8791491ba4..8ae03d8765 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1879,9 +1879,9 @@ def add_forecast( # noqa: C901 "depends_on", multiple=True, type=AutomationIdField(), - help="ID of an automation whose runs each run of this one waits for: the run scheduled for the same time." + help="ID of an automation whose runs each run of this one waits for: its latest run due at or before the run's own time." " A run queues its jobs once the runs it waits for ended, and is skipped if one of them failed (but see --allow-failure-of)." - " The automation waited for has to recur on the same schedule, so --cron and --timezone have to match it." + " The automation waited for may recur on another schedule." " Repeat to wait for several automations.", ) @click.option( @@ -1944,7 +1944,7 @@ def add_automation( # noqa: C901 and configuration stored on it. An automation that consumes what another automation produces can wait for it (--depends-on), - so that each of its runs queues its jobs only once the other automation's run for the same time ended. + so that each of its runs queues its jobs only once the other automation's latest run due at or before that time ended. Every forecaster and pipeline option that `flexmeasures add forecast` accepts is accepted here, too, but is left out of the help text above to keep it focused on the automation itself; diff --git a/flexmeasures/cli/data_edit.py b/flexmeasures/cli/data_edit.py index 30f2cb1a98..b9ea807eed 100644 --- a/flexmeasures/cli/data_edit.py +++ b/flexmeasures/cli/data_edit.py @@ -105,7 +105,7 @@ def fm_edit_data(): "depends_on", multiple=True, type=AutomationIdField(), - help="ID of an automation whose runs each run of this one waits for: the run scheduled for the same time." + help="ID of an automation whose runs each run of this one waits for: its latest run due at or before the run's own time." " Replaces the automations it waited for. Repeat to wait for several automations.", ) @click.option( @@ -132,11 +132,7 @@ def edit_automation( allow_failure_of: tuple[Automation, ...] = (), no_dependencies: bool = False, ): - """Edit the name, recurrence, timezone, activation status or dependencies of an automation. - - The recurrence and timezone of an automation that waits for other automations have to stay those of the automations it waits for, - so pass --depends-on (or --no-dependencies) along with --cron or --timezone to change both. - """ + """Edit the name, recurrence, timezone, activation status or dependencies of an automation.""" if no_dependencies and (depends_on or allow_failure_of): raise click.UsageError( "--no-dependencies cannot be combined with --depends-on or --allow-failure-of." diff --git a/flexmeasures/cli/jobs.py b/flexmeasures/cli/jobs.py index 64ff284d39..cf815c66fb 100644 --- a/flexmeasures/cli/jobs.py +++ b/flexmeasures/cli/jobs.py @@ -101,7 +101,7 @@ def run_automations(): Durable automation run records claim each scheduled run and make queueing resumable. Failed dispatch attempts are retried safely by reusing the original run plan and deterministic job IDs. - A run of an automation that depends on other automations waits for their runs scheduled for the same time, + A run of an automation that depends on other automations waits for their latest runs due at or before its own scheduled time, and queues its jobs as soon as the last of them ended, or is skipped if one of them failed. This command also dispatches the waiting runs whose upstream runs ended without them being released. """ diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 4f24a46221..73c0b1871c 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -3029,12 +3029,11 @@ def test_add_edit_and_show_automation_dependencies( assert shown.exit_code == 0, shown.output assert f"Upstream (ID: {upstream.id}), also when it fails" in shown.output - # The upstream automation keeps its schedule while another one waits for it. - refused = runner.invoke( - edit_automation, ["--id", str(upstream.id), "--cron", "5 * * * *"] + # The upstream automation can move to another schedule while another one waits for it. + moved = runner.invoke( + edit_automation, ["--id", str(upstream.id), "--cron", "*/15 * * * *"] ) - assert refused.exit_code != 0 - assert str(downstream.id) in refused.output + assert moved.exit_code == 0, moved.output cleared = runner.invoke( edit_automation, ["--id", str(downstream.id), "--no-dependencies"] diff --git a/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py index 010156e195..6247b53987 100644 --- a/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py +++ b/flexmeasures/data/migrations/versions/b8e4c2d7f913_add_automation_dependencies.py @@ -1,6 +1,6 @@ """add automation dependencies -Let an automation's runs wait for the run of another automation at the same scheduled time. +Let an automation's runs wait for the latest run of another automation due at or before their own scheduled time. A run that waits is 'waiting' to be dispatched, and a run whose upstream run failed is 'skipped'. Revision ID: b8e4c2d7f913 diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index ad5ef6a5eb..1bd65bbb3e 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -123,7 +123,7 @@ class Automation(db.Model, AuthModelMixin): passive_deletes=True, order_by="desc(AutomationRun.scheduled_at)", ) - # The automations whose run for the same scheduled time each run of this one waits for (see `AutomationDependency`). + # The automations whose latest run due at or before its own scheduled time each run of this one waits for (see `AutomationDependency`). dependencies = db.relationship( "AutomationDependency", foreign_keys="AutomationDependency.automation_id", @@ -196,7 +196,7 @@ def output_sensors(self) -> list: class AutomationDependency(db.Model): """One automation that another automation's runs wait for. - A run of the dependent automation is dispatched only once the upstream automation's run for the same scheduled time has ended, + A run of the dependent automation is dispatched only once the upstream automation's latest run due at or before its own scheduled time has ended, and is skipped if that run failed, unless the dependency allows for failure. """ diff --git a/flexmeasures/data/schemas/automations.py b/flexmeasures/data/schemas/automations.py index c271d4fbf1..44e31ec70d 100644 --- a/flexmeasures/data/schemas/automations.py +++ b/flexmeasures/data/schemas/automations.py @@ -75,7 +75,7 @@ class AutomationDependencySchema(Schema): required=True, data_key="automation", metadata={ - "description": "ID of the automation to wait for. It has to recur on the same schedule (the same cron string, in the same timezone).", + "description": "ID of the automation to wait for. It may recur on another schedule.", "example": 4, }, ) @@ -90,9 +90,9 @@ class AutomationDependencySchema(Schema): DEPENDS_ON_DESCRIPTION = ( - "The automations whose runs each run of this automation waits for: the runs scheduled for the same time." + "The automations whose runs each run of this automation waits for: their latest runs due at or before its own scheduled time." " A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure." - " Upstream automations can be on any asset you can read, and have to recur on the same schedule." + " Upstream automations can be on any asset you can read, and may recur on another schedule." " A run on demand does not wait." ) diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index 2f7332d3ee..e9491ae40e 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -403,33 +403,48 @@ def _upstream_run_state(run: AutomationRun) -> str: return "pending" -def _state_without_upstream_run(upstream: Automation, run: AutomationRun) -> str: - """Say what it means that an upstream automation has no run scheduled for the same time as this run. +def _state_without_upstream_run(upstream: Automation, due_at: datetime) -> str: + """Say what it means that an upstream automation has no run for the time it was due at. - 'pending': the upstream automation is due at that time, but no runner claimed its run yet. - 'passed-over': the upstream automation was due at that time, but moved past it without a run, as missed runs are coalesced or left out (see `get_due_automations`). - 'not-scheduled': the upstream automation was never going to run at that time, - as it is inactive, recurs on another schedule, or did not exist yet. + as it is inactive, or did not exist yet. """ - automation = run.automation if ( not upstream.active - or (upstream.cronstr, upstream.timezone) - != (automation.cronstr, automation.timezone) # An automation's first run can be the one due in the minute it was created (see `get_initial_cursor`). - or floor_to_minute(upstream.created_at) > run.scheduled_at + or floor_to_minute(upstream.created_at) > due_at ): return "not-scheduled" - if upstream.cursor is None or upstream.cursor < run.scheduled_at: + if upstream.cursor is None or upstream.cursor < due_at: return "pending" return "passed-over" +def get_upstream_due_at( + upstream: Automation, scheduled_at: datetime +) -> datetime | None: + """Return when the upstream run that a run scheduled at this time waits for is due. + + That is the latest time the upstream automation is due at, by its own schedule, up to and including the time the waiting run is scheduled at. + For instance, an hourly run at 10:00 waits for the run of a quarter-hourly upstream automation due at 10:00, + and a quarter-hourly run at 10:15 waits for the run of an hourly upstream automation due at 10:00. + + :returns: None if the upstream automation's schedule cannot be read, such as for a stale timezone. + """ + try: + return get_latest_scheduled_run(upstream, scheduled_at) + except (CroniterError, ValueError, ZoneInfoNotFoundError): + return None + + def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: """Decide whether a run can be dispatched yet, going by the runs of the automations it waits for. - Each upstream automation's run for the same scheduled time counts. + Each upstream automation's latest run due at or before the run's own scheduled time counts (see `get_upstream_due_at`), + so the automations need not recur on the same schedule. The run is skipped as soon as one of those failed or will not happen, unless its dependency allows for that, as its jobs would compute with input that was never produced. Otherwise, it waits for as long as one of them has yet to end. @@ -440,25 +455,33 @@ def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: """ found = [] for dependency in run.automation.dependencies: - upstream_run = db.session.scalars( - select(AutomationRun) - .where( - AutomationRun.automation_id == dependency.upstream_id, - AutomationRun.scheduled_at == run.scheduled_at, - ) - .order_by(AutomationRun.schedule_revision.desc()) - .limit(1) - ).first() + due_at = get_upstream_due_at(dependency.upstream, run.scheduled_at) + upstream_run = ( + db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.automation_id == dependency.upstream_id, + AutomationRun.scheduled_at == due_at, + ) + .order_by(AutomationRun.schedule_revision.desc()) + .limit(1) + ).first() + if due_at is not None + else None + ) + if upstream_run is not None: + state = _upstream_run_state(upstream_run) + elif due_at is None: + state = "not-scheduled" + else: + state = _state_without_upstream_run(dependency.upstream, due_at) found.append( { "automation": dependency.upstream_id, "allow-failure": dependency.allow_failure, + "due-at": due_at.isoformat() if due_at is not None else None, "run": upstream_run.id if upstream_run is not None else None, - "state": ( - _upstream_run_state(upstream_run) - if upstream_run is not None - else _state_without_upstream_run(dependency.upstream, run) - ), + "state": state, } ) if any( @@ -578,7 +601,10 @@ def release_automation_runs_waiting_for(upstream_run: AutomationRun) -> None: ) .where( AutomationDependency.upstream_id == upstream_run.automation_id, - AutomationRun.scheduled_at == upstream_run.scheduled_at, + # A run waits for the latest upstream run due at or before its own scheduled time (see `get_upstream_due_at`), + # so the runs that wait for this one are scheduled at the same time or later. + # Those that wait for a later upstream run are checked, too, and left waiting (see `release_waiting_automation_run`). + AutomationRun.scheduled_at >= upstream_run.scheduled_at, AutomationRun.dispatch_state == "waiting", ) .order_by(AutomationRun.id) @@ -2305,11 +2331,7 @@ def create_automation( if dependencies: # Check these first, so that nothing is set up for an automation that is refused. validate_automation_dependencies( - None, - dependencies, - cronstr=cronstr, - timezone=timezone or get_default_automation_timezone(asset), - check_permissions=check_permissions, + None, dependencies, check_permissions=check_permissions ) automation, warnings = get_automation_handler(automation_type).create( asset=asset, @@ -2530,40 +2552,19 @@ def update_automation( Anything that changes which runs are due, namely the recurrence, the timezone and reactivation, also resets the cursor to just before the minute of the change, so that runs from before it are not caught up on, while a run due in that very minute still is. - A run waits for the upstream runs scheduled for the same time, - so the recurrence and the timezone can only change in step with the automations it waits for, - and not at all while other automations wait for it. An audit log record is added to the asset. :param dependencies: the automations whose runs this automation's runs wait for, replacing the ones it had; None leaves them as they are. :param check_permissions: whether to require that the current user may read the upstream automations named in `dependencies`. - :raises marshmallow.ValidationError: if the dependencies, or the new schedule, are not possible. + :raises marshmallow.ValidationError: if the dependencies are not possible. :returns: a list of (human-readable) changes; empty if nothing changed. """ from flexmeasures.data.models.audit_log import AssetAuditLog # Check everything before changing anything, so that a refused update leaves the automation as it was. - validate_dependents_keep_schedule(automation, cronstr, timezone) - schedule_changes = (cronstr is not None and cronstr != automation.cronstr) or ( - timezone is not None and timezone != automation.timezone - ) - if dependencies is not None or schedule_changes: + if dependencies is not None: validate_automation_dependencies( - automation, - ( - dependencies - if dependencies is not None - else [ - { - "upstream": dependency.upstream, - "allow_failure": dependency.allow_failure, - } - for dependency in automation.dependencies - ] - ), - cronstr=cronstr, - timezone=timezone, - check_permissions=check_permissions and dependencies is not None, + automation, dependencies, check_permissions=check_permissions ) changes = [] @@ -2629,29 +2630,21 @@ def _upstream_ids_reachable_from(automation_ids: set[int]) -> set[int]: def validate_automation_dependencies( automation: Automation | None, dependencies: list[dict[str, Any]], - cronstr: str | None = None, - timezone: str | None = None, check_permissions: bool = False, ) -> None: """Check that an automation may wait for these upstream automations. - A run waits for the upstream run scheduled for the very same time, so an upstream automation has to recur on the same schedule: - the same cron string, read in the same timezone. + The automations need not recur on the same schedule, as a run waits for the latest upstream run due at or before its own scheduled time (see `get_upstream_due_at`). An automation waiting for itself, directly or through others, would never run, so such a cycle is refused. :param automation: the automation that is to wait, or None for one that is about to be created. :param dependencies: dicts with the upstream automation under "upstream", and whether its failure is allowed for under "allow_failure". - :param cronstr: the cron string the automation is about to have, if not the one it has. - :param timezone: the timezone the automation is about to have, if not the one it has. :param check_permissions: whether to require that the current user may read each upstream automation (and so its asset). :raises marshmallow.ValidationError: if a dependency is not possible, under the "depends-on" field. :raises werkzeug.exceptions.Forbidden: if the current user may not read an upstream automation. """ from flexmeasures.auth.policy import check_access - if automation is not None: - cronstr = cronstr or automation.cronstr - timezone = timezone or automation.timezone automation_id = automation.id if automation is not None else None errors = [] upstream_ids = [dependency["upstream"].id for dependency in dependencies] @@ -2673,12 +2666,6 @@ def validate_automation_dependencies( if automation_id is not None and upstream.id == automation_id: errors.append(f"Automation {upstream.id} cannot wait for itself.") continue - if upstream.cronstr != cronstr or upstream.timezone != timezone: - errors.append( - f"Automation {upstream.id} recurs per cron string '{upstream.cronstr}' in timezone '{upstream.timezone}'," - f" while this automation recurs per '{cronstr}' in '{timezone}'." - " A run waits for the upstream run scheduled for the same time, so both have to recur on the same schedule." - ) # An automation that is about to be created has nothing waiting for it yet, so it cannot close a cycle. if automation_id is not None and automation_id in _upstream_ids_reachable_from( {upstream_id for upstream_id in upstream_ids if upstream_id != automation_id} @@ -2690,33 +2677,6 @@ def validate_automation_dependencies( raise ValidationError({"depends-on": errors}) -def validate_dependents_keep_schedule( - automation: Automation, cronstr: str | None, timezone: str | None -) -> None: - """Refuse a new schedule for an automation that other automations wait for, as their runs would no longer line up with its runs. - - :raises marshmallow.ValidationError: under the "cron" field, naming the automations that wait for this one. - """ - new_cronstr = cronstr or automation.cronstr - new_timezone = timezone or automation.timezone - if (new_cronstr, new_timezone) == (automation.cronstr, automation.timezone): - return - dependent_ids = db.session.scalars( - select(AutomationDependency.automation_id) - .where(AutomationDependency.upstream_id == automation.id) - .order_by(AutomationDependency.automation_id) - ).all() - if dependent_ids: - raise ValidationError( - { - "cron": [ - f"Automation(s) {', '.join(map(str, dependent_ids))} wait for this automation's runs, so it has to keep recurring per '{automation.cronstr}' in '{automation.timezone}'." - " Remove those dependencies first, or change the schedule of the automations that wait for it, too, once they no longer do." - ] - } - ) - - def set_automation_dependencies( automation: Automation, dependencies: list[dict[str, Any]], diff --git a/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py index 822f268a5f..5d04fc5f7a 100644 --- a/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py +++ b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py @@ -145,6 +145,7 @@ def test_a_dependent_run_waits_and_then_runs_once_its_upstream_run_succeeded( { "automation": snapper.id, "allow-failure": False, + "due-at": SCHEDULED_AT.isoformat(), "run": snapper_run.id, "state": "pending", } @@ -281,6 +282,7 @@ def test_a_run_goes_ahead_without_an_inactive_upstream_automation( { "automation": snapper.id, "allow-failure": False, + "due-at": SCHEDULED_AT.isoformat(), "run": None, "state": "not-scheduled", } @@ -298,60 +300,102 @@ def test_a_run_on_demand_does_not_wait(fresh_db, app, make_automation): assert app.queues["ingestion"].count == 1 -def test_dependencies_must_share_the_schedule(fresh_db, make_automation): - """A run waits for the upstream run scheduled for the same time, so a different cron string or timezone is refused.""" - snapper = make_automation("Snap meter readings") - with pytest.raises(ValidationError) as e: - make_automation("Compute flows", depends_on=[snapper], cronstr="5 * * * *") - assert "same schedule" in str(e.value.messages["depends-on"]) - fresh_db.session.rollback() - with pytest.raises(ValidationError): - make_automation("Compute flows", depends_on=[snapper], tz="Europe/Amsterdam") +def _run_at(db, automation: Automation, scheduled_at: datetime) -> AutomationRun: + db.session.expire_all() + return db.session.scalars( + select(AutomationRun).where( + AutomationRun.automation_id == automation.id, + AutomationRun.scheduled_at == scheduled_at, + ) + ).one() -def test_cycles_and_self_dependencies_are_refused(fresh_db, make_automation): - """An automation that ends up waiting for itself would never run.""" - snapper = make_automation("Snap meter readings") - flows = make_automation("Compute flows", depends_on=[snapper]) - totals = make_automation("Sum flows", depends_on=[flows]) +def test_an_hourly_run_waits_for_the_upstream_run_due_at_its_own_time( + fresh_db, app, make_automation +): + """An hourly automation can wait for one that recurs every 15 minutes, and its run at 01:00 waits for the upstream run due at 01:00.""" + puller = make_automation("Pull meter data", cronstr="*/15 * * * *") + # Only the upstream run at 01:00 is due, rather than those at 00:15, 00:30 and 00:45, too. + puller.cursor = datetime(2026, 8, 5, 0, 45, tzinfo=timezone.utc) + fresh_db.session.commit() + forecast = make_automation("Forecast", depends_on=[puller]) - with pytest.raises(ValidationError) as e: - update_automation(snapper, dependencies=[{"upstream": totals}]) - assert "waiting for itself" in str(e.value.messages["depends-on"]) - with pytest.raises(ValidationError): - update_automation(snapper, dependencies=[{"upstream": snapper}]) - fresh_db.session.rollback() - assert _dependency_ids(fresh_db, snapper) == [] + returns = _tick() + assert returns[forecast.id]["dispatch_state"] == "waiting" + puller_run = _run_at(fresh_db, puller, SCHEDULED_AT) + upstream = _run_at(fresh_db, forecast, SCHEDULED_AT).plan["upstream-runs"][0] + assert (upstream["due-at"], upstream["run"]) == ( + SCHEDULED_AT.isoformat(), + puller_run.id, + ) -def test_an_automation_others_wait_for_keeps_its_schedule(fresh_db, make_automation): - """Changing the schedule of an automation others wait for would leave their runs waiting for runs that never come.""" - snapper = make_automation("Snap meter readings") - flows = make_automation("Compute flows", depends_on=[snapper]) + _work(app, puller_run) - with pytest.raises(ValidationError) as e: - update_automation(snapper, cronstr="5 * * * *") - assert str(flows.id) in str(e.value.messages["cron"]) - assert snapper.cronstr == "0 * * * *" + assert _run_at(fresh_db, forecast, SCHEDULED_AT).dispatch_state == "queued" - # The dependent automation can move along with dropping its dependency, after which the upstream one is free to move, too. - update_automation(flows, cronstr="5 * * * *", dependencies=[]) - fresh_db.session.commit() - update_automation(snapper, cronstr="5 * * * *") + +@pytest.mark.parametrize("upstream_fails", [False, True]) +def test_a_quarter_hourly_run_waits_for_the_latest_hourly_upstream_run( + fresh_db, app, make_automation, failing_value, upstream_fails +): + """A run every 15 minutes waits for the latest run of an hourly upstream automation. + + Its runs at 01:00 and 01:15 both wait for the upstream run due at 01:00, and both are released once that one ended. + """ + quarter_past = datetime(2026, 8, 5, 1, 15, tzinfo=timezone.utc) + puller = make_automation("Pull meter data", value=-1 if upstream_fails else 1) + forecast = make_automation("Forecast", depends_on=[puller], cronstr="*/15 * * * *") + forecast.cursor = datetime(2026, 8, 5, 0, 45, tzinfo=timezone.utc) fresh_db.session.commit() - assert snapper.cronstr == "5 * * * *" + _tick() + returns = _tick(datetime(2026, 8, 5, 1, 15, 30, tzinfo=timezone.utc)) + + assert returns[forecast.id]["dispatch_state"] == "waiting" + later_run = _run_at(fresh_db, forecast, quarter_past) + assert later_run.plan["upstream-runs"][0]["due-at"] == SCHEDULED_AT.isoformat() -def test_a_dependent_automation_cannot_move_away_from_its_upstream_schedule( + _work(app, _run_at(fresh_db, puller, SCHEDULED_AT)) + + expected = "skipped" if upstream_fails else "succeeded" + for scheduled_at in (SCHEDULED_AT, quarter_past): + run = _run_at(fresh_db, forecast, scheduled_at) + if not upstream_fails: + assert run.dispatch_state == "queued" + _work(app, run) + run = _run_at(fresh_db, forecast, scheduled_at) + assert run.execution_state == expected + + +def test_automations_waiting_for_each_other_can_change_their_schedules( fresh_db, make_automation ): - """Changing only the schedule of an automation that waits for others is refused, as its runs would no longer line up.""" + """Runs wait for the latest upstream run due at or before their own time, so either automation can move to another schedule.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + + update_automation(snapper, cronstr="*/15 * * * *") + update_automation(flows, cronstr="5 * * * *") + fresh_db.session.commit() + + assert (snapper.cronstr, flows.cronstr) == ("*/15 * * * *", "5 * * * *") + assert _dependency_ids(fresh_db, flows) == [snapper.id] + + +def test_cycles_and_self_dependencies_are_refused(fresh_db, make_automation): + """An automation that ends up waiting for itself would never run.""" snapper = make_automation("Snap meter readings") flows = make_automation("Compute flows", depends_on=[snapper]) + totals = make_automation("Sum flows", depends_on=[flows]) + with pytest.raises(ValidationError) as e: + update_automation(snapper, dependencies=[{"upstream": totals}]) + assert "waiting for itself" in str(e.value.messages["depends-on"]) with pytest.raises(ValidationError): - update_automation(flows, cronstr="5 * * * *") - assert flows.cronstr == "0 * * * *" + update_automation(snapper, dependencies=[{"upstream": snapper}]) + fresh_db.session.rollback() + assert _dependency_ids(fresh_db, snapper) == [] def test_deleting_an_upstream_automation_drops_the_dependency( diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 8065a2e7f6..77737a3a43 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3666,7 +3666,7 @@ }, "patch": { "summary": "Update an automation's name, cron string, timezone, activation status or dependencies.", - "description": "Any subset of the fields `name`, `cron`, `timezone`, `active` and `depends-on` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nAn automation's runs wait for the runs of the automations in `depends-on` that are scheduled for the same time,\nso the recurrence and the timezone can only change along with `depends-on`, and not while other automations depend on this one.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", + "description": "Any subset of the fields `name`, `cron`, `timezone`, `active` and `depends-on` can be sent.\nChanging the recurrence or the timezone, or reactivating the automation, resets its cursor to just before the minute of the change,\nso runs from before it are not caught up on, while a run due in that very minute still is.\nAn automation's runs wait for the latest runs of the automations in `depends-on` that are due at or before their own scheduled time,\nso the recurrence and the timezone can only change along with `depends-on`, and not while other automations depend on this one.\nOther automation fields cannot be updated; instead, create a new automation.\nRequires permission to add data under the asset.\n", "security": [ { "ApiKeyAuth": [] @@ -3707,7 +3707,7 @@ }, "wait_for_another_automation": { "summary": "Wait for the runs of automation 4", - "description": "Each run waits for the run of automation 4 scheduled for the same time, and is skipped if that one failed.", + "description": "Each run waits for the latest run of automation 4 due at or before its own time, and is skipped if that one failed.", "value": { "depends-on": [ { @@ -3829,7 +3829,7 @@ }, "post": { "summary": "Create an automation on an asset.", - "description": "Create a recurring forecasting, scheduling, reporting or plugin-defined task on the asset.\nThe parameters are validated by the schema matching the automation type:\nforecast parameters for type `forecasting`,\na schedule trigger message (without the asset id) for type `scheduling`,\nreport parameters for type `reporting`, or the registered plugin schema.\nRequires permission to add data under the asset.\n\nAn automation runs again and again, so its parameters cannot fix a moment in time:\na `start`, `end` or `prior` among them is refused.\nSay instead how the period each run covers relates to that run,\nwith two of `start-offset`, `end-offset` and `duration`.\nThe offsets are chains of comma-separated Pandas offsets, plus `DB` (day begin) and `HB` (hour begin),\napplied to the time the run was due, on the automation's own clock.\nLeave the timing out to start at the time of each run,\nor, for a report, to cover the period since the last successful report.\n\nThe automation can only involve sensors that you have access to yourself:\nread access to the sensors it reads data from,\nand permission to record data on the sensors it writes to.\n\nAn automation can wait for other automations that it consumes the results of, listed in `depends-on`.\nEach of its runs then waits for their runs scheduled for the same time, and queues its jobs once those ended,\nor is skipped if one of them failed, unless that dependency has `allow-failure` set.\nThe automations it waits for can be on any asset you can read, and have to recur on the same schedule.\n", + "description": "Create a recurring forecasting, scheduling, reporting or plugin-defined task on the asset.\nThe parameters are validated by the schema matching the automation type:\nforecast parameters for type `forecasting`,\na schedule trigger message (without the asset id) for type `scheduling`,\nreport parameters for type `reporting`, or the registered plugin schema.\nRequires permission to add data under the asset.\n\nAn automation runs again and again, so its parameters cannot fix a moment in time:\na `start`, `end` or `prior` among them is refused.\nSay instead how the period each run covers relates to that run,\nwith two of `start-offset`, `end-offset` and `duration`.\nThe offsets are chains of comma-separated Pandas offsets, plus `DB` (day begin) and `HB` (hour begin),\napplied to the time the run was due, on the automation's own clock.\nLeave the timing out to start at the time of each run,\nor, for a report, to cover the period since the last successful report.\n\nThe automation can only involve sensors that you have access to yourself:\nread access to the sensors it reads data from,\nand permission to record data on the sensors it writes to.\n\nAn automation can wait for other automations that it consumes the results of, listed in `depends-on`.\nEach of its runs then waits for their latest runs due at or before its own scheduled time, and queues its jobs once those ended,\nor is skipped if one of them failed, unless that dependency has `allow-failure` set.\nThe automations it waits for can be on any asset you can read, and may recur on another schedule.\n", "security": [ { "ApiKeyAuth": [] @@ -6828,7 +6828,7 @@ "properties": { "automation": { "type": "integer", - "description": "ID of the automation to wait for. It has to recur on the same schedule (the same cron string, in the same timezone).", + "description": "ID of the automation to wait for. It may recur on another schedule.", "example": 4 }, "allow-failure": { @@ -6901,7 +6901,7 @@ }, "depends-on": { "type": "array", - "description": "The automations whose runs each run of this automation waits for: the runs scheduled for the same time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and have to recur on the same schedule. A run on demand does not wait.", + "description": "The automations whose runs each run of this automation waits for: their latest runs due at or before its own scheduled time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and may recur on another schedule. A run on demand does not wait.", "example": [ { "automation": 4, @@ -6940,7 +6940,7 @@ }, "depends-on": { "type": "array", - "description": "The automations whose runs each run of this automation waits for: the runs scheduled for the same time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and have to recur on the same schedule. A run on demand does not wait. Replaces the automations it depended on; send an empty list to have its runs wait for none.", + "description": "The automations whose runs each run of this automation waits for: their latest runs due at or before its own scheduled time. A run queues its jobs once these have ended, and is skipped if one of them failed, unless its dependency allows for failure. Upstream automations can be on any asset you can read, and may recur on another schedule. A run on demand does not wait. Replaces the automations it depended on; send an empty list to have its runs wait for none.", "example": [ { "automation": 4, diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 407dda3d2e..3a8e524a40 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -195,7 +195,7 @@
Lists the automations on this page. Use the API or the CLI to wait for an automation on another asset.
@@ -296,7 +296,7 @@
Lists the automations on this page. Use the API or the CLI to wait for an automation on another asset.
@@ -829,7 +829,7 @@
Writes to

${sensorLinks(res["output-sensors"], res.source)}

Parameters
${esc(JSON.stringify(res.parameters, null, 4))}
-
Waits for
+
Waits for

${dependsOnText(res["depends-on"])}

Durable runs
${esc(JSON.stringify({ total: runStats.total || 0, dispatch: runStats.dispatch || {}, execution: runStats.execution || {}, "latest-run": latestRunSummary }, null, 4))}
@@ -1120,16 +1120,6 @@
Recently created jobs }); }); - /* A run waits for the upstream runs scheduled for the very same time, so an automation has to recur on the schedule of those it waits for. - Picking one therefore copies its schedule into the form, rather than leaving a mismatch for the server to refuse. */ - $("#automationDependsOn").on("change", function () { - const picked = ($(this).val() || []).map((id) => automationsById.get(Number(id))).filter(Boolean); - if (picked.length) { - $("#automationCron").val(picked[0]["cron"]); - $("#automationTimezone").val(picked[0].timezone); - } - }); - // Opening the form from the New automation button starts from the defaults, not from whatever a copy left in it. $("#newAutomationButton").on("click", () => resetNewAutomationForm()); From a20b557c948177e32ab70b38ff82cf78df237f4e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 30 Oct 2026 03:08:47 +0100 Subject: [PATCH 16/47] Run an automation on demand along with the automations waiting for it flexmeasures jobs run-automation --with-dependents, the trigger endpoint's with-dependents query parameter and the Automations page's Run with dependents start a run of the automation now, and a run for each active automation that waits for it, directly or through others, which waits for the runs started along with it, and is skipped if one of those failed. This is how a chain is re-run, for instance after backfilling the data at its start. A run records whether it was due by schedule or asked for on demand (trigger), so that on-demand runs can share a minute with scheduled runs, and a scheduled run only ever waits for scheduled upstream runs. The runner does not retry on-demand runs, as whoever started them is told when they fail. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- documentation/cli/change_log.rst | 1 + documentation/features/automations.rst | 12 ++ flexmeasures/api/v3_0/assets.py | 102 +++++++++++- ...st_automation_dependencies_api_fresh_db.py | 60 ++++++- flexmeasures/cli/jobs.py | 62 ++++++- flexmeasures/cli/tests/test_automations.py | 35 +++- ...a2e481_record_on_demand_automation_runs.py | 61 +++++++ flexmeasures/data/models/automations.py | 15 +- flexmeasures/data/services/automations.py | 152 +++++++++++++++++- .../test_automation_dependencies_fresh_db.py | 64 ++++++++ flexmeasures/ui/static/openapi-specs.json | 29 +++- .../templates/assets/asset_automations.html | 24 ++- 13 files changed, 603 insertions(+), 16 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/c3d9f6a2e481_record_on_demand_automation_runs.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 83bee5515e..bf67b3efd1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -179,7 +179,7 @@ Automations arrived over several pull requests. This is what each of them contri * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization. A plugin job's result is kept for as long as results on its own queue are kept, and reports the number of beliefs it saved [see `PR #2553 `_ and `PR #2644 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] -* An automation can wait for the automations whose results it uses: each of its runs waits for their latest runs due at or before its own scheduled time, also on another schedule, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page [see `PR #2706 `_] +* An automation can wait for the automations whose results it uses: each of its runs waits for their latest runs due at or before its own scheduled time, also on another schedule, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page, and a chain can be re-run on demand from its start (``--with-dependents``) [see `PR #2706 `_] * Update Readme and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index f0f250f42f..0bf5e1548b 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -13,6 +13,7 @@ since v1.1.0 | September XX, 2026 * Add ``flexmeasures jobs run-automation --automation `` to queue the jobs for a single run of one automation, now, on top of its recurring runs. This leaves the automation's cursor alone, so its next recurring run still happens as scheduled, and inactive automations can be run this way, too. * ``flexmeasures jobs run-automations`` now records a durable run for each scheduled run it claims, and retries the forecast and schedule runs whose queueing did not finish. A run which failed before queueing anything is dispatched again in full, and one which queued only part of its jobs resumes from its stored plan, reusing the job IDs it already queued. Each attempt is recorded with its owner, outcome and error, and the command reports the run and attempt it is working on. A run is only picked up by another runner once the claim lease of the runner holding it has expired, which is how a runner that died mid-queueing hands its work over. * Add ``--depends-on`` and ``--allow-failure-of`` to ``flexmeasures add automation`` and ``flexmeasures edit automation`` (plus ``--no-dependencies`` on edit), to have an automation's runs wait for the latest runs of other automations due at or before their own scheduled time. +* Add ``--with-dependents`` to ``flexmeasures jobs run-automation``, to also run the automations that wait for it, each once the runs it waits for ended, for instance to re-run a chain after a backfill. * ``flexmeasures edit automation`` now counts up the automation's schedule revision whenever it rebases the cursor (on a changed cron string or timezone, or on reactivation), which keeps the durable runs of the old and the new schedule apart, even at the same scheduled UTC time. * ``flexmeasures add automation --type scheduling`` refuses a flex config field which fixes a moment in time, such as ``soc-at-start`` or a ``soc-targets`` entry with a ``datetime``, naming the field: a recurring schedule automation computes a fresh schedule on every run, so such a value would be stale on the next one. * ``flexmeasures add automation`` refuses a fixed ``start``, ``end`` or ``prior`` in the parameters of any automation, as every run would share that moment. The parameters take ``start-offset`` and ``end-offset`` instead, or either one with a ``duration``, applied to the time each run was due, on the automation's own clock, and so do the new ``--start-offset`` and ``--end-offset`` options, next to ``--duration``. diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index dc28bc2c2c..de510fbcc7 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -259,6 +259,18 @@ The API takes the same as ``depends-on``, a list like ``[{"automation": 7, "allo When assets are copied along with their automations, a copy waits for the copy of the automation its original waited for, if that was copied too, and for the original one otherwise. +To re-run a whole chain, for instance once the data at its start was backfilled, run its first automation along with the automations that wait for it: + +.. code-block:: bash + + flexmeasures jobs run-automation --automation 7 --with-dependents + +Automation 7 runs right away, and each active automation that waits for it, directly or through others, gets a run that waits for the runs started along with it, +and is skipped if one of those failed, just like scheduled runs. +These runs on demand do not affect the automations' recurrence, and the runner does not retry them if their jobs failed to queue, as whoever started them is told. +The API does the same with ``POST /api/v3_0/assets//automations//trigger?with-dependents=true``, which needs permission to manage the automations on each of the assets involved, +and the *Automations* page offers *Run with dependents* for an automation that another one listed there waits for. + Running automations ------------------- diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index b7de43c7e2..d04f90da2f 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -64,8 +64,10 @@ get_asset_automations_job_stats, get_automation_job_stats, get_automation_run_stats, + get_dependent_automations, resolve_automation_sensors, run_automation, + run_automation_chain, update_automation, ) from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType @@ -2026,12 +2028,30 @@ def delete_automation(self, id: int, automation_id: int, asset: GenericAsset): }, location="path", ) + @use_kwargs( + { + "with_dependents": fields.Bool( + data_key="with-dependents", + load_default=False, + metadata={ + "description": "Also run the active automations that wait for this one, directly or through others, each once the runs it waits for ended." + }, + ) + }, + location="query", + ) # Running an automation writes data under the asset, which is what create-children means here. # The sensors it writes to were checked against the same permission when the automation was created, # and its output scope is checked again on each run (see validate_automation_output_scope). @permission_required_for_context("manage-automations", ctx_arg_name="asset") @as_json - def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): + def trigger_automation( + self, + id: int, + automation_id: int, + asset: GenericAsset, + with_dependents: bool = False, + ): """ .. :quickref: Assets; Trigger a single run of an automation. @@ -2049,6 +2069,12 @@ def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): An on-demand run does not affect the automation's recurrence: its cursor stays where it was, so the next recurring run still happens as scheduled, and a missed run is still caught up. Inactive automations can be triggered, too, which is how you can try one out before activating it. + + Pass `with-dependents=true` to also run the active automations that wait for this one, directly or through others, + for instance to re-run a chain of automations once the data at its start was backfilled. + Each of them gets a run that waits for the runs triggered along with it, and is skipped if one of those failed. + This needs permission to manage the automations on each of their assets, too. + The response then lists these runs under `dependent-runs`, and its `job` is null if this automation itself had nothing to queue. security: - ApiKeyAuth: [] parameters: @@ -2064,6 +2090,13 @@ def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): description: ID of the automation to run. schema: type: integer + - in: query + name: with-dependents + required: false + description: Also run the active automations that wait for this one, directly or through others. + schema: + type: boolean + default: false responses: 202: description: PROCESSING @@ -2082,6 +2115,18 @@ def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): job-url: "/api/v3_0/jobs/364bfd06-c1fa-430b-8d25-8f5a547651fb" n-jobs: 2 message: "Request has been accepted for processing." + triggered-with-dependents: + summary: Automation run accepted, along with the automations waiting for it + value: + status: ACCEPTED + job: "364bfd06-c1fa-430b-8d25-8f5a547651fb" + job-url: "/api/v3_0/jobs/364bfd06-c1fa-430b-8d25-8f5a547651fb" + n-jobs: 1 + message: "Request has been accepted for processing." + dependent-runs: + - automation: 8 + run: 412 + dispatch-state: waiting 401: description: UNAUTHORIZED 403: @@ -2098,6 +2143,8 @@ def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): return { "message": f"Asset {asset.id} has no automation with id {automation_id}." }, 404 + if with_dependents: + return self._trigger_automation_chain(asset, automation) try: returns = run_automation(automation) except ( @@ -2133,6 +2180,59 @@ def trigger_automation(self, id: int, automation_id: int, asset: GenericAsset): response["n-jobs"] = returns.get("n_jobs") return response, status_code + def _trigger_automation_chain( + self, asset: GenericAsset, automation: Automation + ) -> tuple[dict, int]: + """Run an automation along with the automations that wait for it (see `run_automation_chain`).""" + dependents = get_dependent_automations(automation) + # The automations waiting for this one may be on other assets, where their runs write data, too. + for dependent in dependents: + check_access(dependent.asset, "manage-automations") + try: + returns, dependent_runs = run_automation_chain(automation) + except ( + NotImplementedError, + ValueError, + ValidationError, + AutomationSensorsUnknown, + ) as e: + db.session.rollback() + return unprocessable_entity( + e.messages if isinstance(e, ValidationError) else str(e) + ) + AssetAuditLog.add_record( + asset, + f"Triggered a run of automation '{automation.name}' ({automation.id}) and of the {len(dependent_runs)} automation(s) waiting for it.", + ) + for run in dependent_runs: + AssetAuditLog.add_record( + run.automation.asset, + f"Triggered a run of automation '{run.automation.name}' ({run.automation_id}), waiting for a run of automation {automation.id}.", + ) + db.session.commit() + job_id = returns.get("job_id") + if job_id is not None: + response, status_code = request_accepted_for_processing(job_id) + else: + response, status_code = { + "status": "ACCEPTED", + "job": None, + "message": returns.get("message") + or f"Automation {automation.id} had nothing to queue, so the automations waiting for it go ahead.", + }, 202 + response["n-jobs"] = returns.get("n_jobs") + response["dependent-runs"] = [] + for run in dependent_runs: + db.session.refresh(run) + response["dependent-runs"].append( + { + "automation": run.automation_id, + "run": run.id, + "dispatch-state": run.dispatch_state, + } + ) + return response, status_code + @route("//jobs", methods=["GET"]) @use_kwargs( {"asset": AssetIdField(data_key="id")}, diff --git a/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py index 8f9283055d..808af3139a 100644 --- a/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_automation_dependencies_api_fresh_db.py @@ -4,8 +4,13 @@ import pytest from flask import url_for +from sqlalchemy import select -from flexmeasures.data.models.automations import Automation +from flexmeasures.data.models.automations import ( + Automation, + AutomationDependency, + AutomationRun, +) from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor @@ -194,3 +199,56 @@ def test_patch_automation_refuses_to_wait_for_an_unreadable_automation( assert response.status_code == 403, response.json fresh_db.session.expire_all() assert fresh_db.session.get(Automation, upstream["own"].id).dependencies == [] + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_automation_with_dependents( + app, fresh_db, upstream_automations, requesting_user +): + """Triggering with dependents runs the automations waiting for this one, too, and lists their runs.""" + battery, sensor, upstream = upstream_automations + upstream["own"].parameters = {"sensor": sensor.id} + fresh_db.session.commit() + with app.test_client() as client: + created = _post(client, battery, sensor, [{"automation": upstream["own"].id}]) + assert created.status_code == 201, created.json + response = client.post( + url_for( + "AssetAPI:trigger_automation", + id=battery.id, + automation_id=upstream["own"].id, + **{"with-dependents": "true"}, + ) + ) + assert response.status_code == 202, response.json + assert [ + (run["automation"], run["dispatch-state"]) + for run in response.json["dependent-runs"] + ] == [(created.json["id"], "waiting")] + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_automation_with_dependents_needs_access_to_each_of_them( + app, fresh_db, upstream_automations, requesting_user +): + """An automation waiting for this one on an asset the user may not manage is not run on their say, and neither is this one.""" + battery, sensor, upstream = upstream_automations + fresh_db.session.add( + AutomationDependency(automation=upstream["foreign"], upstream=upstream["own"]) + ) + fresh_db.session.commit() + with app.test_client() as client: + response = client.post( + url_for( + "AssetAPI:trigger_automation", + id=battery.id, + automation_id=upstream["own"].id, + **{"with-dependents": "true"}, + ) + ) + assert response.status_code == 403, response.json + assert fresh_db.session.scalars(select(AutomationRun)).all() == [] diff --git a/flexmeasures/cli/jobs.py b/flexmeasures/cli/jobs.py index cf815c66fb..1f43e1ee9d 100644 --- a/flexmeasures/cli/jobs.py +++ b/flexmeasures/cli/jobs.py @@ -47,6 +47,7 @@ floor_to_minute, get_dispatchable_automation_runs, run_automation, + run_automation_chain, ) from flexmeasures.data.services.data_sources import ( find_duplicate_sources, @@ -159,7 +160,14 @@ def run_automations(): required=True, help="ID of the automation to run.", ) -def run_one_automation(automation: Automation): +@click.option( + "--with-dependents", + "with_dependents", + is_flag=True, + help="Also run the active automations that wait for this one, directly or through others," + " each once the runs it waits for ended, for instance to re-run a chain after backfilling the data at its start.", +) +def run_one_automation(automation: Automation, with_dependents: bool = False): """ Queue the jobs for a single run of one automation, now. @@ -175,7 +183,16 @@ def run_one_automation(automation: Automation): so the next recurring run still happens as scheduled (see `flexmeasures jobs run-automations`). Inactive automations can be run this way, too, which is how you can try one out before activating it. A run on demand does not wait for the automations this one depends on, as it is meant to run now. + + With --with-dependents, the automations that wait for this one get a run, too, + which waits for the runs asked for along with it, and is skipped if one of those failed: + + \b + flexmeasures jobs run-automation --automation 4 --with-dependents """ + if with_dependents: + _run_automation_chain(automation) + return try: returns = run_automation(automation) except ( @@ -211,6 +228,49 @@ def run_one_automation(automation: Automation): ) +def _run_automation_chain(automation: Automation): + """Run an automation along with the automations that wait for it, and say what each of them does.""" + try: + returns, dependent_runs = run_automation_chain(automation) + except ( + NotImplementedError, + ValueError, + ValidationError, + AutomationSensorsUnknown, + Forbidden, + ) as e: + db.session.rollback() + click.secho( + f"Automation {automation.id} ('{automation.name}') failed to queue jobs, so the automations waiting for it are skipped: {e}", + **MsgStyle.ERROR, + ) + raise click.Abort() + AssetAuditLog.add_record( + automation.asset, + f"Triggered a run of automation '{automation.name}' ({automation.id}) and of the {len(dependent_runs)} automation(s) waiting for it via CLI.", + ) + for run in dependent_runs: + AssetAuditLog.add_record( + run.automation.asset, + f"Triggered a run of automation '{run.automation.name}' ({run.automation_id}), waiting for a run of automation {automation.id}, via CLI.", + ) + db.session.commit() + click.secho( + f"Automation {automation.id} ('{automation.name}') run {returns['run_id']} queued {returns['n_jobs']} {automation.type} job(s) for asset {automation.asset_id}.", + **MsgStyle.SUCCESS, + ) + for run in dependent_runs: + db.session.refresh(run) + click.secho( + f"Automation {run.automation_id} ('{run.automation.name}') run {run.id} is {run.dispatch_state}" + + (f" ({run.execution_state})" if run.execution_state == "skipped" else "") + + ".", + **(MsgStyle.WARN if run.execution_state == "skipped" else MsgStyle.SUCCESS), + ) + if not dependent_runs: + click.secho("No active automations wait for this one.", **MsgStyle.WARN) + + @fm_jobs.command("stats") @with_appcontext @click.option( diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 73c0b1871c..fec19717af 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -11,7 +11,7 @@ from flexmeasures import Sensor from flexmeasures.data.models.audit_log import AssetAuditLog -from flexmeasures.data.models.automations import Automation +from flexmeasures.data.models.automations import Automation, AutomationRun from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.cli.tests.utils import to_flags from flexmeasures.data.services.generator_results import ( @@ -3043,6 +3043,39 @@ def test_add_edit_and_show_automation_dependencies( assert fresh_db.session.get(Automation, downstream.id).dependencies == [] +def test_run_automation_with_dependents( + app, fresh_db, setup_dummy_data_fresh_db, clean_redis +): + """--with-dependents runs the automations waiting for this one, too, each holding back until the run it waits for ended.""" + from flexmeasures.cli.jobs import run_one_automation + + runner = app.test_cli_runner() + sensor_id = setup_dummy_data_fresh_db[0] + assert _add_hourly_automation(runner, sensor_id, "Upstream").exit_code == 0 + upstream = fresh_db.session.scalars( + select(Automation).filter_by(name="Upstream") + ).one() + added = _add_hourly_automation( + runner, sensor_id, "Downstream", "--depends-on", str(upstream.id) + ) + assert added.exit_code == 0, added.output + + result = runner.invoke( + run_one_automation, ["--automation", str(upstream.id), "--with-dependents"] + ) + + assert result.exit_code == 0, result.output + assert "('Downstream') run" in result.output and "is waiting" in result.output + fresh_db.session.expire_all() + runs = fresh_db.session.scalars( + select(AutomationRun).order_by(AutomationRun.id) + ).all() + assert [(run.automation.name, run.trigger, run.dispatch_state) for run in runs] == [ + ("Upstream", "on-demand", "queued"), + ("Downstream", "on-demand", "waiting"), + ] + + def test_add_automation_allow_failure_of_needs_depends_on( app, fresh_db, setup_dummy_data_fresh_db ): diff --git a/flexmeasures/data/migrations/versions/c3d9f6a2e481_record_on_demand_automation_runs.py b/flexmeasures/data/migrations/versions/c3d9f6a2e481_record_on_demand_automation_runs.py new file mode 100644 index 0000000000..3720bc89c1 --- /dev/null +++ b/flexmeasures/data/migrations/versions/c3d9f6a2e481_record_on_demand_automation_runs.py @@ -0,0 +1,61 @@ +"""record on-demand automation runs + +Say whether an automation run was due by its schedule or asked for on demand, +so that a chain of automations run on demand can wait for each other's runs, too. +Only one run per scheduled occurrence is allowed, as before, while on-demand runs can share a time with any other run. + +Revision ID: c3d9f6a2e481 +Revises: b8e4c2d7f913 +Create Date: 2026-10-30 03:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "c3d9f6a2e481" +down_revision = "b8e4c2d7f913" +branch_labels = None +depends_on = None + +TRIGGER_CK = "automation_run_automation_run_trigger_ck" + + +def upgrade(): + op.add_column( + "automation_run", + sa.Column( + "trigger", + sa.String(16), + nullable=False, + server_default="schedule", + ), + ) + op.alter_column("automation_run", "trigger", server_default=None) + op.create_check_constraint( + op.f(TRIGGER_CK), + "automation_run", + "trigger IN ('schedule', 'on-demand')", + ) + op.drop_constraint("automation_run_occurrence_uq", "automation_run", type_="unique") + op.create_index( + "automation_run_occurrence_uq", + "automation_run", + ["automation_id", "scheduled_at", "schedule_revision"], + unique=True, + postgresql_where=sa.text("trigger = 'schedule'"), + ) + + +def downgrade(): + # Without the trigger, on-demand runs could clash with scheduled runs at the same time, so they go. + op.execute("DELETE FROM automation_run WHERE trigger = 'on-demand'") + op.drop_index("automation_run_occurrence_uq", table_name="automation_run") + op.create_unique_constraint( + "automation_run_occurrence_uq", + "automation_run", + ["automation_id", "scheduled_at", "schedule_revision"], + ) + op.drop_constraint(op.f(TRIGGER_CK), "automation_run", type_="check") + op.drop_column("automation_run", "trigger") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index 1bd65bbb3e..87d3d9a6a9 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -240,15 +240,22 @@ def __repr__(self): class AutomationRun(db.Model): - """Durable execution record for one scheduled automation occurrence.""" + """Durable execution record for one run of an automation, scheduled or on demand.""" __tablename__ = "automation_run" __table_args__ = ( - db.UniqueConstraint( + # One run per scheduled occurrence, while on-demand runs can share a time with any other run. + db.Index( + "automation_run_occurrence_uq", "automation_id", "scheduled_at", "schedule_revision", - name="automation_run_occurrence_uq", + unique=True, + postgresql_where=db.text("trigger = 'schedule'"), + ), + db.CheckConstraint( + "trigger IN ('schedule', 'on-demand')", + name="automation_run_trigger_ck", ), db.CheckConstraint( "dispatch_state IN ('pending', 'claimed', 'waiting', 'partially_queued', 'queued', 'failed')", @@ -277,6 +284,8 @@ class AutomationRun(db.Model): ) scheduled_at = db.Column(db.DateTime(timezone=True), nullable=False) schedule_revision = db.Column(db.Integer, nullable=False) + # Whether the run was due by the automation's schedule, or asked for on demand along with a chain of runs (see `run_automation_chain`). + trigger = db.Column(db.String(16), nullable=False, default="schedule") automation_type = db.Column(db.String(80), nullable=False) generator_id = db.Column(db.Integer, nullable=True) # A run that is 'waiting' holds back its jobs until the upstream runs it depends on have ended (see `AutomationDependency`). diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index e9491ae40e..2f584308c6 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -338,6 +338,8 @@ def get_dispatchable_automation_runs( .join(Automation) .where( Automation.active.is_(True), + # Whoever asked for a run on demand is told when it fails, and can ask again. + AutomationRun.trigger == "schedule", AutomationRun.automation_type.in_(AUTOMATION_RUN_RETRIED_TYPES), AutomationRun.dispatch_state.in_(AUTOMATION_RUN_RESUMABLE_DISPATCH_STATES), AutomationRun.dispatch_completed_at.is_(None), @@ -384,7 +386,8 @@ def _dispatch_was_given_up(run: AutomationRun) -> bool: # A runner that is still queueing the jobs of a run holds a claim on it, and may yet finish. and run.claim_owner is None and ( - run.automation_type not in AUTOMATION_RUN_RETRIED_TYPES + run.trigger == "on-demand" + or run.automation_type not in AUTOMATION_RUN_RETRIED_TYPES or run.attempt_count >= AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS ) ) @@ -450,9 +453,14 @@ def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: Otherwise, it waits for as long as one of them has yet to end. An upstream automation that was never going to run at that time is left out, and the run goes ahead without it. + A run on demand only waits for the runs that were asked for along with it (see `run_automation_chain`). + :returns: 'wait', 'skip' or 'go', and for each dependency, what was found: the upstream automation, whether its failure is allowed for, its run (if any) and the state of that run. """ + if run.trigger == "on-demand": + found = _find_chain_upstream_runs(run) + return _upstream_verdict(found), found found = [] for dependency in run.automation.dependencies: due_at = get_upstream_due_at(dependency.upstream, run.scheduled_at) @@ -461,6 +469,7 @@ def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: select(AutomationRun) .where( AutomationRun.automation_id == dependency.upstream_id, + AutomationRun.trigger == "schedule", AutomationRun.scheduled_at == due_at, ) .order_by(AutomationRun.schedule_revision.desc()) @@ -484,14 +493,45 @@ def check_upstream_runs(run: AutomationRun) -> tuple[str, list[dict[str, Any]]]: "state": state, } ) + return _upstream_verdict(found), found + + +def _find_chain_upstream_runs(run: AutomationRun) -> list[dict[str, Any]]: + """Find the runs asked for on demand along with this run, which it waits for, as recorded when they were created.""" + allow_failure = { + dependency.upstream_id: dependency.allow_failure + for dependency in run.automation.dependencies + } + found = [] + for upstream in (run.plan or {}).get("chain-upstream-runs", []): + upstream_run = db.session.get(AutomationRun, upstream["run"]) + found.append( + { + "automation": upstream["automation"], + "allow-failure": allow_failure.get(upstream["automation"], False), + "due-at": run.scheduled_at.isoformat(), + "run": upstream["run"], + # A run that is gone, along with its automation, will not produce anything. + "state": ( + _upstream_run_state(upstream_run) + if upstream_run is not None + else "failed" + ), + } + ) + return found + + +def _upstream_verdict(found: list[dict[str, Any]]) -> str: + """Say whether a run should 'skip', 'wait' or 'go', given the state of each upstream run it waits for.""" if any( upstream["state"] in ("failed", "passed-over") and not upstream["allow-failure"] for upstream in found ): - return "skip", found + return "skip" if any(upstream["state"] == "pending" for upstream in found): - return "wait", found - return "go", found + return "wait" + return "go" def _describe_upstream_runs( @@ -2758,6 +2798,7 @@ def serialize_automation_run(run: AutomationRun) -> dict[str, Any]: "id": run.id, "scheduled-at": run.scheduled_at.isoformat(), "schedule-revision": run.schedule_revision, + "trigger": run.trigger, "dispatch-state": run.dispatch_state, "execution-state": run.execution_state, "attempt-count": run.attempt_count, @@ -2988,6 +3029,109 @@ def dispatch_automation_run( } +def get_dependent_automations(automation: Automation) -> list[Automation]: + """Return the active automations that wait for this one, directly or through others, those nearest to it first. + + An inactive automation is left out, and so are the ones that only wait for it, as it does not run. + """ + reached = {automation.id} + frontier = [automation.id] + dependents: list[Automation] = [] + while frontier: + found = db.session.scalars( + select(Automation) + .join( + AutomationDependency, + AutomationDependency.automation_id == Automation.id, + ) + .where( + AutomationDependency.upstream_id.in_(frontier), + Automation.active.is_(True), + ) + .order_by(Automation.id) + ).unique() + frontier = [] + for dependent in found: + if dependent.id not in reached: + reached.add(dependent.id) + dependents.append(dependent) + frontier.append(dependent.id) + return dependents + + +def _new_on_demand_run( + automation: Automation, scheduled_at: datetime, dispatch_state: str +) -> AutomationRun: + """Make a durable record for a run asked for on demand (not committed yet).""" + run = AutomationRun( + automation=automation, + scheduled_at=scheduled_at, + schedule_revision=automation.schedule_revision, + trigger="on-demand", + automation_type=automation.type, + generator_id=automation.generator_id, + dispatch_state=dispatch_state, + execution_state="pending", + attempt_count=0, + parameters=dict(automation.parameters or {}), + plan=_run_snapshot(automation, scheduled_at), + ) + db.session.add(run) + return run + + +def run_automation_chain( + automation: Automation, + now: datetime | None = None, + owner: str | None = None, +) -> tuple[dict[str, Any], list[AutomationRun]]: + """Run an automation now, along with the automations that wait for it, each of which runs once the runs it waits for ended. + + This is how a chain is re-run, for instance once the data at its start was backfilled. + The automation itself runs right away, without waiting for the automations it depends on, as a run on demand does. + Each automation that waits for it, directly or through others (see `get_dependent_automations`), gets a run that waits, + for the runs asked for along with it only, and that is skipped if one of those failed, as a scheduled run is. + None of these runs affect the automations' recurrence, and the runner does not retry them if their jobs failed to queue. + + :returns: what the automation's own run dispatched (see `dispatch_automation_run`), + and the runs that wait for it, in the order of `get_dependent_automations`. + """ + owner = owner or _runner_owner() + claimed_at = _now_utc() + scheduled_at = floor_to_minute(now or claimed_at) + root_run = _new_on_demand_run(automation, scheduled_at, "claimed") + root_run.claim_owner = owner + root_run.claimed_at = claimed_at + root_run.claim_expires_at = _claim_expires_at( + claimed_at, AUTOMATION_RUN_CLAIM_LEASE + ) + root_run.attempt_count = 1 + runs = {automation.id: root_run} + dependents = get_dependent_automations(automation) + for dependent in dependents: + runs[dependent.id] = _new_on_demand_run(dependent, scheduled_at, "waiting") + db.session.flush() + for dependent in dependents: + run = runs[dependent.id] + run.plan = { + **dict(run.plan or {}), + "chain-upstream-runs": [ + { + "automation": dependency.upstream_id, + "run": runs[dependency.upstream_id].id, + } + for dependency in dependent.dependencies + if dependency.upstream_id in runs + ], + } + attempt = _new_attempt(root_run, owner, claimed_at) + db.session.commit() + returns = dispatch_automation_run( + ClaimedAutomationRun(run=root_run, attempt=attempt) + ) + return returns, [runs[dependent.id] for dependent in dependents] + + def run_automation( automation: Automation, automation_run: AutomationRun | None = None, diff --git a/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py index 5d04fc5f7a..509de07958 100644 --- a/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py +++ b/flexmeasures/data/tests/test_automation_dependencies_fresh_db.py @@ -19,6 +19,7 @@ dispatch_automation_run, get_dispatchable_automation_runs, run_automation, + run_automation_chain, update_automation, ) from flexmeasures.data.tests.plugins.mock_ingestion import ( @@ -368,6 +369,69 @@ def test_a_quarter_hourly_run_waits_for_the_latest_hourly_upstream_run( assert run.execution_state == expected +@pytest.mark.parametrize("upstream_fails", [False, True]) +def test_running_a_chain_on_demand( + fresh_db, app, make_automation, failing_value, upstream_fails +): + """Run an automation on demand along with what waits for it: each run waits for the one before it, and a failure skips the rest.""" + snapper = make_automation("Snap meter readings", value=-1 if upstream_fails else 1) + flows = make_automation("Compute flows", depends_on=[snapper]) + totals = make_automation("Sum flows", depends_on=[flows]) + # An inactive automation does not run, and neither does what only waits for it. + paused = make_automation("Paused", depends_on=[snapper]) + paused.active = False + make_automation("Waits for paused", depends_on=[paused]) + fresh_db.session.commit() + + returns, dependent_runs = run_automation_chain(snapper, now=TICK) + + assert [run.automation_id for run in dependent_runs] == [flows.id, totals.id] + assert [run.dispatch_state for run in dependent_runs] == ["waiting", "waiting"] + snapper_run = fresh_db.session.get(AutomationRun, returns["run_id"]) + assert snapper_run.trigger == "on-demand" + assert dependent_runs[1].plan["chain-upstream-runs"] == [ + {"automation": flows.id, "run": dependent_runs[0].id} + ] + # Their recurrence is left alone. + assert {snapper.cursor, flows.cursor, totals.cursor} == {CREATED_AT} + + _work(app, snapper_run) + if upstream_fails: + for run in dependent_runs: + assert _run_at(fresh_db, run.automation, SCHEDULED_AT).execution_state == ( + "skipped" + ) + return + flows_run = _run_at(fresh_db, flows, SCHEDULED_AT) + assert flows_run.dispatch_state == "queued" + assert _run_at(fresh_db, totals, SCHEDULED_AT).dispatch_state == "waiting" + _work(app, flows_run) + totals_run = _run_at(fresh_db, totals, SCHEDULED_AT) + assert totals_run.dispatch_state == "queued" + _work(app, totals_run) + assert _run_at(fresh_db, totals, SCHEDULED_AT).execution_state == "succeeded" + + +def test_a_chain_run_on_demand_and_a_scheduled_run_keep_apart( + fresh_db, app, make_automation +): + """Runs on demand can share a minute with the scheduled runs, and a scheduled run only ever waits for a scheduled upstream run.""" + snapper = make_automation("Snap meter readings") + flows = make_automation("Compute flows", depends_on=[snapper]) + _tick() + + returns, dependent_runs = run_automation_chain(snapper, now=TICK) + on_demand_snapper_run = fresh_db.session.get(AutomationRun, returns["run_id"]) + _work(app, on_demand_snapper_run) + + fresh_db.session.expire_all() + runs = fresh_db.session.scalars( + select(AutomationRun).where(AutomationRun.automation_id == flows.id) + ).all() + states = {run.trigger: run.dispatch_state for run in runs} + assert states == {"schedule": "waiting", "on-demand": "queued"} + + def test_automations_waiting_for_each_other_can_change_their_schedules( fresh_db, make_automation ): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 77737a3a43..cfa676fa23 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4930,7 +4930,7 @@ "/api/v3_0/assets/{id}/automations/{automation-id}/trigger": { "post": { "summary": "Trigger a single run of an automation.", - "description": "Run one automation now, once, in addition to its recurring runs.\nThis is useful to try out a new automation, to re-run one after fixing what made it fail,\nor to refresh its results after late input data arrived.\n\nThe automation runs with the parameters it was created with,\nand the jobs it queues are recorded as its jobs, just like the jobs of a recurring run.\n\nAn on-demand run does not affect the automation's recurrence: its cursor stays where it was,\nso the next recurring run still happens as scheduled, and a missed run is still caught up.\nInactive automations can be triggered, too, which is how you can try one out before activating it.\n", + "description": "Run one automation now, once, in addition to its recurring runs.\nThis is useful to try out a new automation, to re-run one after fixing what made it fail,\nor to refresh its results after late input data arrived.\n\nThe automation runs with the parameters it was created with,\nand the jobs it queues are recorded as its jobs, just like the jobs of a recurring run.\n\nAn on-demand run does not affect the automation's recurrence: its cursor stays where it was,\nso the next recurring run still happens as scheduled, and a missed run is still caught up.\nInactive automations can be triggered, too, which is how you can try one out before activating it.\n\nPass `with-dependents=true` to also run the active automations that wait for this one, directly or through others,\nfor instance to re-run a chain of automations once the data at its start was backfilled.\nEach of them gets a run that waits for the runs triggered along with it, and is skipped if one of those failed.\nThis needs permission to manage the automations on each of their assets, too.\nThe response then lists these runs under `dependent-runs`, and its `job` is null if this automation itself had nothing to queue.\n", "security": [ { "ApiKeyAuth": [] @@ -4954,6 +4954,16 @@ "schema": { "type": "integer" } + }, + { + "in": "query", + "name": "with-dependents", + "required": false, + "description": "Also run the active automations that wait for this one, directly or through others.", + "schema": { + "type": "boolean", + "default": false + } } ], "responses": { @@ -4972,6 +4982,23 @@ "n-jobs": 2, "message": "Request has been accepted for processing." } + }, + "triggered-with-dependents": { + "summary": "Automation run accepted, along with the automations waiting for it", + "value": { + "status": "ACCEPTED", + "job": "364bfd06-c1fa-430b-8d25-8f5a547651fb", + "job-url": "/api/v3_0/jobs/364bfd06-c1fa-430b-8d25-8f5a547651fb", + "n-jobs": 1, + "message": "Request has been accepted for processing.", + "dependent-runs": [ + { + "automation": 8, + "run": 412, + "dispatch-state": "waiting" + } + ] + } } } } diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 3a8e524a40..a5ebdc6e97 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -624,10 +624,22 @@ title="Run this automation once, now. Its recurrence is not affected: the next recurring run still happens as scheduled.">Run now`; } + // Whether an automation listed here waits for this one, which is when running it along with its dependents does more than running it. + function hasListedDependents(automation) { + return [...automationsById.values()].some( + (other) => (other["depends-on"] || []).some((dependency) => dependency.automation === automation.id) + ); + } + function actionButtons(automation) { + const runWithDependentsItem = hasListedDependents(automation) ? ` +
  • + ` : ""; const runItem = userCanRunAutomations ? `
  • + ${runWithDependentsItem} ` : ""; const manageItems = userCanManage ? `
  • @@ -1306,16 +1318,22 @@
    Recently created jobs $(document).on("click", ".run-automation", function () { const button = $(this); const automationId = button.data("automation-id"); + const withDependents = button.data("with-dependents") === true; + const label = button.text(); button.prop("disabled", true).text("Queueing ..."); $.ajax({ - url: automationUrl(automationId, "/trigger"), + url: automationUrl(automationId, withDependents ? "/trigger?with-dependents=true" : "/trigger"), method: "POST", success: function (res) { $("#automations_err").addClass("d-none"); const nJobs = res["n-jobs"]; const queued = nJobs == null ? "Queued the jobs" : `Queued ${nJobs} job(s)`; + const dependentRuns = res["dependent-runs"] || []; + const dependents = dependentRuns.length + ? ` Automation(s) ${dependentRuns.map((run) => run.automation).join(", ")} run once the runs they wait for ended.` + : ""; showAutomationsMessage( - `${queued} for automation ${automationId}. Recent jobs are listed on the status page.` + `${queued} for automation ${automationId}.${dependents} Recent jobs are listed on the status page.` ); loadAutomations({ rebuild: false }); // the listing carries the job counts }, @@ -1326,7 +1344,7 @@
    Recently created jobs ); }, complete: function () { - button.prop("disabled", false).text("Run now"); + button.prop("disabled", false).text(label); }, }); }); From d54531efa8422a05cd946fb42b8ed13537542496 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 30 Oct 2026 03:11:05 +0100 Subject: [PATCH 17/47] ui: draw the chain an automation is part of in its details The details panel on the Automations page draws the automations listed there that this one waits for, on the left, and those that wait for it, on the right, directly or through others, with a dashed arrow where the waiting automation runs even when the one it waits for failed. Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- documentation/features/automations.rst | 3 + .../templates/assets/asset_automations.html | 81 +++++++++++++++++++ .../ui/tests/js/test_automation_chain.py | 47 +++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 flexmeasures/ui/tests/js/test_automation_chain.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index bf67b3efd1..6a1264e3a0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -179,7 +179,7 @@ Automations arrived over several pull requests. This is what each of them contri * Copying an asset now copies the automations on it and on its descendant assets as well, pointing them at the copied sensors and leaving them switched off, so they can be checked before they start running; an automation that cannot be copied safely is skipped and reported, rather than holding up the copy [see `PR #2531 `_] * Plugins can register automation types with their own validated configuration, parameters and worker queue, with data provenance and sensor authorization. A plugin job's result is kept for as long as results on its own queue are kept, and reports the number of beliefs it saved [see `PR #2553 `_ and `PR #2644 `_] * The *Automations* page lists the automations of the assets below an asset too, says when each automation runs next, keeps itself up to date, and shows the configuration of an automation's data source, which the *New automation* form now also lets you set [see `PR #2554 `_ and `PR #2584 `_] -* An automation can wait for the automations whose results it uses: each of its runs waits for their latest runs due at or before its own scheduled time, also on another schedule, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page, and a chain can be re-run on demand from its start (``--with-dependents``) [see `PR #2706 `_] +* An automation can wait for the automations whose results it uses: each of its runs waits for their latest runs due at or before its own scheduled time, also on another schedule, and is skipped if one of them failed, from the CLI (``--depends-on``), the API (``depends-on``) and the *Automations* page, and a chain can be re-run on demand from its start (``--with-dependents``), which the *Automations* page also draws [see `PR #2706 `_] * Update Readme and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] and docs index w.r.t. FlexMeasures being described more as scheduling and optimization engine, not per se a whole EMS (see [see `PR #2751 `_] diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index de510fbcc7..08df913fbc 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -271,6 +271,9 @@ These runs on demand do not affect the automations' recurrence, and the runner d The API does the same with ``POST /api/v3_0/assets//automations//trigger?with-dependents=true``, which needs permission to manage the automations on each of the assets involved, and the *Automations* page offers *Run with dependents* for an automation that another one listed there waits for. +The details of an automation on the *Automations* page draw the chain it is part of, among the automations listed there: +what it waits for on the left, what waits for it on the right, and a dashed arrow where the waiting automation runs even when the one it waits for failed. + Running automations ------------------- diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index a5ebdc6e97..f04f7cf370 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -787,6 +787,85 @@