diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9f0ccb3aa9..739ad10ab7 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ v1.2.0 | December xx, 2026 New features ------------- +* Hosts can be emailed a digest of the automation runs that failed, got stuck or could not be queued, with ``flexmeasures monitor automations``, run from cron; each run is reported once, repeated failures are collapsed per automation, and with the new ``FLEXMEASURES_BASE_URL`` setting each automation links to its *Automations* page [see `PR #2671 `_] Infrastructure / Support ------------------------- diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 68cc8cf7ae..0f2bc76e38 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -4,6 +4,11 @@ FlexMeasures CLI Changelog ************************** +since v1.2.0 | December xx, 2026 +================================= + +* Add ``flexmeasures monitor automations`` to email a digest of the automation runs that failed since the last check: runs whose execution failed, which have been executing for too long (``--stuck-after-minutes``), or whose dispatch failed for good. Each run is reported once, and repeated failures are collapsed per automation. + since v1.1.0 | November 05, 2026 ================================= diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index c37693ae3c..095c125dc0 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -107,6 +107,7 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures monitor latest-run`` Check if the given task's last successful execution happened less than the allowed time ago. ``flexmeasures monitor last-seen`` Check if given users last contact (via a request) happened less than the allowed time ago. +``flexmeasures monitor automations`` Email a digest of the automation runs that failed since the last check. ================================================= ======================================= diff --git a/documentation/configuration.rst b/documentation/configuration.rst index ebdbdd863f..82347bfcf7 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -878,11 +878,34 @@ Default: ``None`` FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run`` and ``flexmeasures monitor last-seen`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. +E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run``, ``flexmeasures monitor last-seen`` and ``flexmeasures monitor automations`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. Default: ``[]`` +.. _monitor_automations_stuck_after: + +FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +How long an automation run may execute before ``flexmeasures monitor automations`` reports it as stuck, for instance because the worker running its job died. +Its ``--stuck-after-minutes`` option overrides this setting. + +Default: ``timedelta(hours=6)`` + +.. _base_url: + +FLEXMEASURES_BASE_URL +^^^^^^^^^^^^^^^^^^^^^ + +Where this server is reached on the web, for instance ``"https://ems.example.com"``. +Emails sent from the CLI, outside of any request, use it to link to the web pages they are about. +For instance, the digest of ``flexmeasures monitor automations`` links each failing automation to the Automations page of its asset. +Without it, these emails carry no links. + +Default: ``None`` + + .. _redis-config: Redis diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index 03df6cf34a..95b2249430 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -287,6 +287,28 @@ This is what an operator needs to tell a run which failed before queueing anythi Editing an automation's cron string or timezone, or reactivating it, counts up its revision. Runs of the old and the new schedule therefore stay distinct, even when they fall on the same scheduled UTC time. +.. _monitoring_automations: + +Monitoring automation runs +-------------------------- + +As a host, you can have FlexMeasures email you a digest of the automation runs that failed, by running ``flexmeasures monitor automations`` regularly, for instance every hour: + +.. code-block:: bash + + 0 * * * * flexmeasures monitor automations + +A run is reported when its execution failed, when it has been executing for longer than six hours, or when its dispatch failed and will not be attempted again. +A forecast run therefore counts once its five dispatch attempts are used up, and a run of any other type after its first failed attempt. +The six hours are set by :ref:`monitor_automations_stuck_after`, and ``--stuck-after-minutes`` overrides that setting. + +Each run is reported once, so if one monitoring run is missed, the next one still reports what failed in the meantime. +Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it last did and with which error. +No email is sent when nothing failed. + +The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). +The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. + Running one automation on demand -------------------------------- diff --git a/documentation/host/error-monitoring.rst b/documentation/host/error-monitoring.rst index 1ff8d049cf..ac98a8ec1f 100644 --- a/documentation/host/error-monitoring.rst +++ b/documentation/host/error-monitoring.rst @@ -36,6 +36,17 @@ If you run distinct filters, such as separate checks per account, account group .. todo:: Adding roles and assigning them to accounts is not supported by the UI yet (user roles can be added in the UI). Account roles can be added with ``flexmeasures add account-role``. +Monitoring automations +----------------------- + +The CLI task ``flexmeasures monitor automations`` emails a digest of the automation runs that failed since its last check, with repeated failures collapsed per automation. +Run it from cron, for instance every hour; see :ref:`monitoring_automations` for what it reports. + +.. code-block:: bash + + $ flexmeasures monitor automations --recipient alerts@example.com + + Monitoring task runs --------------------- diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index 2c4edebbe0..f8fdaafe85 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -21,10 +21,15 @@ from flexmeasures.data import db from flexmeasures.data.models.task_runs import LatestTaskRun +from flexmeasures.data.services.automation_monitoring import ( + AutomationFailures, + get_unalerted_failed_automation_runs, + group_failed_runs_by_automation, +) from flexmeasures.data.models.user import Account, User from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.api.common.schemas.users import UserIdField -from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.time_utils import exact_duration_str, server_now from flexmeasures.cli.utils import LoggedClickExceptionGroup, MsgStyle @@ -505,4 +510,127 @@ def monitor_last_seen( db.session.commit() +# Error messages can carry a whole traceback, which does not help a digest that lists many automations. +MAX_ERROR_LENGTH_IN_DIGEST = 500 + + +def _format_utc(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def _times(count: int) -> str: + return "once" if count == 1 else f"{count} times" + + +def automation_page_url(automation) -> str | None: + """Link to the Automations page of the automation's asset, opening that automation's info, if the server knows its own address.""" + base_url = app.config.get("FLEXMEASURES_BASE_URL") + if not base_url: + return None + return f"{base_url.rstrip('/')}/assets/{automation.asset_id}/automations?automation={automation.id}" + + +def describe_automation_failures(failures: AutomationFailures) -> str: + """Describe the failed runs of one automation in a few lines, for the monitoring digest.""" + automation = failures.automation + asset = automation.asset + if asset.owner is None: + where = f"public asset {asset.id} '{asset.name}'" + else: + where = f"asset {asset.id} '{asset.name}' of organisation '{asset.owner.name}'" + reasons = ", ".join( + f"{reason}: {count}" for reason, count in failures.reason_counts.items() + ) + last_run = failures.last_run + lines = [ + f"Automation {automation.id} '{automation.name}' ({automation.type}) on {where}:", + f" failed {_times(len(failures.failed_runs))} ({reasons})," + f" last at {_format_utc(last_run.scheduled_at)} (run {last_run.id})", + ] + last_error = failures.last_error + if last_error: + if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: + last_error = last_error[:MAX_ERROR_LENGTH_IN_DIGEST] + " ..." + lines.append(f" last error: {last_error}") + url = automation_page_url(automation) + if url: + lines.append(f" runs and jobs: {url}") + return "\n".join(lines) + + +@fm_monitor.command("automations") +@with_appcontext +@click.option( + "--stuck-after-minutes", + type=click.IntRange(min=1), + default=None, + help="Report a run as stuck once it has been executing for this many minutes. Defaults to the FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER setting (6 hours).", +) +@click.option( + "--recipient", + type=str, + multiple=True, + help="User ID or email address to send the digest to. Use multiple times if needed. If not used, the default monitoring mail recipients are informed.", +) +def monitor_automations( + stuck_after_minutes: int | None = None, + recipient: tuple[str, ...] = (), +): + """ + Email a digest of the automation runs that failed since the last check. + + A run is reported when its execution failed, when it has been executing for too long (see --stuck-after-minutes), + or when its dispatch failed and will not be attempted again. + Each run is reported once, so a monitoring run that was missed is made up for by the next one. + Repeated failures are collapsed per automation, and no email is sent when nothing failed. + + Meant to be run regularly, for instance hourly from cron. + """ + email_recipients = get_monitoring_email_recipients(recipient) + if not email_recipients: + # Marking runs as reported without telling anyone would hide them from the next monitoring run. + raise click.UsageError( + "Nobody to send the digest to. Set FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS or use --recipient." + ) + if stuck_after_minutes is not None: + stuck_after = timedelta(minutes=stuck_after_minutes) + else: + stuck_after = app.config["FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER"] + + now = server_now() + failed_runs = get_unalerted_failed_automation_runs(stuck_after, now=now) + if not failed_runs: + click.secho( + "All good ― no automation runs failed since the last check.", + **MsgStyle.SUCCESS, + ) + return + failures = group_failed_runs_by_automation(failed_runs) + + subject = f"Failed runs of {len(failures)} automation(s)" + body = ( + f"These automations have runs that failed, or that have been executing for longer than {exact_duration_str(stuck_after)}, which were not reported before:\n\n" + + "\n\n".join(describe_automation_failures(f) for f in failures) + + ( + "\n\nWe suggest to check the logs." + if app.config.get("FLEXMEASURES_BASE_URL") + else "\n\nEach automation's runs are listed on the Automations page of its asset.\nWe suggest to check the logs." + ) + ) + capture_message_for_sentry(subject) + email = Message(subject=subject, bcc=email_recipients) + email.body = body + app.mail.send(email) + app.logger.error(body) + + # Only once the digest is out, so that a failure to send it leaves the runs to the next monitoring run. + for failed_run in failed_runs: + failed_run.run.alerted_at = now + db.session.commit() + click.secho( + f"Reported {len(failed_runs)} failed run(s) of {len(failures)} automation(s) to {', '.join(email_recipients)}.", + **MsgStyle.WARN, + ) + + app.cli.add_command(fm_monitor) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 5f0d44255a..545b777c56 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1869,6 +1869,58 @@ def test_report_automation_refuses_a_sensor_nobody_checked( assert not app.redis_connection.get(f"automation-last-run:{automation.id}") +def test_a_failed_report_run_is_reported_by_monitoring( + app, + fresh_db, + setup_dummy_data_fresh_db, + clean_redis, + tmp_path, + freeze_server_now, + mocker, +): + """A report run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.data_add import add_automation + from flexmeasures.cli.jobs import run_automations + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.models.reporting.pandas_reporter import PandasReporter + from flexmeasures.utils.job_utils import work_on_rq + + sensor1_id, sensor2_id, report_sensor_id, _ = setup_dummy_data_fresh_db + report_sensor = fresh_db.session.get(Sensor, report_sensor_id) + cli_input = _report_automation_cli_input( + tmp_path, + sensor1_id, + sensor2_id, + report_sensor_id, + parameters_extra={"start-offset": "DB"}, + asset_id=report_sensor.generic_asset_id, + ) + cli_input[cli_input.index("0 1 * * *")] = "* * * * *" # due every minute + cli_input += ["--timezone", "UTC"] + freeze_server_now(datetime(2023, 4, 10, 10, 0, 30, tzinfo=timezone.utc)) + runner = app.test_cli_runner() + result = runner.invoke(add_automation, cli_input) + assert "Successfully created" in result.output, result.output + automation = fresh_db.session.execute(select(Automation)).scalar_one() + result = runner.invoke(run_automations) + assert "queued 1 reporting job(s)" in result.output, result.output + + mocker.patch.object( + PandasReporter, "compute", side_effect=KeyError("df_agg is missing") + ) + work_on_rq(app.queues["reporting"]) + + with app.mail.record_messages() as outbox: + result = runner.invoke(monitor_automations, ["--recipient", "ops@example.test"]) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' (reporting)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: KeyError: 'df_agg is missing'" in body + + def _two_output_report_automation_cli_input( tmp_path, sensor1_id, diff --git a/flexmeasures/cli/tests/test_monitor.py b/flexmeasures/cli/tests/test_monitor.py index caa18fdd6f..ebcfc83da0 100644 --- a/flexmeasures/cli/tests/test_monitor.py +++ b/flexmeasures/cli/tests/test_monitor.py @@ -10,6 +10,7 @@ def test_monitor_help(app): from flexmeasures.cli.monitor import ( fm_monitor, + monitor_automations, monitor_last_seen, monitor_latest_run, ) @@ -19,7 +20,7 @@ def test_monitor_help(app): result = runner.invoke(fm_monitor, ["--help"]) assert result.exit_code == 0 - for command in (monitor_last_seen, monitor_latest_run): + for command in (monitor_last_seen, monitor_latest_run, monitor_automations): result = runner.invoke(command, ["--help"]) assert result.exit_code == 0 diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py new file mode 100644 index 0000000000..7859b8d022 --- /dev/null +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -0,0 +1,381 @@ +"""Tests for `flexmeasures monitor automations`, which emails a digest of failed automation runs.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from flexmeasures.cli.tests.utils import to_flags +from flexmeasures.data.models.automations import Automation, AutomationRun + +RECIPIENT = "ops@example.test" +DUE_AT = datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc) + + +@pytest.fixture(scope="function") +def clean_redis(app): + app.redis_connection.flushdb() + yield + app.redis_connection.flushdb() + + +@pytest.fixture() +def forecast_automation( + app, fresh_db, clean_redis, setup_fresh_test_forecast_data, freeze_server_now +): + """Create a forecast automation which is due at the frozen minute.""" + from flexmeasures.cli.data_add import add_automation + + freeze_server_now(DUE_AT - timedelta(minutes=2)) + sensor = setup_fresh_test_forecast_data["solar-sensor"] + result = app.test_cli_runner().invoke( + add_automation, + to_flags( + { + "asset": sensor.generic_asset_id, + "name": "Monitored forecasts", + "cron": "0 1 * * *", + "timezone": "UTC", + "sensor": sensor.id, + "duration": "PT2H", + "forecast-frequency": "PT1H", + "max-forecast-horizon": "PT2H", + "retrain-frequency": "PT1H", + } + ), + ) + assert result.exit_code == 0, result.output + freeze_server_now(DUE_AT) + return fresh_db.session.scalars(select(Automation)).one() + + +def run_automations(app, exit_code: int = 0): + from flexmeasures.cli.jobs import run_automations as run_automations_command + + result = app.test_cli_runner().invoke(run_automations_command) + assert result.exit_code == exit_code, result.output + + +def monitor(app, *args: str): + """Run the monitor, returning its result and the emails it sent.""" + from flexmeasures.cli.monitor import monitor_automations + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", RECIPIENT, *args] + ) + return result, outbox + + +def perform_job(app, intent): + from rq import SimpleWorker + + queue = app.queues["forecasting"] + worker = SimpleWorker([queue], connection=queue.connection) + worker.perform_job(queue.fetch_job(intent.rq_job_id), queue) + + +def add_run( + db, automation: Automation, scheduled_at: datetime, **columns +) -> AutomationRun: + """Record a run of the automation in whatever state the test needs, dispatched by default.""" + columns = { + "automation_type": automation.type, + "dispatch_state": "queued", + "dispatch_completed_at": scheduled_at, + "execution_state": "pending", + "attempt_count": 1, + **columns, + } + run = AutomationRun( + automation=automation, + scheduled_at=scheduled_at, + schedule_revision=automation.schedule_revision, + generator_id=automation.generator_id, + parameters={}, + plan={}, + **columns, + ) + db.session.add(run) + db.session.commit() + return run + + +def test_a_failed_forecast_is_reported_once(app, fresh_db, forecast_automation): + """A forecast run whose job failed on the worker is emailed about, and only the first time the monitor runs.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + queue = app.queues["forecasting"] + wrap_up = next(i for i in run.job_intents if i.kind == "forecast-wrap-up") + cycle = next(i for i in run.job_intents if i.kind == "forecast-cycle") + # Make the wrap-up job fail by taking away one of the cycle jobs it reports on. + queue.fetch_job(cycle.rq_job_id).delete() + perform_job(app, wrap_up) + fresh_db.session.refresh(run) + assert run.execution_state == "failed" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert outbox[0].bcc == [RECIPIENT] + assert outbox[0].subject == "Failed runs of 1 automation(s)" + body = outbox[0].body + assert ( + f"Automation {forecast_automation.id} 'Monitored forecasts' (forecasting)" + in body + ) + assert "failed once (execution failed: 1), last at 2026-08-05 01:00 UTC" in body + assert f"last error: {run.last_error_type}" in body + fresh_db.session.refresh(run) + assert run.alerted_at == DUE_AT + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + assert "All good" in result.output + + +def test_a_running_forecast_is_reported_once_it_is_stuck( + app, fresh_db, forecast_automation, freeze_server_now +): + """A forecast run which started executing is reported once it has been at it for longer than the configured threshold.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + # The wrap-up job succeeds while the cycle jobs have not run, so the run keeps executing. + perform_job(app, next(i for i in run.job_intents if i.kind == "forecast-wrap-up")) + fresh_db.session.refresh(run) + assert run.execution_state == "running" + + freeze_server_now(DUE_AT + timedelta(hours=5, minutes=59)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + freeze_server_now(DUE_AT + timedelta(hours=6)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (stuck: 1)" in outbox[0].body + assert "last error" not in outbox[0].body + + +def test_the_stuck_threshold_comes_from_the_config_or_the_command_line( + app, fresh_db, forecast_automation, freeze_server_now, monkeypatch +): + """The command-line option overrides the configured threshold, which overrides the default of six hours.""" + stuck_since = DUE_AT - timedelta(hours=2) + add_run( + fresh_db, + forecast_automation, + stuck_since, + execution_state="running", + execution_started_at=stuck_since, + ) + + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER", timedelta(hours=1) + ) + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "longer than 1 hour" in outbox[0].body + + +def test_a_forecast_dispatch_is_reported_once_its_attempts_are_used_up( + app, fresh_db, forecast_automation, freeze_server_now, mocker +): + """A forecast run whose dispatch failed is retried, so it is only reported once it will not be tried again.""" + from flexmeasures.data.services.automations import ( + AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + + mocker.patch.object( + app.queues["forecasting"], + "enqueue_job", + side_effect=RuntimeError("redis unavailable"), + ) + run_automations(app, exit_code=1) + run = fresh_db.session.scalars(select(AutomationRun)).one() + assert run.dispatch_state == "failed" + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + # Each attempt waits longer than the last, so stepping an hour ahead each time takes them all. + for attempt_no in range(2, AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + 1): + freeze_server_now(DUE_AT + timedelta(hours=attempt_no)) + result, outbox = monitor(app) + assert outbox == [], f"reported before attempt {attempt_no}" + run_automations(app, exit_code=1) + fresh_db.session.refresh(run) + assert run.attempt_count == AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (dispatch failed: 1)" in outbox[0].body + assert "last error: RuntimeError: redis unavailable" in outbox[0].body + + +def test_which_runs_count_as_failed(fresh_db, forecast_automation): + """Failures are judged the same way for every automation type, except that only a forecast run is dispatched again.""" + from flexmeasures.data.services.automation_monitoring import ( + get_unalerted_failed_automation_runs, + ) + + def add(minute: int, **columns) -> AutomationRun: + return add_run( + fresh_db, forecast_automation, DUE_AT - timedelta(minutes=minute), **columns + ) + + expected = { + add(1, execution_state="failed").id: "execution failed", + add( + 2, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=7), + ).id: "stuck", + # A schedule run is never dispatched again, so its first failed attempt is its last. + add( + 3, + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + claim_expires_at=DUE_AT + timedelta(minutes=1), + ).id: "dispatch failed", + # A runner which died mid-dispatch leaves its claim behind, until its lease runs out. + add( + 4, + automation_type="reporting", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:dead", + claim_expires_at=DUE_AT - timedelta(minutes=1), + ).id: "dispatch failed", + } + not_failed = [ + add(5, execution_state="succeeded"), + add( + 6, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=5), + ), + # A forecast run with attempts to spare will be dispatched again. + add(7, dispatch_state="failed", dispatch_completed_at=None), + # A runner is still dispatching this one. + add( + 8, + automation_type="scheduling", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:alive", + claim_expires_at=DUE_AT + timedelta(minutes=5), + ), + add(9, execution_state="failed", alerted_at=DUE_AT - timedelta(hours=1)), + ] + + failed_runs = get_unalerted_failed_automation_runs(timedelta(hours=6), now=DUE_AT) + + assert {f.run.id: f.reason for f in failed_runs} == expected + assert not {run.id for run in not_failed} & {f.run.id for f in failed_runs} + + +def test_repeated_failures_are_collapsed_per_automation( + app, fresh_db, forecast_automation +): + """An automation which failed several times takes up one entry, naming its latest failure and error.""" + for hours_ago, error in ((3, "first"), (1, "latest"), (2, "middle")): + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=hours_ago), + execution_state="failed", + last_error_type="ValueError", + last_error_message=error, + ) + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(minutes=30), + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert body.count(f"Automation {forecast_automation.id} ") == 1 + assert ( + "failed 4 times (execution failed: 3, dispatch failed: 1), last at 2026-08-05 00:30 UTC" + in body + ) + # The dispatch failure recorded no error, so the latest error is that of the latest failed execution. + assert "last error: ValueError: latest" in body + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + +def test_without_recipients_nothing_is_marked_as_reported( + app, fresh_db, forecast_automation, monkeypatch +): + """Without anyone to email, the monitor refuses to run, rather than marking runs as reported that nobody heard of.""" + from flexmeasures.cli.monitor import monitor_automations + + monkeypatch.setitem( + app.config, "FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS", [] + ) + monkeypatch.setitem(app.config, "FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", []) + run = add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result = app.test_cli_runner().invoke(monitor_automations) + + assert result.exit_code == 2 + assert "Nobody to send the digest to" in result.output + fresh_db.session.refresh(run) + assert run.alerted_at is None + + +def test_the_digest_links_to_the_automation_once_the_base_url_is_set( + app, fresh_db, forecast_automation, monkeypatch +): + """With the server's own address configured, each automation in the digest links to its info on the Automations page.""" + add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + link = f"/assets/{forecast_automation.asset_id}/automations?automation={forecast_automation.id}" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert link not in outbox[0].body + + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=1), + execution_state="failed", + ) + monkeypatch.setitem( + app.config, "FLEXMEASURES_BASE_URL", "https://ems.example.test/" + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert f" runs and jobs: https://ems.example.test{link}" in outbox[0].body diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py new file mode 100644 index 0000000000..74230d5642 --- /dev/null +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -0,0 +1,57 @@ +"""Record when an automation run was reported by `flexmeasures monitor automations` + +Revision ID: d4b7e9a1c362 +Revises: bf22b21d0990 +Create Date: 2026-10-07 10:00:00.000000 + +The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. +The upgrade marks the runs that had already failed for good as reported, so that the first monitoring run does not report every failure on record. +It judges them by the monitor's rules as they stand at this revision, with the default stuck threshold of 6 hours, +and leaves out any run that may still be retried or finish. + +The partial index holds only the runs the monitor has to look at: +those not reported yet which failed, are still running, or never finished their dispatch. +Every other run, which is nearly all of them, stays out of it. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "d4b7e9a1c362" +down_revision = "bf22b21d0990" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "automation_run", + sa.Column("alerted_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "automation_run_unalerted_idx", + "automation_run", + ["automation_id"], + postgresql_where=sa.text( + "alerted_at IS NULL" + " AND (execution_state = 'failed' OR execution_state = 'running' OR dispatch_completed_at IS NULL)" + ), + ) + # Copied from the monitor's query rather than imported, so this migration keeps doing what it did when the query changes. + op.execute(""" + UPDATE automation_run + SET alerted_at = now() + WHERE execution_state = 'failed' + OR (execution_state = 'running' AND execution_started_at <= now() - interval '6 hours') + OR ( + dispatch_completed_at IS NULL + AND (claim_owner IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= now()) + AND NOT (automation_type = 'forecasting' AND attempt_count < 5) + ) + """) + + +def downgrade(): + op.drop_index("automation_run_unalerted_idx", table_name="automation_run") + op.drop_column("automation_run", "alerted_at") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index cbba34f811..e4199607b1 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -245,6 +245,8 @@ class AutomationRun(db.Model): execution_completed_at = db.Column(db.DateTime(timezone=True), nullable=True) last_error_type = db.Column(db.String(160), nullable=True) last_error_message = db.Column(db.Text, nullable=True) + # When `flexmeasures monitor automations` reported this run as failed or stuck, so that it is reported once. + alerted_at = db.Column(db.DateTime(timezone=True), nullable=True) parameters = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) plan = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) @@ -274,6 +276,7 @@ class AutomationRun(db.Model): "dispatch_completed_at", "execution_started_at", "execution_completed_at", + "alerted_at", ) def validate_datetime_is_aware( self, key: str, value: datetime | None diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py new file mode 100644 index 0000000000..0b82190887 --- /dev/null +++ b/flexmeasures/data/services/automation_monitoring.py @@ -0,0 +1,150 @@ +"""Find the automation runs that failed, so that monitoring can report them.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from sqlalchemy import and_, not_, or_, select +from sqlalchemy.orm import selectinload + +from flexmeasures.data import db +from flexmeasures.data.models.automations import Automation, AutomationRun +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.services.automations import AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS +from flexmeasures.utils.time_utils import server_now + +EXECUTION_FAILED = "execution failed" +STUCK = "stuck" +DISPATCH_FAILED = "dispatch failed" + + +@dataclass(frozen=True) +class FailedAutomationRun: + """An automation run that failed, with the way it failed.""" + + run: AutomationRun + reason: str + + +@dataclass +class AutomationFailures: + """The failed runs of one automation, collapsed into what a digest says about them.""" + + automation: Automation + failed_runs: list[FailedAutomationRun] = field(default_factory=list) + + @property + def reason_counts(self) -> Counter: + """How many runs failed in each way.""" + return Counter(failed_run.reason for failed_run in self.failed_runs) + + @property + def last_run(self) -> AutomationRun: + """The failed run that was due most recently.""" + return max( + (failed_run.run for failed_run in self.failed_runs), + key=lambda run: (run.scheduled_at, run.id), + ) + + @property + def last_error(self) -> str | None: + """The error of the most recent failed run that recorded one, if any did.""" + runs_with_an_error = [ + failed_run.run + for failed_run in self.failed_runs + if failed_run.run.last_error_type or failed_run.run.last_error_message + ] + if not runs_with_an_error: + return None + run = max(runs_with_an_error, key=lambda run: (run.scheduled_at, run.id)) + return ": ".join( + part for part in (run.last_error_type, run.last_error_message) if part + ) + + +def _dispatch_failed_for_good(now: datetime): + """Return the criterion for a run whose dispatch did not finish and will not be attempted again. + + No runner may hold a live claim on it, or it is still being dispatched. + Only a forecast run is dispatched again, until its attempts are used up (see `get_dispatchable_automation_runs`), + so a run of any other type has failed for good after its first attempt. + """ + return and_( + AutomationRun.dispatch_completed_at.is_(None), + or_( + AutomationRun.claim_owner.is_(None), + AutomationRun.claim_expires_at.is_(None), + AutomationRun.claim_expires_at <= now, + ), + not_( + and_( + AutomationRun.automation_type == "forecasting", + AutomationRun.attempt_count < AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + ), + ) + + +def get_unalerted_failed_automation_runs( + stuck_after: timedelta, + now: datetime | None = None, +) -> list[FailedAutomationRun]: + """Return the automation runs that failed and were not reported yet, oldest first. + + A run failed when its execution failed, when its execution has been running for longer than ``stuck_after``, + or when its dispatch failed and will not be attempted again. + This holds for every automation type, although only the types that record their execution can fail on it. + + :param stuck_after: How long a run may execute before it counts as stuck. + :param now: The moment to judge by, which defaults to the server's current time. + """ + if now is None: + now = server_now() + now = now.astimezone(timezone.utc) + execution_failed = AutomationRun.execution_state == "failed" + stuck = and_( + AutomationRun.execution_state == "running", + AutomationRun.execution_started_at <= now - stuck_after, + ) + runs = db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.alerted_at.is_(None), + or_(execution_failed, stuck, _dispatch_failed_for_good(now)), + ) + .options( + selectinload(AutomationRun.automation) + .joinedload(Automation.asset) + .joinedload(GenericAsset.owner) + ) + .order_by( + AutomationRun.automation_id, AutomationRun.scheduled_at, AutomationRun.id + ) + ).all() + failed_runs = [] + for run in runs: + if run.execution_state == "failed": + reason = EXECUTION_FAILED + elif ( + run.execution_state == "running" + and run.execution_started_at <= now - stuck_after + ): + reason = STUCK + else: + reason = DISPATCH_FAILED + failed_runs.append(FailedAutomationRun(run=run, reason=reason)) + return failed_runs + + +def group_failed_runs_by_automation( + failed_runs: list[FailedAutomationRun], +) -> list[AutomationFailures]: + """Collapse failed runs per automation, keeping the order in which the automations first appear.""" + failures: dict[int, AutomationFailures] = {} + for failed_run in failed_runs: + automation = failed_run.run.automation + failures.setdefault(automation.id, AutomationFailures(automation=automation)) + failures[automation.id].failed_runs.append(failed_run) + return list(failures.values()) diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index ac36ef3ad4..ef61434dde 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -333,6 +333,50 @@ def refuse_to_save(*args, **kwargs): assert run.execution_completed_at is not None +def test_a_failed_schedule_run_is_reported_by_monitoring( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, +): + """A schedule run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services import scheduling + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_generator_results", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} 'Nightly schedules' (scheduling)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ValueError: The database is read-only today." in body + + def test_a_schedule_job_queued_outside_a_run_is_not_recorded( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 14f546c647..478f8ee0c9 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -535,6 +535,44 @@ def test_a_plugin_run_records_how_its_job_ended( assert run.execution_state == "succeeded" +def test_a_failed_plugin_run_is_reported_by_monitoring( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A plugin run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + + def fail_to_ingest(generator, value): + raise ConnectionError("The meter portal is down.") + + monkeypatch.setattr(MockIngestionGenerator, "_compute", fail_to_ingest) + result = run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' ({automation.type})" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ConnectionError: The meter portal is down." in body + + def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch ): diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 1a7c23a74f..c69b50fd5e 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -824,6 +824,26 @@
Recently created jobs // The types the page keeps a tab and a table for. const AUTOMATION_TYPES = Object.keys(automationTypes); + /* A link can point at one automation (?automation=), as the monitoring digest's links do. + Once the listing is in, its tab and its info panel are opened, which show its latest run, its error and its jobs. */ + let automationToOpen = Number(new URLSearchParams(window.location.search).get("automation")) || null; + + function openLinkedAutomation() { + const automation = automationToOpen === null ? undefined : automationsById.get(automationToOpen); + automationToOpen = null; // only on the first load, not on every refresh + if (automation === undefined) { + return; + } + const tab = document.getElementById(`${automation.type}-tab`); + if (tab) { + bootstrap.Tab.getOrCreateInstance(tab).show(); + } + const modal = document.getElementById(`AutomationInfoModal-${automation.id}`); + if (modal) { + bootstrap.Modal.getOrCreateInstance(modal).show(); + } + } + function loadAutomations({ rebuild = false } = {}) { // One refresh at a time: a slow response must not pile up behind the next tick. if (loadInFlight) { @@ -846,6 +866,7 @@
Recently created jobs } } markAutomationsRefreshed(); + openLinkedAutomation(); // The listing counts the jobs itself, so it is also where it says when Redis could not be reached. if (res["redis-connection-err"]) { showAutomationsError(res["redis-connection-err"]); diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index d4ced83728..b35291c4eb 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -125,6 +125,10 @@ class Config(object): [] ) # Deprecated. Use FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS instead. FLEXMEASURES_MONITORING_MAIL_RECIPIENTS: list[str] = [] + # How long an automation run may execute before `flexmeasures monitor automations` reports it as stuck. + FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER: timedelta = timedelta(hours=6) + # Where this server is reached on the web, e.g. "https://ems.example.com", for links in emails sent outside of a request, such as monitoring digests. + FLEXMEASURES_BASE_URL: str | None = None FLEXMEASURES_PLATFORM_NAME: str | list[str | tuple[str, list[str]]] = "FlexMeasures" FLEXMEASURES_MODE: str = "" diff --git a/flexmeasures/utils/tests/test_time_utils.py b/flexmeasures/utils/tests/test_time_utils.py index 9c637acdf7..ac75c877c5 100644 --- a/flexmeasures/utils/tests/test_time_utils.py +++ b/flexmeasures/utils/tests/test_time_utils.py @@ -10,6 +10,7 @@ from flexmeasures.utils.time_utils import ( as_utc_isoformat, duration_isoformat, + exact_duration_str, server_now, naturalized_datetime_str, get_most_recent_clocktime_window, @@ -309,3 +310,17 @@ def test_to_utc_timestamp(input_value, expected_output): assert result is None else: assert result == pytest.approx(expected_output) + + +@pytest.mark.parametrize( + "duration, expected", + [ + (timedelta(hours=6), "6 hours"), + (timedelta(hours=1), "1 hour"), + (timedelta(minutes=90), "90 minutes"), + (timedelta(minutes=1), "1 minute"), + (timedelta(days=1), "24 hours"), + ], +) +def test_exact_duration_str(duration: timedelta, expected: str): + assert exact_duration_str(duration) == expected diff --git a/flexmeasures/utils/time_utils.py b/flexmeasures/utils/time_utils.py index 16f8a52f23..42822d2d1c 100644 --- a/flexmeasures/utils/time_utils.py +++ b/flexmeasures/utils/time_utils.py @@ -305,6 +305,19 @@ def duration_isoformat(duration: timedelta): return re.sub("%P", repl, "P%P") +def exact_duration_str(duration: timedelta) -> str: + """Say how long a duration is in whole hours or minutes, exactly, which a rounded duration such as humanize's would not. + + For example, 6 hours is "6 hours" and 90 minutes is "90 minutes". + Seconds are left out. + """ + minutes = int(duration.total_seconds() // 60) + if minutes % 60 == 0: + hours = minutes // 60 + return "1 hour" if hours == 1 else f"{hours} hours" + return "1 minute" if minutes == 1 else f"{minutes} minutes" + + def determine_minimum_resampling_resolution( event_resolutions: list[timedelta], fallback_resolution: timedelta = timedelta(0),