diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9f0ccb3aa9..5aed8ccc80 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ v1.2.0 | December xx, 2026 New features ------------- +* Hosts can be emailed a digest of the automation runs that failed, got stuck or could not be queued, with ``flexmeasures monitor automations``, run from cron; each run is reported once, repeated failures are collapsed per automation, with the new ``FLEXMEASURES_BASE_URL`` setting each automation links to its *Automations* page, and hosts can have the admins and consultants of each organisation emailed, too [see `PR #2671 `_ and `PR #2703 `_] Infrastructure / Support ------------------------- diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 68cc8cf7ae..99accdceca 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -4,6 +4,11 @@ FlexMeasures CLI Changelog ************************** +since v1.2.0 | December xx, 2026 +================================= + +* Add ``flexmeasures monitor automations`` to email a digest of the automation runs that failed since the last check: runs whose execution failed, which have been executing for too long (``--stuck-after-minutes``), or whose dispatch failed for good. Each run is reported once, and repeated failures are collapsed per automation. With ``FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS`` switched on, which it is not by default, the admins and consultants of each asset's organisation also receive a digest of the automations they can see. + since v1.1.0 | November 05, 2026 ================================= diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index c37693ae3c..095c125dc0 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -107,6 +107,7 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures monitor latest-run`` Check if the given task's last successful execution happened less than the allowed time ago. ``flexmeasures monitor last-seen`` Check if given users last contact (via a request) happened less than the allowed time ago. +``flexmeasures monitor automations`` Email a digest of the automation runs that failed since the last check. ================================================= ======================================= diff --git a/documentation/configuration.rst b/documentation/configuration.rst index ebdbdd863f..013ac2f7ad 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -878,11 +878,49 @@ Default: ``None`` FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run`` and ``flexmeasures monitor last-seen`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. +E-mail addresses to send monitoring alerts to from the CLI tasks ``flexmeasures monitor latest-run``, ``flexmeasures monitor last-seen`` and ``flexmeasures monitor automations`` if no explicit user recipients are given. For example ``["fred@one.com", "wilma@two.com"]``. Default: ``[]`` +.. _monitor_automations_stuck_after: + +FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +How long an automation run may execute before ``flexmeasures monitor automations`` reports it as stuck, for instance because the worker running its job died. +Its ``--stuck-after-minutes`` option overrides this setting. + +Default: ``timedelta(hours=6)`` + +.. _base_url: + +FLEXMEASURES_BASE_URL +^^^^^^^^^^^^^^^^^^^^^ + +Where this server is reached on the web, for instance ``"https://ems.example.com"``. +Emails sent from the CLI, outside of any request, use it to link to the web pages they are about. +For instance, the digest of ``flexmeasures monitor automations`` links each failing automation to the Automations page of its asset. +Without it, these emails carry no links. + +Default: ``None`` + + +.. _monitor_automations_notify_users: + +FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Whether ``flexmeasures monitor automations`` also emails the people responsible for the assets whose automations failed: +the users with the ``account-admin`` role in the asset's organisation, and the users with the ``consultant`` role in its consultancy organisation. +Each of them receives one digest per monitoring run, listing only the automations they can see. +The host's digest is sent regardless, and still lists every failed run. + +Switch this on once you want your users to start receiving these emails. + +Default: ``False`` + + .. _redis-config: Redis diff --git a/documentation/features/automations.rst b/documentation/features/automations.rst index 03df6cf34a..3c63f54b38 100644 --- a/documentation/features/automations.rst +++ b/documentation/features/automations.rst @@ -287,6 +287,35 @@ This is what an operator needs to tell a run which failed before queueing anythi Editing an automation's cron string or timezone, or reactivating it, counts up its revision. Runs of the old and the new schedule therefore stay distinct, even when they fall on the same scheduled UTC time. +.. _monitoring_automations: + +Monitoring automation runs +-------------------------- + +As a host, you can have FlexMeasures email you a digest of the automation runs that failed, by running ``flexmeasures monitor automations`` regularly, for instance every hour: + +.. code-block:: bash + + 0 * * * * flexmeasures monitor automations + +A run is reported when its execution failed, when it has been executing for longer than six hours, or when its dispatch failed and will not be attempted again. +A forecast run therefore counts once its five dispatch attempts are used up, and a run of any other type after its first failed attempt. +The six hours are set by :ref:`monitor_automations_stuck_after`, and ``--stuck-after-minutes`` overrides that setting. + +Each run is reported once, so if one monitoring run is missed, the next one still reports what failed in the meantime. +Repeated failures of an automation are collapsed into one entry, saying how often it failed, when it last did and with which error. +No email is sent when nothing failed. + +The digest goes to :ref:`default_monitoring_mail_recipients`, or to whoever you name with ``--recipient`` (a user ID or an email address, as often as needed). +The command stays out of ``flexmeasures jobs run-automations``, so that a problem with monitoring, such as an unreachable mail server, never holds up the automations themselves. + +The people responsible for the assets can receive a digest of their own, once you switch on :ref:`monitor_automations_notify_users`. +Each user with the ``account-admin`` role in an asset's organisation, and each user with the ``consultant`` role in its consultancy organisation, then receives one email per monitoring run, +listing only the automations they can see. +Inactive users and users without an email address are skipped. +Automations on public assets belong to no organisation, so they only appear in your digest, which keeps listing every failed run. +A run counts as reported once your digest is sent, so a user digest that cannot be delivered is logged, and not sent again. + Running one automation on demand -------------------------------- diff --git a/documentation/host/error-monitoring.rst b/documentation/host/error-monitoring.rst index 1ff8d049cf..ac98a8ec1f 100644 --- a/documentation/host/error-monitoring.rst +++ b/documentation/host/error-monitoring.rst @@ -36,6 +36,17 @@ If you run distinct filters, such as separate checks per account, account group .. todo:: Adding roles and assigning them to accounts is not supported by the UI yet (user roles can be added in the UI). Account roles can be added with ``flexmeasures add account-role``. +Monitoring automations +----------------------- + +The CLI task ``flexmeasures monitor automations`` emails a digest of the automation runs that failed since its last check, with repeated failures collapsed per automation. +Run it from cron, for instance every hour; see :ref:`monitoring_automations` for what it reports. + +.. code-block:: bash + + $ flexmeasures monitor automations --recipient alerts@example.com + + Monitoring task runs --------------------- diff --git a/flexmeasures/cli/monitor.py b/flexmeasures/cli/monitor.py index 2c4edebbe0..0ec8393331 100644 --- a/flexmeasures/cli/monitor.py +++ b/flexmeasures/cli/monitor.py @@ -21,10 +21,16 @@ from flexmeasures.data import db from flexmeasures.data.models.task_runs import LatestTaskRun +from flexmeasures.data.services.automation_monitoring import ( + AutomationFailures, + get_unalerted_failed_automation_runs, + get_users_to_notify, + group_failed_runs_by_automation, +) from flexmeasures.data.models.user import Account, User from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.api.common.schemas.users import UserIdField -from flexmeasures.utils.time_utils import server_now +from flexmeasures.utils.time_utils import exact_duration_str, server_now from flexmeasures.cli.utils import LoggedClickExceptionGroup, MsgStyle @@ -505,4 +511,188 @@ def monitor_last_seen( db.session.commit() +# Error messages can carry a whole traceback, which does not help a digest that lists many automations. +MAX_ERROR_LENGTH_IN_DIGEST = 500 + + +def _format_utc(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def _times(count: int) -> str: + return "once" if count == 1 else f"{count} times" + + +def automation_page_url(automation) -> str | None: + """Link to the Automations page of the automation's asset, opening that automation's info, if the server knows its own address.""" + base_url = app.config.get("FLEXMEASURES_BASE_URL") + if not base_url: + return None + return f"{base_url.rstrip('/')}/assets/{automation.asset_id}/automations?automation={automation.id}" + + +def describe_automation_failures(failures: AutomationFailures) -> str: + """Describe the failed runs of one automation in a few lines, for the monitoring digest.""" + automation = failures.automation + asset = automation.asset + if asset.owner is None: + where = f"public asset {asset.id} '{asset.name}'" + else: + where = f"asset {asset.id} '{asset.name}' of organisation '{asset.owner.name}'" + reasons = ", ".join( + f"{reason}: {count}" for reason, count in failures.reason_counts.items() + ) + last_run = failures.last_run + lines = [ + f"Automation {automation.id} '{automation.name}' ({automation.type}) on {where}:", + f" failed {_times(len(failures.failed_runs))} ({reasons})," + f" last at {_format_utc(last_run.scheduled_at)} (run {last_run.id})", + ] + last_error = failures.last_error + if last_error: + if len(last_error) > MAX_ERROR_LENGTH_IN_DIGEST: + last_error = last_error[:MAX_ERROR_LENGTH_IN_DIGEST] + " ..." + lines.append(f" last error: {last_error}") + url = automation_page_url(automation) + if url: + lines.append(f" runs and jobs: {url}") + return "\n".join(lines) + + +def compose_automation_digest( + failures: list[AutomationFailures], stuck_after: timedelta, for_host: bool = True +) -> tuple[str, str]: + """Return the subject and body of a digest about these failed automations. + + :param for_host: Whether the digest goes to the host, who can check the logs, rather than to a user of the assets' organisations. + """ + subject = f"Failed runs of {len(failures)} automation(s)" + automations = ( + "These automations" + if for_host + else "These automations on assets you look after" + ) + closing_lines = [] + # With the server's address configured, each automation links to its page instead. + if not app.config.get("FLEXMEASURES_BASE_URL"): + closing_lines.append( + "Each automation's runs are listed on the Automations page of its asset." + ) + if for_host: + closing_lines.append("We suggest to check the logs.") + body = ( + f"{automations} have runs that failed, or that have been executing for longer than {exact_duration_str(stuck_after)}, which were not reported before:\n\n" + + "\n\n".join(describe_automation_failures(f) for f in failures) + ) + if closing_lines: + body += "\n\n" + "\n".join(closing_lines) + return subject, body + + +def send_automation_digests_to_users( + failures: list[AutomationFailures], stuck_after: timedelta +): + """Email each account admin and consultant a digest of the failed automations they may read. + + The runs are already marked as reported, so a digest that cannot be sent is logged rather than tried again. + """ + users_to_notify = get_users_to_notify(failures) + undelivered = [] + for user, user_failures in users_to_notify: + subject, body = compose_automation_digest( + user_failures, stuck_after, for_host=False + ) + email = Message(subject=subject, recipients=[user.email]) + email.body = body + try: + app.mail.send(email) + except OSError as exc: # SMTP errors are OSErrors, too + app.logger.error( + f"Could not email user {user.id} about failed automation runs: {exc}" + ) + undelivered.append(user.email) + delivered = len(users_to_notify) - len(undelivered) + click.secho( + f"Sent a digest of failed automation runs to {delivered} user(s).", + **MsgStyle.WARN, + ) + if undelivered: + click.secho( + f"Could not send a digest to {', '.join(undelivered)}.", + **MsgStyle.ERROR, + ) + + +@fm_monitor.command("automations") +@with_appcontext +@click.option( + "--stuck-after-minutes", + type=click.IntRange(min=1), + default=None, + help="Report a run as stuck once it has been executing for this many minutes. Defaults to the FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER setting (6 hours).", +) +@click.option( + "--recipient", + type=str, + multiple=True, + help="User ID or email address to send the digest to. Use multiple times if needed. If not used, the default monitoring mail recipients are informed.", +) +def monitor_automations( + stuck_after_minutes: int | None = None, + recipient: tuple[str, ...] = (), +): + """ + Email a digest of the automation runs that failed since the last check. + + A run is reported when its execution failed, when it has been executing for too long (see --stuck-after-minutes), + or when its dispatch failed and will not be attempted again. + Each run is reported once, so a monitoring run that was missed is made up for by the next one. + Repeated failures are collapsed per automation, and no email is sent when nothing failed. + + With FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS switched on, the admins of each asset's organisation and its consultants also receive a digest, + listing only the automations they may read. + A run counts as reported once the host's digest is sent. + + Meant to be run regularly, for instance hourly from cron. + """ + email_recipients = get_monitoring_email_recipients(recipient) + if not email_recipients: + # Marking runs as reported without telling anyone would hide them from the next monitoring run. + raise click.UsageError( + "Nobody to send the digest to. Set FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS or use --recipient." + ) + if stuck_after_minutes is not None: + stuck_after = timedelta(minutes=stuck_after_minutes) + else: + stuck_after = app.config["FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER"] + + now = server_now() + failed_runs = get_unalerted_failed_automation_runs(stuck_after, now=now) + if not failed_runs: + click.secho( + "All good ― no automation runs failed since the last check.", + **MsgStyle.SUCCESS, + ) + return + failures = group_failed_runs_by_automation(failed_runs) + + subject, body = compose_automation_digest(failures, stuck_after) + capture_message_for_sentry(subject) + email = Message(subject=subject, bcc=email_recipients) + email.body = body + app.mail.send(email) + app.logger.error(body) + + # Only once the digest is out, so that a failure to send it leaves the runs to the next monitoring run. + for failed_run in failed_runs: + failed_run.run.alerted_at = now + db.session.commit() + click.secho( + f"Reported {len(failed_runs)} failed run(s) of {len(failures)} automation(s) to {', '.join(email_recipients)}.", + **MsgStyle.WARN, + ) + if app.config["FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS"]: + send_automation_digests_to_users(failures, stuck_after) + + app.cli.add_command(fm_monitor) diff --git a/flexmeasures/cli/tests/test_automations.py b/flexmeasures/cli/tests/test_automations.py index 5f0d44255a..545b777c56 100644 --- a/flexmeasures/cli/tests/test_automations.py +++ b/flexmeasures/cli/tests/test_automations.py @@ -1869,6 +1869,58 @@ def test_report_automation_refuses_a_sensor_nobody_checked( assert not app.redis_connection.get(f"automation-last-run:{automation.id}") +def test_a_failed_report_run_is_reported_by_monitoring( + app, + fresh_db, + setup_dummy_data_fresh_db, + clean_redis, + tmp_path, + freeze_server_now, + mocker, +): + """A report run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.data_add import add_automation + from flexmeasures.cli.jobs import run_automations + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.models.reporting.pandas_reporter import PandasReporter + from flexmeasures.utils.job_utils import work_on_rq + + sensor1_id, sensor2_id, report_sensor_id, _ = setup_dummy_data_fresh_db + report_sensor = fresh_db.session.get(Sensor, report_sensor_id) + cli_input = _report_automation_cli_input( + tmp_path, + sensor1_id, + sensor2_id, + report_sensor_id, + parameters_extra={"start-offset": "DB"}, + asset_id=report_sensor.generic_asset_id, + ) + cli_input[cli_input.index("0 1 * * *")] = "* * * * *" # due every minute + cli_input += ["--timezone", "UTC"] + freeze_server_now(datetime(2023, 4, 10, 10, 0, 30, tzinfo=timezone.utc)) + runner = app.test_cli_runner() + result = runner.invoke(add_automation, cli_input) + assert "Successfully created" in result.output, result.output + automation = fresh_db.session.execute(select(Automation)).scalar_one() + result = runner.invoke(run_automations) + assert "queued 1 reporting job(s)" in result.output, result.output + + mocker.patch.object( + PandasReporter, "compute", side_effect=KeyError("df_agg is missing") + ) + work_on_rq(app.queues["reporting"]) + + with app.mail.record_messages() as outbox: + result = runner.invoke(monitor_automations, ["--recipient", "ops@example.test"]) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' (reporting)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: KeyError: 'df_agg is missing'" in body + + def _two_output_report_automation_cli_input( tmp_path, sensor1_id, diff --git a/flexmeasures/cli/tests/test_monitor.py b/flexmeasures/cli/tests/test_monitor.py index caa18fdd6f..ebcfc83da0 100644 --- a/flexmeasures/cli/tests/test_monitor.py +++ b/flexmeasures/cli/tests/test_monitor.py @@ -10,6 +10,7 @@ def test_monitor_help(app): from flexmeasures.cli.monitor import ( fm_monitor, + monitor_automations, monitor_last_seen, monitor_latest_run, ) @@ -19,7 +20,7 @@ def test_monitor_help(app): result = runner.invoke(fm_monitor, ["--help"]) assert result.exit_code == 0 - for command in (monitor_last_seen, monitor_latest_run): + for command in (monitor_last_seen, monitor_latest_run, monitor_automations): result = runner.invoke(command, ["--help"]) assert result.exit_code == 0 diff --git a/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py new file mode 100644 index 0000000000..05b6be8990 --- /dev/null +++ b/flexmeasures/cli/tests/test_monitor_automations_fresh_db.py @@ -0,0 +1,541 @@ +"""Tests for `flexmeasures monitor automations`, which emails a digest of failed automation runs.""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from flexmeasures.cli.tests.utils import to_flags +from flexmeasures.data.models.automations import Automation, AutomationRun + +RECIPIENT = "ops@example.test" +DUE_AT = datetime(2026, 8, 5, 1, 0, tzinfo=timezone.utc) + + +@pytest.fixture(scope="function") +def clean_redis(app): + app.redis_connection.flushdb() + yield + app.redis_connection.flushdb() + + +@pytest.fixture() +def forecast_automation( + app, fresh_db, clean_redis, setup_fresh_test_forecast_data, freeze_server_now +): + """Create a forecast automation which is due at the frozen minute.""" + from flexmeasures.cli.data_add import add_automation + + freeze_server_now(DUE_AT - timedelta(minutes=2)) + sensor = setup_fresh_test_forecast_data["solar-sensor"] + result = app.test_cli_runner().invoke( + add_automation, + to_flags( + { + "asset": sensor.generic_asset_id, + "name": "Monitored forecasts", + "cron": "0 1 * * *", + "timezone": "UTC", + "sensor": sensor.id, + "duration": "PT2H", + "forecast-frequency": "PT1H", + "max-forecast-horizon": "PT2H", + "retrain-frequency": "PT1H", + } + ), + ) + assert result.exit_code == 0, result.output + freeze_server_now(DUE_AT) + return fresh_db.session.scalars(select(Automation)).one() + + +def run_automations(app, exit_code: int = 0): + from flexmeasures.cli.jobs import run_automations as run_automations_command + + result = app.test_cli_runner().invoke(run_automations_command) + assert result.exit_code == exit_code, result.output + + +def monitor(app, *args: str): + """Run the monitor, returning its result and the emails it sent.""" + from flexmeasures.cli.monitor import monitor_automations + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", RECIPIENT, *args] + ) + return result, outbox + + +def perform_job(app, intent): + from rq import SimpleWorker + + queue = app.queues["forecasting"] + worker = SimpleWorker([queue], connection=queue.connection) + worker.perform_job(queue.fetch_job(intent.rq_job_id), queue) + + +def add_run( + db, automation: Automation, scheduled_at: datetime, **columns +) -> AutomationRun: + """Record a run of the automation in whatever state the test needs, dispatched by default.""" + columns = { + "automation_type": automation.type, + "dispatch_state": "queued", + "dispatch_completed_at": scheduled_at, + "execution_state": "pending", + "attempt_count": 1, + **columns, + } + run = AutomationRun( + automation=automation, + scheduled_at=scheduled_at, + schedule_revision=automation.schedule_revision, + generator_id=automation.generator_id, + parameters={}, + plan={}, + **columns, + ) + db.session.add(run) + db.session.commit() + return run + + +def test_a_failed_forecast_is_reported_once(app, fresh_db, forecast_automation): + """A forecast run whose job failed on the worker is emailed about, and only the first time the monitor runs.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + queue = app.queues["forecasting"] + wrap_up = next(i for i in run.job_intents if i.kind == "forecast-wrap-up") + cycle = next(i for i in run.job_intents if i.kind == "forecast-cycle") + # Make the wrap-up job fail by taking away one of the cycle jobs it reports on. + queue.fetch_job(cycle.rq_job_id).delete() + perform_job(app, wrap_up) + fresh_db.session.refresh(run) + assert run.execution_state == "failed" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert outbox[0].bcc == [RECIPIENT] + assert outbox[0].subject == "Failed runs of 1 automation(s)" + body = outbox[0].body + assert ( + f"Automation {forecast_automation.id} 'Monitored forecasts' (forecasting)" + in body + ) + assert "failed once (execution failed: 1), last at 2026-08-05 01:00 UTC" in body + assert f"last error: {run.last_error_type}" in body + fresh_db.session.refresh(run) + assert run.alerted_at == DUE_AT + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + assert "All good" in result.output + + +def test_a_running_forecast_is_reported_once_it_is_stuck( + app, fresh_db, forecast_automation, freeze_server_now +): + """A forecast run which started executing is reported once it has been at it for longer than the configured threshold.""" + run_automations(app) + run = fresh_db.session.scalars(select(AutomationRun)).one() + # The wrap-up job succeeds while the cycle jobs have not run, so the run keeps executing. + perform_job(app, next(i for i in run.job_intents if i.kind == "forecast-wrap-up")) + fresh_db.session.refresh(run) + assert run.execution_state == "running" + + freeze_server_now(DUE_AT + timedelta(hours=5, minutes=59)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + freeze_server_now(DUE_AT + timedelta(hours=6)) + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (stuck: 1)" in outbox[0].body + assert "last error" not in outbox[0].body + + +def test_the_stuck_threshold_comes_from_the_config_or_the_command_line( + app, fresh_db, forecast_automation, freeze_server_now, monkeypatch +): + """The command-line option overrides the configured threshold, which overrides the default of six hours.""" + stuck_since = DUE_AT - timedelta(hours=2) + add_run( + fresh_db, + forecast_automation, + stuck_since, + execution_state="running", + execution_started_at=stuck_since, + ) + + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER", timedelta(hours=1) + ) + result, outbox = monitor(app, "--stuck-after-minutes", "180") + assert result.exit_code == 0, result.output + assert outbox == [] + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "longer than 1 hour" in outbox[0].body + + +def test_a_forecast_dispatch_is_reported_once_its_attempts_are_used_up( + app, fresh_db, forecast_automation, freeze_server_now, mocker +): + """A forecast run whose dispatch failed is retried, so it is only reported once it will not be tried again.""" + from flexmeasures.data.services.automations import ( + AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + + mocker.patch.object( + app.queues["forecasting"], + "enqueue_job", + side_effect=RuntimeError("redis unavailable"), + ) + run_automations(app, exit_code=1) + run = fresh_db.session.scalars(select(AutomationRun)).one() + assert run.dispatch_state == "failed" + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert outbox == [] + + # Each attempt waits longer than the last, so stepping an hour ahead each time takes them all. + for attempt_no in range(2, AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + 1): + freeze_server_now(DUE_AT + timedelta(hours=attempt_no)) + result, outbox = monitor(app) + assert outbox == [], f"reported before attempt {attempt_no}" + run_automations(app, exit_code=1) + fresh_db.session.refresh(run) + assert run.attempt_count == AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS + + result, outbox = monitor(app) + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert "failed once (dispatch failed: 1)" in outbox[0].body + assert "last error: RuntimeError: redis unavailable" in outbox[0].body + + +def test_which_runs_count_as_failed(fresh_db, forecast_automation): + """Failures are judged the same way for every automation type, except that only a forecast run is dispatched again.""" + from flexmeasures.data.services.automation_monitoring import ( + get_unalerted_failed_automation_runs, + ) + + def add(minute: int, **columns) -> AutomationRun: + return add_run( + fresh_db, forecast_automation, DUE_AT - timedelta(minutes=minute), **columns + ) + + expected = { + add(1, execution_state="failed").id: "execution failed", + add( + 2, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=7), + ).id: "stuck", + # A schedule run is never dispatched again, so its first failed attempt is its last. + add( + 3, + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + claim_expires_at=DUE_AT + timedelta(minutes=1), + ).id: "dispatch failed", + # A runner which died mid-dispatch leaves its claim behind, until its lease runs out. + add( + 4, + automation_type="reporting", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:dead", + claim_expires_at=DUE_AT - timedelta(minutes=1), + ).id: "dispatch failed", + } + not_failed = [ + add(5, execution_state="succeeded"), + add( + 6, + execution_state="running", + execution_started_at=DUE_AT - timedelta(hours=5), + ), + # A forecast run with attempts to spare will be dispatched again. + add(7, dispatch_state="failed", dispatch_completed_at=None), + # A runner is still dispatching this one. + add( + 8, + automation_type="scheduling", + dispatch_state="claimed", + dispatch_completed_at=None, + claim_owner="runner:alive", + claim_expires_at=DUE_AT + timedelta(minutes=5), + ), + add(9, execution_state="failed", alerted_at=DUE_AT - timedelta(hours=1)), + ] + + failed_runs = get_unalerted_failed_automation_runs(timedelta(hours=6), now=DUE_AT) + + assert {f.run.id: f.reason for f in failed_runs} == expected + assert not {run.id for run in not_failed} & {f.run.id for f in failed_runs} + + +def test_repeated_failures_are_collapsed_per_automation( + app, fresh_db, forecast_automation +): + """An automation which failed several times takes up one entry, naming its latest failure and error.""" + for hours_ago, error in ((3, "first"), (1, "latest"), (2, "middle")): + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=hours_ago), + execution_state="failed", + last_error_type="ValueError", + last_error_message=error, + ) + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(minutes=30), + automation_type="scheduling", + dispatch_state="failed", + dispatch_completed_at=None, + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert body.count(f"Automation {forecast_automation.id} ") == 1 + assert ( + "failed 4 times (execution failed: 3, dispatch failed: 1), last at 2026-08-05 00:30 UTC" + in body + ) + # The dispatch failure recorded no error, so the latest error is that of the latest failed execution. + assert "last error: ValueError: latest" in body + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + +def test_without_recipients_nothing_is_marked_as_reported( + app, fresh_db, forecast_automation, monkeypatch +): + """Without anyone to email, the monitor refuses to run, rather than marking runs as reported that nobody heard of.""" + from flexmeasures.cli.monitor import monitor_automations + + monkeypatch.setitem( + app.config, "FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS", [] + ) + monkeypatch.setitem(app.config, "FLEXMEASURES_MONITORING_MAIL_RECIPIENTS", []) + run = add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + + result = app.test_cli_runner().invoke(monitor_automations) + + assert result.exit_code == 2 + assert "Nobody to send the digest to" in result.output + fresh_db.session.refresh(run) + assert run.alerted_at is None + + +@pytest.fixture() +def automations_per_organisation(fresh_db, setup_roles_users_fresh_db): + """Create one automation on an asset of the prosumer organisation, one on an asset of the consultancy client, and one on a public asset. + + Also add the users who should not hear about any of them: an inactive account admin and one without an email address. + """ + from flexmeasures.data.models.data_sources import DataSource + from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType + from flexmeasures.data.models.user import Account, Role, User + + accounts = { + account.name: account for account in fresh_db.session.scalars(select(Account)) + } + account_admin = fresh_db.session.scalars( + select(Role).filter_by(name="account-admin") + ).one() + users = {user.email: user for user in fresh_db.session.scalars(select(User))} + # The consultancy client gets an account admin of its own. + users["test_consultant_client@seita.nl"].roles.append(account_admin) + for email, active in ( + ("inactive_prosumer_admin@seita.nl", False), + (None, True), + ("", True), + ): + fresh_db.session.add( + User( + username=f"Prosumer admin {email!r} {active}", + email=email, + active=active, + account=accounts["Test Prosumer Account"], + roles=[account_admin], + fs_uniquifier=uuid.uuid4().hex, + ) + ) + asset_type = GenericAssetType(name="site") + generator = DataSource(name="Monitored forecaster", type="forecaster") + automations = {} + for key, owner in ( + ("prosumer", accounts["Test Prosumer Account"]), + ("client", accounts["Test ConsultancyClient Account"]), + ("public", None), + ): + asset = GenericAsset( + name=f"{key} site", generic_asset_type=asset_type, owner=owner + ) + automations[key] = Automation( + asset=asset, + name=f"{key} forecasts", + cronstr="0 1 * * *", + generator=generator, + ) + fresh_db.session.add(automations[key]) + fresh_db.session.commit() + for automation in automations.values(): + add_run(fresh_db, automation, DUE_AT, execution_state="failed") + return automations + + +def emails_by_user(outbox) -> dict[str, str]: + """Map each user who got their own digest to its body, leaving out the host's digest.""" + return { + recipient: message.body + for message in outbox + if not message.bcc + for recipient in message.recipients + } + + +def test_users_hear_only_about_the_automations_they_may_read( + app, fresh_db, automations_per_organisation, monkeypatch +): + """Account admins and consultants each get one digest, about their organisations' automations, while the host hears about all of them.""" + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS", True + ) + prosumer, client, public = ( + automations_per_organisation[key] for key in ("prosumer", "client", "public") + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + host_digests = [message for message in outbox if message.bcc] + assert len(host_digests) == 1 + assert host_digests[0].bcc == [RECIPIENT] + for automation in (prosumer, client, public): + assert f"Automation {automation.id} " in host_digests[0].body + digests = emails_by_user(outbox) + # Plain users, inactive users, users without an email address, + # account admins of other organisations and consultancy users without the consultant role hear nothing. + assert set(digests) == { + "test_prosumer_user_2@seita.nl", + "test_consultant_client@seita.nl", + "test_consultant@seita.nl", + } + assert len(outbox) == 1 + len(digests) + assert f"Automation {prosumer.id} " in digests["test_prosumer_user_2@seita.nl"] + for email in ("test_consultant_client@seita.nl", "test_consultant@seita.nl"): + assert f"Automation {client.id} " in digests[email] + for body in digests.values(): + assert body.count("Automation ") == 1 + assert f"Automation {public.id} " not in body + assert "These automations on assets you look after" in body + assert "check the logs" not in body + assert "Sent a digest of failed automation runs to 3 user(s)." in result.output + + +def test_user_digests_are_off_by_default(app, fresh_db, automations_per_organisation): + """Without the setting, only the host is told, and the runs still count as reported.""" + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + assert outbox[0].bcc == [RECIPIENT] + assert emails_by_user(outbox) == {} + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + +def test_a_user_digest_that_cannot_be_sent_is_not_sent_again( + app, fresh_db, automations_per_organisation, monkeypatch +): + """Runs count as reported once the host is told, so a user digest that fails is logged, while the other users still get theirs.""" + from smtplib import SMTPRecipientsRefused + + monkeypatch.setitem( + app.config, "FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS", True + ) + send = app.mail.send + + def refuse_the_consultant(message): + if message.recipients == ["test_consultant@seita.nl"]: + raise SMTPRecipientsRefused({"test_consultant@seita.nl": (550, b"no")}) + send(message) + + monkeypatch.setattr(app.mail, "send", refuse_the_consultant) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert set(emails_by_user(outbox)) == { + "test_prosumer_user_2@seita.nl", + "test_consultant_client@seita.nl", + } + assert "Sent a digest of failed automation runs to 2 user(s)." in result.output + assert "Could not send a digest to test_consultant@seita.nl." in result.output + assert all( + run.alerted_at is not None + for run in fresh_db.session.scalars(select(AutomationRun)) + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert outbox == [] + + +def test_the_digest_links_to_the_automation_once_the_base_url_is_set( + app, fresh_db, forecast_automation, monkeypatch +): + """With the server's own address configured, each automation in the digest links to its info on the Automations page.""" + add_run(fresh_db, forecast_automation, DUE_AT, execution_state="failed") + link = f"/assets/{forecast_automation.asset_id}/automations?automation={forecast_automation.id}" + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert link not in outbox[0].body + + add_run( + fresh_db, + forecast_automation, + DUE_AT - timedelta(hours=1), + execution_state="failed", + ) + monkeypatch.setitem( + app.config, "FLEXMEASURES_BASE_URL", "https://ems.example.test/" + ) + + result, outbox = monitor(app) + + assert result.exit_code == 0, result.output + assert f" runs and jobs: https://ems.example.test{link}" in outbox[0].body diff --git a/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py new file mode 100644 index 0000000000..74230d5642 --- /dev/null +++ b/flexmeasures/data/migrations/versions/d4b7e9a1c362_add_alerted_at_to_automation_runs.py @@ -0,0 +1,57 @@ +"""Record when an automation run was reported by `flexmeasures monitor automations` + +Revision ID: d4b7e9a1c362 +Revises: bf22b21d0990 +Create Date: 2026-10-07 10:00:00.000000 + +The monitor reports each failed or stuck run once, and marks it in ``alerted_at``. +The upgrade marks the runs that had already failed for good as reported, so that the first monitoring run does not report every failure on record. +It judges them by the monitor's rules as they stand at this revision, with the default stuck threshold of 6 hours, +and leaves out any run that may still be retried or finish. + +The partial index holds only the runs the monitor has to look at: +those not reported yet which failed, are still running, or never finished their dispatch. +Every other run, which is nearly all of them, stays out of it. +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "d4b7e9a1c362" +down_revision = "bf22b21d0990" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "automation_run", + sa.Column("alerted_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "automation_run_unalerted_idx", + "automation_run", + ["automation_id"], + postgresql_where=sa.text( + "alerted_at IS NULL" + " AND (execution_state = 'failed' OR execution_state = 'running' OR dispatch_completed_at IS NULL)" + ), + ) + # Copied from the monitor's query rather than imported, so this migration keeps doing what it did when the query changes. + op.execute(""" + UPDATE automation_run + SET alerted_at = now() + WHERE execution_state = 'failed' + OR (execution_state = 'running' AND execution_started_at <= now() - interval '6 hours') + OR ( + dispatch_completed_at IS NULL + AND (claim_owner IS NULL OR claim_expires_at IS NULL OR claim_expires_at <= now()) + AND NOT (automation_type = 'forecasting' AND attempt_count < 5) + ) + """) + + +def downgrade(): + op.drop_index("automation_run_unalerted_idx", table_name="automation_run") + op.drop_column("automation_run", "alerted_at") diff --git a/flexmeasures/data/models/automations.py b/flexmeasures/data/models/automations.py index cbba34f811..e4199607b1 100644 --- a/flexmeasures/data/models/automations.py +++ b/flexmeasures/data/models/automations.py @@ -245,6 +245,8 @@ class AutomationRun(db.Model): execution_completed_at = db.Column(db.DateTime(timezone=True), nullable=True) last_error_type = db.Column(db.String(160), nullable=True) last_error_message = db.Column(db.Text, nullable=True) + # When `flexmeasures monitor automations` reported this run as failed or stuck, so that it is reported once. + alerted_at = db.Column(db.DateTime(timezone=True), nullable=True) parameters = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) plan = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default=dict) @@ -274,6 +276,7 @@ class AutomationRun(db.Model): "dispatch_completed_at", "execution_started_at", "execution_completed_at", + "alerted_at", ) def validate_datetime_is_aware( self, key: str, value: datetime | None diff --git a/flexmeasures/data/services/automation_monitoring.py b/flexmeasures/data/services/automation_monitoring.py new file mode 100644 index 0000000000..539e1f5f74 --- /dev/null +++ b/flexmeasures/data/services/automation_monitoring.py @@ -0,0 +1,218 @@ +"""Find the automation runs that failed, so that monitoring can report them.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from sqlalchemy import and_, not_, or_, select +from sqlalchemy.orm import selectinload + +from flexmeasures.auth.policy import ( + ACCOUNT_ADMIN_ROLE, + CONSULTANT_ROLE, + user_has_scoped_permission, +) +from flexmeasures.data import db +from flexmeasures.data.models.automations import Automation, AutomationRun +from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.models.user import Role, User +from flexmeasures.data.services.automations import AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS +from flexmeasures.utils.time_utils import server_now + +EXECUTION_FAILED = "execution failed" +STUCK = "stuck" +DISPATCH_FAILED = "dispatch failed" + + +@dataclass(frozen=True) +class FailedAutomationRun: + """An automation run that failed, with the way it failed.""" + + run: AutomationRun + reason: str + + +@dataclass +class AutomationFailures: + """The failed runs of one automation, collapsed into what a digest says about them.""" + + automation: Automation + failed_runs: list[FailedAutomationRun] = field(default_factory=list) + + @property + def reason_counts(self) -> Counter: + """How many runs failed in each way.""" + return Counter(failed_run.reason for failed_run in self.failed_runs) + + @property + def last_run(self) -> AutomationRun: + """The failed run that was due most recently.""" + return max( + (failed_run.run for failed_run in self.failed_runs), + key=lambda run: (run.scheduled_at, run.id), + ) + + @property + def last_error(self) -> str | None: + """The error of the most recent failed run that recorded one, if any did.""" + runs_with_an_error = [ + failed_run.run + for failed_run in self.failed_runs + if failed_run.run.last_error_type or failed_run.run.last_error_message + ] + if not runs_with_an_error: + return None + run = max(runs_with_an_error, key=lambda run: (run.scheduled_at, run.id)) + return ": ".join( + part for part in (run.last_error_type, run.last_error_message) if part + ) + + +def _dispatch_failed_for_good(now: datetime): + """Return the criterion for a run whose dispatch did not finish and will not be attempted again. + + No runner may hold a live claim on it, or it is still being dispatched. + Only a forecast run is dispatched again, until its attempts are used up (see `get_dispatchable_automation_runs`), + so a run of any other type has failed for good after its first attempt. + """ + return and_( + AutomationRun.dispatch_completed_at.is_(None), + or_( + AutomationRun.claim_owner.is_(None), + AutomationRun.claim_expires_at.is_(None), + AutomationRun.claim_expires_at <= now, + ), + not_( + and_( + AutomationRun.automation_type == "forecasting", + AutomationRun.attempt_count < AUTOMATION_RUN_MAX_DISPATCH_ATTEMPTS, + ) + ), + ) + + +def get_unalerted_failed_automation_runs( + stuck_after: timedelta, + now: datetime | None = None, +) -> list[FailedAutomationRun]: + """Return the automation runs that failed and were not reported yet, oldest first. + + A run failed when its execution failed, when its execution has been running for longer than ``stuck_after``, + or when its dispatch failed and will not be attempted again. + This holds for every automation type, although only the types that record their execution can fail on it. + + :param stuck_after: How long a run may execute before it counts as stuck. + :param now: The moment to judge by, which defaults to the server's current time. + """ + if now is None: + now = server_now() + now = now.astimezone(timezone.utc) + execution_failed = AutomationRun.execution_state == "failed" + stuck = and_( + AutomationRun.execution_state == "running", + AutomationRun.execution_started_at <= now - stuck_after, + ) + runs = db.session.scalars( + select(AutomationRun) + .where( + AutomationRun.alerted_at.is_(None), + or_(execution_failed, stuck, _dispatch_failed_for_good(now)), + ) + .options( + selectinload(AutomationRun.automation) + .joinedload(Automation.asset) + .joinedload(GenericAsset.owner) + ) + .order_by( + AutomationRun.automation_id, AutomationRun.scheduled_at, AutomationRun.id + ) + ).all() + failed_runs = [] + for run in runs: + if run.execution_state == "failed": + reason = EXECUTION_FAILED + elif ( + run.execution_state == "running" + and run.execution_started_at <= now - stuck_after + ): + reason = STUCK + else: + reason = DISPATCH_FAILED + failed_runs.append(FailedAutomationRun(run=run, reason=reason)) + return failed_runs + + +def group_failed_runs_by_automation( + failed_runs: list[FailedAutomationRun], +) -> list[AutomationFailures]: + """Collapse failed runs per automation, keeping the order in which the automations first appear.""" + failures: dict[int, AutomationFailures] = {} + for failed_run in failed_runs: + automation = failed_run.run.automation + failures.setdefault(automation.id, AutomationFailures(automation=automation)) + failures[automation.id].failed_runs.append(failed_run) + return list(failures.values()) + + +def get_users_to_notify( + failures: list[AutomationFailures], +) -> list[tuple[User, list[AutomationFailures]]]: + """Return who should hear about which failed automations, in a digest of their own. + + Those are the account admins of each asset's organisation and the consultants of its consultancy organisation, + as long as they are active, have an email address and may read the automation. + Automations on public assets have no organisation, so only the host hears about them. + + :param failures: The failed runs, collapsed per automation. + :returns: Each user to notify, ordered by ID, with the failures that user may read, in the given order. + """ + owners = { + failure.automation.asset.owner + for failure in failures + if failure.automation.asset.owner is not None + } + admin_account_ids = {owner.id for owner in owners} + consultancy_account_ids = { + owner.consultancy_account_id + for owner in owners + if owner.consultancy_account_id is not None + } + if not admin_account_ids: + return [] + users = db.session.scalars( + select(User) + .join(User.roles) + .where( + User.active.is_(True), + # In SQL, this also leaves out users whose email is NULL. + User.email != "", + or_( + and_( + User.account_id.in_(admin_account_ids), + Role.name == ACCOUNT_ADMIN_ROLE, + ), + and_( + User.account_id.in_(consultancy_account_ids), + Role.name == CONSULTANT_ROLE, + ), + ), + ) + .distinct() + .order_by(User.id) + ).all() + users_to_notify = [] + for user in users: + # The roles above bring these users to the right organisations, and the ACL decides which automations they see there. + readable_failures = [ + failure + for failure in failures + if failure.automation.asset.owner is not None + and user_has_scoped_permission( + user, "read", failure.automation.__acl__()["read"] + ) + ] + if readable_failures: + users_to_notify.append((user, readable_failures)) + return users_to_notify diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index ac36ef3ad4..ef61434dde 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -333,6 +333,50 @@ def refuse_to_save(*args, **kwargs): assert run.execution_completed_at is not None +def test_a_failed_schedule_run_is_reported_by_monitoring( + fresh_db, + app, + add_battery_assets_fresh_db, + add_market_prices_fresh_db, + clean_scheduling_redis, + monkeypatch, +): + """A schedule run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services import scheduling + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + from flexmeasures.utils.job_utils import work_on_rq + + automation, run = _planned_schedule_run( + fresh_db, add_battery_assets_fresh_db["Test battery"], sequential=False + ) + run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + + def refuse_to_save(*args, **kwargs): + raise ValueError("The database is read-only today.") + + monkeypatch.setattr(scheduling, "save_generator_results", refuse_to_save) + work_on_rq( + app.queues["scheduling"], exc_handler=scheduling.handle_scheduling_exception + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} 'Nightly schedules' (scheduling)" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ValueError: The database is read-only today." in body + + def test_a_schedule_job_queued_outside_a_run_is_not_recorded( fresh_db, app, diff --git a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py index 14f546c647..478f8ee0c9 100644 --- a/flexmeasures/data/tests/test_plugin_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_plugin_automations_fresh_db.py @@ -535,6 +535,44 @@ def test_a_plugin_run_records_how_its_job_ended( assert run.execution_state == "succeeded" +def test_a_failed_plugin_run_is_reported_by_monitoring( + fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch +): + """A plugin run whose job failed on the worker shows up in the host's monitoring digest, with the job's error.""" + from flexmeasures.cli.monitor import monitor_automations + from flexmeasures.data.services.automations import ( + mark_automation_run_dispatch_queued, + ) + + root, sensors = ingestion_assets + automation = make_automation(fresh_db, root, sensors[0], parameters={"value": 3}) + run = _claimed_run(fresh_db, automation) + + def fail_to_ingest(generator, value): + raise ConnectionError("The meter portal is down.") + + monkeypatch.setattr(MockIngestionGenerator, "_compute", fail_to_ingest) + result = run_automation(automation, automation_run=run) + # Its dispatch went through, as a runner would record, so only its execution can fail. + mark_automation_run_dispatch_queued(run.id) + work_on_rq( + app.queues["ingestion"], + job=Job.fetch(result["job_id"], connection=app.redis_connection), + ) + + with app.mail.record_messages() as outbox: + result = app.test_cli_runner().invoke( + monitor_automations, ["--recipient", "ops@example.test"] + ) + + assert result.exit_code == 0, result.output + assert len(outbox) == 1 + body = outbox[0].body + assert f"Automation {automation.id} '{automation.name}' ({automation.type})" in body + assert "failed once (execution failed: 1)" in body + assert "last error: ConnectionError: The meter portal is down." in body + + def test_a_job_that_ended_before_its_run_recorded_it_is_caught_up_on( fresh_db, app, ingestion_plugin, ingestion_assets, monkeypatch ): diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index 1a7c23a74f..c69b50fd5e 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -824,6 +824,26 @@
Recently created jobs // The types the page keeps a tab and a table for. const AUTOMATION_TYPES = Object.keys(automationTypes); + /* A link can point at one automation (?automation=), as the monitoring digest's links do. + Once the listing is in, its tab and its info panel are opened, which show its latest run, its error and its jobs. */ + let automationToOpen = Number(new URLSearchParams(window.location.search).get("automation")) || null; + + function openLinkedAutomation() { + const automation = automationToOpen === null ? undefined : automationsById.get(automationToOpen); + automationToOpen = null; // only on the first load, not on every refresh + if (automation === undefined) { + return; + } + const tab = document.getElementById(`${automation.type}-tab`); + if (tab) { + bootstrap.Tab.getOrCreateInstance(tab).show(); + } + const modal = document.getElementById(`AutomationInfoModal-${automation.id}`); + if (modal) { + bootstrap.Modal.getOrCreateInstance(modal).show(); + } + } + function loadAutomations({ rebuild = false } = {}) { // One refresh at a time: a slow response must not pile up behind the next tick. if (loadInFlight) { @@ -846,6 +866,7 @@
Recently created jobs } } markAutomationsRefreshed(); + openLinkedAutomation(); // The listing counts the jobs itself, so it is also where it says when Redis could not be reached. if (res["redis-connection-err"]) { showAutomationsError(res["redis-connection-err"]); diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index d4ced83728..7491f8f599 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -125,6 +125,12 @@ class Config(object): [] ) # Deprecated. Use FLEXMEASURES_DEFAULT_MONITORING_MAIL_RECIPIENTS instead. FLEXMEASURES_MONITORING_MAIL_RECIPIENTS: list[str] = [] + # How long an automation run may execute before `flexmeasures monitor automations` reports it as stuck. + FLEXMEASURES_MONITOR_AUTOMATIONS_STUCK_AFTER: timedelta = timedelta(hours=6) + # Whether `flexmeasures monitor automations` also emails account admins and consultants about the failed runs on their organisations' assets. + FLEXMEASURES_MONITOR_AUTOMATIONS_NOTIFY_USERS: bool = False + # Where this server is reached on the web, e.g. "https://ems.example.com", for links in emails sent outside of a request, such as monitoring digests. + FLEXMEASURES_BASE_URL: str | None = None FLEXMEASURES_PLATFORM_NAME: str | list[str | tuple[str, list[str]]] = "FlexMeasures" FLEXMEASURES_MODE: str = "" diff --git a/flexmeasures/utils/tests/test_time_utils.py b/flexmeasures/utils/tests/test_time_utils.py index 9c637acdf7..ac75c877c5 100644 --- a/flexmeasures/utils/tests/test_time_utils.py +++ b/flexmeasures/utils/tests/test_time_utils.py @@ -10,6 +10,7 @@ from flexmeasures.utils.time_utils import ( as_utc_isoformat, duration_isoformat, + exact_duration_str, server_now, naturalized_datetime_str, get_most_recent_clocktime_window, @@ -309,3 +310,17 @@ def test_to_utc_timestamp(input_value, expected_output): assert result is None else: assert result == pytest.approx(expected_output) + + +@pytest.mark.parametrize( + "duration, expected", + [ + (timedelta(hours=6), "6 hours"), + (timedelta(hours=1), "1 hour"), + (timedelta(minutes=90), "90 minutes"), + (timedelta(minutes=1), "1 minute"), + (timedelta(days=1), "24 hours"), + ], +) +def test_exact_duration_str(duration: timedelta, expected: str): + assert exact_duration_str(duration) == expected diff --git a/flexmeasures/utils/time_utils.py b/flexmeasures/utils/time_utils.py index 16f8a52f23..42822d2d1c 100644 --- a/flexmeasures/utils/time_utils.py +++ b/flexmeasures/utils/time_utils.py @@ -305,6 +305,19 @@ def duration_isoformat(duration: timedelta): return re.sub("%P", repl, "P%P") +def exact_duration_str(duration: timedelta) -> str: + """Say how long a duration is in whole hours or minutes, exactly, which a rounded duration such as humanize's would not. + + For example, 6 hours is "6 hours" and 90 minutes is "90 minutes". + Seconds are left out. + """ + minutes = int(duration.total_seconds() // 60) + if minutes % 60 == 0: + hours = minutes // 60 + return "1 hour" if hours == 1 else f"{hours} hours" + return "1 minute" if minutes == 1 else f"{minutes} minutes" + + def determine_minimum_resampling_resolution( event_resolutions: list[timedelta], fallback_resolution: timedelta = timedelta(0),