Auto-apply high-confidence frontend-triage results and report them to Slack

agents/frontend-triage/hackbot_agents/frontend_triage/config.py

@@ -33,6 +33,11 @@
 # and plans only: it records a comment with its findings/plan and, at high
 # confidence, may propose field updates (e.g. keyword/severity). It never
 # creates bugs or attaches files.
+#
+# `bugzilla.update_bug` needs `editbugs` on the account the apply step uses (the
+# same one `bug-fix` writes with). Note the apply step coalesces a same-bug field
+# change with the nearest comment into one PUT, so if that privilege were ever
+# lost, a rejected field change would take the analysis comment down with it.
 ENABLED_ACTION_TYPES = [
     "bugzilla.add_comment",
     "bugzilla.update_bug",

agents/frontend-triage/hackbot_agents/frontend_triage/rules/frontend-triage.md

@@ -34,6 +34,11 @@ restate the whole bug. Do not claim the fix is verified — you did not run it.
 
 ## Confidence and field changes
 
+Your `confidence` decides whether your actions reach the bug: a **high**-confidence
+run is applied to Bugzilla automatically, while **medium** and **low** are held for
+a human to review first. Reserve `high` for when you have actually localized the
+cause in specific code — not for a plausible-sounding hypothesis.
+
 - **High** (you found the specific code and the cause is clear): record the
   plan comment. If a rule or convention clearly applies, you may also record a
   `bugzilla_update_bug` for an obviously-correct field (e.g. adding a relevant

services/hackbot-api/alembic/versions/d2b3c4e5f6a7_run_notified_at.py

@@ -0,0 +1,30 @@
+"""Run notified_at.
+
+Revision ID: d2b3c4e5f6a7
+Revises: c1a2f3e4b5d6
+Create Date: 2026-07-28 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d2b3c4e5f6a7"
+down_revision: Union[str, Sequence[str], None] = "c1a2f3e4b5d6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    op.add_column(
+        "runs", sa.Column("notified_at", sa.DateTime(timezone=True), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_column("runs", "notified_at")

services/hackbot-api/app/actions_applier.py

@@ -28,8 +28,15 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app import gcs
-from app.agents import AGENT_REGISTRY
+from app.agents import AGENT_REGISTRY, AgentSpec
 from app.database.models import Run, RunAction
+from app.notify import (
+    FAILED,
+    HELD,
+    NO_ACTIONS,
+    POSTED,
+    notify_run_completed,
+)
 from app.schemas import RunStatus
 
 log = logging.getLogger(__name__)
@@ -86,6 +93,32 @@ def _sub(match: re.Match) -> str:
     return value
 
 
+def _reported_confidence(run: Run) -> str | None:
+    """The run's self-reported confidence, normalised, or None if unusable.
+
+    ``confidence`` reaches us from the agent's free-form JSON block, so tolerate
+    casing and stray whitespace rather than reading "High" as "no confidence".
+    """
+    value = ((run.summary or {}).get("findings") or {}).get("confidence")
+    if not isinstance(value, str):
+        return None
+    return value.strip().lower() or None
+
+
+def _should_auto_apply(spec: AgentSpec | None, run: Run) -> bool:
+    """Whether `run`'s recorded actions may be applied without a human.
+
+    Pure, so the policy is testable on its own. Two independent gates: the
+    agent's `auto_apply_actions` opt-in, then — for agents that report a
+    confidence — whether that confidence is one the spec admits.
+    """
+    if spec is None or not spec.auto_apply_actions:
+        return False
+    if spec.auto_apply_confidence is None:
+        return True
+    return _reported_confidence(run) in spec.auto_apply_confidence
+
+
 async def ensure_action_rows(
     db: AsyncSession, run: Run
 ) -> list[tuple[RunAction, list[dict]]]:
@@ -223,12 +256,26 @@ async def _apply_pending_rows(
                     results_by_ref[member.ref] = outcome.result
 
 
+def _apply_outcome(rows: list[tuple[RunAction, list[dict]]]) -> str:
+    """Summarise what an apply pass actually achieved, for the notification."""
+    if not rows:
+        return NO_ACTIONS
+    if all(row.status == "applied" for row, _ in rows):
+        return POSTED
+    return FAILED
+
+
 async def on_run_completed(db: AsyncSession, run: Run) -> None:
-    """Record a completed run's actions, and auto-apply them if the agent opts in.
+    """Record a completed run's actions, auto-apply them, then notify.
 
     Called from the `apply-run-actions` push route. Actions are always recorded
     (so the UI can show/manually apply them); they're applied automatically only
-    when the run's agent has `auto_apply_actions=True`.
+    when `_should_auto_apply` says so.
+
+    The notification runs here rather than off its own `run.completed`
+    subscription because it has to report *whether* Bugzilla was written — a
+    second consumer would race this one. It fires either way: a result too
+    unconfident to post is still worth a human's glance.
     """
     # Defense-in-depth: only a succeeded run's actions are recorded/applied. A
     # failed/timed-out run may have recorded actions before erroring, but acting
@@ -243,16 +290,24 @@ async def on_run_completed(db: AsyncSession, run: Run) -> None:
     await db.commit()
 
     spec = AGENT_REGISTRY.get(run.agent)
-    if spec and spec.auto_apply_actions:
+    if _should_auto_apply(spec, run):
         await _apply_pending_rows(db, run, rows)
+        outcome = _apply_outcome(rows)
     else:
+        outcome = HELD if rows else NO_ACTIONS
         log.info(
-            "Recorded %d action(s) for run %s; auto-apply off for agent %s",
+            "Recorded %d action(s) for run %s; not auto-applying "
+            "(agent %s, confidence %s)",
             len(rows),
             run.run_id,
             run.agent,
+            _reported_confidence(run),
         )
 
+    if spec and spec.notify_completion:
+        await notify_run_completed(run, outcome)
+        await db.commit()
+
 
 async def apply_all_pending(db: AsyncSession, run: Run) -> None:
     """Apply all of a run's not-yet-`applied` actions on demand (manual).

services/hackbot-api/app/agents.py

@@ -27,6 +27,15 @@ class AgentSpec:
     # succeeds. Off by default: actions are still recorded and can always be
     # applied manually from the UI; only opted-in agents auto-apply.
     auto_apply_actions: bool = False
+    # Confidence levels (from ``summary["findings"]["confidence"]``) whose actions
+    # may be auto-applied, for agents that report one. ``None`` means no
+    # confidence restriction. Fail closed: when this is set, a run whose findings
+    # carry no usable confidence never qualifies. Widening the policy (e.g. to
+    # also accept "medium") is an edit to this set, not to the applier.
+    auto_apply_confidence: frozenset[str] | None = None
+    # Whether a succeeded run gets a completion notification (see app/notify.py).
+    # Off by default; the destination is deployment config, not a per-agent choice.
+    notify_completion: bool = False
 
 
 def model_to_env(inputs: BaseModel) -> dict[str, str]:
@@ -79,6 +88,12 @@ def model_to_env(inputs: BaseModel) -> dict[str, str]:
         description="Triage a Firefox desktop frontend bug (read-only) and produce a root-cause analysis and proposed fix plan.",
         job_name="hackbot-agent-frontend-triage",
         input_schema=FrontendTriageInputs,
+        # Triage results go to the bug unattended, so only the ones the agent
+        # localized to specific code qualify. A medium/low result is still
+        # recorded for manual apply from the UI.
+        auto_apply_actions=True,
+        auto_apply_confidence=frozenset({"high"}),
+        notify_completion=True,
     ),
     "test-repair": AgentSpec(
         name="test-repair",

services/hackbot-api/app/config.py

@@ -70,6 +70,21 @@ class Settings(BaseSettings):
     push_auth_audience: str = ""
     push_auth_service_account: str = ""
 
+    # Run-completion notification (email; the recipients are Slack channel
+    # addresses via Slack's "Email to channel"). Unset => notification skipped.
+    sendgrid_api_key: str | None = None
+    notification_sender: str | None = None
+    # Where to report a triaged bug, keyed by its `"<Product> :: <Component>"`.
+    # A channel belongs to the team that owns the component, so this is a map
+    # rather than one address; the optional `"default"` key catches components
+    # with no channel of their own. JSON-encoded in the environment, e.g.
+    # {"Firefox :: New Tab Page": "hnt-…@mozilla.org.slack.com"}.
+    notification_slack_emails: dict[str, str] = {}
+
+    # Link targets used in notifications.
+    bugzilla_url: str = "https://bugzilla.mozilla.org"
+    hackbot_ui_url: str = ""
+
     # Server
     port: int = 8080
     environment: str = "development"

services/hackbot-api/app/database/models.py

@@ -39,6 +39,12 @@ class Run(Base):
     finalized_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True
     )
+    # When the completion notification was sent. Unlike an action row, an email
+    # is not idempotent, so this is what stops a redelivered `run.completed`
+    # event from reposting to the channel (see app/notify.py).
+    notified_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True
+    )
 
 
 class RunAction(Base):