From 05d0891e80080c4a2b0a94a87dade4924f2e64ef Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:27:03 +0000 Subject: [PATCH 1/8] Surface workflow tasks that keep losing their lease (#1231) A pcntl queue:work kills a job at --timeout (60 s by default). A workflow task killed there rolled back, stayed leased until its 300 s lease expired, and task repair returned it to ready for the next worker to repeat. A task that always needs longer than the timeout retried forever: attempt_count grew, no failure was recorded and nothing told the operator why. - Task repair counts each expired workflow-task lease on the task (lost_lease_count, last_lease_lost_at; additive migration, written only where the columns exist). The next claim logs a typed workflow_task_lease_lost diagnostic. No history is written and the run behaves as before. - At workflows.v2.workflow_task_lost_leases.attention_threshold (3) lost leases, RepairBlockedReason reports workflow_task_lease_lost ("Task Timing Out", badge visible), so the run summary, describe and Waterline show repair attention. Operator metrics add runs.workflow_task_lease_lost, tasks.lease_lost_repeatedly and tasks.max_lost_lease_count. - Opt-in workflow_task_lost_leases.fail_run (off by default): at the threshold the next claim fails the run with WorkflowTaskLeaseLostException (task_failure) before workflow code runs. - A queue worker starting with pcntl and --timeout below the workflow task lease logs one warning (WorkerStarting). The README documents running workflow workers with --timeout=0 or above the longest workflow task. V2WorkflowTaskLostLeaseTest covers counting, attention, describe, metrics, the claim diagnostic, unchanged history, the opt-in failure and its threshold, and the worker warning. Replay-regression evidence workflow-task-lease-lost-fails-run-opt-in records the opt-in failure; its consumer reproduces that history only with this change. Co-Authored-By: Claude Opus 5.5 --- .github/feature-test-timings.json | 2 + CHANGELOG.md | 20 ++ README.md | 14 +- docs/architecture/operational-liveness.md | 11 +- src/Providers/WorkflowServiceProvider.php | 14 ++ .../WorkflowTaskLeaseLostException.php | 29 +++ src/V2/Jobs/RunWorkflowTask.php | 16 ++ src/V2/Models/WorkflowTask.php | 2 + src/V2/Support/FailureFactory.php | 5 +- src/V2/Support/OperatorMetrics.php | 32 +++ src/V2/Support/RepairBlockedReason.php | 13 + src/V2/Support/TaskRepair.php | 2 + src/V2/Support/WorkerTimeoutAdvisory.php | 74 ++++++ src/V2/Support/WorkflowExecutor.php | 10 + src/V2/Support/WorkflowTaskLeaseLoss.php | 210 ++++++++++++++++ src/config/workflows.php | 11 + ..._lost_lease_tracking_to_workflow_tasks.php | 43 ++++ .../V2EmbeddedReplayRegressionCorpusTest.php | 71 ++++++ .../V2/V2WorkflowTaskLostLeaseTest.php | 231 ++++++++++++++++++ ...flow-task-lease-lost-fails-run-opt-in.json | 90 +++++++ 20 files changed, 896 insertions(+), 4 deletions(-) create mode 100644 src/V2/Exceptions/WorkflowTaskLeaseLostException.php create mode 100644 src/V2/Support/WorkerTimeoutAdvisory.php create mode 100644 src/V2/Support/WorkflowTaskLeaseLoss.php create mode 100644 src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php create mode 100644 tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php create mode 100644 tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index e876481ea..31fdc3ab7 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -241,6 +241,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.653342, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 12.837294, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 269.332131, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 15.0, "tests/Feature/V2/V2WorkflowTest.php": 161.306516, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 11.990609, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.888561, @@ -494,6 +495,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.571702, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 10.15931, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 149.713579, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 12.0, "tests/Feature/V2/V2WorkflowTest.php": 115.227937, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 6.892443, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.853134, diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a4867cac..c3a13a2b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +- A workflow task that keeps losing its lease without completing no longer + retries silently forever (#1231). This usually happens when a pcntl + `queue:work` kills it at the job timeout, which is 60 s by default. + - Task repair counts each lost lease on the task (`lost_lease_count`, + `last_lease_lost_at`; new migration). The next claim logs a typed + `workflow_task_lease_lost` diagnostic. No history is written, and run + semantics are unchanged. + - After three consecutive lost leases + (`workflows.v2.workflow_task_lost_leases.attention_threshold`), the run + reports `repair_blocked_reason` `workflow_task_lease_lost` ("Task Timing + Out") with repair attention in the summary, describe and Waterline. Operator + metrics add `runs.workflow_task_lease_lost`, `tasks.lease_lost_repeatedly` + and `tasks.max_lost_lease_count`. + - Opt-in (`workflows.v2.workflow_task_lost_leases.fail_run`, off by default): + at the threshold, the next claim fails the run with a typed + `WorkflowTaskLeaseLostException` (`task_failure`) instead of retrying. + - A queue worker that starts with pcntl and a `--timeout` below the workflow + task lease logs one warning. The README documents running workflow + workers with `--timeout=0` or above the longest workflow task. + - Workflow definition and `await()` condition fingerprints are versioned, and a new opt-in version 2 makes comment-only and whitespace-only edits replay-safe (#1227). The prefix names the version: diff --git a/README.md b/README.md index 63c323e4f..b18e360df 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,21 @@ existing tables. Run a Laravel queue worker or Horizon to execute workflows and activities: ```bash -php artisan queue:work +php artisan queue:work --timeout=0 ``` +With the `pcntl` extension, `queue:work` kills a job after `--timeout` seconds +(60 by default). A workflow task that runs longer, such as one that schedules or +finishes a wide `all()`, is rolled back and retried after its lease expires, so it +never finishes. Run workers with `--timeout=0` or a value above your longest +workflow task, and keep `workflows.v2.workflow_task_lease_seconds` (300 by +default) above it; Horizon's `timeout` option is the same setting. A worker +started with a shorter timeout logs a warning. Each lease a workflow task loses +is counted, and after three in a row (`DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD`) +the run reports `repair_blocked_reason` `workflow_task_lease_lost` with repair +attention. Set `DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN=true` to fail such a run +with a typed task failure instead. + For the default database queue on SQLite, failed reservation writes preserve unclaimed work after a proven rollback. See [SQLite queue reservations](docs/sqlite-queue-reservations.md) for the supported framework versions and custom-connector boundary. diff --git a/docs/architecture/operational-liveness.md b/docs/architecture/operational-liveness.md index bebd81351..40797e081 100644 --- a/docs/architecture/operational-liveness.md +++ b/docs/architecture/operational-liveness.md @@ -350,12 +350,19 @@ Repair: batch path. The repair path MUST be explicit about why it cannot act. The -`RepairBlockedReason::catalog()` enumerates the five frozen reason -codes that a caller may observe: +`RepairBlockedReason::catalog()` enumerates the six reason codes that +a caller may observe: - `unsupported_history` — the run's history contains events the current engine build cannot interpret; the operator must roll forward a compatible worker or manually resolve. +- `workflow_task_lease_lost` — repair keeps returning the open + workflow task to ready, but it has lost its lease without completing + `workflows.v2.workflow_task_lost_leases.attention_threshold` times in + a row (3 by default), usually because the queue worker kills it at its + job timeout. The operator must raise the worker's `--timeout` (or use + `--timeout=0`). With `workflow_task_lost_leases.fail_run` enabled, the + next claim fails the run with a typed task failure instead (#1231). - `waiting_for_compatible_worker` — a task exists, but no live worker advertises the required compatibility marker. - `selected_run_not_current` — the caller selected a historical diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index 92d3bd383..a03b28972 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -79,6 +79,7 @@ use Workflow\V2\Support\SqliteSavepointRelease; use Workflow\V2\Support\TaskFairnessState; use Workflow\V2\Support\TypeRegistry; +use Workflow\V2\Support\WorkerTimeoutAdvisory; use Workflow\V2\Support\WorkflowModeGuard; use Workflow\Watchdog; @@ -237,6 +238,19 @@ public function boot(): void // Validate cache backend for multi-node deployments $this->validateCacheBackend(); + if (class_exists(\Illuminate\Queue\Events\WorkerStarting::class)) { + // A job timeout below the workflow task lease can kill a workflow + // task on every retry; warn once per worker (#1231). + Event::listen( + \Illuminate\Queue\Events\WorkerStarting::class, + static function (object $event): void { + if (isset($event->workerOptions) && is_object($event->workerOptions)) { + WorkerTimeoutAdvisory::workerStarting($event->workerOptions); + } + }, + ); + } + Event::listen(Looping::class, static function (Looping $event) use ($v1Enabled): void { if ($v1Enabled) { Watchdog::wake($event->connectionName, $event->queue); diff --git a/src/V2/Exceptions/WorkflowTaskLeaseLostException.php b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php new file mode 100644 index 000000000..7e165b262 --- /dev/null +++ b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php @@ -0,0 +1,29 @@ +disable('workflow_task_lease_lost'); + $executor->failAfterLostLeases($run, $task); + + return null; + } + if ($continuation !== null && ! $continuation->hasAuthority($run, $task)) { $continuation->disable('lost_task_authority'); return null; @@ -275,6 +285,12 @@ private function claimTask(): array ServiceCallWorkflowDelivery::recordForClaim($task, $run); $this->projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures'])); + // A retry after lost leases records its typed diagnostic (#1231). + $leaseLoss = WorkflowTaskLeaseLoss::claimDiagnostic($task); + if ($leaseLoss !== null) { + WorkflowTaskLeaseLoss::reportClaim($leaseLoss); + } + return [$task->setRelation('run', $run), null, $claimedAttempt]; }); } diff --git a/src/V2/Models/WorkflowTask.php b/src/V2/Models/WorkflowTask.php index 8dbf2ca66..a7235ac22 100644 --- a/src/V2/Models/WorkflowTask.php +++ b/src/V2/Models/WorkflowTask.php @@ -39,6 +39,7 @@ class WorkflowTask extends Model 'payload' => 'array', 'queue_delivery_reference' => 'array', 'repair_count' => 'integer', + 'lost_lease_count' => 'integer', 'priority' => 'integer', 'fairness_weight' => 'integer', 'sticky_until' => 'datetime', @@ -50,6 +51,7 @@ class WorkflowTask extends Model 'last_dispatched_at' => 'datetime', 'last_claim_failed_at' => 'datetime', 'repair_available_at' => 'datetime', + 'last_lease_lost_at' => 'datetime', ]; public function run(): BelongsTo diff --git a/src/V2/Support/FailureFactory.php b/src/V2/Support/FailureFactory.php index 4144bc4f1..03c6becf0 100644 --- a/src/V2/Support/FailureFactory.php +++ b/src/V2/Support/FailureFactory.php @@ -27,6 +27,7 @@ use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Exceptions\UnresolvedWorkflowFailureException; use Workflow\V2\Exceptions\UnsupportedWorkflowYieldException; +use Workflow\V2\Exceptions\WorkflowTaskLeaseLostException; final class FailureFactory { @@ -361,10 +362,12 @@ private static function classifyFromThrowable(?Throwable $throwable): FailureCat return FailureCategory::StructuralLimit; } - // Task-level failures: determinism violations, unsupported yields, replay shape errors. + // Task-level failures: determinism violations, unsupported yields, replay shape errors, + // and a workflow task that keeps losing its lease (#1231). if ( $throwable instanceof UnsupportedWorkflowYieldException || $throwable instanceof StraightLineWorkflowRequiredException + || $throwable instanceof WorkflowTaskLeaseLostException ) { return FailureCategory::TaskFailure; } diff --git a/src/V2/Support/OperatorMetrics.php b/src/V2/Support/OperatorMetrics.php index 93b791556..252c73f70 100644 --- a/src/V2/Support/OperatorMetrics.php +++ b/src/V2/Support/OperatorMetrics.php @@ -132,6 +132,10 @@ private static function runMetrics(CarbonInterface $now, ?string $namespace): ar ? 0 : (int) $oldestReplayBlockedAt->diffInMilliseconds($now), 'claim_failed' => self::claimFailedRuns($namespace), + // Open runs whose workflow task kept losing its lease (#1231). + 'workflow_task_lease_lost' => self::summaryQuery($namespace) + ->where('repair_blocked_reason', WorkflowTaskLeaseLoss::CODE) + ->count(), 'compatibility_blocked' => self::compatibilityBlockedRuns($namespace), 'waiting' => self::waitingRuns($namespace), 'oldest_wait_started_at' => $oldestWaitStartedAt?->toJSON(), @@ -190,6 +194,9 @@ private static function taskMetrics(CarbonInterface $now, ?string $namespace): a : (int) $oldestDispatchFailedAt->diffInMilliseconds($now), 'max_attempt_count' => self::maxOpenTaskAttemptCount($namespace), 'max_repair_count' => self::maxOpenTaskRepairCount($namespace), + // Open workflow tasks at or over the lost-lease attention threshold (#1231). + 'lease_lost_repeatedly' => self::repeatedlyLostLeaseTasks($namespace), + 'max_lost_lease_count' => self::maxOpenTaskLostLeaseCount($namespace), 'unhealthy' => self::dispatchFailedTasks($namespace) + self::claimFailedTasks($namespace) + self::dispatchOverdueTasks($now, $namespace) @@ -1557,6 +1564,31 @@ private static function maxOpenTaskAttemptCount(?string $namespace): int * active repair burn rather than accumulating against historical * rows. */ + private static function repeatedlyLostLeaseTasks(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::tracked()) { + return 0; + } + + return self::scopedRunModelQuery(self::taskModel(), $namespace) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', WorkflowTaskLeaseLoss::attentionThreshold()) + ->count(); + } + + private static function maxOpenTaskLostLeaseCount(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::tracked()) { + return 0; + } + + return (int) self::scopedRunModelQuery(self::taskModel(), $namespace) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->max('lost_lease_count'); + } + private static function maxOpenTaskRepairCount(?string $namespace): int { return (int) self::scopedRunModelQuery(self::taskModel(), $namespace) diff --git a/src/V2/Support/RepairBlockedReason.php b/src/V2/Support/RepairBlockedReason.php index ff581b54e..8fb963aec 100644 --- a/src/V2/Support/RepairBlockedReason.php +++ b/src/V2/Support/RepairBlockedReason.php @@ -26,6 +26,14 @@ public static function catalog(): array 'tone' => 'dark', 'badge_visible' => true, ], + WorkflowTaskLeaseLoss::CODE => [ + 'label' => 'Task Timing Out', + 'description' => 'The workflow task keeps losing its lease without completing; its queue worker ' + . 'may be killing it at the job timeout. Run workers with --timeout=0 or above the longest ' + . 'workflow task.', + 'tone' => 'warning', + 'badge_visible' => true, + ], 'waiting_for_compatible_worker' => [ 'label' => 'Compat Blocked', 'description' => 'Repair is blocked because the task is waiting for a compatible worker.', @@ -128,6 +136,11 @@ public static function forRun( return 'run_closed'; } + // Repair keeps redispatching a task that never completes (#1231). + if (WorkflowTaskLeaseLoss::runNeedsAttention($run)) { + return WorkflowTaskLeaseLoss::CODE; + } + if ($livenessState === 'repair_needed') { return null; } diff --git a/src/V2/Support/TaskRepair.php b/src/V2/Support/TaskRepair.php index 13c954f19..07b51b56b 100644 --- a/src/V2/Support/TaskRepair.php +++ b/src/V2/Support/TaskRepair.php @@ -102,6 +102,8 @@ private static function recoverLeasedTask(WorkflowTask $task, WorkflowRun $run): 'repair_count' => $task->repair_count + 1, 'repair_available_at' => null, 'last_error' => null, + // A workflow task's expired lease is work that never completed (#1231). + ...WorkflowTaskLeaseLoss::repairedAttributes($task), ])->save(); return $task; diff --git a/src/V2/Support/WorkerTimeoutAdvisory.php b/src/V2/Support/WorkerTimeoutAdvisory.php new file mode 100644 index 000000000..695ffa345 --- /dev/null +++ b/src/V2/Support/WorkerTimeoutAdvisory.php @@ -0,0 +1,74 @@ += $leaseSeconds) { + return null; + } + + return sprintf( + '[Durable Workflow] This queue worker kills a job after %d s (--timeout), before the %d s workflow task ' + . 'lease. A workflow task that runs longer, such as a wide all() fan-out, is killed, rolled back and ' + . 'retried after its lease expires, and can never finish. Run workers with --timeout=0 or a value above ' + . 'your longest workflow task, and keep workflows.v2.workflow_task_lease_seconds above --timeout.', + $timeoutSeconds, + $leaseSeconds, + ); + } + + public static function workerStarting(object $workerOptions): void + { + if (self::$warned || ! property_exists($workerOptions, 'timeout')) { + return; + } + + $message = self::message( + (int) $workerOptions->timeout, + extension_loaded('pcntl'), + WorkflowTaskLease::seconds(), + ); + + if ($message === null) { + return; + } + + self::$warned = true; + Log::warning($message, [ + 'code' => 'worker_timeout_below_workflow_task_lease', + 'worker_timeout_seconds' => (int) $workerOptions->timeout, + 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), + ]); + } + + /** + * @internal For tests that start more than one worker in a process. + */ + public static function reset(): void + { + self::$warned = false; + } +} diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 5ae5c4567..e81ca56d5 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -94,6 +94,16 @@ public function run( } } + /** + * Fail a run whose claimed workflow task kept losing its lease, when + * workflows.v2.workflow_task_lost_leases.fail_run is enabled (#1231). + * Called under the task and run locks, before any workflow code runs. + */ + public function failAfterLostLeases(WorkflowRun $run, WorkflowTask $task): void + { + $this->failRun($run, $task, WorkflowTaskLeaseLoss::exceptionFor($task), 'workflow_run', $run->id); + } + /** * Close a run whose server-enforced execution or run deadline elapsed. * diff --git a/src/V2/Support/WorkflowTaskLeaseLoss.php b/src/V2/Support/WorkflowTaskLeaseLoss.php new file mode 100644 index 000000000..62c6ca89d --- /dev/null +++ b/src/V2/Support/WorkflowTaskLeaseLoss.php @@ -0,0 +1,210 @@ + + */ + private static array $trackedSchemas = []; + + public static function attentionThreshold(): int + { + $value = config(self::CONFIG_KEY . '.attention_threshold', self::DEFAULT_ATTENTION_THRESHOLD); + + return is_numeric($value) && (int) $value >= 1 + ? (int) $value + : self::DEFAULT_ATTENTION_THRESHOLD; + } + + public static function failsRun(): bool + { + return (bool) config(self::CONFIG_KEY . '.fail_run', false); + } + + public static function lostLeases(WorkflowTask $task): int + { + return max(0, (int) ($task->getAttribute('lost_lease_count') ?? 0)); + } + + /** + * Count one lost lease on a workflow task that task repair returns to + * ready. A schema without the tracking columns keeps the old behavior. + * + * @return array + */ + public static function repairedAttributes(WorkflowTask $task): array + { + if ($task->task_type !== TaskType::Workflow + || ! array_key_exists('lost_lease_count', $task->getAttributes())) { + return []; + } + + return [ + 'lost_lease_count' => self::lostLeases($task) + 1, + 'last_lease_lost_at' => now(), + ]; + } + + /** + * The typed diagnostic for a claim that retries a task after lost leases, + * or null for a task that never lost one. + * + * @return array{ + * code: string, + * workflow_run_id: string, + * workflow_task_id: string, + * lost_leases: int, + * attention_threshold: int, + * needs_attention: bool, + * last_lease_lost_at: string|null, + * workflow_task_lease_seconds: int, + * claim_attempt: int + * }|null + */ + public static function claimDiagnostic(WorkflowTask $task): ?array + { + $lostLeases = self::lostLeases($task); + + if ($task->task_type !== TaskType::Workflow || $lostLeases === 0) { + return null; + } + + $lastLostAt = $task->getAttribute('last_lease_lost_at'); + + return [ + 'code' => self::CODE, + 'workflow_run_id' => (string) $task->workflow_run_id, + 'workflow_task_id' => (string) $task->id, + 'lost_leases' => $lostLeases, + 'attention_threshold' => self::attentionThreshold(), + 'needs_attention' => $lostLeases >= self::attentionThreshold(), + 'last_lease_lost_at' => $lastLostAt instanceof \DateTimeInterface + ? $lastLostAt->format('Y-m-d\TH:i:s.up') + : null, + 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), + 'claim_attempt' => (int) $task->attempt_count, + ]; + } + + /** + * Report a claim that retries a task after lost leases. + * + * @param array $diagnostic + */ + public static function reportClaim(array $diagnostic): void + { + Log::warning( + sprintf( + '[Durable Workflow] Workflow task %s lost its lease %d time(s) without completing; ' + . 'its queue worker may be killing it at the job timeout. Run workers with ' + . '--timeout=0 or above your longest workflow task.', + $diagnostic['workflow_task_id'], + $diagnostic['lost_leases'], + ), + $diagnostic, + ); + } + + /** + * Whether a claimed workflow task must fail its run instead of executing. + */ + public static function failsRunAt(WorkflowTask $task): bool + { + return self::failsRun() + && $task->task_type === TaskType::Workflow + && self::lostLeases($task) >= self::attentionThreshold(); + } + + public static function exceptionFor(WorkflowTask $task): WorkflowTaskLeaseLostException + { + return new WorkflowTaskLeaseLostException( + (string) $task->id, + self::lostLeases($task), + WorkflowTaskLease::seconds(), + ); + } + + /** + * Whether the run's open workflow task has reached the attention threshold. + */ + public static function runNeedsAttention(WorkflowRun $run): bool + { + $threshold = self::attentionThreshold(); + + if ($run->relationLoaded('tasks')) { + foreach ($run->tasks as $task) { + if ($task instanceof WorkflowTask + && $task->task_type === TaskType::Workflow + && in_array($task->status, [TaskStatus::Ready, TaskStatus::Leased], true) + && self::lostLeases($task) >= $threshold) { + return true; + } + } + + return false; + } + + if (! $run->exists) { + return false; + } + + if (! self::tracked()) { + return false; + } + + $query = ConfiguredV2Models::query('task_model', WorkflowTask::class); + + return $query + ->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', $threshold) + ->exists(); + } + + /** + * Whether the configured task table has the lost-lease columns. A schema + * that has not run the migration has no lost leases to report. + */ + public static function tracked(): bool + { + $model = ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel(); + $schemaKey = $model->getConnectionName() . '|' . $model->getTable(); + + return self::$trackedSchemas[$schemaKey] ??= $model->getConnection() + ->getSchemaBuilder() + ->hasColumn($model->getTable(), 'lost_lease_count'); + } +} diff --git a/src/config/workflows.php b/src/config/workflows.php index c35968c10..bfb8a91a2 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -375,6 +375,17 @@ Workflow\V2\Support\WorkflowTaskLease::DEFAULT_SECONDS, ), + // A workflow task whose lease expires without completing, typically + // because the queue worker killed it at its job timeout, is retried + // after task repair. After this many consecutive lost leases the run + // shows as needing attention (repair_blocked_reason + // workflow_task_lease_lost). With fail_run enabled, the next claim + // instead fails the run with a typed task failure. Off by default. + 'workflow_task_lost_leases' => [ + 'attention_threshold' => (int) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD', 3), + 'fail_run' => (bool) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN', false), + ], + // Runtime ownership, not an activity execution timeout. Valid values // are 1..86400 seconds; invalid values resolve to the 300-second default. // A live runtime renews the same attempt without inventing user progress. diff --git a/src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php b/src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php new file mode 100644 index 000000000..274c8e02a --- /dev/null +++ b/src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php @@ -0,0 +1,43 @@ +getConnection()); + foreach (array_unique(['workflow_tasks', (new $model())->getTable()]) as $tableName) { + if ($schema->hasTable($tableName) && ! $schema->hasColumn($tableName, 'lost_lease_count')) { + $schema->table($tableName, static function (Blueprint $table): void { + $table->unsignedInteger('lost_lease_count') + ->default(0); + $table->timestamp('last_lease_lost_at', 6) + ->nullable(); + }); + } + } + } + + public function down(): void + { + $model = ConfiguredV2Models::resolve('task_model', WorkflowTask::class); + $schema = Schema::connection($this->getConnection()); + foreach (array_unique(['workflow_tasks', (new $model())->getTable()]) as $tableName) { + if ($schema->hasTable($tableName) && $schema->hasColumn($tableName, 'lost_lease_count')) { + $schema->table($tableName, static function (Blueprint $table): void { + $table->dropColumn(['lost_lease_count', 'last_lease_lost_at']); + }); + } + } + } +}; diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index b081bfba8..2f605d36a 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -500,6 +500,77 @@ public function testOversizedFanOutMemberCorpusIsRefusedBeforeFirstWrite(): void $this->assertSame(1, (new WorkflowReplayer())->replay($run->fresh(['historyEvents']))->sequence); } + /** + * A workflow task that lost its lease three times, as when a pcntl worker + * kills it at the job timeout, fails its run with the typed lease-lost + * failure on the next claim when fail_run is enabled (#1231). The run's + * history is exactly the evidence: no lost lease writes history. + */ + public function testWorkflowTaskLeaseLostCorpusFailsTheRunWhenOptedIn(): void + { + $path = self::FIXTURE_DIR . '/workflow-task-lease-lost-fails-run-opt-in.json'; + if (! is_file($path)) { + $this->markTestSkipped('The workflow task lease-lost evidence is absent.'); + } + $fixture = json_decode(file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + self::stopWorkers(); + config([ + 'queue.default' => 'redis', + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => $fixture['workflow_task_lease_lost']['attention_threshold'], + ]); + Queue::fake(); + + $workflow = WorkflowStub::make($fixture['workflow']['type'], 'regression-workflow-task-lease-lost'); + $workflow->start(...$fixture['workflow']['arguments']); + $runId = $workflow->runId(); + + for ($lost = 0; $lost < $fixture['workflow_task_lease_lost']['attention_threshold']; $lost++) { + $task = WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value) + ->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'leased_at' => now() + ->subMinutes(10), + 'lease_owner' => $task->id, + 'lease_expires_at' => now() + ->subMinute(), + 'attempt_count' => $task->attempt_count + 1, + ])->save(); + Cache::forget(TaskWatchdog::LOOP_THROTTLE_KEY); + TaskWatchdog::runPass(connection: 'redis', queue: 'default', respectThrottle: false); + } + + $task = WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value) + ->sole(); + $this->app->call([new RunWorkflowTask($task->id), 'handle']); + + $this->assertSame( + array_map(static fn (array $event): array => [ + $event['event_type'], + $event['payload']['failure_category'] ?? null, + $event['payload']['exception_class'] ?? null, + ], $fixture['history']), + WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->where('event_type', '!=', HistoryEventType::StartAccepted->value) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): array => [ + $event->event_type->value, + $event->payload['failure_category'] ?? null, + $event->payload['exception_class'] ?? null, + ]) + ->all(), + ); + } + /** * The partial-group history 2.32.0 recorded for the same refusal still * replays through the embedded replayer to the all(), while a new run of diff --git a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php new file mode 100644 index 000000000..b43f1a03b --- /dev/null +++ b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php @@ -0,0 +1,231 @@ +start('Taylor'); + $runId = $workflow->runId(); + $historyBefore = $this->historyTypes($runId); + + $this->loseLease($runId); + $this->loseLease($runId); + + $task = $this->workflowTask($runId); + $this->assertSame(TaskStatus::Ready, $task->status); + $this->assertSame(2, $task->lost_lease_count); + $this->assertNotNull($task->last_lease_lost_at); + $this->assertNotSame(WorkflowTaskLeaseLoss::CODE, $this->summary($runId)->repair_blocked_reason); + $this->assertFalse((bool) $this->summary($runId)->repair_attention); + + $this->loseLease($runId); + + $summary = $this->summary($runId); + $this->assertSame(WorkflowTaskLeaseLoss::CODE, $summary->repair_blocked_reason); + $this->assertTrue((bool) $summary->repair_attention); + + $detail = RunDetailView::forRun(WorkflowRun::query()->findOrFail($runId)); + $this->assertSame(WorkflowTaskLeaseLoss::CODE, $detail['repair_blocked_reason']); + $this->assertTrue($detail['repair_attention']); + $this->assertSame('Task Timing Out', $detail['repair_blocked']['label']); + + $metrics = OperatorMetrics::snapshot(); + $this->assertSame(1, $metrics['runs']['workflow_task_lease_lost']); + $this->assertSame(1, $metrics['tasks']['lease_lost_repeatedly']); + $this->assertSame(3, $metrics['tasks']['max_lost_lease_count']); + + // Counting and reporting lost leases writes no history. + $this->assertSame($historyBefore, $this->historyTypes($runId)); + + // The next claim reports the typed diagnostic and still runs the task. + Log::spy(); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + Log::shouldHaveReceived('warning') + ->withArgs( + static fn (string $message, array $context): bool => $context['code'] === WorkflowTaskLeaseLoss::CODE + && $context['workflow_run_id'] === $runId + && $context['lost_leases'] === 3 + && $context['needs_attention'] === true + ) + ->once(); + $this->assertSame([ + HistoryEventType::StartAccepted->value, + HistoryEventType::WorkflowStarted->value, + HistoryEventType::ActivityScheduled->value, + ], $this->historyTypes($runId)); + + // The completed task's replacement starts with no lost leases. + $summary = $this->summary($runId); + $this->assertNotSame(WorkflowTaskLeaseLoss::CODE, $summary->repair_blocked_reason); + $this->assertFalse((bool) $summary->repair_attention); + } + + public function testFailingTheRunAfterLostLeasesIsOptIn(): void + { + Queue::fake(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + ]); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-fail-run'); + $workflow->start('Taylor'); + $runId = $workflow->runId(); + + $this->loseLease($runId); + $this->loseLease($runId); + $this->loseLease($runId); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + $this->assertTrue($workflow->refresh()->failed()); + $failure = WorkflowFailure::query() + ->where('workflow_run_id', $runId) + ->sole(); + $this->assertSame(WorkflowTaskLeaseLostException::class, $failure->exception_class); + $this->assertSame( + FailureCategory::TaskFailure->value, + $failure->failure_category?->value ?? $failure->failure_category, + ); + $this->assertSame([ + HistoryEventType::StartAccepted->value, + HistoryEventType::WorkflowStarted->value, + HistoryEventType::WorkflowFailed->value, + ], $this->historyTypes($runId)); + } + + public function testFailingTheRunWaitsForTheAttentionThreshold(): void + { + Queue::fake(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 3, + ]); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-below-threshold'); + $workflow->start('Taylor'); + $runId = $workflow->runId(); + + $this->loseLease($runId); + $this->loseLease($runId); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + $this->assertFalse($workflow->refresh()->failed()); + $this->assertContains(HistoryEventType::ActivityScheduled->value, $this->historyTypes($runId)); + } + + public function testWorkerStartWarnsWhenTheJobTimeoutIsBelowTheTaskLease(): void + { + WorkerTimeoutAdvisory::reset(); + Log::spy(); + $options = new WorkerOptions(timeout: 60); + + Event::dispatch(new WorkerStarting('redis', 'default', $options)); + Event::dispatch(new WorkerStarting('redis', 'default', $options)); + + if (extension_loaded('pcntl')) { + Log::shouldHaveReceived('warning') + ->withArgs(static fn (string $message, array $context): bool => + $context['code'] === 'worker_timeout_below_workflow_task_lease' + && $context['worker_timeout_seconds'] === 60 + && str_contains($message, '--timeout=0')) + ->once(); + } else { + Log::shouldNotHaveReceived('warning'); + } + + $this->assertNull(WorkerTimeoutAdvisory::message(0, true, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(300, true, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(60, false, 300)); + $this->assertNotNull(WorkerTimeoutAdvisory::message(60, true, 300)); + WorkerTimeoutAdvisory::reset(); + } + + protected function startQueueWorkersAutomatically(): bool + { + return false; + } + + /** + * Claim the run's workflow task as a worker would, let its lease expire + * without completing, and run task repair. + */ + private function loseLease(string $runId): void + { + $task = $this->workflowTask($runId); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'leased_at' => Carbon::now()->subMinutes(10), + 'lease_owner' => $task->id, + 'lease_expires_at' => Carbon::now()->subMinute(), + 'attempt_count' => $task->attempt_count + 1, + ])->save(); + + Cache::forget(TaskWatchdog::LOOP_THROTTLE_KEY); + TaskWatchdog::runPass(connection: 'redis', queue: 'default', respectThrottle: false); + + $this->assertSame(TaskStatus::Ready, $this->workflowTask($runId)->status); + } + + private function workflowTask(string $runId): WorkflowTask + { + return WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->sole(); + } + + private function summary(string $runId): WorkflowRunSummary + { + return WorkflowRunSummary::query()->findOrFail($runId); + } + + /** + * @return list + */ + private function historyTypes(string $runId): array + { + return WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): string => $event->event_type->value) + ->all(); + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json new file mode 100644 index 000000000..9a9701bb9 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "workflow-task-lease-lost-fails-run-opt-in", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "arguments": [ + "Taylor" + ], + "payload_codec": "avro" + }, + "workflow_task_lease_lost": { + "issue": "durable-workflow/workflow#1231", + "attention_threshold": 3, + "fail_run": true, + "note": "The run's only workflow task lost its lease three times without completing, as when a pcntl queue worker kills it at the job timeout. With workflows.v2.workflow_task_lost_leases.fail_run enabled, the next claim fails the run with the typed WorkflowTaskLeaseLostException (task_failure) before any workflow code runs. Lost leases themselves write no history; with fail_run off the same run keeps retrying and only shows workflow_task_lease_lost repair attention." + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "memo": null, + "business_key": null, + "workflow_type": "test-greeting-workflow", + "workflow_class": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "run_deadline_at": null, + "workflow_run_id": "lost-lease-run", + "declared_queries": [], + "declared_signals": [], + "declared_updates": [], + "search_attributes": null, + "visibility_labels": null, + "declared_entry_mode": "canonical", + "run_timeout_seconds": null, + "workflow_command_id": "lost-lease-start-command", + "workflow_instance_id": "lost-lease-instance", + "declared_entry_method": "handle", + "execution_deadline_at": null, + "declared_query_contracts": [], + "declared_signal_contracts": [], + "declared_update_contracts": [], + "execution_timeout_seconds": null, + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "workflow_definition_fingerprint": "sha256:d938f655e9a8d2d1074d252bdd2004c611304acb5db649367a21552784f7fa2c" + }, + "recorded_at": "2026-11-19T00:00:00.000000Z" + }, + { + "sequence": 2, + "event_type": "WorkflowFailed", + "payload": { + "message": "Workflow task lost-lease-task lost its lease 3 times without completing. Its queue worker may be killing it at the job timeout; run workers with --timeout=0 or above the longest workflow task.", + "exception": { + "code": 0, + "file": "/app/src/V2/Support/WorkflowTaskLeaseLoss.php", + "line": 152, + "class": "Workflow\\V2\\Exceptions\\WorkflowTaskLeaseLostException", + "trace": [], + "message": "Workflow task lost-lease-task lost its lease 3 times without completing. Its queue worker may be killing it at the job timeout; run workers with --timeout=0 or above the longest workflow task." + }, + "source_id": "lost-lease-run", + "failure_id": "lost-lease-failure", + "source_kind": "workflow_run", + "non_retryable": false, + "exception_type": null, + "exception_class": "Workflow\\V2\\Exceptions\\WorkflowTaskLeaseLostException", + "failure_category": "task_failure" + }, + "recorded_at": "2026-11-19T00:00:01.000000Z" + } + ], + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity" + } + ] + } +} From c258599cc79094232eec6a334cf49fcdc7a38075 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:31:33 +0000 Subject: [PATCH 2/8] Drop the issue reference from the operational liveness guide The public boundary scan forbids cross-tracker issue references in docs. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/operational-liveness.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/operational-liveness.md b/docs/architecture/operational-liveness.md index 40797e081..b15f17ab2 100644 --- a/docs/architecture/operational-liveness.md +++ b/docs/architecture/operational-liveness.md @@ -362,7 +362,7 @@ a caller may observe: a row (3 by default), usually because the queue worker kills it at its job timeout. The operator must raise the worker's `--timeout` (or use `--timeout=0`). With `workflow_task_lost_leases.fail_run` enabled, the - next claim fails the run with a typed task failure instead (#1231). + next claim fails the run with a typed task failure instead. - `waiting_for_compatible_worker` — a task exists, but no live worker advertises the required compatibility marker. - `selected_run_not_current` — the caller selected a historical From 748c47f9537ab397a9a558de519dd6773b9fbb8a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:39:30 +0000 Subject: [PATCH 3/8] Date the lost-lease migration before the example application's migration Co-Authored-By: Claude Opus 5.5 --- ...26_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/migrations/{2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php => 2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php} (100%) diff --git a/src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php b/src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php similarity index 100% rename from src/migrations/2026_11_19_000100_add_lost_lease_tracking_to_workflow_tasks.php rename to src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php From 6f1fd08dcaa61e31e7c201250dfb67aa0d58f21f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:40:35 +0000 Subject: [PATCH 4/8] Aim the upgrade journeys' refused rollback at the ledger's batch The state-machine journey upgrades from 2.33.0, which already has the compensation ledger, so a candidate that adds a migration runs it in a new batch. A plain migrate:rollback then undid only that newer batch and never reached the ledger's refusal, failing the journey for any added migration. expect_rollback_refused now rolls back the batch that holds the ledger (migrate:rollback --batch), which the rollback preflight already checks, so the journeys keep proving that retained scopes refuse a schema rollback. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/upgrade-journey-lib.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/ci/upgrade-journey-lib.sh b/scripts/ci/upgrade-journey-lib.sh index f60c65a3c..5bc751fc1 100644 --- a/scripts/ci/upgrade-journey-lib.sh +++ b/scripts/ci/upgrade-journey-lib.sh @@ -145,10 +145,27 @@ artisan_output() { # Runs migrate:rollback, which must be refused because retained compensation # scopes exist. Fails unless the refusal comes from the package preflight. +# The migration batch that holds the compensation ledger. A candidate that adds +# its own migrations runs them in a later batch, which a plain rollback would +# undo first without reaching the ledger. +ledger_batch() { + COLUMNS=1000 php artisan migrate:status --no-ansi --no-interaction 2>/dev/null \ + | grep -E '_create_compensation_scopes' \ + | grep -oE '\[[0-9]+\]' \ + | tr -d '[]' \ + | head -n 1 || true +} + expect_rollback_refused() { - local rollback_output + local rollback_output batch + local -a batch_option=() + + batch=$(ledger_batch) + if [ -n "$batch" ]; then + batch_option=("--batch=$batch") + fi - if rollback_output=$(COLUMNS=1000 php artisan migrate:rollback --force --no-interaction 2>&1 | tr -s ' \n' ' '; exit "${PIPESTATUS[0]}"); then + if rollback_output=$(COLUMNS=1000 php artisan migrate:rollback "${batch_option[@]}" --force --no-interaction 2>&1 | tr -s ' \n' ' '; exit "${PIPESTATUS[0]}"); then printf '%s\n' "$rollback_output" >&2 echo "$1" >&2 exit 1 From 392c18de2bdba762c542aa910b9eed3963743731 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:09:20 +0000 Subject: [PATCH 5/8] Report lost leases additively and give workflow tasks their own job timeout Review of the lost-lease change: - M1, migration lock: the columns are added outside the migrator transaction, each under the schema lock budget, and repeatably (2026_11_18_000100_add_lost_lease_tracking, as for the #863 column), so an ALTER behind an open task transaction fails fast instead of queueing reads. - M2, describe contract: the five repair_blocked_reason codes stay frozen, and can_repair and repair_attention are unchanged. Lost leases are reported in a separate additive task_attention field ({reason, consecutive, threshold, workflow_task_id, last_lease_lost_at}) on the run summary, list items and describe. Operator metrics add runs.task_attention (alongside tasks.lease_lost_repeatedly and tasks.max_lost_lease_count). With fail_run off, every field an existing consumer reads is identical at the threshold. - M3, timeouts: no blanket --timeout=0. RunWorkflowTask carries its own job timeout (WorkflowTaskJobTimeout): the lease less 30 s, or the configured workflow_task_job_timeout_seconds if lower, always 10 s below the connection's retry_after. A short worker --timeout no longer kills workflow tasks, and activities keep the worker's bounded --timeout. The worker-start warning now fires only for workers on the package's workflow queue, and only when retry_after caps workflow tasks. The README explains the lease, retry_after and timeout relationship and recommends a dedicated workflow-task connection. - Lows: neutral wording (the worker was killed, timed out, or lost its connection); fail_run refuses a threshold below 2 at boot; the schema check is re-done every 60 s and on any missing-column error (in its own savepoint, so it never aborts the caller's transaction), and degrades to untracked after a schema rollback, including SQLite's quirk of reading an unknown quoted column as a string. Co-Authored-By: Claude Opus 5.5 --- .github/feature-test-timings.json | 6 +- CHANGELOG.md | 36 +-- README.md | 52 +++- docs/architecture/operational-liveness.md | 19 +- src/Providers/WorkflowServiceProvider.php | 13 +- .../WorkflowTaskLeaseLostException.php | 10 +- src/V2/Jobs/RunWorkflowTask.php | 7 + src/V2/Models/WorkflowRunSummary.php | 1 + src/V2/Support/OperatorMetrics.php | 53 ++-- src/V2/Support/RepairBlockedReason.php | 13 - src/V2/Support/RunDetailView.php | 5 + src/V2/Support/RunListItemView.php | 5 + src/V2/Support/RunSummaryProjector.php | 3 + src/V2/Support/TaskDispatcher.php | 5 + src/V2/Support/WorkerTimeoutAdvisory.php | 85 ++++-- src/V2/Support/WorkflowTaskJobTimeout.php | 83 ++++++ src/V2/Support/WorkflowTaskLeaseLoss.php | 248 ++++++++++++++---- src/config/workflows.php | 18 +- ...6_11_18_000100_add_lost_lease_tracking.php | 103 ++++++++ ..._lost_lease_tracking_to_workflow_tasks.php | 43 --- .../V2/V2LostLeaseTrackingMigrationTest.php | 82 ++++++ .../V2/V2WorkflowTaskLostLeaseTest.php | 208 +++++++++++++-- ...flow-task-lease-lost-fails-run-opt-in.json | 6 +- 23 files changed, 883 insertions(+), 221 deletions(-) create mode 100644 src/V2/Support/WorkflowTaskJobTimeout.php create mode 100644 src/migrations/2026_11_18_000100_add_lost_lease_tracking.php delete mode 100644 src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php create mode 100644 tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index 31fdc3ab7..1215aebef 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -136,6 +136,7 @@ "tests/Feature/V2/V2LifecycleEventTest.php": 3.430998, "tests/Feature/V2/V2LocalActivityTest.php": 2.806176, "tests/Feature/V2/V2LocalEntityReferenceAuthorityTest.php": 9.066242, + "tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php": 10.0, "tests/Feature/V2/V2MachineDefinitionTest.php": 13.210704, "tests/Feature/V2/V2MemoUpsertTest.php": 10.058037, "tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 4.673203, @@ -241,7 +242,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.653342, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 12.837294, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 269.332131, - "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 15.0, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 20.0, "tests/Feature/V2/V2WorkflowTest.php": 161.306516, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 11.990609, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.888561, @@ -390,6 +391,7 @@ "tests/Feature/V2/V2LifecycleEventTest.php": 2.681959, "tests/Feature/V2/V2LocalActivityTest.php": 1.202231, "tests/Feature/V2/V2LocalEntityReferenceAuthorityTest.php": 10.707134, + "tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php": 3.0, "tests/Feature/V2/V2MachineDefinitionTest.php": 9.882914, "tests/Feature/V2/V2MemoUpsertTest.php": 5.262701, "tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 1.910908, @@ -495,7 +497,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.571702, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 10.15931, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 149.713579, - "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 12.0, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 15.0, "tests/Feature/V2/V2WorkflowTest.php": 115.227937, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 6.892443, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.853134, diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3113ce9..140de6fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,25 +13,31 @@ Docblocks and documentation only; no behavior changes. - A workflow task that keeps losing its lease without completing no longer - retries silently forever (#1231). This usually happens when a pcntl - `queue:work` kills it at the job timeout, which is 60 s by default. + retries silently forever (#1231). A lease is lost when the worker running + the task is killed, times out, or loses its connection. + - Workflow task jobs carry their own queue timeout: the workflow task lease + less 30 s (`workflows.v2.workflow_task_job_timeout_seconds` can lower it), + kept 10 s below the connection's `retry_after`. A short worker `--timeout` + no longer kills workflow tasks, and activity jobs keep the worker's + bounded `--timeout`. A worker on the workflow queue logs one warning at + start when `retry_after` caps workflow tasks below that. - Task repair counts each lost lease on the task (`lost_lease_count`, - `last_lease_lost_at`; new migration). The next claim logs a typed + `last_lease_lost_at`). The next claim logs a typed `workflow_task_lease_lost` diagnostic. No history is written, and run semantics are unchanged. - - After three consecutive lost leases + - At three consecutive lost leases (`workflows.v2.workflow_task_lost_leases.attention_threshold`), the run - reports `repair_blocked_reason` `workflow_task_lease_lost` ("Task Timing - Out") with repair attention in the summary, describe and Waterline. Operator - metrics add `runs.workflow_task_lease_lost`, `tasks.lease_lost_repeatedly` - and `tasks.max_lost_lease_count`. - - Opt-in (`workflows.v2.workflow_task_lost_leases.fail_run`, off by default): - at the threshold, the next claim fails the run with a typed - `WorkflowTaskLeaseLostException` (`task_failure`) instead of retrying. - - A queue worker that starts with pcntl and a `--timeout` below the workflow - task lease logs one warning. The README documents running workflow - workers with `--timeout=0` or above the longest workflow task. - + summary, list items and describe carry an additive `task_attention` field + (`reason`, `consecutive`, `threshold`), and operator metrics add + `runs.task_attention`, `tasks.lease_lost_repeatedly` and + `tasks.max_lost_lease_count`. Repair-blocked reasons, repair attention + and `can_repair` are unchanged. + - Opt-in (`workflows.v2.workflow_task_lost_leases.fail_run`, off by default; + boot refuses a threshold below 2): at the threshold, the next claim fails + the run with a typed `WorkflowTaskLeaseLostException` (`task_failure`) + instead of retrying. + - The migration adds the columns outside a transaction under the schema + lock budget. - Workflow definition and `await()` condition fingerprints are versioned, and a new opt-in version 2 makes comment-only and whitespace-only edits replay-safe (#1227). The prefix names the version: diff --git a/README.md b/README.md index b18e360df..914e95c2e 100644 --- a/README.md +++ b/README.md @@ -61,20 +61,48 @@ existing tables. Run a Laravel queue worker or Horizon to execute workflows and activities: ```bash -php artisan queue:work --timeout=0 +php artisan queue:work ``` -With the `pcntl` extension, `queue:work` kills a job after `--timeout` seconds -(60 by default). A workflow task that runs longer, such as one that schedules or -finishes a wide `all()`, is rolled back and retried after its lease expires, so it -never finishes. Run workers with `--timeout=0` or a value above your longest -workflow task, and keep `workflows.v2.workflow_task_lease_seconds` (300 by -default) above it; Horizon's `timeout` option is the same setting. A worker -started with a shorter timeout logs a warning. Each lease a workflow task loses -is counted, and after three in a row (`DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD`) -the run reports `repair_blocked_reason` `workflow_task_lease_lost` with repair -attention. Set `DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN=true` to fail such a run -with a typed task failure instead. +With the `pcntl` extension, a queue worker kills any job that runs longer than +its timeout. A workflow task that is killed loses its lease, and it is retried +only after the lease expires (`workflows.v2.workflow_task_lease_seconds`, 300 +seconds by default). Workflow task jobs therefore carry their own timeout in +place of the worker's `--timeout`: the lease less 30 seconds, or +`DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS` if that is lower. Activity jobs keep +the worker's bounded `--timeout`, so a hung activity cannot hold a worker +forever. + +Laravel requires a queue connection's `retry_after` to exceed every job's +timeout, so a workflow task's timeout is also kept 10 seconds below its +connection's `retry_after`. With Laravel's default `retry_after` of 90 seconds, +workflow tasks are limited to 80 seconds, and a worker on that queue logs a +warning when it starts. Wide `all()` fan-outs need longer. Give workflow tasks +a queue connection whose `retry_after` exceeds the lease, and run dedicated +workers for it. Activities run on their workflow's queue unless the activity +class sets its own `$connection` or `$queue`: + +```php +// config/queue.php: a connection for workflow tasks +'workflows' => [ + 'driver' => 'redis', + 'connection' => 'default', + 'queue' => 'workflows', + 'retry_after' => 330, // above the 300-second workflow task lease +], +``` + +```bash +# DW_V2_CONNECTION=workflows DW_V2_QUEUE=workflows +php artisan queue:work workflows --queue=workflows +php artisan queue:work --queue=default --timeout=60 # activities, bounded +``` + +If a workflow task loses its lease three times in a row +(`DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD`), the run summary and +describe carry an additive `task_attention` field, and operator metrics count +it. Set `DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN=true` to fail such a run with a +typed task failure instead; that requires a threshold of at least 2. For the default database queue on SQLite, failed reservation writes preserve unclaimed work after a proven rollback. See [SQLite queue reservations](docs/sqlite-queue-reservations.md) diff --git a/docs/architecture/operational-liveness.md b/docs/architecture/operational-liveness.md index b15f17ab2..7faa7d15f 100644 --- a/docs/architecture/operational-liveness.md +++ b/docs/architecture/operational-liveness.md @@ -350,19 +350,12 @@ Repair: batch path. The repair path MUST be explicit about why it cannot act. The -`RepairBlockedReason::catalog()` enumerates the six reason codes that -a caller may observe: +`RepairBlockedReason::catalog()` enumerates the five frozen reason +codes that a caller may observe: - `unsupported_history` — the run's history contains events the current engine build cannot interpret; the operator must roll forward a compatible worker or manually resolve. -- `workflow_task_lease_lost` — repair keeps returning the open - workflow task to ready, but it has lost its lease without completing - `workflows.v2.workflow_task_lost_leases.attention_threshold` times in - a row (3 by default), usually because the queue worker kills it at its - job timeout. The operator must raise the worker's `--timeout` (or use - `--timeout=0`). With `workflow_task_lost_leases.fail_run` enabled, the - next claim fails the run with a typed task failure instead. - `waiting_for_compatible_worker` — a task exists, but no live worker advertises the required compatibility marker. - `selected_run_not_current` — the caller selected a historical @@ -375,6 +368,14 @@ a caller may observe: Adding a blocked reason is a protocol change; deleting one is a protocol change. +A workflow task that keeps losing its lease without completing (its +worker was killed, timed out, or lost its connection) is not a blocked +repair: repair keeps returning it to ready and `can_repair` is unchanged. +It is reported in the separate, additive `task_attention` field of the +run summary, list items and describe, set once the open workflow task has +lost `workflows.v2.workflow_task_lost_leases.attention_threshold` leases +in a row (3 by default) and null otherwise. + ## Repair cadence and bounded backoff The repair path is bounded. An automated recovery pass cannot diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index a03b28972..e159ad04a 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -81,6 +81,7 @@ use Workflow\V2\Support\TypeRegistry; use Workflow\V2\Support\WorkerTimeoutAdvisory; use Workflow\V2\Support\WorkflowModeGuard; +use Workflow\V2\Support\WorkflowTaskLeaseLoss; use Workflow\Watchdog; final class WorkflowServiceProvider extends ServiceProvider @@ -223,6 +224,7 @@ public function boot(): void // Refuse an unsupported definition fingerprint version before any // workflow can run; at runtime the setting never fails a run. SourceFingerprint::validateConfiguration(); + WorkflowTaskLeaseLoss::validateConfiguration(); $this->configureSqliteStorageConnection(); // Register long-poll wake signal observers @@ -239,14 +241,15 @@ public function boot(): void $this->validateCacheBackend(); if (class_exists(\Illuminate\Queue\Events\WorkerStarting::class)) { - // A job timeout below the workflow task lease can kill a workflow - // task on every retry; warn once per worker (#1231). + // A low retry_after caps how long a workflow task may run on the + // workflow queue; warn that queue's workers once. Event::listen( \Illuminate\Queue\Events\WorkerStarting::class, static function (object $event): void { - if (isset($event->workerOptions) && is_object($event->workerOptions)) { - WorkerTimeoutAdvisory::workerStarting($event->workerOptions); - } + WorkerTimeoutAdvisory::workerStarting( + is_string($event->connectionName ?? null) ? $event->connectionName : null, + is_string($event->queue ?? null) ? $event->queue : null, + ); }, ); } diff --git a/src/V2/Exceptions/WorkflowTaskLeaseLostException.php b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php index 7e165b262..f9ad232d0 100644 --- a/src/V2/Exceptions/WorkflowTaskLeaseLostException.php +++ b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php @@ -8,9 +8,9 @@ /** * A run failed because its workflow task kept losing its lease without - * completing, with workflows.v2.workflow_task_lost_leases.fail_run enabled - * (#1231). The usual cause is a queue worker job timeout shorter than the - * workflow task. + * completing, with workflows.v2.workflow_task_lost_leases.fail_run enabled. + * A lease is lost when the worker running the task is killed, times out, or + * loses its connection before the task completes. */ final class WorkflowTaskLeaseLostException extends RuntimeException { @@ -20,8 +20,8 @@ public function __construct( public readonly int $leaseSeconds, ) { parent::__construct(sprintf( - 'Workflow task %s lost its lease %d times without completing. Its queue worker may be killing it ' - . 'at the job timeout; run workers with --timeout=0 or above the longest workflow task.', + 'Workflow task %s lost its lease %d times in a row without completing (its worker was killed, ' + . 'timed out, or lost its connection).', $taskId, $lostLeases, )); diff --git a/src/V2/Jobs/RunWorkflowTask.php b/src/V2/Jobs/RunWorkflowTask.php index c81d8fa77..4dec3aacf 100644 --- a/src/V2/Jobs/RunWorkflowTask.php +++ b/src/V2/Jobs/RunWorkflowTask.php @@ -34,6 +34,7 @@ use Workflow\V2\Support\WorkflowContinuationCache; use Workflow\V2\Support\WorkflowExecutor; use Workflow\V2\Support\WorkflowSignalReadScope; +use Workflow\V2\Support\WorkflowTaskJobTimeout; use Workflow\V2\Support\WorkflowTaskLease; use Workflow\V2\Support\WorkflowTaskLeaseLoss; @@ -46,6 +47,12 @@ final class RunWorkflowTask implements ShouldQueue public int $tries = 1; + /** + * The job's own queue timeout ({@see WorkflowTaskJobTimeout}), set when + * the task is dispatched; null leaves the worker's --timeout in force. + */ + public ?int $timeout = null; + // Mixed so malformed telemetry cannot prevent an older job from loading. public mixed $dispatchObservabilityContext = null; diff --git a/src/V2/Models/WorkflowRunSummary.php b/src/V2/Models/WorkflowRunSummary.php index bfd76516e..9b7d7bf1a 100644 --- a/src/V2/Models/WorkflowRunSummary.php +++ b/src/V2/Models/WorkflowRunSummary.php @@ -42,6 +42,7 @@ class WorkflowRunSummary extends Model protected $casts = [ 'is_current_run' => 'bool', 'repair_attention' => 'bool', + 'task_attention' => 'array', 'task_problem' => 'bool', 'visibility_labels' => 'array', 'projection_schema_version' => 'integer', diff --git a/src/V2/Support/OperatorMetrics.php b/src/V2/Support/OperatorMetrics.php index 252c73f70..970013be1 100644 --- a/src/V2/Support/OperatorMetrics.php +++ b/src/V2/Support/OperatorMetrics.php @@ -132,10 +132,8 @@ private static function runMetrics(CarbonInterface $now, ?string $namespace): ar ? 0 : (int) $oldestReplayBlockedAt->diffInMilliseconds($now), 'claim_failed' => self::claimFailedRuns($namespace), - // Open runs whose workflow task kept losing its lease (#1231). - 'workflow_task_lease_lost' => self::summaryQuery($namespace) - ->where('repair_blocked_reason', WorkflowTaskLeaseLoss::CODE) - ->count(), + // Additive: open runs whose workflow task keeps losing its lease. + 'task_attention' => self::taskAttentionRuns($namespace), 'compatibility_blocked' => self::compatibilityBlockedRuns($namespace), 'waiting' => self::waitingRuns($namespace), 'oldest_wait_started_at' => $oldestWaitStartedAt?->toJSON(), @@ -194,7 +192,7 @@ private static function taskMetrics(CarbonInterface $now, ?string $namespace): a : (int) $oldestDispatchFailedAt->diffInMilliseconds($now), 'max_attempt_count' => self::maxOpenTaskAttemptCount($namespace), 'max_repair_count' => self::maxOpenTaskRepairCount($namespace), - // Open workflow tasks at or over the lost-lease attention threshold (#1231). + // Additive: open workflow tasks at or over the lost-lease threshold. 'lease_lost_repeatedly' => self::repeatedlyLostLeaseTasks($namespace), 'max_lost_lease_count' => self::maxOpenTaskLostLeaseCount($namespace), 'unhealthy' => self::dispatchFailedTasks($namespace) @@ -1564,29 +1562,52 @@ private static function maxOpenTaskAttemptCount(?string $namespace): int * active repair burn rather than accumulating against historical * rows. */ + private static function taskAttentionRuns(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::summaryTracked(fresh: true)) { + return 0; + } + + $query = self::summaryQuery($namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => $query->whereNotNull('task_attention') + ->count(), + ); + } + private static function repeatedlyLostLeaseTasks(?string $namespace): int { - if (! WorkflowTaskLeaseLoss::tracked()) { + if (! WorkflowTaskLeaseLoss::tracked(fresh: true)) { return 0; } - return self::scopedRunModelQuery(self::taskModel(), $namespace) - ->where('task_type', TaskType::Workflow->value) - ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) - ->where('lost_lease_count', '>=', WorkflowTaskLeaseLoss::attentionThreshold()) - ->count(); + $query = self::scopedRunModelQuery(self::taskModel(), $namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => $query->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', WorkflowTaskLeaseLoss::attentionThreshold()) + ->count(), + ); } private static function maxOpenTaskLostLeaseCount(?string $namespace): int { - if (! WorkflowTaskLeaseLoss::tracked()) { + if (! WorkflowTaskLeaseLoss::tracked(fresh: true)) { return 0; } - return (int) self::scopedRunModelQuery(self::taskModel(), $namespace) - ->where('task_type', TaskType::Workflow->value) - ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) - ->max('lost_lease_count'); + $query = self::scopedRunModelQuery(self::taskModel(), $namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => (int) $query->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->max('lost_lease_count'), + ); } private static function maxOpenTaskRepairCount(?string $namespace): int diff --git a/src/V2/Support/RepairBlockedReason.php b/src/V2/Support/RepairBlockedReason.php index 8fb963aec..ff581b54e 100644 --- a/src/V2/Support/RepairBlockedReason.php +++ b/src/V2/Support/RepairBlockedReason.php @@ -26,14 +26,6 @@ public static function catalog(): array 'tone' => 'dark', 'badge_visible' => true, ], - WorkflowTaskLeaseLoss::CODE => [ - 'label' => 'Task Timing Out', - 'description' => 'The workflow task keeps losing its lease without completing; its queue worker ' - . 'may be killing it at the job timeout. Run workers with --timeout=0 or above the longest ' - . 'workflow task.', - 'tone' => 'warning', - 'badge_visible' => true, - ], 'waiting_for_compatible_worker' => [ 'label' => 'Compat Blocked', 'description' => 'Repair is blocked because the task is waiting for a compatible worker.', @@ -136,11 +128,6 @@ public static function forRun( return 'run_closed'; } - // Repair keeps redispatching a task that never completes (#1231). - if (WorkflowTaskLeaseLoss::runNeedsAttention($run)) { - return WorkflowTaskLeaseLoss::CODE; - } - if ($livenessState === 'repair_needed') { return null; } diff --git a/src/V2/Support/RunDetailView.php b/src/V2/Support/RunDetailView.php index d5e05d9fd..bf3369443 100644 --- a/src/V2/Support/RunDetailView.php +++ b/src/V2/Support/RunDetailView.php @@ -265,6 +265,11 @@ public static function forRun(WorkflowRun $run, ?int $timelineLimit = null): arr $repairBlockedReason )), 'repair_blocked' => RepairBlockedReason::metadata($repairBlockedReason), + // Additive: a workflow task that keeps losing its lease. Repair + // still works and can_repair is unchanged. + 'task_attention' => $summary !== null && array_key_exists('task_attention', $summary->getAttributes()) + ? $summary->task_attention + : WorkflowTaskLeaseLoss::taskAttention($run), 'can_archive' => $canArchive, 'archive_blocked_reason' => $archiveBlockedReason, 'can_redrive' => $redriveBlockedReason === null, diff --git a/src/V2/Support/RunListItemView.php b/src/V2/Support/RunListItemView.php index f6ba3aa61..751a78402 100644 --- a/src/V2/Support/RunListItemView.php +++ b/src/V2/Support/RunListItemView.php @@ -75,6 +75,10 @@ public static function fromSummary(WorkflowRunSummary $summary): array 'repair_blocked' => RepairBlockedReason::metadata( is_string($summary->repair_blocked_reason) ? $summary->repair_blocked_reason : null, ), + // Additive: a workflow task that keeps losing its lease. + 'task_attention' => array_key_exists('task_attention', $summary->getAttributes()) + ? $summary->task_attention + : null, 'task_problem' => (bool) $summary->task_problem, 'task_problem_badge' => WorkflowTaskProblem::metadata( (bool) $summary->task_problem, @@ -144,6 +148,7 @@ public static function fields(): array 'repair_attention', 'repair_blocked_reason', 'repair_blocked', + 'task_attention', 'task_problem', 'task_problem_badge', 'declared_entry_mode', diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 8e71ae833..401ccabce 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -510,6 +510,7 @@ public static function project(WorkflowRun $run): WorkflowRunSummary 'next_task_lease_expires_at' => $selectedNextTask?->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run), 'task_problem' => $taskProblem, 'exception_count' => count($failureSnapshots), 'history_event_count' => $historyBudget['history_event_count'], @@ -704,6 +705,7 @@ private static function projectOpenWorkflowTask( 'next_task_lease_expires_at' => $task->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run), 'task_problem' => $replayBlockedTask instanceof WorkflowTask || $hasPendingChildResolution || self::claimHasWorkflowTaskProblem($run) @@ -1258,6 +1260,7 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow 'next_task_lease_expires_at' => $nextTask->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run), 'task_problem' => false, 'exception_count' => 0, 'history_event_count' => $historyEventCount, diff --git a/src/V2/Support/TaskDispatcher.php b/src/V2/Support/TaskDispatcher.php index f08a67cde..06f215a8e 100644 --- a/src/V2/Support/TaskDispatcher.php +++ b/src/V2/Support/TaskDispatcher.php @@ -188,6 +188,11 @@ private static function makeJob(WorkflowTask $task): RunWorkflowTask|RunActivity $job->onQueue($task->queue); } + if ($job instanceof RunWorkflowTask) { + // A short worker --timeout must not kill workflow tasks. + $job->timeout = WorkflowTaskJobTimeout::seconds($task->connection); + } + if ($task->available_at !== null && $task->available_at->isFuture()) { // Queue backends store second-resolution deadlines. Never round a wakeup down. $availableAt = $task->available_at->copy() diff --git a/src/V2/Support/WorkerTimeoutAdvisory.php b/src/V2/Support/WorkerTimeoutAdvisory.php index 695ffa345..356cd4279 100644 --- a/src/V2/Support/WorkerTimeoutAdvisory.php +++ b/src/V2/Support/WorkerTimeoutAdvisory.php @@ -7,12 +7,15 @@ use Illuminate\Support\Facades\Log; /** - * Warn once per worker process when Laravel's job timeout can kill a - * workflow task before its lease ends (#1231). + * Warn once per worker process when its queue connection limits how long a + * workflow task can run. * - * With pcntl loaded, `queue:work` kills a job after `--timeout` seconds (60 by - * default). A workflow task killed there rolls back its work and is retried - * after its lease expires, so a task longer than the timeout never finishes. + * A workflow task's job carries its own timeout ({@see WorkflowTaskJobTimeout}), + * so the worker's `--timeout` no longer applies to it. That timeout is kept + * below the connection's `retry_after`, so a low `retry_after` (90 seconds in + * Laravel's default configuration) still caps workflow tasks well below their + * lease. Only workers that serve the queue the package dispatches workflow + * tasks to are warned, and only where pcntl enforces job timeouts. * * @internal */ @@ -21,35 +24,71 @@ final class WorkerTimeoutAdvisory private static bool $warned = false; /** - * The warning for a worker started with $timeoutSeconds, or null when the - * job timeout cannot end a workflow task before its lease. + * The warning for a worker on $connection, or null when workflow tasks + * there can run up to their uncapped timeout. */ - public static function message(int $timeoutSeconds, bool $pcntlLoaded, int $leaseSeconds): ?string - { - if (! $pcntlLoaded || $timeoutSeconds <= 0 || $timeoutSeconds >= $leaseSeconds) { + public static function message( + bool $pcntlLoaded, + ?int $workflowTaskTimeout, + int $uncappedTimeout, + ?int $retryAfter, + string $connection, + ): ?string { + if (! $pcntlLoaded || ($workflowTaskTimeout !== null && $workflowTaskTimeout >= $uncappedTimeout)) { return null; } return sprintf( - '[Durable Workflow] This queue worker kills a job after %d s (--timeout), before the %d s workflow task ' - . 'lease. A workflow task that runs longer, such as a wide all() fan-out, is killed, rolled back and ' - . 'retried after its lease expires, and can never finish. Run workers with --timeout=0 or a value above ' - . 'your longest workflow task, and keep workflows.v2.workflow_task_lease_seconds above --timeout.', - $timeoutSeconds, - $leaseSeconds, + '[Durable Workflow] Queue connection "%s" has retry_after = %s s, so a workflow task on it can run ' + . 'for at most %s before the worker kills it and the task is retried after its lease expires. ' + . 'Set this connection\'s retry_after above %d s (the workflow task timeout plus %d s), ideally on ' + . 'a connection dedicated to workflow tasks, so workflow tasks can run for %d s.', + $connection, + $retryAfter === null ? 'unset' : (string) $retryAfter, + $workflowTaskTimeout === null ? 'the worker --timeout' : $workflowTaskTimeout . ' s', + $uncappedTimeout + WorkflowTaskJobTimeout::RETRY_AFTER_MARGIN_SECONDS, + WorkflowTaskJobTimeout::RETRY_AFTER_MARGIN_SECONDS, + $uncappedTimeout, ); } - public static function workerStarting(object $workerOptions): void + /** + * Whether a worker started for $connection and $queues serves the queue + * the package dispatches workflow tasks to by default. + */ + public static function servesWorkflowQueue(?string $connection, ?string $queues): bool + { + $workflowConnection = config('workflows.v2.connection'); + $workflowConnection = is_string($workflowConnection) && $workflowConnection !== '' + ? $workflowConnection + : config('queue.default'); + + if (! is_string($workflowConnection) || $connection !== $workflowConnection) { + return false; + } + + $workflowQueue = config('workflows.v2.queue'); + $workflowQueue = is_string($workflowQueue) && $workflowQueue !== '' + ? $workflowQueue + : config('queue.connections.' . $workflowConnection . '.queue', 'default'); + + $served = array_map('trim', explode(',', (string) ($queues ?? ''))); + + return in_array((string) $workflowQueue, $served, true); + } + + public static function workerStarting(?string $connection, ?string $queues): void { - if (self::$warned || ! property_exists($workerOptions, 'timeout')) { + if (self::$warned || $connection === null || ! self::servesWorkflowQueue($connection, $queues)) { return; } $message = self::message( - (int) $workerOptions->timeout, extension_loaded('pcntl'), - WorkflowTaskLease::seconds(), + WorkflowTaskJobTimeout::seconds($connection), + WorkflowTaskJobTimeout::uncappedSeconds(), + WorkflowTaskJobTimeout::retryAfter($connection), + $connection, ); if ($message === null) { @@ -58,8 +97,10 @@ public static function workerStarting(object $workerOptions): void self::$warned = true; Log::warning($message, [ - 'code' => 'worker_timeout_below_workflow_task_lease', - 'worker_timeout_seconds' => (int) $workerOptions->timeout, + 'code' => 'workflow_task_timeout_capped_by_retry_after', + 'connection' => $connection, + 'retry_after_seconds' => WorkflowTaskJobTimeout::retryAfter($connection), + 'workflow_task_timeout_seconds' => WorkflowTaskJobTimeout::seconds($connection), 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), ]); } diff --git a/src/V2/Support/WorkflowTaskJobTimeout.php b/src/V2/Support/WorkflowTaskJobTimeout.php new file mode 100644 index 000000000..6c8b1f3f6 --- /dev/null +++ b/src/V2/Support/WorkflowTaskJobTimeout.php @@ -0,0 +1,83 @@ += 1 ? $timeout : null; + } + + /** + * The timeout before any `retry_after` cap: the configured value, or the + * lease less its margin. + */ + public static function uncappedSeconds(): int + { + $configured = config(self::CONFIG_KEY); + $ceiling = max(1, WorkflowTaskLease::seconds() - self::LEASE_MARGIN_SECONDS); + + if (is_numeric($configured) && (int) $configured >= 1) { + return min((int) $configured, $ceiling); + } + + return $ceiling; + } + + /** + * The queue connection's `retry_after`, or null when it has none. + */ + public static function retryAfter(?string $connection = null): ?int + { + $connection ??= config('queue.default'); + + if (! is_string($connection) || $connection === '') { + return null; + } + + $retryAfter = config("queue.connections.{$connection}.retry_after"); + + return is_numeric($retryAfter) && (int) $retryAfter > 0 ? (int) $retryAfter : null; + } +} diff --git a/src/V2/Support/WorkflowTaskLeaseLoss.php b/src/V2/Support/WorkflowTaskLeaseLoss.php index 62c6ca89d..994cc746a 100644 --- a/src/V2/Support/WorkflowTaskLeaseLoss.php +++ b/src/V2/Support/WorkflowTaskLeaseLoss.php @@ -4,27 +4,31 @@ namespace Workflow\V2\Support; +use Illuminate\Database\Eloquent\Model; +use Illuminate\Database\QueryException; use Illuminate\Support\Facades\Log; +use InvalidArgumentException; use Workflow\V2\Enums\TaskStatus; use Workflow\V2\Enums\TaskType; use Workflow\V2\Exceptions\WorkflowTaskLeaseLostException; use Workflow\V2\Models\WorkflowRun; +use Workflow\V2\Models\WorkflowRunSummary; use Workflow\V2\Models\WorkflowTask; /** - * Workflow tasks that lose their lease without completing (#1231). + * Workflow tasks that lose their lease without completing. * - * A queue worker that kills a workflow task at its job timeout rolls back the - * task's work. The task stays leased until its lease expires, task repair - * returns it to ready, and the next worker repeats the same work. Without a - * record of the lost leases a run that can never finish inside the worker's - * timeout retried forever with no diagnosis. + * A worker that is killed, times out or loses its connection while running a + * workflow task rolls back the task's work. The task stays leased until its + * lease expires, task repair returns it to ready, and the next worker repeats + * the same work. Without a record of the lost leases a task that can never + * complete retried forever with no diagnosis. * - * Task repair counts each lost lease on the task. The next claim records a - * typed diagnostic. After the attention threshold, the run summary reports - * repair_blocked_reason {@see self::CODE} with repair attention. Failing the - * run instead is opt-in. None of this writes history or changes how a run - * that completes its task behaves. + * Task repair counts each lost lease on the task. The next claim logs a typed + * diagnostic. At the attention threshold the run summary and describe carry + * the additive `task_attention` field. Repair-blocked reasons and repair + * actions are unchanged. Failing the run instead is opt-in. None of this + * writes history or changes how a run whose task completes behaves. */ final class WorkflowTaskLeaseLoss { @@ -35,9 +39,20 @@ final class WorkflowTaskLeaseLoss public const DEFAULT_ATTENTION_THRESHOLD = 3; /** - * @var array + * A failing threshold of one would fail a run on its first lost lease. */ - private static array $trackedSchemas = []; + public const MINIMUM_FAILING_THRESHOLD = 2; + + /** + * Seconds a cached schema answer is trusted before the columns are looked + * up again, so a process notices a schema rollback. + */ + private const SCHEMA_RECHECK_SECONDS = 60; + + /** + * @var array + */ + private static array $schemas = []; public static function attentionThreshold(): int { @@ -53,6 +68,28 @@ public static function failsRun(): bool return (bool) config(self::CONFIG_KEY . '.fail_run', false); } + /** + * Refuse, at boot, a run-failing configuration that would fail a run on + * too few lost leases. + */ + public static function validateConfiguration(): void + { + if (! self::failsRun()) { + return; + } + + $value = config(self::CONFIG_KEY . '.attention_threshold', self::DEFAULT_ATTENTION_THRESHOLD); + + if (! is_numeric($value) || (int) $value < self::MINIMUM_FAILING_THRESHOLD) { + throw new InvalidArgumentException(sprintf( + '%s.attention_threshold must be at least %d when %s.fail_run is enabled.', + self::CONFIG_KEY, + self::MINIMUM_FAILING_THRESHOLD, + self::CONFIG_KEY, + )); + } + } + public static function lostLeases(WorkflowTask $task): int { return max(0, (int) ($task->getAttribute('lost_lease_count') ?? 0)); @@ -60,7 +97,7 @@ public static function lostLeases(WorkflowTask $task): int /** * Count one lost lease on a workflow task that task repair returns to - * ready. A schema without the tracking columns keeps the old behavior. + * ready. A task row without the tracking columns keeps the old behavior. * * @return array */ @@ -101,8 +138,6 @@ public static function claimDiagnostic(WorkflowTask $task): ?array return null; } - $lastLostAt = $task->getAttribute('last_lease_lost_at'); - return [ 'code' => self::CODE, 'workflow_run_id' => (string) $task->workflow_run_id, @@ -110,9 +145,7 @@ public static function claimDiagnostic(WorkflowTask $task): ?array 'lost_leases' => $lostLeases, 'attention_threshold' => self::attentionThreshold(), 'needs_attention' => $lostLeases >= self::attentionThreshold(), - 'last_lease_lost_at' => $lastLostAt instanceof \DateTimeInterface - ? $lastLostAt->format('Y-m-d\TH:i:s.up') - : null, + 'last_lease_lost_at' => self::timestamp($task->getAttribute('last_lease_lost_at')), 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), 'claim_attempt' => (int) $task->attempt_count, ]; @@ -127,9 +160,8 @@ public static function reportClaim(array $diagnostic): void { Log::warning( sprintf( - '[Durable Workflow] Workflow task %s lost its lease %d time(s) without completing; ' - . 'its queue worker may be killing it at the job timeout. Run workers with ' - . '--timeout=0 or above your longest workflow task.', + '[Durable Workflow] Workflow task %s lost its lease %d time(s) without completing ' + . '(its worker was killed, timed out, or lost its connection); retrying it.', $diagnostic['workflow_task_id'], $diagnostic['lost_leases'], ), @@ -144,7 +176,7 @@ public static function failsRunAt(WorkflowTask $task): bool { return self::failsRun() && $task->task_type === TaskType::Workflow - && self::lostLeases($task) >= self::attentionThreshold(); + && self::lostLeases($task) >= max(self::MINIMUM_FAILING_THRESHOLD, self::attentionThreshold()); } public static function exceptionFor(WorkflowTask $task): WorkflowTaskLeaseLostException @@ -157,54 +189,170 @@ public static function exceptionFor(WorkflowTask $task): WorkflowTaskLeaseLostEx } /** - * Whether the run's open workflow task has reached the attention threshold. + * The run's additive `task_attention` value: its open workflow task's + * consecutive lost leases once they reach the threshold, otherwise null. + * + * @return array{reason: string, consecutive: int, threshold: int, workflow_task_id: string, last_lease_lost_at: string|null}|null */ - public static function runNeedsAttention(WorkflowRun $run): bool + public static function taskAttention(WorkflowRun $run): ?array { + if ($run->status->isTerminal()) { + return null; + } + $threshold = self::attentionThreshold(); + $task = null; if ($run->relationLoaded('tasks')) { - foreach ($run->tasks as $task) { - if ($task instanceof WorkflowTask - && $task->task_type === TaskType::Workflow - && in_array($task->status, [TaskStatus::Ready, TaskStatus::Leased], true) - && self::lostLeases($task) >= $threshold) { - return true; + foreach ($run->tasks as $candidate) { + if ($candidate instanceof WorkflowTask + && $candidate->task_type === TaskType::Workflow + && in_array($candidate->status, [TaskStatus::Ready, TaskStatus::Leased], true) + && self::lostLeases($candidate) >= $threshold + && ($task === null || self::lostLeases($candidate) > self::lostLeases($task))) { + $task = $candidate; } } + } elseif ($run->exists && self::tracked()) { + $task = self::untrackedOnMissingColumn( + self::taskModel(), + static fn (): ?WorkflowTask => ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', $threshold) + ->orderByDesc('lost_lease_count') + ->first(), + ); + } - return false; + if (! $task instanceof WorkflowTask) { + return null; } - if (! $run->exists) { - return false; + // SQLite reads an unknown double-quoted column name as a string, so a + // removed column can match instead of failing. A row without the + // column means the schema was rolled back. + if (! array_key_exists('lost_lease_count', $task->getAttributes())) { + self::$schemas = []; + + return null; } - if (! self::tracked()) { - return false; + return [ + 'reason' => self::CODE, + 'consecutive' => self::lostLeases($task), + 'threshold' => $threshold, + 'workflow_task_id' => (string) $task->id, + 'last_lease_lost_at' => self::timestamp($task->getAttribute('last_lease_lost_at')), + ]; + } + + /** + * The summary attributes that carry `task_attention`, or none when the + * summary table has no such column. + * + * @return array + */ + public static function summaryAttributes(WorkflowRun $run): array + { + if (! self::summaryTracked()) { + return []; } - $query = ConfiguredV2Models::query('task_model', WorkflowTask::class); + return [ + 'task_attention' => self::taskAttention($run), + ]; + } + + /** + * Whether the configured task table has the lost-lease columns. + */ + public static function tracked(bool $fresh = false): bool + { + return self::hasColumn(self::taskModel(), 'lost_lease_count', $fresh); + } - return $query - ->where('workflow_run_id', $run->id) - ->where('task_type', TaskType::Workflow->value) - ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) - ->where('lost_lease_count', '>=', $threshold) - ->exists(); + /** + * Whether the configured run-summary table has `task_attention`. + */ + public static function summaryTracked(bool $fresh = false): bool + { + return self::hasColumn(self::summaryModel(), 'task_attention', $fresh); } /** - * Whether the configured task table has the lost-lease columns. A schema - * that has not run the migration has no lost leases to report. + * Run a query that reads the lost-lease columns. If a schema rollback + * removed them, degrade to untracked instead of failing. Inside a + * transaction the query runs in its own savepoint, so the failed + * statement cannot abort the caller's transaction. + * + * @template T + * @param callable(): T $query + * @return T|null */ - public static function tracked(): bool + public static function untrackedOnMissingColumn(Model $model, callable $query): mixed { - $model = ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel(); - $schemaKey = $model->getConnectionName() . '|' . $model->getTable(); + $connection = $model->getConnection(); + + try { + return $connection->transactionLevel() > 0 + ? $connection->transaction(static fn () => $query()) + : $query(); + } catch (QueryException $exception) { + if (! self::missingColumn($exception)) { + throw $exception; + } - return self::$trackedSchemas[$schemaKey] ??= $model->getConnection() + self::$schemas = []; + + return null; + } + } + + private static function hasColumn(Model $model, string $column, bool $fresh = false): bool + { + $key = $model->getConnectionName() . '|' . $model->getTable() . '|' . $column; + $cached = self::$schemas[$key] ?? null; + $now = time(); + + if (! $fresh && $cached !== null && $now - $cached['checked_at'] < self::SCHEMA_RECHECK_SECONDS) { + return $cached['tracked']; + } + + $tracked = $model->getConnection() ->getSchemaBuilder() - ->hasColumn($model->getTable(), 'lost_lease_count'); + ->hasColumn($model->getTable(), $column); + self::$schemas[$key] = [ + 'tracked' => $tracked, + 'checked_at' => $now, + ]; + + return $tracked; + } + + private static function missingColumn(QueryException $exception): bool + { + $message = $exception->getMessage(); + + return in_array((string) $exception->getCode(), ['42S22', '42703'], true) + || str_contains($message, 'no such column') + || str_contains($message, 'Unknown column') + || str_contains($message, 'does not exist'); + } + + private static function taskModel(): Model + { + return ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel(); + } + + private static function summaryModel(): Model + { + return ConfiguredV2Models::query('run_summary_model', WorkflowRunSummary::class)->getModel(); + } + + private static function timestamp(mixed $value): ?string + { + return $value instanceof \DateTimeInterface ? $value->format('Y-m-d\TH:i:s.up') : null; } } diff --git a/src/config/workflows.php b/src/config/workflows.php index bfb8a91a2..7c43d282e 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -375,12 +375,18 @@ Workflow\V2\Support\WorkflowTaskLease::DEFAULT_SECONDS, ), - // A workflow task whose lease expires without completing, typically - // because the queue worker killed it at its job timeout, is retried - // after task repair. After this many consecutive lost leases the run - // shows as needing attention (repair_blocked_reason - // workflow_task_lease_lost). With fail_run enabled, the next claim - // instead fails the run with a typed task failure. Off by default. + // A workflow task whose lease expires without completing (its worker + // was killed, timed out, or lost its connection) is retried after task + // repair. After this many consecutive lost leases the run summary and + // describe carry task_attention. With fail_run enabled (off by + // default; requires a threshold of at least 2), the next claim fails + // the run with a typed task failure instead. + // The queue job timeout every workflow task job carries, replacing + // the worker's --timeout for workflow tasks only. Unset, it is the + // workflow task lease less 30 s. It is always kept below the lease + // and 10 s below the queue connection's retry_after. + 'workflow_task_job_timeout_seconds' => env('DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS'), + 'workflow_task_lost_leases' => [ 'attention_threshold' => (int) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD', 3), 'fail_run' => (bool) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN', false), diff --git a/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php b/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php new file mode 100644 index 000000000..9217d0462 --- /dev/null +++ b/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php @@ -0,0 +1,103 @@ +getConnection()); + + foreach (self::taskTables() as $table) { + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'lost_lease_count')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->unsignedInteger('lost_lease_count') + ->default(0); + }); + }); + } + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'last_lease_lost_at')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->timestamp('last_lease_lost_at', 6) + ->nullable(); + }); + }); + } + } + + foreach (self::summaryTables() as $table) { + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'task_attention')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->json('task_attention') + ->nullable(); + }); + }); + } + } + } + + public function down(): void + { + $schema = Schema::connection($this->getConnection()); + + foreach (self::summaryTables() as $table) { + if ($schema->hasTable($table) && $schema->hasColumn($table, 'task_attention')) { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->dropColumn('task_attention'); + }); + } + } + + foreach (self::taskTables() as $table) { + foreach (['last_lease_lost_at', 'lost_lease_count'] as $column) { + if ($schema->hasTable($table) && $schema->hasColumn($table, $column)) { + $schema->table($table, static function (Blueprint $blueprint) use ($column): void { + $blueprint->dropColumn($column); + }); + } + } + } + } + + /** + * @return list + */ + private static function taskTables(): array + { + $model = ConfiguredV2Models::resolve('task_model', WorkflowTask::class); + + return array_values(array_unique(['workflow_tasks', (new $model())->getTable()])); + } + + /** + * @return list + */ + private static function summaryTables(): array + { + $model = ConfiguredV2Models::resolve('run_summary_model', WorkflowRunSummary::class); + + return array_values(array_unique(['workflow_run_summaries', (new $model())->getTable()])); + } +}; diff --git a/src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php b/src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php deleted file mode 100644 index 274c8e02a..000000000 --- a/src/migrations/2026_11_18_000100_add_lost_lease_tracking_to_workflow_tasks.php +++ /dev/null @@ -1,43 +0,0 @@ -getConnection()); - foreach (array_unique(['workflow_tasks', (new $model())->getTable()]) as $tableName) { - if ($schema->hasTable($tableName) && ! $schema->hasColumn($tableName, 'lost_lease_count')) { - $schema->table($tableName, static function (Blueprint $table): void { - $table->unsignedInteger('lost_lease_count') - ->default(0); - $table->timestamp('last_lease_lost_at', 6) - ->nullable(); - }); - } - } - } - - public function down(): void - { - $model = ConfiguredV2Models::resolve('task_model', WorkflowTask::class); - $schema = Schema::connection($this->getConnection()); - foreach (array_unique(['workflow_tasks', (new $model())->getTable()]) as $tableName) { - if ($schema->hasTable($tableName) && $schema->hasColumn($tableName, 'lost_lease_count')) { - $schema->table($tableName, static function (Blueprint $table): void { - $table->dropColumn(['lost_lease_count', 'last_lease_lost_at']); - }); - } - } - } -}; diff --git a/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php b/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php new file mode 100644 index 000000000..f73a17846 --- /dev/null +++ b/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php @@ -0,0 +1,82 @@ +migration(); + $this->assertFalse($migration->withinTransaction); + + $source = (string) file_get_contents(self::path()); + $this->assertSame(3, substr_count($source, 'withSchemaLockBudget')); + // No backfill: existing tasks have lost no lease and need no attention. + $this->assertDoesNotMatchRegularExpression('/->\s*(update|insert|upsert|delete)\s*\(/', $source); + } + + public function testItAddsTheColumnsToPopulatedTablesAndCanRunAgain(): void + { + $migration = $this->migration(); + $migration->down(); + $this->assertFalse(Schema::hasColumn('workflow_tasks', 'lost_lease_count')); + $this->assertFalse(Schema::hasColumn('workflow_tasks', 'last_lease_lost_at')); + $this->assertFalse(Schema::hasColumn('workflow_run_summaries', 'task_attention')); + + $ids = []; + foreach (range(1, 5) as $number) { + $ids[] = $id = (string) Str::ulid(); + DB::table('workflow_tasks')->insert([ + 'id' => $id, + 'workflow_run_id' => (string) Str::ulid(), + 'task_type' => 'workflow', + 'status' => 'ready', + 'attempt_count' => 0, + 'repair_count' => 0, + 'created_at' => now(), + 'updated_at' => now(), + ]); + } + + $migration->up(); + // An interrupted or repeated run skips work already done. + $migration->up(); + + $this->assertTrue(Schema::hasColumn('workflow_tasks', 'lost_lease_count')); + $this->assertTrue(Schema::hasColumn('workflow_tasks', 'last_lease_lost_at')); + $this->assertTrue(Schema::hasColumn('workflow_run_summaries', 'task_attention')); + $this->assertSame( + 5, + DB::table('workflow_tasks') + ->whereIn('id', $ids) + ->where('lost_lease_count', 0) + ->whereNull('last_lease_lost_at') + ->count(), + ); + } + + private function migration(): WorkflowMigration + { + return require self::path(); + } + + private static function path(): string + { + return dirname(__DIR__, 3) . '/src/migrations/' . self::NAME . '.php'; + } +} diff --git a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php index b43f1a03b..48cf829f8 100644 --- a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php +++ b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php @@ -4,13 +4,17 @@ namespace Tests\Feature\V2; +use Illuminate\Database\Schema\Blueprint; use Illuminate\Queue\Events\WorkerStarting; use Illuminate\Queue\WorkerOptions; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Queue; +use Illuminate\Support\Facades\Schema; +use InvalidArgumentException; use Tests\Fixtures\V2\TestGreetingWorkflow; use Tests\TestCase; use Workflow\V2\Enums\FailureCategory; @@ -27,6 +31,7 @@ use Workflow\V2\Support\OperatorMetrics; use Workflow\V2\Support\RunDetailView; use Workflow\V2\Support\WorkerTimeoutAdvisory; +use Workflow\V2\Support\WorkflowTaskJobTimeout; use Workflow\V2\Support\WorkflowTaskLeaseLoss; use Workflow\V2\TaskWatchdog; use Workflow\V2\WorkflowStub; @@ -53,22 +58,32 @@ public function testRepeatedLostLeasesNeedAttentionWithoutChangingTheRun(): void $this->assertSame(TaskStatus::Ready, $task->status); $this->assertSame(2, $task->lost_lease_count); $this->assertNotNull($task->last_lease_lost_at); - $this->assertNotSame(WorkflowTaskLeaseLoss::CODE, $this->summary($runId)->repair_blocked_reason); - $this->assertFalse((bool) $this->summary($runId)->repair_attention); + $this->assertNull($this->summary($runId)->task_attention); + $summaryBelow = $this->consumerFields($this->summary($runId)->getAttributes()); + $detailBelow = $this->repairFields(RunDetailView::forRun(WorkflowRun::query()->findOrFail($runId))); $this->loseLease($runId); + // At the threshold only the additive task_attention field changes: + // repair-blocked reasons, repair attention and can_repair do not. $summary = $this->summary($runId); - $this->assertSame(WorkflowTaskLeaseLoss::CODE, $summary->repair_blocked_reason); - $this->assertTrue((bool) $summary->repair_attention); + // MySQL's JSON column may reorder keys. + $this->assertEquals([ + 'reason' => WorkflowTaskLeaseLoss::CODE, + 'consecutive' => 3, + 'threshold' => 3, + 'workflow_task_id' => $task->id, + ], array_diff_key($summary->task_attention, [ + 'last_lease_lost_at' => true, + ])); + $this->assertSame($summaryBelow, $this->consumerFields($summary->getAttributes())); $detail = RunDetailView::forRun(WorkflowRun::query()->findOrFail($runId)); - $this->assertSame(WorkflowTaskLeaseLoss::CODE, $detail['repair_blocked_reason']); - $this->assertTrue($detail['repair_attention']); - $this->assertSame('Task Timing Out', $detail['repair_blocked']['label']); + $this->assertSame($summary->task_attention, $detail['task_attention']); + $this->assertSame($detailBelow, $this->repairFields($detail)); $metrics = OperatorMetrics::snapshot(); - $this->assertSame(1, $metrics['runs']['workflow_task_lease_lost']); + $this->assertSame(1, $metrics['runs']['task_attention']); $this->assertSame(1, $metrics['tasks']['lease_lost_repeatedly']); $this->assertSame(3, $metrics['tasks']['max_lost_lease_count']); @@ -94,9 +109,7 @@ public function testRepeatedLostLeasesNeedAttentionWithoutChangingTheRun(): void ], $this->historyTypes($runId)); // The completed task's replacement starts with no lost leases. - $summary = $this->summary($runId); - $this->assertNotSame(WorkflowTaskLeaseLoss::CODE, $summary->repair_blocked_reason); - $this->assertFalse((bool) $summary->repair_attention); + $this->assertNull($this->summary($runId)->task_attention); } public function testFailingTheRunAfterLostLeasesIsOptIn(): void @@ -149,33 +162,138 @@ public function testFailingTheRunWaitsForTheAttentionThreshold(): void $this->assertContains(HistoryEventType::ActivityScheduled->value, $this->historyTypes($runId)); } - public function testWorkerStartWarnsWhenTheJobTimeoutIsBelowTheTaskLease(): void + public function testWorkflowTaskJobsCarryTheirOwnTimeoutBelowLeaseAndRetryAfter(): void { + config([ + 'workflows.v2.workflow_task_lease_seconds' => 300, + 'queue.connections.redis.retry_after' => 600, + ]); + $this->assertSame(270, WorkflowTaskJobTimeout::seconds('redis')); + + // Never at or past retry_after, so the queue never hands a running + // job to another worker. + config([ + 'queue.connections.redis.retry_after' => 90, + ]); + $this->assertSame(80, WorkflowTaskJobTimeout::seconds('redis')); + + config([ + 'queue.connections.redis.retry_after' => 600, + 'workflows.v2.workflow_task_job_timeout_seconds' => 120, + ]); + $this->assertSame(120, WorkflowTaskJobTimeout::seconds('redis')); + config([ + 'workflows.v2.workflow_task_job_timeout_seconds' => 900, + ]); + $this->assertSame(270, WorkflowTaskJobTimeout::seconds('redis')); + + // The dispatched job carries it; Laravel's worker uses it in place of + // the worker's --timeout. + config([ + 'workflows.v2.workflow_task_job_timeout_seconds' => null, + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout'); + $workflow->start('Taylor'); + Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === 270); + } + + public function testWorkerStartWarnsOnlyWorkflowQueueWorkersCappedByRetryAfter(): void + { + config([ + 'queue.default' => 'redis', + 'workflows.v2.connection' => null, + 'workflows.v2.queue' => null, + 'queue.connections.redis.queue' => 'default', + 'queue.connections.redis.retry_after' => 90, + 'workflows.v2.workflow_task_lease_seconds' => 300, + ]); WorkerTimeoutAdvisory::reset(); Log::spy(); $options = new WorkerOptions(timeout: 60); - Event::dispatch(new WorkerStarting('redis', 'default', $options)); + // A worker that serves only other queues is never warned. + Event::dispatch(new WorkerStarting('redis', 'activities', $options)); + Log::shouldNotHaveReceived('warning'); + + Event::dispatch(new WorkerStarting('redis', 'high,default', $options)); Event::dispatch(new WorkerStarting('redis', 'default', $options)); if (extension_loaded('pcntl')) { Log::shouldHaveReceived('warning') ->withArgs(static fn (string $message, array $context): bool => - $context['code'] === 'worker_timeout_below_workflow_task_lease' - && $context['worker_timeout_seconds'] === 60 - && str_contains($message, '--timeout=0')) + $context['code'] === 'workflow_task_timeout_capped_by_retry_after' + && $context['retry_after_seconds'] === 90 + && $context['workflow_task_timeout_seconds'] === 80 + && str_contains($message, 'retry_after above 280 s')) ->once(); } else { Log::shouldNotHaveReceived('warning'); } - $this->assertNull(WorkerTimeoutAdvisory::message(0, true, 300)); - $this->assertNull(WorkerTimeoutAdvisory::message(300, true, 300)); - $this->assertNull(WorkerTimeoutAdvisory::message(60, false, 300)); - $this->assertNotNull(WorkerTimeoutAdvisory::message(60, true, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(true, 270, 270, 600, 'redis')); + $this->assertNull(WorkerTimeoutAdvisory::message(false, 80, 270, 90, 'redis')); + $this->assertNotNull(WorkerTimeoutAdvisory::message(true, 80, 270, 90, 'redis')); WorkerTimeoutAdvisory::reset(); } + public function testFailingRunsRequiresAThresholdOfAtLeastTwo(): void + { + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 1, + ]); + + try { + WorkflowTaskLeaseLoss::validateConfiguration(); + $this->fail('A failing threshold of one must be refused.'); + } catch (InvalidArgumentException $exception) { + $this->assertStringContainsString('at least 2', $exception->getMessage()); + } + + config([ + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 2, + ]); + WorkflowTaskLeaseLoss::validateConfiguration(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => false, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 1, + ]); + WorkflowTaskLeaseLoss::validateConfiguration(); + $this->addToAssertionCount(2); + } + + public function testASchemaRollbackDegradesToUntracked(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-schema-rollback'); + $workflow->start('Taylor'); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $this->assertTrue(WorkflowTaskLeaseLoss::tracked()); + + // The process still believes the columns exist. + Schema::table('workflow_tasks', static function (Blueprint $table): void { + $table->dropColumn(['last_lease_lost_at', 'lost_lease_count']); + }); + + try { + DB::transaction(function () use ($run): void { + $this->assertNull(WorkflowTaskLeaseLoss::taskAttention($run)); + // The caller's transaction is still usable. + $this->assertSame(1, WorkflowRun::query()->whereKey($run->id)->count()); + }); + $this->assertFalse(WorkflowTaskLeaseLoss::tracked()); + $this->assertSame(0, OperatorMetrics::snapshot()['tasks']['lease_lost_repeatedly']); + } finally { + Schema::table('workflow_tasks', static function (Blueprint $table): void { + $table->unsignedInteger('lost_lease_count') + ->default(0); + $table->timestamp('last_lease_lost_at', 6) + ->nullable(); + }); + } + } + protected function startQueueWorkersAutomatically(): bool { return false; @@ -202,6 +320,56 @@ private function loseLease(string $runId): void $this->assertSame(TaskStatus::Ready, $this->workflowTask($runId)->status); } + /** + * The fields an existing consumer reads, without the additive field and + * the timestamps a repair pass refreshes. + * + * @param array $fields + * @return array + */ + private function consumerFields(array $fields): array + { + return array_diff_key($fields, array_flip([ + 'task_attention', + 'updated_at', + 'next_task_at', + 'next_task_lease_expires_at', + 'wait_started_at', + 'wait_deadline_at', + 'tasks', + 'timeline', + 'waits', + 'liveness_reason', + 'last_progress_at', + 'sort_timestamp', + 'sort_key', + ])); + } + + /** + * The describe fields a repair decision reads. + * + * @param array $detail + * @return array + */ + private function repairFields(array $detail): array + { + return array_intersect_key($detail, array_flip([ + 'status', + 'wait_kind', + 'liveness_state', + 'can_repair', + 'repair_blocked_reason', + 'repair_attention', + 'repair_blocked', + 'task_problem', + 'can_cancel', + 'can_terminate', + 'can_signal', + 'can_update', + ])); + } + private function workflowTask(string $runId): WorkflowTask { return WorkflowTask::query() diff --git a/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json index 9a9701bb9..8ef2ba2b6 100644 --- a/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json +++ b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json @@ -20,7 +20,7 @@ "issue": "durable-workflow/workflow#1231", "attention_threshold": 3, "fail_run": true, - "note": "The run's only workflow task lost its lease three times without completing, as when a pcntl queue worker kills it at the job timeout. With workflows.v2.workflow_task_lost_leases.fail_run enabled, the next claim fails the run with the typed WorkflowTaskLeaseLostException (task_failure) before any workflow code runs. Lost leases themselves write no history; with fail_run off the same run keeps retrying and only shows workflow_task_lease_lost repair attention." + "note": "The run's only workflow task lost its lease three times without completing, as when its worker is killed, times out or loses its connection. With workflows.v2.workflow_task_lost_leases.fail_run enabled, the next claim fails the run with the typed WorkflowTaskLeaseLostException (task_failure) before any workflow code runs. Lost leases themselves write no history; with fail_run off the same run keeps retrying and its summary carries the additive task_attention field." }, "history": [ { @@ -57,14 +57,14 @@ "sequence": 2, "event_type": "WorkflowFailed", "payload": { - "message": "Workflow task lost-lease-task lost its lease 3 times without completing. Its queue worker may be killing it at the job timeout; run workers with --timeout=0 or above the longest workflow task.", + "message": "Workflow task lost-lease-task lost its lease 3 times in a row without completing (its worker was killed, timed out, or lost its connection).", "exception": { "code": 0, "file": "/app/src/V2/Support/WorkflowTaskLeaseLoss.php", "line": 152, "class": "Workflow\\V2\\Exceptions\\WorkflowTaskLeaseLostException", "trace": [], - "message": "Workflow task lost-lease-task lost its lease 3 times without completing. Its queue worker may be killing it at the job timeout; run workers with --timeout=0 or above the longest workflow task." + "message": "Workflow task lost-lease-task lost its lease 3 times in a row without completing (its worker was killed, timed out, or lost its connection)." }, "source_id": "lost-lease-run", "failure_id": "lost-lease-failure", From f5b81dc76436b3d35ced1deb7f39f79e29f6f454 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:22:12 +0000 Subject: [PATCH 6/8] Keep bounded summary projections at their query budget The bounded claim, repair and fresh-run projections already hold the run's open workflow task, so they derive task_attention from it instead of querying the task table again. A bridge test pins that budget. Co-Authored-By: Claude Opus 5.5 --- src/V2/Support/RunSummaryProjector.php | 4 ++-- src/V2/Support/WorkflowTaskLeaseLoss.php | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 401ccabce..c4bf6134f 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -705,7 +705,7 @@ private static function projectOpenWorkflowTask( 'next_task_lease_expires_at' => $task->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), - ...WorkflowTaskLeaseLoss::summaryAttributes($run), + ...WorkflowTaskLeaseLoss::summaryAttributes($run, $task), 'task_problem' => $replayBlockedTask instanceof WorkflowTask || $hasPendingChildResolution || self::claimHasWorkflowTaskProblem($run) @@ -1260,7 +1260,7 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow 'next_task_lease_expires_at' => $nextTask->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), - ...WorkflowTaskLeaseLoss::summaryAttributes($run), + ...WorkflowTaskLeaseLoss::summaryAttributes($run, $nextTask), 'task_problem' => false, 'exception_count' => 0, 'history_event_count' => $historyEventCount, diff --git a/src/V2/Support/WorkflowTaskLeaseLoss.php b/src/V2/Support/WorkflowTaskLeaseLoss.php index 994cc746a..dcfc2a7b4 100644 --- a/src/V2/Support/WorkflowTaskLeaseLoss.php +++ b/src/V2/Support/WorkflowTaskLeaseLoss.php @@ -194,7 +194,7 @@ public static function exceptionFor(WorkflowTask $task): WorkflowTaskLeaseLostEx * * @return array{reason: string, consecutive: int, threshold: int, workflow_task_id: string, last_lease_lost_at: string|null}|null */ - public static function taskAttention(WorkflowRun $run): ?array + public static function taskAttention(WorkflowRun $run, ?WorkflowTask $openWorkflowTask = null): ?array { if ($run->status->isTerminal()) { return null; @@ -203,7 +203,12 @@ public static function taskAttention(WorkflowRun $run): ?array $threshold = self::attentionThreshold(); $task = null; - if ($run->relationLoaded('tasks')) { + if ($openWorkflowTask instanceof WorkflowTask + && $openWorkflowTask->task_type === TaskType::Workflow + && in_array($openWorkflowTask->status, [TaskStatus::Ready, TaskStatus::Leased], true)) { + // A bounded projection already holds the run's open workflow task. + $task = self::lostLeases($openWorkflowTask) >= $threshold ? $openWorkflowTask : null; + } elseif ($run->relationLoaded('tasks')) { foreach ($run->tasks as $candidate) { if ($candidate instanceof WorkflowTask && $candidate->task_type === TaskType::Workflow @@ -254,14 +259,14 @@ public static function taskAttention(WorkflowRun $run): ?array * * @return array */ - public static function summaryAttributes(WorkflowRun $run): array + public static function summaryAttributes(WorkflowRun $run, ?WorkflowTask $openWorkflowTask = null): array { if (! self::summaryTracked()) { return []; } return [ - 'task_attention' => self::taskAttention($run), + 'task_attention' => self::taskAttention($run, $openWorkflowTask), ]; } From e6db477145bcc0392e41f21a80c4cd0d0da87415 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:06:29 +0000 Subject: [PATCH 7/8] Make the workflow task job timeout opt-in Workflow task jobs carry their own queue timeout only when workflows.v2.workflow_task_job_timeout_seconds is configured; unset, the worker's --timeout applies as before. The worker-start warning now fires when the timeout workflow tasks run under is below the lease, and says such tasks are retried after the lease expires and counted as lost leases. Summary writes tolerate a task_attention column dropped after the cached schema check. The README example routes activities to the bounded worker. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +-- README.md | 55 ++++++---- src/Providers/WorkflowServiceProvider.php | 7 +- src/V2/Jobs/RunWorkflowTask.php | 6 +- src/V2/Support/RunSummaryProjector.php | 100 +++++++++++------- src/V2/Support/TaskDispatcher.php | 2 +- src/V2/Support/WorkerTimeoutAdvisory.php | 69 ++++++------ src/V2/Support/WorkflowTaskJobTimeout.php | 51 ++++----- src/V2/Support/WorkflowTaskLeaseLoss.php | 41 +++++++ src/config/workflows.php | 12 +-- .../V2EmbeddedReplayRegressionCorpusTest.php | 2 +- .../V2/V2WorkflowTaskLostLeaseTest.php | 89 ++++++++++++---- 12 files changed, 276 insertions(+), 171 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 140de6fe9..a5cf7089c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,12 +15,13 @@ - A workflow task that keeps losing its lease without completing no longer retries silently forever (#1231). A lease is lost when the worker running the task is killed, times out, or loses its connection. - - Workflow task jobs carry their own queue timeout: the workflow task lease - less 30 s (`workflows.v2.workflow_task_job_timeout_seconds` can lower it), - kept 10 s below the connection's `retry_after`. A short worker `--timeout` - no longer kills workflow tasks, and activity jobs keep the worker's - bounded `--timeout`. A worker on the workflow queue logs one warning at - start when `retry_after` caps workflow tasks below that. + - Opt-in `workflows.v2.workflow_task_job_timeout_seconds` + (`DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS`, unset by default) gives workflow + task jobs their own queue timeout, kept 10 s below the connection's + `retry_after`, in place of the worker's `--timeout`. Unset, the worker's + `--timeout` applies as before. A worker on the workflow queue logs one + warning at start when the timeout its workflow tasks run under is below + the workflow task lease. - Task repair counts each lost lease on the task (`lost_lease_count`, `last_lease_lost_at`). The next claim logs a typed `workflow_task_lease_lost` diagnostic. No history is written, and run diff --git a/README.md b/README.md index 914e95c2e..58035e13b 100644 --- a/README.md +++ b/README.md @@ -65,39 +65,48 @@ php artisan queue:work ``` With the `pcntl` extension, a queue worker kills any job that runs longer than -its timeout. A workflow task that is killed loses its lease, and it is retried -only after the lease expires (`workflows.v2.workflow_task_lease_seconds`, 300 -seconds by default). Workflow task jobs therefore carry their own timeout in -place of the worker's `--timeout`: the lease less 30 seconds, or -`DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS` if that is lower. Activity jobs keep -the worker's bounded `--timeout`, so a hung activity cannot hold a worker -forever. - -Laravel requires a queue connection's `retry_after` to exceed every job's -timeout, so a workflow task's timeout is also kept 10 seconds below its -connection's `retry_after`. With Laravel's default `retry_after` of 90 seconds, -workflow tasks are limited to 80 seconds, and a worker on that queue logs a -warning when it starts. Wide `all()` fan-outs need longer. Give workflow tasks -a queue connection whose `retry_after` exceeds the lease, and run dedicated -workers for it. Activities run on their workflow's queue unless the activity -class sets its own `$connection` or `$queue`: +its timeout: the worker's `--timeout`, 60 seconds by default. A workflow task +killed this way is retried only after its lease expires +(`workflows.v2.workflow_task_lease_seconds`, 300 seconds by default) and is +counted as a lost lease. A worker that serves the workflow queue logs a warning +when it starts if its timeout is below the lease. Wide `all()` fan-outs can need +longer than 60 seconds for one workflow task. + +Raise `--timeout` on the workers that serve workflow tasks to cover your +longest workflow task, keeping the queue connection's `retry_after` above it; +Laravel requires `retry_after` to exceed every job's timeout. To keep a +bounded timeout for activities, give them their own queue and workers. +Activities run on their workflow's connection and queue unless the activity +class sets `$connection` or `$queue`, or the workflow passes `ActivityOptions`: ```php -// config/queue.php: a connection for workflow tasks -'workflows' => [ +// config/queue.php +'redis' => [ 'driver' => 'redis', 'connection' => 'default', - 'queue' => 'workflows', - 'retry_after' => 330, // above the 300-second workflow task lease + 'queue' => 'default', + 'retry_after' => 330, // above the workflow task worker's --timeout ], ``` +```php +final class ChargeCard extends Activity +{ + public ?string $queue = 'activities'; +} +``` + ```bash -# DW_V2_CONNECTION=workflows DW_V2_QUEUE=workflows -php artisan queue:work workflows --queue=workflows -php artisan queue:work --queue=default --timeout=60 # activities, bounded +php artisan queue:work redis --queue=default --timeout=300 # workflow tasks +php artisan queue:work redis --queue=activities --timeout=60 # activities, bounded ``` +From Workflow 2.34.0, `DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS` can instead give +workflow task jobs their own timeout, which Laravel's worker uses in place of +its `--timeout` for those jobs, while activities on the same workers keep the +worker's `--timeout`. It is kept 10 seconds below the connection's +`retry_after`, so raise `retry_after` above it. It is unset by default. + If a workflow task loses its lease three times in a row (`DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD`), the run summary and describe carry an additive `task_attention` field, and operator metrics count diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index e159ad04a..195e389c6 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -241,14 +241,17 @@ public function boot(): void $this->validateCacheBackend(); if (class_exists(\Illuminate\Queue\Events\WorkerStarting::class)) { - // A low retry_after caps how long a workflow task may run on the - // workflow queue; warn that queue's workers once. + // A timeout below the workflow task lease kills workflow tasks that + // are then retried only after the lease; warn workflow-queue + // workers once. Event::listen( \Illuminate\Queue\Events\WorkerStarting::class, static function (object $event): void { + $options = $event->workerOptions ?? null; WorkerTimeoutAdvisory::workerStarting( is_string($event->connectionName ?? null) ? $event->connectionName : null, is_string($event->queue ?? null) ? $event->queue : null, + is_object($options) && is_numeric($options->timeout ?? null) ? (int) $options->timeout : null, ); }, ); diff --git a/src/V2/Jobs/RunWorkflowTask.php b/src/V2/Jobs/RunWorkflowTask.php index 4dec3aacf..2834883e9 100644 --- a/src/V2/Jobs/RunWorkflowTask.php +++ b/src/V2/Jobs/RunWorkflowTask.php @@ -48,8 +48,10 @@ final class RunWorkflowTask implements ShouldQueue public int $tries = 1; /** - * The job's own queue timeout ({@see WorkflowTaskJobTimeout}), set when - * the task is dispatched; null leaves the worker's --timeout in force. + * The job's own queue timeout, set at dispatch only when + * workflows.v2.workflow_task_job_timeout_seconds is configured + * ({@see WorkflowTaskJobTimeout}); null leaves the worker's --timeout in + * force. Workers of earlier releases ignore the serialized value. */ public ?int $timeout = null; diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index c4bf6134f..7172a9bd6 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -573,25 +573,30 @@ private static function upsertFullSummary( array $values, array $failureIds, ): WorkflowRunSummary { - return SqliteTransaction::run( - static function () use ($summaryModel, $run, $values, $failureIds): WorkflowRunSummary { - /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); + // The write sits outside the transaction so a dropped task_attention + // column can be retried in a fresh transaction. + return WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => SqliteTransaction::run( + static function () use ($summaryModel, $run, $values, $failureIds): WorkflowRunSummary { + /** @var WorkflowRunSummary $summary */ + $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ + 'id' => $run->id, + ], $values,); - ChildProjectionRepairStore::markSnapshotFailuresCounted($run, $failureIds); + ChildProjectionRepairStore::markSnapshotFailuresCounted($run, $failureIds); - return $summary; - }, - admission: static function () use ($summaryModel, $run): void { - // Acquire SQLite's writer before updateOrCreate reads or model hooks run. - // A rejected admission may retry; the projection body must execute once. - $summaryModel::query()->where('id', $run->id)->toBase()->update([ - 'id' => $run->id, - ]); - }, - connection: $run->getConnection(), + return $summary; + }, + admission: static function () use ($summaryModel, $run): void { + // Acquire SQLite's writer before updateOrCreate reads or model hooks run. + // A rejected admission may retry; the projection body must execute once. + $summaryModel::query()->where('id', $run->id)->toBase()->update([ + 'id' => $run->id, + ]); + }, + connection: $run->getConnection(), + ), ); } @@ -745,30 +750,40 @@ private static function projectOpenWorkflowTask( if ($exceptionCount['event_ids'] === []) { /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); + $summary = WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => IdempotentProjectionUpsert::upsert( + $summaryModel, + [ + 'id' => $run->id, + ], + $values, + ), + ); } else { // Best-effort child projection may catch a later projection error. // Keep the count and its event marker in one transaction so neither // half can survive alone before the repair is retried. /** @var WorkflowRunSummary $summary */ - $summary = $run->getConnection() - ->transaction(static function () use ( - $summaryModel, - $run, - $values, - $exceptionCount - ): WorkflowRunSummary { - /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); - - ChildProjectionRepairStore::markFailedChildrenCounted($run, $exceptionCount['event_ids']); - - return $summary; - }); + $summary = WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => $run->getConnection() + ->transaction(static function () use ( + $summaryModel, + $run, + $values, + $exceptionCount + ): WorkflowRunSummary { + /** @var WorkflowRunSummary $summary */ + $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ + 'id' => $run->id, + ], $values,); + + ChildProjectionRepairStore::markFailedChildrenCounted($run, $exceptionCount['event_ids']); + + return $summary; + }), + ); } foreach ($childResolutionEvents as $childResolutionEvent) { @@ -1207,11 +1222,7 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow $summaryModel = self::summaryModel(); /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert( - $summaryModel, - [ - 'id' => $run->id, - ], + $summary = WorkflowTaskLeaseLoss::writeSummary( [ 'workflow_instance_id' => $run->workflow_instance_id, 'run_number' => $run->run_number, @@ -1271,6 +1282,13 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow 'created_at' => $run->created_at, 'updated_at' => $run->closed_at ?? $run->last_progress_at ?? $run->updated_at, ], + static fn (array $values): WorkflowRunSummary => IdempotentProjectionUpsert::upsert( + $summaryModel, + [ + 'id' => $run->id, + ], + $values, + ), ); return $summary; diff --git a/src/V2/Support/TaskDispatcher.php b/src/V2/Support/TaskDispatcher.php index 06f215a8e..faac69421 100644 --- a/src/V2/Support/TaskDispatcher.php +++ b/src/V2/Support/TaskDispatcher.php @@ -189,7 +189,7 @@ private static function makeJob(WorkflowTask $task): RunWorkflowTask|RunActivity } if ($job instanceof RunWorkflowTask) { - // A short worker --timeout must not kill workflow tasks. + // Opt-in: workflow tasks get their own timeout only when configured. $job->timeout = WorkflowTaskJobTimeout::seconds($task->connection); } diff --git a/src/V2/Support/WorkerTimeoutAdvisory.php b/src/V2/Support/WorkerTimeoutAdvisory.php index 356cd4279..8494accbb 100644 --- a/src/V2/Support/WorkerTimeoutAdvisory.php +++ b/src/V2/Support/WorkerTimeoutAdvisory.php @@ -7,15 +7,15 @@ use Illuminate\Support\Facades\Log; /** - * Warn once per worker process when its queue connection limits how long a - * workflow task can run. + * Warn once per worker process when the timeout its workflow tasks run under + * is shorter than the workflow task lease. * - * A workflow task's job carries its own timeout ({@see WorkflowTaskJobTimeout}), - * so the worker's `--timeout` no longer applies to it. That timeout is kept - * below the connection's `retry_after`, so a low `retry_after` (90 seconds in - * Laravel's default configuration) still caps workflow tasks well below their - * lease. Only workers that serve the queue the package dispatches workflow - * tasks to are warned, and only where pcntl enforces job timeouts. + * With pcntl, a worker kills a job that outlives its timeout: the worker's + * `--timeout` (60 seconds by default), or the workflow task job timeout when + * `workflows.v2.workflow_task_job_timeout_seconds` is configured. A workflow + * task killed that way is retried only after its lease expires and is counted + * as a lost lease. Only workers that serve the queue the package dispatches + * workflow tasks to are warned. * * @internal */ @@ -24,31 +24,25 @@ final class WorkerTimeoutAdvisory private static bool $warned = false; /** - * The warning for a worker on $connection, or null when workflow tasks - * there can run up to their uncapped timeout. + * The warning, or null when workflow tasks on this worker are not killed + * before their lease ends. */ - public static function message( - bool $pcntlLoaded, - ?int $workflowTaskTimeout, - int $uncappedTimeout, - ?int $retryAfter, - string $connection, - ): ?string { - if (! $pcntlLoaded || ($workflowTaskTimeout !== null && $workflowTaskTimeout >= $uncappedTimeout)) { + public static function message(bool $pcntlLoaded, int $effectiveTimeout, bool $jobTimeout, int $leaseSeconds): ?string + { + if (! $pcntlLoaded || $effectiveTimeout <= 0 || $effectiveTimeout >= $leaseSeconds) { return null; } return sprintf( - '[Durable Workflow] Queue connection "%s" has retry_after = %s s, so a workflow task on it can run ' - . 'for at most %s before the worker kills it and the task is retried after its lease expires. ' - . 'Set this connection\'s retry_after above %d s (the workflow task timeout plus %d s), ideally on ' - . 'a connection dedicated to workflow tasks, so workflow tasks can run for %d s.', - $connection, - $retryAfter === null ? 'unset' : (string) $retryAfter, - $workflowTaskTimeout === null ? 'the worker --timeout' : $workflowTaskTimeout . ' s', - $uncappedTimeout + WorkflowTaskJobTimeout::RETRY_AFTER_MARGIN_SECONDS, - WorkflowTaskJobTimeout::RETRY_AFTER_MARGIN_SECONDS, - $uncappedTimeout, + '[Durable Workflow] This worker kills a workflow task after %d s (%s), less than the %d s workflow task ' + . 'lease. A workflow task killed this way is retried only after its lease expires and is counted as a ' + . 'lost lease. Raise %s to cover your longest workflow task, keeping the queue connection\'s ' + . 'retry_after above it, or set DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS to give workflow tasks their ' + . 'own timeout.', + $effectiveTimeout, + $jobTimeout ? 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS' : 'the worker --timeout', + $leaseSeconds, + $jobTimeout ? 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS' : '--timeout', ); } @@ -77,19 +71,15 @@ public static function servesWorkflowQueue(?string $connection, ?string $queues) return in_array((string) $workflowQueue, $served, true); } - public static function workerStarting(?string $connection, ?string $queues): void + public static function workerStarting(?string $connection, ?string $queues, ?int $workerTimeout): void { if (self::$warned || $connection === null || ! self::servesWorkflowQueue($connection, $queues)) { return; } - $message = self::message( - extension_loaded('pcntl'), - WorkflowTaskJobTimeout::seconds($connection), - WorkflowTaskJobTimeout::uncappedSeconds(), - WorkflowTaskJobTimeout::retryAfter($connection), - $connection, - ); + $jobTimeout = WorkflowTaskJobTimeout::seconds($connection); + $effective = $jobTimeout ?? (int) ($workerTimeout ?? 0); + $message = self::message(extension_loaded('pcntl'), $effective, $jobTimeout !== null, WorkflowTaskLease::seconds()); if ($message === null) { return; @@ -97,11 +87,12 @@ public static function workerStarting(?string $connection, ?string $queues): voi self::$warned = true; Log::warning($message, [ - 'code' => 'workflow_task_timeout_capped_by_retry_after', + 'code' => 'workflow_task_timeout_below_lease', 'connection' => $connection, - 'retry_after_seconds' => WorkflowTaskJobTimeout::retryAfter($connection), - 'workflow_task_timeout_seconds' => WorkflowTaskJobTimeout::seconds($connection), + 'workflow_task_timeout_seconds' => $effective, + 'workflow_task_timeout_source' => $jobTimeout !== null ? 'workflow_task_job_timeout' : 'worker_timeout', 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), + 'retry_after_seconds' => WorkflowTaskJobTimeout::retryAfter($connection), ]); } diff --git a/src/V2/Support/WorkflowTaskJobTimeout.php b/src/V2/Support/WorkflowTaskJobTimeout.php index 6c8b1f3f6..6063e6beb 100644 --- a/src/V2/Support/WorkflowTaskJobTimeout.php +++ b/src/V2/Support/WorkflowTaskJobTimeout.php @@ -5,29 +5,24 @@ namespace Workflow\V2\Support; /** - * The queue job timeout a workflow task carries. + * An optional queue job timeout for workflow tasks. * * With pcntl, Laravel's worker kills a job that runs longer than its timeout, - * and a job's own timeout replaces the worker's `--timeout`. A workflow task - * killed that way loses its lease and is retried, so a short worker timeout - * (60 seconds by default) used to make long workflow tasks impossible. + * and a job's own timeout replaces the worker's `--timeout` for that job. By + * default workflow task jobs carry no timeout, so the worker's `--timeout` + * applies to them as it does to every other job. * - * A workflow task's job therefore carries its own timeout: the configured - * `workflows.v2.workflow_task_job_timeout_seconds`, or by default the - * workflow task lease less a margin, because work past the lease cannot - * complete. It is also kept below the queue connection's `retry_after`, so - * the queue never hands a still-running job to another worker. Activity jobs - * keep the worker's own bounded `--timeout`. + * When `workflows.v2.workflow_task_job_timeout_seconds` is configured, every + * workflow task job carries that timeout instead, so workflow tasks can run + * longer (or shorter) than the activities on the same workers. It is kept + * 10 seconds below the queue connection's `retry_after`, which Laravel + * requires to exceed a job's timeout so the queue never hands a running job + * to another worker. */ final class WorkflowTaskJobTimeout { public const CONFIG_KEY = 'workflows.v2.workflow_task_job_timeout_seconds'; - /** - * Seconds kept between the job timeout and the task lease. - */ - public const LEASE_MARGIN_SECONDS = 30; - /** * Seconds kept between `retry_after` and the job timeout. */ @@ -35,34 +30,32 @@ final class WorkflowTaskJobTimeout /** * The job timeout, in seconds, for a workflow task on $connection, or - * null to leave the worker's `--timeout` in force. + * null (the default) to leave the worker's `--timeout` in force. */ public static function seconds(?string $connection = null): ?int { - $timeout = self::uncappedSeconds(); - $retryAfter = self::retryAfter($connection); + $configured = self::configuredSeconds(); - if ($retryAfter !== null) { - $timeout = min($timeout, $retryAfter - self::RETRY_AFTER_MARGIN_SECONDS); + if ($configured === null) { + return null; } + $retryAfter = self::retryAfter($connection); + $timeout = $retryAfter === null + ? $configured + : min($configured, $retryAfter - self::RETRY_AFTER_MARGIN_SECONDS); + return $timeout >= 1 ? $timeout : null; } /** - * The timeout before any `retry_after` cap: the configured value, or the - * lease less its margin. + * The explicitly configured timeout, or null when none is configured. */ - public static function uncappedSeconds(): int + public static function configuredSeconds(): ?int { $configured = config(self::CONFIG_KEY); - $ceiling = max(1, WorkflowTaskLease::seconds() - self::LEASE_MARGIN_SECONDS); - - if (is_numeric($configured) && (int) $configured >= 1) { - return min((int) $configured, $ceiling); - } - return $ceiling; + return is_numeric($configured) && (int) $configured >= 1 ? (int) $configured : null; } /** diff --git a/src/V2/Support/WorkflowTaskLeaseLoss.php b/src/V2/Support/WorkflowTaskLeaseLoss.php index dcfc2a7b4..e0b10ae8e 100644 --- a/src/V2/Support/WorkflowTaskLeaseLoss.php +++ b/src/V2/Support/WorkflowTaskLeaseLoss.php @@ -315,6 +315,46 @@ public static function untrackedOnMissingColumn(Model $model, callable $query): } } + /** + * Write a run summary whose values may carry task_attention. + * + * A schema rollback can drop the column while the cached check still + * reports it. On that error the cache is forgotten and $write runs again + * without task_attention. A failed statement aborts a caller's open + * PostgreSQL transaction, so there the error is rethrown instead and the + * caller's retry sees the current schema. + * + * @template T + * @param array $values + * @param callable(array): T $write + * @return T + */ + public static function writeSummary(array $values, callable $write): mixed + { + if (! array_key_exists('task_attention', $values)) { + return $write($values); + } + + try { + return $write($values); + } catch (QueryException $exception) { + if (! self::missingColumn($exception)) { + throw $exception; + } + + self::$schemas = []; + $connection = self::summaryModel()->getConnection(); + + if ($connection->transactionLevel() > 0 && $connection->getDriverName() === 'pgsql') { + throw $exception; + } + + unset($values['task_attention']); + + return $write($values); + } + } + private static function hasColumn(Model $model, string $column, bool $fresh = false): bool { $key = $model->getConnectionName() . '|' . $model->getTable() . '|' . $column; @@ -342,6 +382,7 @@ private static function missingColumn(QueryException $exception): bool return in_array((string) $exception->getCode(), ['42S22', '42703'], true) || str_contains($message, 'no such column') + || str_contains($message, 'has no column named') || str_contains($message, 'Unknown column') || str_contains($message, 'does not exist'); } diff --git a/src/config/workflows.php b/src/config/workflows.php index 7c43d282e..ff747f062 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -375,18 +375,18 @@ Workflow\V2\Support\WorkflowTaskLease::DEFAULT_SECONDS, ), + // Opt-in queue job timeout for workflow task jobs. Unset (the default), + // the worker's --timeout applies to workflow tasks like any other job. + // Set, every workflow task job carries this timeout instead, kept 10 s + // below its queue connection's retry_after. + 'workflow_task_job_timeout_seconds' => env('DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS'), + // A workflow task whose lease expires without completing (its worker // was killed, timed out, or lost its connection) is retried after task // repair. After this many consecutive lost leases the run summary and // describe carry task_attention. With fail_run enabled (off by // default; requires a threshold of at least 2), the next claim fails // the run with a typed task failure instead. - // The queue job timeout every workflow task job carries, replacing - // the worker's --timeout for workflow tasks only. Unset, it is the - // workflow task lease less 30 s. It is always kept below the lease - // and 10 s below the queue connection's retry_after. - 'workflow_task_job_timeout_seconds' => env('DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS'), - 'workflow_task_lost_leases' => [ 'attention_threshold' => (int) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD', 3), 'fail_run' => (bool) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN', false), diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 2f605d36a..ec215a0e9 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -502,7 +502,7 @@ public function testOversizedFanOutMemberCorpusIsRefusedBeforeFirstWrite(): void /** * A workflow task that lost its lease three times, as when a pcntl worker - * kills it at the job timeout, fails its run with the typed lease-lost + * kills it at its timeout, fails its run with the typed lease-lost * failure on the next claim when fail_run is enabled (#1231). The run's * history is exactly the evidence: no lost lease writes history. */ diff --git a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php index 48cf829f8..aeaa4b059 100644 --- a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php +++ b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php @@ -30,6 +30,7 @@ use Workflow\V2\Models\WorkflowTask; use Workflow\V2\Support\OperatorMetrics; use Workflow\V2\Support\RunDetailView; +use Workflow\V2\Support\RunSummaryProjector; use Workflow\V2\Support\WorkerTimeoutAdvisory; use Workflow\V2\Support\WorkflowTaskJobTimeout; use Workflow\V2\Support\WorkflowTaskLeaseLoss; @@ -38,7 +39,7 @@ /** * A workflow task that keeps losing its lease, as when the queue worker kills - * it at its job timeout, is counted, diagnosed and surfaced instead of being + * it at its timeout, is counted, diagnosed and surfaced instead of being * retried forever without a signal (#1231). */ final class V2WorkflowTaskLostLeaseTest extends TestCase @@ -162,13 +163,26 @@ public function testFailingTheRunWaitsForTheAttentionThreshold(): void $this->assertContains(HistoryEventType::ActivityScheduled->value, $this->historyTypes($runId)); } - public function testWorkflowTaskJobsCarryTheirOwnTimeoutBelowLeaseAndRetryAfter(): void + public function testWorkflowTaskJobsCarryAJobTimeoutOnlyWhenConfigured(): void { config([ 'workflows.v2.workflow_task_lease_seconds' => 300, + 'queue.connections.redis.retry_after' => 90, + 'workflows.v2.workflow_task_job_timeout_seconds' => null, + ]); + + // Unset, the worker's --timeout applies to workflow tasks unchanged. + $this->assertNull(WorkflowTaskJobTimeout::seconds('redis')); + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout-unset'); + $workflow->start('Taylor'); + Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === null); + + config([ 'queue.connections.redis.retry_after' => 600, + 'workflows.v2.workflow_task_job_timeout_seconds' => 120, ]); - $this->assertSame(270, WorkflowTaskJobTimeout::seconds('redis')); + $this->assertSame(120, WorkflowTaskJobTimeout::seconds('redis')); // Never at or past retry_after, so the queue never hands a running // job to another worker. @@ -179,26 +193,22 @@ public function testWorkflowTaskJobsCarryTheirOwnTimeoutBelowLeaseAndRetryAfter( config([ 'queue.connections.redis.retry_after' => 600, - 'workflows.v2.workflow_task_job_timeout_seconds' => 120, - ]); - $this->assertSame(120, WorkflowTaskJobTimeout::seconds('redis')); - config([ 'workflows.v2.workflow_task_job_timeout_seconds' => 900, ]); - $this->assertSame(270, WorkflowTaskJobTimeout::seconds('redis')); + $this->assertSame(590, WorkflowTaskJobTimeout::seconds('redis')); - // The dispatched job carries it; Laravel's worker uses it in place of - // the worker's --timeout. + // Set, the dispatched job carries it; Laravel's worker uses it in + // place of the worker's --timeout. config([ - 'workflows.v2.workflow_task_job_timeout_seconds' => null, + 'workflows.v2.workflow_task_job_timeout_seconds' => 120, ]); Queue::fake(); - $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout'); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout-set'); $workflow->start('Taylor'); - Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === 270); + Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === 120); } - public function testWorkerStartWarnsOnlyWorkflowQueueWorkersCappedByRetryAfter(): void + public function testWorkerStartWarnsWorkflowQueueWorkersWhoseTimeoutIsBelowTheLease(): void { config([ 'queue.default' => 'redis', @@ -207,6 +217,7 @@ public function testWorkerStartWarnsOnlyWorkflowQueueWorkersCappedByRetryAfter() 'queue.connections.redis.queue' => 'default', 'queue.connections.redis.retry_after' => 90, 'workflows.v2.workflow_task_lease_seconds' => 300, + 'workflows.v2.workflow_task_job_timeout_seconds' => null, ]); WorkerTimeoutAdvisory::reset(); Log::spy(); @@ -222,18 +233,27 @@ public function testWorkerStartWarnsOnlyWorkflowQueueWorkersCappedByRetryAfter() if (extension_loaded('pcntl')) { Log::shouldHaveReceived('warning') ->withArgs(static fn (string $message, array $context): bool => - $context['code'] === 'workflow_task_timeout_capped_by_retry_after' - && $context['retry_after_seconds'] === 90 - && $context['workflow_task_timeout_seconds'] === 80 - && str_contains($message, 'retry_after above 280 s')) + $context['code'] === 'workflow_task_timeout_below_lease' + && $context['workflow_task_timeout_source'] === 'worker_timeout' + && $context['workflow_task_timeout_seconds'] === 60 + && $context['workflow_task_lease_seconds'] === 300 + && str_contains($message, 'retried only after its lease expires') + && str_contains($message, 'counted as a lost lease') + && str_contains($message, 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS')) ->once(); } else { Log::shouldNotHaveReceived('warning'); } - $this->assertNull(WorkerTimeoutAdvisory::message(true, 270, 270, 600, 'redis')); - $this->assertNull(WorkerTimeoutAdvisory::message(false, 80, 270, 90, 'redis')); - $this->assertNotNull(WorkerTimeoutAdvisory::message(true, 80, 270, 90, 'redis')); + // No warning without pcntl, for --timeout=0, or at or above the lease. + $this->assertNull(WorkerTimeoutAdvisory::message(false, 60, false, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(true, 0, false, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(true, 300, false, 300)); + $this->assertStringContainsString('the worker --timeout', (string) WorkerTimeoutAdvisory::message(true, 60, false, 300)); + $this->assertStringContainsString( + 'after 80 s (DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS)', + (string) WorkerTimeoutAdvisory::message(true, 80, true, 300), + ); WorkerTimeoutAdvisory::reset(); } @@ -294,6 +314,33 @@ public function testASchemaRollbackDegradesToUntracked(): void } } + public function testASummaryColumnRollbackDegradesSummaryWrites(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-summary-rollback'); + $workflow->start('Taylor'); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $this->assertTrue(WorkflowTaskLeaseLoss::summaryTracked()); + + // The process still believes the summary column exists. + Schema::table('workflow_run_summaries', static function (Blueprint $table): void { + $table->dropColumn('task_attention'); + }); + + try { + $summary = RunSummaryProjector::project($run->refresh()); + $this->assertSame($run->id, $summary->id); + $this->assertFalse(array_key_exists('task_attention', $summary->getAttributes())); + $this->assertFalse(WorkflowTaskLeaseLoss::summaryTracked()); + $this->assertNull(RunDetailView::forRun($run->refresh())['task_attention']); + } finally { + Schema::table('workflow_run_summaries', static function (Blueprint $table): void { + $table->json('task_attention') + ->nullable(); + }); + } + } + protected function startQueueWorkersAutomatically(): bool { return false; From 2a46d5f9c943adf39956450d076ca35e4064a883 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:21:22 +0000 Subject: [PATCH 8/8] Apply coding standard fixes Co-Authored-By: Claude Opus 5.5 --- src/V2/Support/RunSummaryProjector.php | 2 +- src/V2/Support/WorkerTimeoutAdvisory.php | 15 ++++++++++++--- tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php | 5 ++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 969278cb7..41f6ecb41 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -778,7 +778,7 @@ private static function projectOpenWorkflowTask( /** @var WorkflowRunSummary $summary */ $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ 'id' => $run->id, - ], $values,); + ], $values, ); ChildProjectionRepairStore::markFailedChildrenCounted($run, $exceptionCount['event_ids']); diff --git a/src/V2/Support/WorkerTimeoutAdvisory.php b/src/V2/Support/WorkerTimeoutAdvisory.php index 8494accbb..0614f6d14 100644 --- a/src/V2/Support/WorkerTimeoutAdvisory.php +++ b/src/V2/Support/WorkerTimeoutAdvisory.php @@ -27,8 +27,12 @@ final class WorkerTimeoutAdvisory * The warning, or null when workflow tasks on this worker are not killed * before their lease ends. */ - public static function message(bool $pcntlLoaded, int $effectiveTimeout, bool $jobTimeout, int $leaseSeconds): ?string - { + public static function message( + bool $pcntlLoaded, + int $effectiveTimeout, + bool $jobTimeout, + int $leaseSeconds + ): ?string { if (! $pcntlLoaded || $effectiveTimeout <= 0 || $effectiveTimeout >= $leaseSeconds) { return null; } @@ -79,7 +83,12 @@ public static function workerStarting(?string $connection, ?string $queues, ?int $jobTimeout = WorkflowTaskJobTimeout::seconds($connection); $effective = $jobTimeout ?? (int) ($workerTimeout ?? 0); - $message = self::message(extension_loaded('pcntl'), $effective, $jobTimeout !== null, WorkflowTaskLease::seconds()); + $message = self::message( + extension_loaded('pcntl'), + $effective, + $jobTimeout !== null, + WorkflowTaskLease::seconds() + ); if ($message === null) { return; diff --git a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php index aeaa4b059..bf333a217 100644 --- a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php +++ b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php @@ -249,7 +249,10 @@ public function testWorkerStartWarnsWorkflowQueueWorkersWhoseTimeoutIsBelowTheLe $this->assertNull(WorkerTimeoutAdvisory::message(false, 60, false, 300)); $this->assertNull(WorkerTimeoutAdvisory::message(true, 0, false, 300)); $this->assertNull(WorkerTimeoutAdvisory::message(true, 300, false, 300)); - $this->assertStringContainsString('the worker --timeout', (string) WorkerTimeoutAdvisory::message(true, 60, false, 300)); + $this->assertStringContainsString( + 'the worker --timeout', + (string) WorkerTimeoutAdvisory::message(true, 60, false, 300) + ); $this->assertStringContainsString( 'after 80 s (DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS)', (string) WorkerTimeoutAdvisory::message(true, 80, true, 300),