diff --git a/.github/feature-test-timings.json b/.github/feature-test-timings.json index 0b6164c6b..9206108fb 100644 --- a/.github/feature-test-timings.json +++ b/.github/feature-test-timings.json @@ -139,6 +139,7 @@ "tests/Feature/V2/V2LifecycleEventTest.php": 3.430998, "tests/Feature/V2/V2LocalActivityTest.php": 2.806176, "tests/Feature/V2/V2LocalEntityReferenceAuthorityTest.php": 9.066242, + "tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php": 10.0, "tests/Feature/V2/V2MachineDefinitionTest.php": 13.210704, "tests/Feature/V2/V2MemoUpsertTest.php": 10.058037, "tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 4.673203, @@ -244,6 +245,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.653342, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 12.837294, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 269.332131, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 20.0, "tests/Feature/V2/V2WorkflowTest.php": 161.306516, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 11.990609, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.888561, @@ -395,6 +397,7 @@ "tests/Feature/V2/V2LifecycleEventTest.php": 2.681959, "tests/Feature/V2/V2LocalActivityTest.php": 1.202231, "tests/Feature/V2/V2LocalEntityReferenceAuthorityTest.php": 10.707134, + "tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php": 3.0, "tests/Feature/V2/V2MachineDefinitionTest.php": 9.882914, "tests/Feature/V2/V2MemoUpsertTest.php": 5.262701, "tests/Feature/V2/V2MessageCursorContinueAsNewTest.php": 1.910908, @@ -500,6 +503,7 @@ "tests/Feature/V2/V2WorkflowStubContractTest.php": 0.571702, "tests/Feature/V2/V2WorkflowStubFakeTest.php": 10.15931, "tests/Feature/V2/V2WorkflowTaskBridgeTest.php": 149.713579, + "tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php": 15.0, "tests/Feature/V2/V2WorkflowTest.php": 115.227937, "tests/Feature/V2/V2WorkflowTimeoutTest.php": 6.892443, "tests/Feature/V2/V2WorkflowVersionPinningTest.php": 0.853134, diff --git a/CHANGELOG.md b/CHANGELOG.md index bfe2f2264..48b7e6bea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,33 @@ definitions, rules, budgets, child bindings and native child start in 2.29.0. Docblocks and documentation only; no behavior changes. +- A workflow task that keeps losing its lease without completing no longer + retries silently forever (#1231). A lease is lost when the worker running + the task is killed, times out, or loses its connection. + - Opt-in `workflows.v2.workflow_task_job_timeout_seconds` + (`DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS`, unset by default) gives workflow + task jobs their own queue timeout, kept 10 s below the connection's + `retry_after`, in place of the worker's `--timeout`. Unset, the worker's + `--timeout` applies as before. A worker on the workflow queue logs one + warning at start when the timeout its workflow tasks run under is below + the workflow task lease. + - Task repair counts each lost lease on the task (`lost_lease_count`, + `last_lease_lost_at`). The next claim logs a typed + `workflow_task_lease_lost` diagnostic. No history is written, and run + semantics are unchanged. + - At three consecutive lost leases + (`workflows.v2.workflow_task_lost_leases.attention_threshold`), the run + summary, list items and describe carry an additive `task_attention` field + (`reason`, `consecutive`, `threshold`), and operator metrics add + `runs.task_attention`, `tasks.lease_lost_repeatedly` and + `tasks.max_lost_lease_count`. Repair-blocked reasons, repair attention + and `can_repair` are unchanged. + - Opt-in (`workflows.v2.workflow_task_lost_leases.fail_run`, off by default; + boot refuses a threshold below 2): at the threshold, the next claim fails + the run with a typed `WorkflowTaskLeaseLostException` (`task_failure`) + instead of retrying. + - The migration adds the columns outside a transaction under the schema + lock budget. - Workflow definition and `await()` condition fingerprints are versioned, and a new opt-in version 2 makes comment-only and whitespace-only edits replay-safe (#1227). The prefix names the version: diff --git a/README.md b/README.md index 63c323e4f..58035e13b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,55 @@ Run a Laravel queue worker or Horizon to execute workflows and activities: php artisan queue:work ``` +With the `pcntl` extension, a queue worker kills any job that runs longer than +its timeout: the worker's `--timeout`, 60 seconds by default. A workflow task +killed this way is retried only after its lease expires +(`workflows.v2.workflow_task_lease_seconds`, 300 seconds by default) and is +counted as a lost lease. A worker that serves the workflow queue logs a warning +when it starts if its timeout is below the lease. Wide `all()` fan-outs can need +longer than 60 seconds for one workflow task. + +Raise `--timeout` on the workers that serve workflow tasks to cover your +longest workflow task, keeping the queue connection's `retry_after` above it; +Laravel requires `retry_after` to exceed every job's timeout. To keep a +bounded timeout for activities, give them their own queue and workers. +Activities run on their workflow's connection and queue unless the activity +class sets `$connection` or `$queue`, or the workflow passes `ActivityOptions`: + +```php +// config/queue.php +'redis' => [ + 'driver' => 'redis', + 'connection' => 'default', + 'queue' => 'default', + 'retry_after' => 330, // above the workflow task worker's --timeout +], +``` + +```php +final class ChargeCard extends Activity +{ + public ?string $queue = 'activities'; +} +``` + +```bash +php artisan queue:work redis --queue=default --timeout=300 # workflow tasks +php artisan queue:work redis --queue=activities --timeout=60 # activities, bounded +``` + +From Workflow 2.34.0, `DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS` can instead give +workflow task jobs their own timeout, which Laravel's worker uses in place of +its `--timeout` for those jobs, while activities on the same workers keep the +worker's `--timeout`. It is kept 10 seconds below the connection's +`retry_after`, so raise `retry_after` above it. It is unset by default. + +If a workflow task loses its lease three times in a row +(`DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD`), the run summary and +describe carry an additive `task_attention` field, and operator metrics count +it. Set `DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN=true` to fail such a run with a +typed task failure instead; that requires a threshold of at least 2. + For the default database queue on SQLite, failed reservation writes preserve unclaimed work after a proven rollback. See [SQLite queue reservations](docs/sqlite-queue-reservations.md) for the supported framework versions and custom-connector boundary. diff --git a/docs/architecture/operational-liveness.md b/docs/architecture/operational-liveness.md index bebd81351..7faa7d15f 100644 --- a/docs/architecture/operational-liveness.md +++ b/docs/architecture/operational-liveness.md @@ -368,6 +368,14 @@ codes that a caller may observe: Adding a blocked reason is a protocol change; deleting one is a protocol change. +A workflow task that keeps losing its lease without completing (its +worker was killed, timed out, or lost its connection) is not a blocked +repair: repair keeps returning it to ready and `can_repair` is unchanged. +It is reported in the separate, additive `task_attention` field of the +run summary, list items and describe, set once the open workflow task has +lost `workflows.v2.workflow_task_lost_leases.attention_threshold` leases +in a row (3 by default) and null otherwise. + ## Repair cadence and bounded backoff The repair path is bounded. An automated recovery pass cannot diff --git a/scripts/ci/upgrade-journey-lib.sh b/scripts/ci/upgrade-journey-lib.sh index f60c65a3c..5bc751fc1 100644 --- a/scripts/ci/upgrade-journey-lib.sh +++ b/scripts/ci/upgrade-journey-lib.sh @@ -145,10 +145,27 @@ artisan_output() { # Runs migrate:rollback, which must be refused because retained compensation # scopes exist. Fails unless the refusal comes from the package preflight. +# The migration batch that holds the compensation ledger. A candidate that adds +# its own migrations runs them in a later batch, which a plain rollback would +# undo first without reaching the ledger. +ledger_batch() { + COLUMNS=1000 php artisan migrate:status --no-ansi --no-interaction 2>/dev/null \ + | grep -E '_create_compensation_scopes' \ + | grep -oE '\[[0-9]+\]' \ + | tr -d '[]' \ + | head -n 1 || true +} + expect_rollback_refused() { - local rollback_output + local rollback_output batch + local -a batch_option=() + + batch=$(ledger_batch) + if [ -n "$batch" ]; then + batch_option=("--batch=$batch") + fi - if rollback_output=$(COLUMNS=1000 php artisan migrate:rollback --force --no-interaction 2>&1 | tr -s ' \n' ' '; exit "${PIPESTATUS[0]}"); then + if rollback_output=$(COLUMNS=1000 php artisan migrate:rollback "${batch_option[@]}" --force --no-interaction 2>&1 | tr -s ' \n' ' '; exit "${PIPESTATUS[0]}"); then printf '%s\n' "$rollback_output" >&2 echo "$1" >&2 exit 1 diff --git a/src/Providers/WorkflowServiceProvider.php b/src/Providers/WorkflowServiceProvider.php index 92d3bd383..195e389c6 100644 --- a/src/Providers/WorkflowServiceProvider.php +++ b/src/Providers/WorkflowServiceProvider.php @@ -79,7 +79,9 @@ use Workflow\V2\Support\SqliteSavepointRelease; use Workflow\V2\Support\TaskFairnessState; use Workflow\V2\Support\TypeRegistry; +use Workflow\V2\Support\WorkerTimeoutAdvisory; use Workflow\V2\Support\WorkflowModeGuard; +use Workflow\V2\Support\WorkflowTaskLeaseLoss; use Workflow\Watchdog; final class WorkflowServiceProvider extends ServiceProvider @@ -222,6 +224,7 @@ public function boot(): void // Refuse an unsupported definition fingerprint version before any // workflow can run; at runtime the setting never fails a run. SourceFingerprint::validateConfiguration(); + WorkflowTaskLeaseLoss::validateConfiguration(); $this->configureSqliteStorageConnection(); // Register long-poll wake signal observers @@ -237,6 +240,23 @@ public function boot(): void // Validate cache backend for multi-node deployments $this->validateCacheBackend(); + if (class_exists(\Illuminate\Queue\Events\WorkerStarting::class)) { + // A timeout below the workflow task lease kills workflow tasks that + // are then retried only after the lease; warn workflow-queue + // workers once. + Event::listen( + \Illuminate\Queue\Events\WorkerStarting::class, + static function (object $event): void { + $options = $event->workerOptions ?? null; + WorkerTimeoutAdvisory::workerStarting( + is_string($event->connectionName ?? null) ? $event->connectionName : null, + is_string($event->queue ?? null) ? $event->queue : null, + is_object($options) && is_numeric($options->timeout ?? null) ? (int) $options->timeout : null, + ); + }, + ); + } + Event::listen(Looping::class, static function (Looping $event) use ($v1Enabled): void { if ($v1Enabled) { Watchdog::wake($event->connectionName, $event->queue); diff --git a/src/V2/Exceptions/WorkflowTaskLeaseLostException.php b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php new file mode 100644 index 000000000..f9ad232d0 --- /dev/null +++ b/src/V2/Exceptions/WorkflowTaskLeaseLostException.php @@ -0,0 +1,29 @@ +disable('workflow_task_lease_lost'); + $executor->failAfterLostLeases($run, $task); + + return null; + } + if ($continuation !== null && ! $continuation->hasAuthority($run, $task)) { $continuation->disable('lost_task_authority'); return null; @@ -286,6 +305,12 @@ private function claimTask(): array ServiceCallWorkflowDelivery::recordForClaim($task, $run); $this->projectRun($run->fresh(['instance', 'tasks', 'activityExecutions', 'failures'])); + // A retry after lost leases records its typed diagnostic (#1231). + $leaseLoss = WorkflowTaskLeaseLoss::claimDiagnostic($task); + if ($leaseLoss !== null) { + WorkflowTaskLeaseLoss::reportClaim($leaseLoss); + } + return [$task->setRelation('run', $run), null, $claimedAttempt]; }); } diff --git a/src/V2/Models/WorkflowRunSummary.php b/src/V2/Models/WorkflowRunSummary.php index bfd76516e..9b7d7bf1a 100644 --- a/src/V2/Models/WorkflowRunSummary.php +++ b/src/V2/Models/WorkflowRunSummary.php @@ -42,6 +42,7 @@ class WorkflowRunSummary extends Model protected $casts = [ 'is_current_run' => 'bool', 'repair_attention' => 'bool', + 'task_attention' => 'array', 'task_problem' => 'bool', 'visibility_labels' => 'array', 'projection_schema_version' => 'integer', diff --git a/src/V2/Models/WorkflowTask.php b/src/V2/Models/WorkflowTask.php index 8dbf2ca66..a7235ac22 100644 --- a/src/V2/Models/WorkflowTask.php +++ b/src/V2/Models/WorkflowTask.php @@ -39,6 +39,7 @@ class WorkflowTask extends Model 'payload' => 'array', 'queue_delivery_reference' => 'array', 'repair_count' => 'integer', + 'lost_lease_count' => 'integer', 'priority' => 'integer', 'fairness_weight' => 'integer', 'sticky_until' => 'datetime', @@ -50,6 +51,7 @@ class WorkflowTask extends Model 'last_dispatched_at' => 'datetime', 'last_claim_failed_at' => 'datetime', 'repair_available_at' => 'datetime', + 'last_lease_lost_at' => 'datetime', ]; public function run(): BelongsTo diff --git a/src/V2/Support/FailureFactory.php b/src/V2/Support/FailureFactory.php index 4144bc4f1..03c6becf0 100644 --- a/src/V2/Support/FailureFactory.php +++ b/src/V2/Support/FailureFactory.php @@ -27,6 +27,7 @@ use Workflow\V2\Exceptions\StructuralLimitExceededException; use Workflow\V2\Exceptions\UnresolvedWorkflowFailureException; use Workflow\V2\Exceptions\UnsupportedWorkflowYieldException; +use Workflow\V2\Exceptions\WorkflowTaskLeaseLostException; final class FailureFactory { @@ -361,10 +362,12 @@ private static function classifyFromThrowable(?Throwable $throwable): FailureCat return FailureCategory::StructuralLimit; } - // Task-level failures: determinism violations, unsupported yields, replay shape errors. + // Task-level failures: determinism violations, unsupported yields, replay shape errors, + // and a workflow task that keeps losing its lease (#1231). if ( $throwable instanceof UnsupportedWorkflowYieldException || $throwable instanceof StraightLineWorkflowRequiredException + || $throwable instanceof WorkflowTaskLeaseLostException ) { return FailureCategory::TaskFailure; } diff --git a/src/V2/Support/OperatorMetrics.php b/src/V2/Support/OperatorMetrics.php index 93b791556..970013be1 100644 --- a/src/V2/Support/OperatorMetrics.php +++ b/src/V2/Support/OperatorMetrics.php @@ -132,6 +132,8 @@ private static function runMetrics(CarbonInterface $now, ?string $namespace): ar ? 0 : (int) $oldestReplayBlockedAt->diffInMilliseconds($now), 'claim_failed' => self::claimFailedRuns($namespace), + // Additive: open runs whose workflow task keeps losing its lease. + 'task_attention' => self::taskAttentionRuns($namespace), 'compatibility_blocked' => self::compatibilityBlockedRuns($namespace), 'waiting' => self::waitingRuns($namespace), 'oldest_wait_started_at' => $oldestWaitStartedAt?->toJSON(), @@ -190,6 +192,9 @@ private static function taskMetrics(CarbonInterface $now, ?string $namespace): a : (int) $oldestDispatchFailedAt->diffInMilliseconds($now), 'max_attempt_count' => self::maxOpenTaskAttemptCount($namespace), 'max_repair_count' => self::maxOpenTaskRepairCount($namespace), + // Additive: open workflow tasks at or over the lost-lease threshold. + 'lease_lost_repeatedly' => self::repeatedlyLostLeaseTasks($namespace), + 'max_lost_lease_count' => self::maxOpenTaskLostLeaseCount($namespace), 'unhealthy' => self::dispatchFailedTasks($namespace) + self::claimFailedTasks($namespace) + self::dispatchOverdueTasks($now, $namespace) @@ -1557,6 +1562,54 @@ private static function maxOpenTaskAttemptCount(?string $namespace): int * active repair burn rather than accumulating against historical * rows. */ + private static function taskAttentionRuns(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::summaryTracked(fresh: true)) { + return 0; + } + + $query = self::summaryQuery($namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => $query->whereNotNull('task_attention') + ->count(), + ); + } + + private static function repeatedlyLostLeaseTasks(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::tracked(fresh: true)) { + return 0; + } + + $query = self::scopedRunModelQuery(self::taskModel(), $namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => $query->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', WorkflowTaskLeaseLoss::attentionThreshold()) + ->count(), + ); + } + + private static function maxOpenTaskLostLeaseCount(?string $namespace): int + { + if (! WorkflowTaskLeaseLoss::tracked(fresh: true)) { + return 0; + } + + $query = self::scopedRunModelQuery(self::taskModel(), $namespace); + + return (int) WorkflowTaskLeaseLoss::untrackedOnMissingColumn( + $query->getModel(), + static fn (): int => (int) $query->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->max('lost_lease_count'), + ); + } + private static function maxOpenTaskRepairCount(?string $namespace): int { return (int) self::scopedRunModelQuery(self::taskModel(), $namespace) diff --git a/src/V2/Support/RunDetailView.php b/src/V2/Support/RunDetailView.php index d5e05d9fd..bf3369443 100644 --- a/src/V2/Support/RunDetailView.php +++ b/src/V2/Support/RunDetailView.php @@ -265,6 +265,11 @@ public static function forRun(WorkflowRun $run, ?int $timelineLimit = null): arr $repairBlockedReason )), 'repair_blocked' => RepairBlockedReason::metadata($repairBlockedReason), + // Additive: a workflow task that keeps losing its lease. Repair + // still works and can_repair is unchanged. + 'task_attention' => $summary !== null && array_key_exists('task_attention', $summary->getAttributes()) + ? $summary->task_attention + : WorkflowTaskLeaseLoss::taskAttention($run), 'can_archive' => $canArchive, 'archive_blocked_reason' => $archiveBlockedReason, 'can_redrive' => $redriveBlockedReason === null, diff --git a/src/V2/Support/RunListItemView.php b/src/V2/Support/RunListItemView.php index f6ba3aa61..751a78402 100644 --- a/src/V2/Support/RunListItemView.php +++ b/src/V2/Support/RunListItemView.php @@ -75,6 +75,10 @@ public static function fromSummary(WorkflowRunSummary $summary): array 'repair_blocked' => RepairBlockedReason::metadata( is_string($summary->repair_blocked_reason) ? $summary->repair_blocked_reason : null, ), + // Additive: a workflow task that keeps losing its lease. + 'task_attention' => array_key_exists('task_attention', $summary->getAttributes()) + ? $summary->task_attention + : null, 'task_problem' => (bool) $summary->task_problem, 'task_problem_badge' => WorkflowTaskProblem::metadata( (bool) $summary->task_problem, @@ -144,6 +148,7 @@ public static function fields(): array 'repair_attention', 'repair_blocked_reason', 'repair_blocked', + 'task_attention', 'task_problem', 'task_problem_badge', 'declared_entry_mode', diff --git a/src/V2/Support/RunSummaryProjector.php b/src/V2/Support/RunSummaryProjector.php index 9bff88a11..41f6ecb41 100644 --- a/src/V2/Support/RunSummaryProjector.php +++ b/src/V2/Support/RunSummaryProjector.php @@ -511,6 +511,7 @@ public static function project(WorkflowRun $run): WorkflowRunSummary 'next_task_lease_expires_at' => $selectedNextTask?->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run), 'task_problem' => $taskProblem, 'exception_count' => count($failureSnapshots), 'history_event_count' => $historyBudget['history_event_count'], @@ -573,25 +574,30 @@ private static function upsertFullSummary( array $values, array $failureIds, ): WorkflowRunSummary { - return SqliteTransaction::run( - static function () use ($summaryModel, $run, $values, $failureIds): WorkflowRunSummary { - /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); + // The write sits outside the transaction so a dropped task_attention + // column can be retried in a fresh transaction. + return WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => SqliteTransaction::run( + static function () use ($summaryModel, $run, $values, $failureIds): WorkflowRunSummary { + /** @var WorkflowRunSummary $summary */ + $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ + 'id' => $run->id, + ], $values,); - ChildProjectionRepairStore::markSnapshotFailuresCounted($run, $failureIds); + ChildProjectionRepairStore::markSnapshotFailuresCounted($run, $failureIds); - return $summary; - }, - admission: static function () use ($summaryModel, $run): void { - // Acquire SQLite's writer before updateOrCreate reads or model hooks run. - // A rejected admission may retry; the projection body must execute once. - $summaryModel::query()->where('id', $run->id)->toBase()->update([ - 'id' => $run->id, - ]); - }, - connection: $run->getConnection(), + return $summary; + }, + admission: static function () use ($summaryModel, $run): void { + // Acquire SQLite's writer before updateOrCreate reads or model hooks run. + // A rejected admission may retry; the projection body must execute once. + $summaryModel::query()->where('id', $run->id)->toBase()->update([ + 'id' => $run->id, + ]); + }, + connection: $run->getConnection(), + ), ); } @@ -705,6 +711,7 @@ private static function projectOpenWorkflowTask( 'next_task_lease_expires_at' => $task->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run, $task), 'task_problem' => $replayBlockedTask instanceof WorkflowTask || $hasPendingChildResolution || self::claimHasWorkflowTaskProblem($run) @@ -744,30 +751,40 @@ private static function projectOpenWorkflowTask( if ($exceptionCount['event_ids'] === []) { /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); + $summary = WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => IdempotentProjectionUpsert::upsert( + $summaryModel, + [ + 'id' => $run->id, + ], + $values, + ), + ); } else { // Best-effort child projection may catch a later projection error. // Keep the count and its event marker in one transaction so neither // half can survive alone before the repair is retried. /** @var WorkflowRunSummary $summary */ - $summary = $run->getConnection() - ->transaction(static function () use ( - $summaryModel, - $run, - $values, - $exceptionCount - ): WorkflowRunSummary { - /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ - 'id' => $run->id, - ], $values,); - - ChildProjectionRepairStore::markFailedChildrenCounted($run, $exceptionCount['event_ids']); - - return $summary; - }); + $summary = WorkflowTaskLeaseLoss::writeSummary( + $values, + static fn (array $values): WorkflowRunSummary => $run->getConnection() + ->transaction(static function () use ( + $summaryModel, + $run, + $values, + $exceptionCount + ): WorkflowRunSummary { + /** @var WorkflowRunSummary $summary */ + $summary = IdempotentProjectionUpsert::upsert($summaryModel, [ + 'id' => $run->id, + ], $values, ); + + ChildProjectionRepairStore::markFailedChildrenCounted($run, $exceptionCount['event_ids']); + + return $summary; + }), + ); } foreach ($childResolutionEvents as $childResolutionEvent) { @@ -1206,11 +1223,7 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow $summaryModel = self::summaryModel(); /** @var WorkflowRunSummary $summary */ - $summary = IdempotentProjectionUpsert::upsert( - $summaryModel, - [ - 'id' => $run->id, - ], + $summary = WorkflowTaskLeaseLoss::writeSummary( [ 'workflow_instance_id' => $run->workflow_instance_id, 'run_number' => $run->run_number, @@ -1259,6 +1272,7 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow 'next_task_lease_expires_at' => $nextTask->lease_expires_at, 'repair_blocked_reason' => $repairBlockedReason, 'repair_attention' => RepairBlockedReason::needsAttention($repairBlockedReason), + ...WorkflowTaskLeaseLoss::summaryAttributes($run, $nextTask), 'task_problem' => false, 'exception_count' => 0, 'history_event_count' => $historyEventCount, @@ -1269,6 +1283,13 @@ private static function projectFreshWorkflowTaskRun(WorkflowRun $run): ?Workflow 'created_at' => $run->created_at, 'updated_at' => $run->closed_at ?? $run->last_progress_at ?? $run->updated_at, ], + static fn (array $values): WorkflowRunSummary => IdempotentProjectionUpsert::upsert( + $summaryModel, + [ + 'id' => $run->id, + ], + $values, + ), ); return $summary; diff --git a/src/V2/Support/TaskDispatcher.php b/src/V2/Support/TaskDispatcher.php index 2ade4846f..30c0c3978 100644 --- a/src/V2/Support/TaskDispatcher.php +++ b/src/V2/Support/TaskDispatcher.php @@ -582,6 +582,11 @@ private static function makeJob(WorkflowTask $task): RunWorkflowTask|RunActivity $job->onQueue($task->queue); } + if ($job instanceof RunWorkflowTask) { + // Opt-in: workflow tasks get their own timeout only when configured. + $job->timeout = WorkflowTaskJobTimeout::seconds($task->connection); + } + if ($task->available_at !== null && $task->available_at->isFuture()) { // Queue backends store second-resolution deadlines. Never round a wakeup down. $availableAt = $task->available_at->copy() diff --git a/src/V2/Support/TaskRepair.php b/src/V2/Support/TaskRepair.php index 13c954f19..07b51b56b 100644 --- a/src/V2/Support/TaskRepair.php +++ b/src/V2/Support/TaskRepair.php @@ -102,6 +102,8 @@ private static function recoverLeasedTask(WorkflowTask $task, WorkflowRun $run): 'repair_count' => $task->repair_count + 1, 'repair_available_at' => null, 'last_error' => null, + // A workflow task's expired lease is work that never completed (#1231). + ...WorkflowTaskLeaseLoss::repairedAttributes($task), ])->save(); return $task; diff --git a/src/V2/Support/WorkerTimeoutAdvisory.php b/src/V2/Support/WorkerTimeoutAdvisory.php new file mode 100644 index 000000000..0614f6d14 --- /dev/null +++ b/src/V2/Support/WorkerTimeoutAdvisory.php @@ -0,0 +1,115 @@ += $leaseSeconds) { + return null; + } + + return sprintf( + '[Durable Workflow] This worker kills a workflow task after %d s (%s), less than the %d s workflow task ' + . 'lease. A workflow task killed this way is retried only after its lease expires and is counted as a ' + . 'lost lease. Raise %s to cover your longest workflow task, keeping the queue connection\'s ' + . 'retry_after above it, or set DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS to give workflow tasks their ' + . 'own timeout.', + $effectiveTimeout, + $jobTimeout ? 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS' : 'the worker --timeout', + $leaseSeconds, + $jobTimeout ? 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS' : '--timeout', + ); + } + + /** + * Whether a worker started for $connection and $queues serves the queue + * the package dispatches workflow tasks to by default. + */ + public static function servesWorkflowQueue(?string $connection, ?string $queues): bool + { + $workflowConnection = config('workflows.v2.connection'); + $workflowConnection = is_string($workflowConnection) && $workflowConnection !== '' + ? $workflowConnection + : config('queue.default'); + + if (! is_string($workflowConnection) || $connection !== $workflowConnection) { + return false; + } + + $workflowQueue = config('workflows.v2.queue'); + $workflowQueue = is_string($workflowQueue) && $workflowQueue !== '' + ? $workflowQueue + : config('queue.connections.' . $workflowConnection . '.queue', 'default'); + + $served = array_map('trim', explode(',', (string) ($queues ?? ''))); + + return in_array((string) $workflowQueue, $served, true); + } + + public static function workerStarting(?string $connection, ?string $queues, ?int $workerTimeout): void + { + if (self::$warned || $connection === null || ! self::servesWorkflowQueue($connection, $queues)) { + return; + } + + $jobTimeout = WorkflowTaskJobTimeout::seconds($connection); + $effective = $jobTimeout ?? (int) ($workerTimeout ?? 0); + $message = self::message( + extension_loaded('pcntl'), + $effective, + $jobTimeout !== null, + WorkflowTaskLease::seconds() + ); + + if ($message === null) { + return; + } + + self::$warned = true; + Log::warning($message, [ + 'code' => 'workflow_task_timeout_below_lease', + 'connection' => $connection, + 'workflow_task_timeout_seconds' => $effective, + 'workflow_task_timeout_source' => $jobTimeout !== null ? 'workflow_task_job_timeout' : 'worker_timeout', + 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), + 'retry_after_seconds' => WorkflowTaskJobTimeout::retryAfter($connection), + ]); + } + + /** + * @internal For tests that start more than one worker in a process. + */ + public static function reset(): void + { + self::$warned = false; + } +} diff --git a/src/V2/Support/WorkflowExecutor.php b/src/V2/Support/WorkflowExecutor.php index 5cf51fc16..44caf230f 100644 --- a/src/V2/Support/WorkflowExecutor.php +++ b/src/V2/Support/WorkflowExecutor.php @@ -94,6 +94,16 @@ public function run( } } + /** + * Fail a run whose claimed workflow task kept losing its lease, when + * workflows.v2.workflow_task_lost_leases.fail_run is enabled (#1231). + * Called under the task and run locks, before any workflow code runs. + */ + public function failAfterLostLeases(WorkflowRun $run, WorkflowTask $task): void + { + $this->failRun($run, $task, WorkflowTaskLeaseLoss::exceptionFor($task), 'workflow_run', $run->id); + } + /** * Close a run whose server-enforced execution or run deadline elapsed. * diff --git a/src/V2/Support/WorkflowTaskJobTimeout.php b/src/V2/Support/WorkflowTaskJobTimeout.php new file mode 100644 index 000000000..6063e6beb --- /dev/null +++ b/src/V2/Support/WorkflowTaskJobTimeout.php @@ -0,0 +1,76 @@ += 1 ? $timeout : null; + } + + /** + * The explicitly configured timeout, or null when none is configured. + */ + public static function configuredSeconds(): ?int + { + $configured = config(self::CONFIG_KEY); + + return is_numeric($configured) && (int) $configured >= 1 ? (int) $configured : null; + } + + /** + * The queue connection's `retry_after`, or null when it has none. + */ + public static function retryAfter(?string $connection = null): ?int + { + $connection ??= config('queue.default'); + + if (! is_string($connection) || $connection === '') { + return null; + } + + $retryAfter = config("queue.connections.{$connection}.retry_after"); + + return is_numeric($retryAfter) && (int) $retryAfter > 0 ? (int) $retryAfter : null; + } +} diff --git a/src/V2/Support/WorkflowTaskLeaseLoss.php b/src/V2/Support/WorkflowTaskLeaseLoss.php new file mode 100644 index 000000000..e0b10ae8e --- /dev/null +++ b/src/V2/Support/WorkflowTaskLeaseLoss.php @@ -0,0 +1,404 @@ + + */ + private static array $schemas = []; + + public static function attentionThreshold(): int + { + $value = config(self::CONFIG_KEY . '.attention_threshold', self::DEFAULT_ATTENTION_THRESHOLD); + + return is_numeric($value) && (int) $value >= 1 + ? (int) $value + : self::DEFAULT_ATTENTION_THRESHOLD; + } + + public static function failsRun(): bool + { + return (bool) config(self::CONFIG_KEY . '.fail_run', false); + } + + /** + * Refuse, at boot, a run-failing configuration that would fail a run on + * too few lost leases. + */ + public static function validateConfiguration(): void + { + if (! self::failsRun()) { + return; + } + + $value = config(self::CONFIG_KEY . '.attention_threshold', self::DEFAULT_ATTENTION_THRESHOLD); + + if (! is_numeric($value) || (int) $value < self::MINIMUM_FAILING_THRESHOLD) { + throw new InvalidArgumentException(sprintf( + '%s.attention_threshold must be at least %d when %s.fail_run is enabled.', + self::CONFIG_KEY, + self::MINIMUM_FAILING_THRESHOLD, + self::CONFIG_KEY, + )); + } + } + + public static function lostLeases(WorkflowTask $task): int + { + return max(0, (int) ($task->getAttribute('lost_lease_count') ?? 0)); + } + + /** + * Count one lost lease on a workflow task that task repair returns to + * ready. A task row without the tracking columns keeps the old behavior. + * + * @return array + */ + public static function repairedAttributes(WorkflowTask $task): array + { + if ($task->task_type !== TaskType::Workflow + || ! array_key_exists('lost_lease_count', $task->getAttributes())) { + return []; + } + + return [ + 'lost_lease_count' => self::lostLeases($task) + 1, + 'last_lease_lost_at' => now(), + ]; + } + + /** + * The typed diagnostic for a claim that retries a task after lost leases, + * or null for a task that never lost one. + * + * @return array{ + * code: string, + * workflow_run_id: string, + * workflow_task_id: string, + * lost_leases: int, + * attention_threshold: int, + * needs_attention: bool, + * last_lease_lost_at: string|null, + * workflow_task_lease_seconds: int, + * claim_attempt: int + * }|null + */ + public static function claimDiagnostic(WorkflowTask $task): ?array + { + $lostLeases = self::lostLeases($task); + + if ($task->task_type !== TaskType::Workflow || $lostLeases === 0) { + return null; + } + + return [ + 'code' => self::CODE, + 'workflow_run_id' => (string) $task->workflow_run_id, + 'workflow_task_id' => (string) $task->id, + 'lost_leases' => $lostLeases, + 'attention_threshold' => self::attentionThreshold(), + 'needs_attention' => $lostLeases >= self::attentionThreshold(), + 'last_lease_lost_at' => self::timestamp($task->getAttribute('last_lease_lost_at')), + 'workflow_task_lease_seconds' => WorkflowTaskLease::seconds(), + 'claim_attempt' => (int) $task->attempt_count, + ]; + } + + /** + * Report a claim that retries a task after lost leases. + * + * @param array $diagnostic + */ + public static function reportClaim(array $diagnostic): void + { + Log::warning( + sprintf( + '[Durable Workflow] Workflow task %s lost its lease %d time(s) without completing ' + . '(its worker was killed, timed out, or lost its connection); retrying it.', + $diagnostic['workflow_task_id'], + $diagnostic['lost_leases'], + ), + $diagnostic, + ); + } + + /** + * Whether a claimed workflow task must fail its run instead of executing. + */ + public static function failsRunAt(WorkflowTask $task): bool + { + return self::failsRun() + && $task->task_type === TaskType::Workflow + && self::lostLeases($task) >= max(self::MINIMUM_FAILING_THRESHOLD, self::attentionThreshold()); + } + + public static function exceptionFor(WorkflowTask $task): WorkflowTaskLeaseLostException + { + return new WorkflowTaskLeaseLostException( + (string) $task->id, + self::lostLeases($task), + WorkflowTaskLease::seconds(), + ); + } + + /** + * The run's additive `task_attention` value: its open workflow task's + * consecutive lost leases once they reach the threshold, otherwise null. + * + * @return array{reason: string, consecutive: int, threshold: int, workflow_task_id: string, last_lease_lost_at: string|null}|null + */ + public static function taskAttention(WorkflowRun $run, ?WorkflowTask $openWorkflowTask = null): ?array + { + if ($run->status->isTerminal()) { + return null; + } + + $threshold = self::attentionThreshold(); + $task = null; + + if ($openWorkflowTask instanceof WorkflowTask + && $openWorkflowTask->task_type === TaskType::Workflow + && in_array($openWorkflowTask->status, [TaskStatus::Ready, TaskStatus::Leased], true)) { + // A bounded projection already holds the run's open workflow task. + $task = self::lostLeases($openWorkflowTask) >= $threshold ? $openWorkflowTask : null; + } elseif ($run->relationLoaded('tasks')) { + foreach ($run->tasks as $candidate) { + if ($candidate instanceof WorkflowTask + && $candidate->task_type === TaskType::Workflow + && in_array($candidate->status, [TaskStatus::Ready, TaskStatus::Leased], true) + && self::lostLeases($candidate) >= $threshold + && ($task === null || self::lostLeases($candidate) > self::lostLeases($task))) { + $task = $candidate; + } + } + } elseif ($run->exists && self::tracked()) { + $task = self::untrackedOnMissingColumn( + self::taskModel(), + static fn (): ?WorkflowTask => ConfiguredV2Models::query('task_model', WorkflowTask::class) + ->where('workflow_run_id', $run->id) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->where('lost_lease_count', '>=', $threshold) + ->orderByDesc('lost_lease_count') + ->first(), + ); + } + + if (! $task instanceof WorkflowTask) { + return null; + } + + // SQLite reads an unknown double-quoted column name as a string, so a + // removed column can match instead of failing. A row without the + // column means the schema was rolled back. + if (! array_key_exists('lost_lease_count', $task->getAttributes())) { + self::$schemas = []; + + return null; + } + + return [ + 'reason' => self::CODE, + 'consecutive' => self::lostLeases($task), + 'threshold' => $threshold, + 'workflow_task_id' => (string) $task->id, + 'last_lease_lost_at' => self::timestamp($task->getAttribute('last_lease_lost_at')), + ]; + } + + /** + * The summary attributes that carry `task_attention`, or none when the + * summary table has no such column. + * + * @return array + */ + public static function summaryAttributes(WorkflowRun $run, ?WorkflowTask $openWorkflowTask = null): array + { + if (! self::summaryTracked()) { + return []; + } + + return [ + 'task_attention' => self::taskAttention($run, $openWorkflowTask), + ]; + } + + /** + * Whether the configured task table has the lost-lease columns. + */ + public static function tracked(bool $fresh = false): bool + { + return self::hasColumn(self::taskModel(), 'lost_lease_count', $fresh); + } + + /** + * Whether the configured run-summary table has `task_attention`. + */ + public static function summaryTracked(bool $fresh = false): bool + { + return self::hasColumn(self::summaryModel(), 'task_attention', $fresh); + } + + /** + * Run a query that reads the lost-lease columns. If a schema rollback + * removed them, degrade to untracked instead of failing. Inside a + * transaction the query runs in its own savepoint, so the failed + * statement cannot abort the caller's transaction. + * + * @template T + * @param callable(): T $query + * @return T|null + */ + public static function untrackedOnMissingColumn(Model $model, callable $query): mixed + { + $connection = $model->getConnection(); + + try { + return $connection->transactionLevel() > 0 + ? $connection->transaction(static fn () => $query()) + : $query(); + } catch (QueryException $exception) { + if (! self::missingColumn($exception)) { + throw $exception; + } + + self::$schemas = []; + + return null; + } + } + + /** + * Write a run summary whose values may carry task_attention. + * + * A schema rollback can drop the column while the cached check still + * reports it. On that error the cache is forgotten and $write runs again + * without task_attention. A failed statement aborts a caller's open + * PostgreSQL transaction, so there the error is rethrown instead and the + * caller's retry sees the current schema. + * + * @template T + * @param array $values + * @param callable(array): T $write + * @return T + */ + public static function writeSummary(array $values, callable $write): mixed + { + if (! array_key_exists('task_attention', $values)) { + return $write($values); + } + + try { + return $write($values); + } catch (QueryException $exception) { + if (! self::missingColumn($exception)) { + throw $exception; + } + + self::$schemas = []; + $connection = self::summaryModel()->getConnection(); + + if ($connection->transactionLevel() > 0 && $connection->getDriverName() === 'pgsql') { + throw $exception; + } + + unset($values['task_attention']); + + return $write($values); + } + } + + private static function hasColumn(Model $model, string $column, bool $fresh = false): bool + { + $key = $model->getConnectionName() . '|' . $model->getTable() . '|' . $column; + $cached = self::$schemas[$key] ?? null; + $now = time(); + + if (! $fresh && $cached !== null && $now - $cached['checked_at'] < self::SCHEMA_RECHECK_SECONDS) { + return $cached['tracked']; + } + + $tracked = $model->getConnection() + ->getSchemaBuilder() + ->hasColumn($model->getTable(), $column); + self::$schemas[$key] = [ + 'tracked' => $tracked, + 'checked_at' => $now, + ]; + + return $tracked; + } + + private static function missingColumn(QueryException $exception): bool + { + $message = $exception->getMessage(); + + return in_array((string) $exception->getCode(), ['42S22', '42703'], true) + || str_contains($message, 'no such column') + || str_contains($message, 'has no column named') + || str_contains($message, 'Unknown column') + || str_contains($message, 'does not exist'); + } + + private static function taskModel(): Model + { + return ConfiguredV2Models::query('task_model', WorkflowTask::class)->getModel(); + } + + private static function summaryModel(): Model + { + return ConfiguredV2Models::query('run_summary_model', WorkflowRunSummary::class)->getModel(); + } + + private static function timestamp(mixed $value): ?string + { + return $value instanceof \DateTimeInterface ? $value->format('Y-m-d\TH:i:s.up') : null; + } +} diff --git a/src/config/workflows.php b/src/config/workflows.php index c1ebeecc1..3ac71c50d 100644 --- a/src/config/workflows.php +++ b/src/config/workflows.php @@ -384,6 +384,23 @@ Workflow\V2\Support\WorkflowTaskLease::DEFAULT_SECONDS, ), + // Opt-in queue job timeout for workflow task jobs. Unset (the default), + // the worker's --timeout applies to workflow tasks like any other job. + // Set, every workflow task job carries this timeout instead, kept 10 s + // below its queue connection's retry_after. + 'workflow_task_job_timeout_seconds' => env('DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS'), + + // A workflow task whose lease expires without completing (its worker + // was killed, timed out, or lost its connection) is retried after task + // repair. After this many consecutive lost leases the run summary and + // describe carry task_attention. With fail_run enabled (off by + // default; requires a threshold of at least 2), the next claim fails + // the run with a typed task failure instead. + 'workflow_task_lost_leases' => [ + 'attention_threshold' => (int) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_ATTENTION_THRESHOLD', 3), + 'fail_run' => (bool) env('DW_V2_WORKFLOW_TASK_LOST_LEASE_FAIL_RUN', false), + ], + // Runtime ownership, not an activity execution timeout. Valid values // are 1..86400 seconds; invalid values resolve to the 300-second default. // A live runtime renews the same attempt without inventing user progress. diff --git a/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php b/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php new file mode 100644 index 000000000..9217d0462 --- /dev/null +++ b/src/migrations/2026_11_18_000100_add_lost_lease_tracking.php @@ -0,0 +1,103 @@ +getConnection()); + + foreach (self::taskTables() as $table) { + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'lost_lease_count')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->unsignedInteger('lost_lease_count') + ->default(0); + }); + }); + } + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'last_lease_lost_at')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->timestamp('last_lease_lost_at', 6) + ->nullable(); + }); + }); + } + } + + foreach (self::summaryTables() as $table) { + if ($schema->hasTable($table) && ! $schema->hasColumn($table, 'task_attention')) { + $this->withSchemaLockBudget(static function () use ($schema, $table): void { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->json('task_attention') + ->nullable(); + }); + }); + } + } + } + + public function down(): void + { + $schema = Schema::connection($this->getConnection()); + + foreach (self::summaryTables() as $table) { + if ($schema->hasTable($table) && $schema->hasColumn($table, 'task_attention')) { + $schema->table($table, static function (Blueprint $blueprint): void { + $blueprint->dropColumn('task_attention'); + }); + } + } + + foreach (self::taskTables() as $table) { + foreach (['last_lease_lost_at', 'lost_lease_count'] as $column) { + if ($schema->hasTable($table) && $schema->hasColumn($table, $column)) { + $schema->table($table, static function (Blueprint $blueprint) use ($column): void { + $blueprint->dropColumn($column); + }); + } + } + } + } + + /** + * @return list + */ + private static function taskTables(): array + { + $model = ConfiguredV2Models::resolve('task_model', WorkflowTask::class); + + return array_values(array_unique(['workflow_tasks', (new $model())->getTable()])); + } + + /** + * @return list + */ + private static function summaryTables(): array + { + $model = ConfiguredV2Models::resolve('run_summary_model', WorkflowRunSummary::class); + + return array_values(array_unique(['workflow_run_summaries', (new $model())->getTable()])); + } +}; diff --git a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php index 8d132c560..8acf40960 100644 --- a/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php +++ b/tests/Feature/V2/V2EmbeddedReplayRegressionCorpusTest.php @@ -506,6 +506,77 @@ public function testOversizedFanOutMemberCorpusIsRefusedBeforeFirstWrite(): void $this->assertSame(1, (new WorkflowReplayer())->replay($run->fresh(['historyEvents']))->sequence); } + /** + * A workflow task that lost its lease three times, as when a pcntl worker + * kills it at its timeout, fails its run with the typed lease-lost + * failure on the next claim when fail_run is enabled (#1231). The run's + * history is exactly the evidence: no lost lease writes history. + */ + public function testWorkflowTaskLeaseLostCorpusFailsTheRunWhenOptedIn(): void + { + $path = self::FIXTURE_DIR . '/workflow-task-lease-lost-fails-run-opt-in.json'; + if (! is_file($path)) { + $this->markTestSkipped('The workflow task lease-lost evidence is absent.'); + } + $fixture = json_decode(file_get_contents($path), true, flags: JSON_THROW_ON_ERROR); + self::stopWorkers(); + config([ + 'queue.default' => 'redis', + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => $fixture['workflow_task_lease_lost']['attention_threshold'], + ]); + Queue::fake(); + + $workflow = WorkflowStub::make($fixture['workflow']['type'], 'regression-workflow-task-lease-lost'); + $workflow->start(...$fixture['workflow']['arguments']); + $runId = $workflow->runId(); + + for ($lost = 0; $lost < $fixture['workflow_task_lease_lost']['attention_threshold']; $lost++) { + $task = WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value) + ->sole(); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'leased_at' => now() + ->subMinutes(10), + 'lease_owner' => $task->id, + 'lease_expires_at' => now() + ->subMinute(), + 'attempt_count' => $task->attempt_count + 1, + ])->save(); + Cache::forget(TaskWatchdog::LOOP_THROTTLE_KEY); + TaskWatchdog::runPass(connection: 'redis', queue: 'default', respectThrottle: false); + } + + $task = WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->where('status', TaskStatus::Ready->value) + ->sole(); + $this->app->call([new RunWorkflowTask($task->id), 'handle']); + + $this->assertSame( + array_map(static fn (array $event): array => [ + $event['event_type'], + $event['payload']['failure_category'] ?? null, + $event['payload']['exception_class'] ?? null, + ], $fixture['history']), + WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->where('event_type', '!=', HistoryEventType::StartAccepted->value) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): array => [ + $event->event_type->value, + $event->payload['failure_category'] ?? null, + $event->payload['exception_class'] ?? null, + ]) + ->all(), + ); + } + /** * The partial-group history 2.32.0 recorded for the same refusal still * replays through the embedded replayer to the all(), while a new run of diff --git a/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php b/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php new file mode 100644 index 000000000..f73a17846 --- /dev/null +++ b/tests/Feature/V2/V2LostLeaseTrackingMigrationTest.php @@ -0,0 +1,82 @@ +migration(); + $this->assertFalse($migration->withinTransaction); + + $source = (string) file_get_contents(self::path()); + $this->assertSame(3, substr_count($source, 'withSchemaLockBudget')); + // No backfill: existing tasks have lost no lease and need no attention. + $this->assertDoesNotMatchRegularExpression('/->\s*(update|insert|upsert|delete)\s*\(/', $source); + } + + public function testItAddsTheColumnsToPopulatedTablesAndCanRunAgain(): void + { + $migration = $this->migration(); + $migration->down(); + $this->assertFalse(Schema::hasColumn('workflow_tasks', 'lost_lease_count')); + $this->assertFalse(Schema::hasColumn('workflow_tasks', 'last_lease_lost_at')); + $this->assertFalse(Schema::hasColumn('workflow_run_summaries', 'task_attention')); + + $ids = []; + foreach (range(1, 5) as $number) { + $ids[] = $id = (string) Str::ulid(); + DB::table('workflow_tasks')->insert([ + 'id' => $id, + 'workflow_run_id' => (string) Str::ulid(), + 'task_type' => 'workflow', + 'status' => 'ready', + 'attempt_count' => 0, + 'repair_count' => 0, + 'created_at' => now(), + 'updated_at' => now(), + ]); + } + + $migration->up(); + // An interrupted or repeated run skips work already done. + $migration->up(); + + $this->assertTrue(Schema::hasColumn('workflow_tasks', 'lost_lease_count')); + $this->assertTrue(Schema::hasColumn('workflow_tasks', 'last_lease_lost_at')); + $this->assertTrue(Schema::hasColumn('workflow_run_summaries', 'task_attention')); + $this->assertSame( + 5, + DB::table('workflow_tasks') + ->whereIn('id', $ids) + ->where('lost_lease_count', 0) + ->whereNull('last_lease_lost_at') + ->count(), + ); + } + + private function migration(): WorkflowMigration + { + return require self::path(); + } + + private static function path(): string + { + return dirname(__DIR__, 3) . '/src/migrations/' . self::NAME . '.php'; + } +} diff --git a/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php new file mode 100644 index 000000000..bf333a217 --- /dev/null +++ b/tests/Feature/V2/V2WorkflowTaskLostLeaseTest.php @@ -0,0 +1,449 @@ +start('Taylor'); + $runId = $workflow->runId(); + $historyBefore = $this->historyTypes($runId); + + $this->loseLease($runId); + $this->loseLease($runId); + + $task = $this->workflowTask($runId); + $this->assertSame(TaskStatus::Ready, $task->status); + $this->assertSame(2, $task->lost_lease_count); + $this->assertNotNull($task->last_lease_lost_at); + $this->assertNull($this->summary($runId)->task_attention); + $summaryBelow = $this->consumerFields($this->summary($runId)->getAttributes()); + $detailBelow = $this->repairFields(RunDetailView::forRun(WorkflowRun::query()->findOrFail($runId))); + + $this->loseLease($runId); + + // At the threshold only the additive task_attention field changes: + // repair-blocked reasons, repair attention and can_repair do not. + $summary = $this->summary($runId); + // MySQL's JSON column may reorder keys. + $this->assertEquals([ + 'reason' => WorkflowTaskLeaseLoss::CODE, + 'consecutive' => 3, + 'threshold' => 3, + 'workflow_task_id' => $task->id, + ], array_diff_key($summary->task_attention, [ + 'last_lease_lost_at' => true, + ])); + $this->assertSame($summaryBelow, $this->consumerFields($summary->getAttributes())); + + $detail = RunDetailView::forRun(WorkflowRun::query()->findOrFail($runId)); + $this->assertSame($summary->task_attention, $detail['task_attention']); + $this->assertSame($detailBelow, $this->repairFields($detail)); + + $metrics = OperatorMetrics::snapshot(); + $this->assertSame(1, $metrics['runs']['task_attention']); + $this->assertSame(1, $metrics['tasks']['lease_lost_repeatedly']); + $this->assertSame(3, $metrics['tasks']['max_lost_lease_count']); + + // Counting and reporting lost leases writes no history. + $this->assertSame($historyBefore, $this->historyTypes($runId)); + + // The next claim reports the typed diagnostic and still runs the task. + Log::spy(); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + Log::shouldHaveReceived('warning') + ->withArgs( + static fn (string $message, array $context): bool => $context['code'] === WorkflowTaskLeaseLoss::CODE + && $context['workflow_run_id'] === $runId + && $context['lost_leases'] === 3 + && $context['needs_attention'] === true + ) + ->once(); + $this->assertSame([ + HistoryEventType::StartAccepted->value, + HistoryEventType::WorkflowStarted->value, + HistoryEventType::ActivityScheduled->value, + ], $this->historyTypes($runId)); + + // The completed task's replacement starts with no lost leases. + $this->assertNull($this->summary($runId)->task_attention); + } + + public function testFailingTheRunAfterLostLeasesIsOptIn(): void + { + Queue::fake(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + ]); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-fail-run'); + $workflow->start('Taylor'); + $runId = $workflow->runId(); + + $this->loseLease($runId); + $this->loseLease($runId); + $this->loseLease($runId); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + $this->assertTrue($workflow->refresh()->failed()); + $failure = WorkflowFailure::query() + ->where('workflow_run_id', $runId) + ->sole(); + $this->assertSame(WorkflowTaskLeaseLostException::class, $failure->exception_class); + $this->assertSame( + FailureCategory::TaskFailure->value, + $failure->failure_category?->value ?? $failure->failure_category, + ); + $this->assertSame([ + HistoryEventType::StartAccepted->value, + HistoryEventType::WorkflowStarted->value, + HistoryEventType::WorkflowFailed->value, + ], $this->historyTypes($runId)); + } + + public function testFailingTheRunWaitsForTheAttentionThreshold(): void + { + Queue::fake(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 3, + ]); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-below-threshold'); + $workflow->start('Taylor'); + $runId = $workflow->runId(); + + $this->loseLease($runId); + $this->loseLease($runId); + $this->app->call([new RunWorkflowTask($this->workflowTask($runId)->id), 'handle']); + + $this->assertFalse($workflow->refresh()->failed()); + $this->assertContains(HistoryEventType::ActivityScheduled->value, $this->historyTypes($runId)); + } + + public function testWorkflowTaskJobsCarryAJobTimeoutOnlyWhenConfigured(): void + { + config([ + 'workflows.v2.workflow_task_lease_seconds' => 300, + 'queue.connections.redis.retry_after' => 90, + 'workflows.v2.workflow_task_job_timeout_seconds' => null, + ]); + + // Unset, the worker's --timeout applies to workflow tasks unchanged. + $this->assertNull(WorkflowTaskJobTimeout::seconds('redis')); + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout-unset'); + $workflow->start('Taylor'); + Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === null); + + config([ + 'queue.connections.redis.retry_after' => 600, + 'workflows.v2.workflow_task_job_timeout_seconds' => 120, + ]); + $this->assertSame(120, WorkflowTaskJobTimeout::seconds('redis')); + + // Never at or past retry_after, so the queue never hands a running + // job to another worker. + config([ + 'queue.connections.redis.retry_after' => 90, + ]); + $this->assertSame(80, WorkflowTaskJobTimeout::seconds('redis')); + + config([ + 'queue.connections.redis.retry_after' => 600, + 'workflows.v2.workflow_task_job_timeout_seconds' => 900, + ]); + $this->assertSame(590, WorkflowTaskJobTimeout::seconds('redis')); + + // Set, the dispatched job carries it; Laravel's worker uses it in + // place of the worker's --timeout. + config([ + 'workflows.v2.workflow_task_job_timeout_seconds' => 120, + ]); + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-job-timeout-set'); + $workflow->start('Taylor'); + Queue::assertPushed(RunWorkflowTask::class, static fn (RunWorkflowTask $job): bool => $job->timeout === 120); + } + + public function testWorkerStartWarnsWorkflowQueueWorkersWhoseTimeoutIsBelowTheLease(): void + { + config([ + 'queue.default' => 'redis', + 'workflows.v2.connection' => null, + 'workflows.v2.queue' => null, + 'queue.connections.redis.queue' => 'default', + 'queue.connections.redis.retry_after' => 90, + 'workflows.v2.workflow_task_lease_seconds' => 300, + 'workflows.v2.workflow_task_job_timeout_seconds' => null, + ]); + WorkerTimeoutAdvisory::reset(); + Log::spy(); + $options = new WorkerOptions(timeout: 60); + + // A worker that serves only other queues is never warned. + Event::dispatch(new WorkerStarting('redis', 'activities', $options)); + Log::shouldNotHaveReceived('warning'); + + Event::dispatch(new WorkerStarting('redis', 'high,default', $options)); + Event::dispatch(new WorkerStarting('redis', 'default', $options)); + + if (extension_loaded('pcntl')) { + Log::shouldHaveReceived('warning') + ->withArgs(static fn (string $message, array $context): bool => + $context['code'] === 'workflow_task_timeout_below_lease' + && $context['workflow_task_timeout_source'] === 'worker_timeout' + && $context['workflow_task_timeout_seconds'] === 60 + && $context['workflow_task_lease_seconds'] === 300 + && str_contains($message, 'retried only after its lease expires') + && str_contains($message, 'counted as a lost lease') + && str_contains($message, 'DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS')) + ->once(); + } else { + Log::shouldNotHaveReceived('warning'); + } + + // No warning without pcntl, for --timeout=0, or at or above the lease. + $this->assertNull(WorkerTimeoutAdvisory::message(false, 60, false, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(true, 0, false, 300)); + $this->assertNull(WorkerTimeoutAdvisory::message(true, 300, false, 300)); + $this->assertStringContainsString( + 'the worker --timeout', + (string) WorkerTimeoutAdvisory::message(true, 60, false, 300) + ); + $this->assertStringContainsString( + 'after 80 s (DW_V2_WORKFLOW_TASK_JOB_TIMEOUT_SECONDS)', + (string) WorkerTimeoutAdvisory::message(true, 80, true, 300), + ); + WorkerTimeoutAdvisory::reset(); + } + + public function testFailingRunsRequiresAThresholdOfAtLeastTwo(): void + { + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => true, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 1, + ]); + + try { + WorkflowTaskLeaseLoss::validateConfiguration(); + $this->fail('A failing threshold of one must be refused.'); + } catch (InvalidArgumentException $exception) { + $this->assertStringContainsString('at least 2', $exception->getMessage()); + } + + config([ + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 2, + ]); + WorkflowTaskLeaseLoss::validateConfiguration(); + config([ + 'workflows.v2.workflow_task_lost_leases.fail_run' => false, + 'workflows.v2.workflow_task_lost_leases.attention_threshold' => 1, + ]); + WorkflowTaskLeaseLoss::validateConfiguration(); + $this->addToAssertionCount(2); + } + + public function testASchemaRollbackDegradesToUntracked(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-schema-rollback'); + $workflow->start('Taylor'); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $this->assertTrue(WorkflowTaskLeaseLoss::tracked()); + + // The process still believes the columns exist. + Schema::table('workflow_tasks', static function (Blueprint $table): void { + $table->dropColumn(['last_lease_lost_at', 'lost_lease_count']); + }); + + try { + DB::transaction(function () use ($run): void { + $this->assertNull(WorkflowTaskLeaseLoss::taskAttention($run)); + // The caller's transaction is still usable. + $this->assertSame(1, WorkflowRun::query()->whereKey($run->id)->count()); + }); + $this->assertFalse(WorkflowTaskLeaseLoss::tracked()); + $this->assertSame(0, OperatorMetrics::snapshot()['tasks']['lease_lost_repeatedly']); + } finally { + Schema::table('workflow_tasks', static function (Blueprint $table): void { + $table->unsignedInteger('lost_lease_count') + ->default(0); + $table->timestamp('last_lease_lost_at', 6) + ->nullable(); + }); + } + } + + public function testASummaryColumnRollbackDegradesSummaryWrites(): void + { + Queue::fake(); + $workflow = WorkflowStub::make(TestGreetingWorkflow::class, 'lost-lease-summary-rollback'); + $workflow->start('Taylor'); + $run = WorkflowRun::query()->findOrFail($workflow->runId()); + $this->assertTrue(WorkflowTaskLeaseLoss::summaryTracked()); + + // The process still believes the summary column exists. + Schema::table('workflow_run_summaries', static function (Blueprint $table): void { + $table->dropColumn('task_attention'); + }); + + try { + $summary = RunSummaryProjector::project($run->refresh()); + $this->assertSame($run->id, $summary->id); + $this->assertFalse(array_key_exists('task_attention', $summary->getAttributes())); + $this->assertFalse(WorkflowTaskLeaseLoss::summaryTracked()); + $this->assertNull(RunDetailView::forRun($run->refresh())['task_attention']); + } finally { + Schema::table('workflow_run_summaries', static function (Blueprint $table): void { + $table->json('task_attention') + ->nullable(); + }); + } + } + + protected function startQueueWorkersAutomatically(): bool + { + return false; + } + + /** + * Claim the run's workflow task as a worker would, let its lease expire + * without completing, and run task repair. + */ + private function loseLease(string $runId): void + { + $task = $this->workflowTask($runId); + $task->forceFill([ + 'status' => TaskStatus::Leased, + 'leased_at' => Carbon::now()->subMinutes(10), + 'lease_owner' => $task->id, + 'lease_expires_at' => Carbon::now()->subMinute(), + 'attempt_count' => $task->attempt_count + 1, + ])->save(); + + Cache::forget(TaskWatchdog::LOOP_THROTTLE_KEY); + TaskWatchdog::runPass(connection: 'redis', queue: 'default', respectThrottle: false); + + $this->assertSame(TaskStatus::Ready, $this->workflowTask($runId)->status); + } + + /** + * The fields an existing consumer reads, without the additive field and + * the timestamps a repair pass refreshes. + * + * @param array $fields + * @return array + */ + private function consumerFields(array $fields): array + { + return array_diff_key($fields, array_flip([ + 'task_attention', + 'updated_at', + 'next_task_at', + 'next_task_lease_expires_at', + 'wait_started_at', + 'wait_deadline_at', + 'tasks', + 'timeline', + 'waits', + 'liveness_reason', + 'last_progress_at', + 'sort_timestamp', + 'sort_key', + ])); + } + + /** + * The describe fields a repair decision reads. + * + * @param array $detail + * @return array + */ + private function repairFields(array $detail): array + { + return array_intersect_key($detail, array_flip([ + 'status', + 'wait_kind', + 'liveness_state', + 'can_repair', + 'repair_blocked_reason', + 'repair_attention', + 'repair_blocked', + 'task_problem', + 'can_cancel', + 'can_terminate', + 'can_signal', + 'can_update', + ])); + } + + private function workflowTask(string $runId): WorkflowTask + { + return WorkflowTask::query() + ->where('workflow_run_id', $runId) + ->where('task_type', TaskType::Workflow->value) + ->whereIn('status', [TaskStatus::Ready->value, TaskStatus::Leased->value]) + ->sole(); + } + + private function summary(string $runId): WorkflowRunSummary + { + return WorkflowRunSummary::query()->findOrFail($runId); + } + + /** + * @return list + */ + private function historyTypes(string $runId): array + { + return WorkflowHistoryEvent::query() + ->where('workflow_run_id', $runId) + ->orderBy('sequence') + ->get() + ->map(static fn (WorkflowHistoryEvent $event): string => $event->event_type->value) + ->all(); + } +} diff --git a/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json new file mode 100644 index 000000000..8ef2ba2b6 --- /dev/null +++ b/tests/Fixtures/V2/ReplayRegression/workflow-task-lease-lost-fails-run-opt-in.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "workflow-task-lease-lost-fails-run-opt-in", + "protocol_version": "1.0", + "bindings": [ + "php" + ], + "consumers": [ + "workflow-fiber-runner" + ], + "workflow": { + "type": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "arguments": [ + "Taylor" + ], + "payload_codec": "avro" + }, + "workflow_task_lease_lost": { + "issue": "durable-workflow/workflow#1231", + "attention_threshold": 3, + "fail_run": true, + "note": "The run's only workflow task lost its lease three times without completing, as when its worker is killed, times out or loses its connection. With workflows.v2.workflow_task_lost_leases.fail_run enabled, the next claim fails the run with the typed WorkflowTaskLeaseLostException (task_failure) before any workflow code runs. Lost leases themselves write no history; with fail_run off the same run keeps retrying and its summary carries the additive task_attention field." + }, + "history": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "memo": null, + "business_key": null, + "workflow_type": "test-greeting-workflow", + "workflow_class": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "run_deadline_at": null, + "workflow_run_id": "lost-lease-run", + "declared_queries": [], + "declared_signals": [], + "declared_updates": [], + "search_attributes": null, + "visibility_labels": null, + "declared_entry_mode": "canonical", + "run_timeout_seconds": null, + "workflow_command_id": "lost-lease-start-command", + "workflow_instance_id": "lost-lease-instance", + "declared_entry_method": "handle", + "execution_deadline_at": null, + "declared_query_contracts": [], + "declared_signal_contracts": [], + "declared_update_contracts": [], + "execution_timeout_seconds": null, + "declared_entry_declaring_class": "Tests\\Fixtures\\V2\\TestGreetingWorkflow", + "workflow_definition_fingerprint": "sha256:d938f655e9a8d2d1074d252bdd2004c611304acb5db649367a21552784f7fa2c" + }, + "recorded_at": "2026-11-19T00:00:00.000000Z" + }, + { + "sequence": 2, + "event_type": "WorkflowFailed", + "payload": { + "message": "Workflow task lost-lease-task lost its lease 3 times in a row without completing (its worker was killed, timed out, or lost its connection).", + "exception": { + "code": 0, + "file": "/app/src/V2/Support/WorkflowTaskLeaseLoss.php", + "line": 152, + "class": "Workflow\\V2\\Exceptions\\WorkflowTaskLeaseLostException", + "trace": [], + "message": "Workflow task lost-lease-task lost its lease 3 times in a row without completing (its worker was killed, timed out, or lost its connection)." + }, + "source_id": "lost-lease-run", + "failure_id": "lost-lease-failure", + "source_kind": "workflow_run", + "non_retryable": false, + "exception_type": null, + "exception_class": "Workflow\\V2\\Exceptions\\WorkflowTaskLeaseLostException", + "failure_category": "task_failure" + }, + "recorded_at": "2026-11-19T00:00:01.000000Z" + } + ], + "expected": { + "completed": false, + "result": null, + "commands": [ + { + "type": "schedule_activity", + "activity_type": "Tests\\Fixtures\\V2\\TestGreetingActivity" + } + ] + } +}