From e079029ec8e04d13e6ae26091b0ce626344c4d6b Mon Sep 17 00:00:00 2001 From: PhiLily <252857470+PhiLily@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:04:13 +0300 Subject: [PATCH] Add the optional `oxpull` app to django-ox Include the optional `oxpull` app in the django-ox wheel and sdist under BSD-3-Clause. All features are free and open source. Provide unique tasks, batches, shared admission rate limits, and task workflows. The workflows API remains provisional. Require Django 6.0+ only when the app is enabled. Projects that do not enable it do not load it. Keep django-ox support for Django 5.2 through the django-tasks backport. Preserve settings, import paths, task data, the app label, table names, the five `oxpull` migrations, and persisted task paths. Align `oxpull.__version__` with `django_ox.__version__`. Earlier separate installs must uninstall the earlier `oxpull` distribution first, then upgrade django-ox, migrate, and restart. Upgrade guide: https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs Add sharded app tests on Django 6.0+ with SQLite, PostgreSQL (plain and pooled), and MySQL. Run long-running hazard tests for relevant changed paths, and in full on release-candidate tags, nightly, and manual dispatch. Add opt-in installed-wheel upgrade tests and a single aggregate result check. Keep running the existing django-ox suite on Django 5.2 to 6.1. Add and extend tests for the Django 5.2 refusal of `oxpull`, CI path selection, release checks, log event documentation, and docs index page URLs. Add type-check settings. --- .github/FUNDING.yml | 1 - .github/workflows/ci.yml | 211 +- AGENTS.md | 162 +- CHANGELOG.md | 53 +- CODE_OF_CONDUCT.md | 2 +- CONTRIBUTING.md | 42 +- README.md | 65 +- SECURITY.md | 22 +- context7.json | 4 +- docs/agents.md | 71 +- docs/choosing.md | 60 +- docs/configuration.md | 54 + docs/features/batches.md | 263 ++ docs/features/index.md | 64 + docs/features/rate-limiting.md | 257 ++ docs/features/unique-tasks.md | 236 + docs/features/workflows.md | 368 ++ docs/index.md | 27 +- docs/llms-full.txt | 1965 +++++++-- docs/llms.txt | 52 +- docs/migrating.md | 82 +- docs/monitoring.md | 86 +- docs/patterns.md | 29 +- docs/pro.md | 121 - docs/production.md | 64 +- docs/requirements.txt | 2 + docs/stability.md | 66 +- docs/stored-schedules.md | 10 +- docs/use-cases.md | 14 +- docs/why-django-ox.md | 10 +- mkdocs.yml | 13 +- pyproject.toml | 11 +- src/django_ox/_run_tasks.py | 2 +- src/oxpull/__init__.py | 5 + src/oxpull/_django_gate.py | 38 + src/oxpull/_requires_django_6.py | 10 + src/oxpull/apps.py | 16 + src/oxpull/backend.py | 1831 ++++++++ src/oxpull/batches.py | 1633 +++++++ src/oxpull/checks.py | 176 + src/oxpull/exceptions.py | 58 + src/oxpull/management/__init__.py | 0 src/oxpull/management/commands/__init__.py | 0 .../management/commands/oxpull_prune.py | 173 + src/oxpull/migrations/0001_initial.py | 25 + .../0002_oxpullbatch_oxpullbatchmember.py | 99 + .../migrations/0003_oxpullratewindow.py | 24 + src/oxpull/migrations/0004_workflows.py | 338 ++ .../0005_workflow_edge_upstream_idx.py | 32 + src/oxpull/migrations/__init__.py | 0 src/oxpull/migrations/_workflow_guard.py | 74 + src/oxpull/models.py | 468 ++ src/oxpull/py.typed | 0 src/oxpull/rate_limits.py | 218 + src/oxpull/routing.py | 99 + src/oxpull/unique.py | 104 + src/oxpull/worker.py | 1119 +++++ src/oxpull/workflows.py | 3926 +++++++++++++++++ tests/settings_typecheck.py | 6 + tests/test_ci_changes.py | 211 + tests/test_claim_fence.py | 18 +- tests/test_claim_recovery.py | 4 +- tests/test_llms_full.py | 9 + tests/test_monitoring_docs.py | 85 + tests/test_oxpull_gate.py | 142 + tests/test_release_check.py | 79 +- tests_oxpull/__init__.py | 25 + tests_oxpull/_historical.py | 54 + tests_oxpull/conftest.py | 235 + tests_oxpull/fixtures/PROVENANCE.md | 23 + .../fixtures/build_oxpull_1_2_0_src.py | 49 + tests_oxpull/fixtures/oxpull-1.2.0-src.tar.gz | Bin 0 -> 42942 bytes tests_oxpull/forms.py | 18 + tests_oxpull/isolation.py | 310 ++ tests_oxpull/policy_tasks.py | 146 + tests_oxpull/racing.py | 84 + tests_oxpull/routers.py | 28 + tests_oxpull/settings.py | 142 + tests_oxpull/settings_mysql.py | 60 + tests_oxpull/settings_postgres.py | 33 + tests_oxpull/settings_postgres_pooled.py | 7 + tests_oxpull/subprocesses.py | 17 + tests_oxpull/tasks.py | 172 + tests_oxpull/test_batches.py | 2353 ++++++++++ tests_oxpull/test_concurrency.py | 2090 +++++++++ tests_oxpull/test_heartbeat_file.py | 548 +++ tests_oxpull/test_keys.py | 78 + tests_oxpull/test_policy.py | 651 +++ tests_oxpull/test_prune.py | 1298 ++++++ tests_oxpull/test_rate_limit_names.py | 980 ++++ tests_oxpull/test_rate_limits.py | 1149 +++++ tests_oxpull/test_run_tasks_integration.py | 94 + tests_oxpull/test_schedule_isolation.py | 685 +++ tests_oxpull/test_unique.py | 1980 +++++++++ tests_oxpull/test_upgrade.py | 146 + tests_oxpull/test_worker_subclass.py | 514 +++ tests_oxpull/test_workflows.py | 1276 ++++++ tests_oxpull/test_workflows_at_scale.py | 150 + tests_oxpull/test_workflows_callers.py | 205 + tests_oxpull/test_workflows_concurrency.py | 224 + tests_oxpull/test_workflows_cost.py | 236 + tests_oxpull/test_workflows_deferred.py | 403 ++ tests_oxpull/test_workflows_guarantees.py | 258 ++ ..._correctly_configured_worker_on_another.py | 612 +++ ...ag_that_runs_longer_than_ox_prune_older.py | 312 ++ ...nstream_node_can_start_while_an_earlier.py | 427 ++ ..._node_that_builds_workflow_structure_is.py | 397 ++ ...stored_admin_edited_reconciler_schedule.py | 566 +++ ...support_engineer_sets_an_abandoned_open.py | 337 ++ ...unique_enqueue_inside_a_transaction_can.py | 501 +++ ...kflow_build_that_raises_part_way_leaves.py | 374 ++ ...a_workflow_callback_that_becomes_unique.py | 324 ++ ...fires_task_enqueued_before_it_validates.py | 242 + .../test_workflows_hazard_add_mysql.py | 265 ++ .../test_workflows_hazard_cancel_cas.py | 401 ++ ...ncellation_locks_the_upstreams_that_are.py | 541 +++ ...defer_blocked_nodes_does_not_take_nodes.py | 377 ++ ...er_blocked_nodes_is_decided_by_the_node.py | 283 ++ ...eting_a_workflow_row_cascades_its_edges.py | 244 + .../test_workflows_hazard_discard_promise.py | 1804 ++++++++ ...o_ox_stats_ox_health_and_the_prometheus.py | 318 ++ ...ry_reconciler_tick_row_locks_every_dead.py | 503 +++ ...wing_w003_s_hint_verbatim_stops_workers.py | 376 ++ ..._flight_fan_in_edges_make_the_gate_cost.py | 533 +++ .../test_workflows_hazard_lost_straggler.py | 568 +++ ...ariadb_after_a_server_upgrade_from_10_6.py | 275 ++ ..._workflows_hazard_mixed_fleet_migration.py | 896 ++++ ...mysql_a_gated_claim_whose_scan_spans_an.py | 361 ++ ...sql_edge_columns_created_on_a_different.py | 555 +++ ...node_name_uniqueness_follows_the_column.py | 156 + ...release_dependents_and_the_cancellation.py | 510 +++ ...mysql_repeatable_read_inside_a_caller_s.py | 291 ++ ...ysql_repeatable_read_the_cancellation_s.py | 934 ++++ ...des_the_reconciler_cancels_emit_no_task.py | 650 +++ ...n_postgresql_a_worker_class_subclass_of.py | 499 +++ ...prune_include_failed_deletes_a_workflow.py | 430 ++ ...stgresql_repeatable_read_concurrent_add.py | 529 +++ .../test_workflows_hazard_prune_membership.py | 441 ++ .../test_workflows_hazard_prune_ungates.py | 500 +++ ...est_workflows_hazard_reconcile_schedule.py | 277 ++ ...ying_a_failed_node_after_one_reconciler.py | 395 ++ ...g_back_migrate_oxpull_0003_or_downgrade.py | 383 ++ .../test_workflows_hazard_seal_lock.py | 416 ++ ...e_cancellation_transaction_s_lock_order.py | 461 ++ ...onciler_s_per_workflow_except_exception.py | 941 ++++ ..._reconciler_s_release_sweep_reads_every.py | 445 ++ ...the_stored_cancelled_count_counts_every.py | 301 ++ ...r_use_tz_false_on_sqlite_or_mysql_every.py | 579 +++ .../test_workflows_hazard_unique_nodes.py | 628 +++ ...es_let_two_workflows_build_a_dependency.py | 253 ++ .../test_workflows_hazard_unknown_status.py | 305 ++ ...th_defer_blocked_nodes_releasing_a_node.py | 234 + .../test_workflows_one_key_statistics.py | 821 ++++ .../test_workflows_outcome_reconnect.py | 1055 +++++ tests_oxpull/test_workflows_outcomes.py | 1997 +++++++++ tests_oxpull/test_workflows_reconciler.py | 1699 +++++++ tests_oxpull/test_workflows_statistics.py | 307 ++ tests_oxpull/test_workflows_waiting.py | 1171 +++++ tests_oxpull/workers.py | 134 + tests_wheel/__init__.py | 0 tests_wheel/conftest.py | 90 + tests_wheel/test_wheel_transitions.py | 341 ++ tests_wheel/wheelkit.py | 427 ++ tools/build_llms_full.py | 10 +- tools/check-policy-pair | 653 --- tools/check_release.py | 67 +- tools/ci/changes.py | 183 + tools/ci/shard.py | 117 + tools/ci/shards.json | 92 + tools/mixed_fleet_policy.py | 9 +- 170 files changed, 63054 insertions(+), 1299 deletions(-) delete mode 100644 .github/FUNDING.yml create mode 100644 docs/features/batches.md create mode 100644 docs/features/index.md create mode 100644 docs/features/rate-limiting.md create mode 100644 docs/features/unique-tasks.md create mode 100644 docs/features/workflows.md delete mode 100644 docs/pro.md create mode 100644 src/oxpull/__init__.py create mode 100644 src/oxpull/_django_gate.py create mode 100644 src/oxpull/_requires_django_6.py create mode 100644 src/oxpull/apps.py create mode 100644 src/oxpull/backend.py create mode 100644 src/oxpull/batches.py create mode 100644 src/oxpull/checks.py create mode 100644 src/oxpull/exceptions.py create mode 100644 src/oxpull/management/__init__.py create mode 100644 src/oxpull/management/commands/__init__.py create mode 100644 src/oxpull/management/commands/oxpull_prune.py create mode 100644 src/oxpull/migrations/0001_initial.py create mode 100644 src/oxpull/migrations/0002_oxpullbatch_oxpullbatchmember.py create mode 100644 src/oxpull/migrations/0003_oxpullratewindow.py create mode 100644 src/oxpull/migrations/0004_workflows.py create mode 100644 src/oxpull/migrations/0005_workflow_edge_upstream_idx.py create mode 100644 src/oxpull/migrations/__init__.py create mode 100644 src/oxpull/migrations/_workflow_guard.py create mode 100644 src/oxpull/models.py create mode 100644 src/oxpull/py.typed create mode 100644 src/oxpull/rate_limits.py create mode 100644 src/oxpull/routing.py create mode 100644 src/oxpull/unique.py create mode 100644 src/oxpull/worker.py create mode 100644 src/oxpull/workflows.py create mode 100644 tests/settings_typecheck.py create mode 100644 tests/test_ci_changes.py create mode 100644 tests/test_oxpull_gate.py create mode 100644 tests_oxpull/__init__.py create mode 100644 tests_oxpull/_historical.py create mode 100644 tests_oxpull/conftest.py create mode 100644 tests_oxpull/fixtures/PROVENANCE.md create mode 100644 tests_oxpull/fixtures/build_oxpull_1_2_0_src.py create mode 100644 tests_oxpull/fixtures/oxpull-1.2.0-src.tar.gz create mode 100644 tests_oxpull/forms.py create mode 100644 tests_oxpull/isolation.py create mode 100644 tests_oxpull/policy_tasks.py create mode 100644 tests_oxpull/racing.py create mode 100644 tests_oxpull/routers.py create mode 100644 tests_oxpull/settings.py create mode 100644 tests_oxpull/settings_mysql.py create mode 100644 tests_oxpull/settings_postgres.py create mode 100644 tests_oxpull/settings_postgres_pooled.py create mode 100644 tests_oxpull/subprocesses.py create mode 100644 tests_oxpull/tasks.py create mode 100644 tests_oxpull/test_batches.py create mode 100644 tests_oxpull/test_concurrency.py create mode 100644 tests_oxpull/test_heartbeat_file.py create mode 100644 tests_oxpull/test_keys.py create mode 100644 tests_oxpull/test_policy.py create mode 100644 tests_oxpull/test_prune.py create mode 100644 tests_oxpull/test_rate_limit_names.py create mode 100644 tests_oxpull/test_rate_limits.py create mode 100644 tests_oxpull/test_run_tasks_integration.py create mode 100644 tests_oxpull/test_schedule_isolation.py create mode 100644 tests_oxpull/test_unique.py create mode 100644 tests_oxpull/test_upgrade.py create mode 100644 tests_oxpull/test_worker_subclass.py create mode 100644 tests_oxpull/test_workflows.py create mode 100644 tests_oxpull/test_workflows_at_scale.py create mode 100644 tests_oxpull/test_workflows_callers.py create mode 100644 tests_oxpull/test_workflows_concurrency.py create mode 100644 tests_oxpull/test_workflows_cost.py create mode 100644 tests_oxpull/test_workflows_deferred.py create mode 100644 tests_oxpull/test_workflows_guarantees.py create mode 100644 tests_oxpull/test_workflows_hazard_a_correctly_configured_worker_on_another.py create mode 100644 tests_oxpull/test_workflows_hazard_a_dag_that_runs_longer_than_ox_prune_older.py create mode 100644 tests_oxpull/test_workflows_hazard_a_downstream_node_can_start_while_an_earlier.py create mode 100644 tests_oxpull/test_workflows_hazard_a_node_that_builds_workflow_structure_is.py create mode 100644 tests_oxpull/test_workflows_hazard_a_stored_admin_edited_reconciler_schedule.py create mode 100644 tests_oxpull/test_workflows_hazard_a_support_engineer_sets_an_abandoned_open.py create mode 100644 tests_oxpull/test_workflows_hazard_a_unique_enqueue_inside_a_transaction_can.py create mode 100644 tests_oxpull/test_workflows_hazard_a_workflow_build_that_raises_part_way_leaves.py create mode 100644 tests_oxpull/test_workflows_hazard_a_workflow_callback_that_becomes_unique.py create mode 100644 tests_oxpull/test_workflows_hazard_add_fires_task_enqueued_before_it_validates.py create mode 100644 tests_oxpull/test_workflows_hazard_add_mysql.py create mode 100644 tests_oxpull/test_workflows_hazard_cancel_cas.py create mode 100644 tests_oxpull/test_workflows_hazard_cancellation_locks_the_upstreams_that_are.py create mode 100644 tests_oxpull/test_workflows_hazard_defer_blocked_nodes_does_not_take_nodes.py create mode 100644 tests_oxpull/test_workflows_hazard_defer_blocked_nodes_is_decided_by_the_node.py create mode 100644 tests_oxpull/test_workflows_hazard_deleting_a_workflow_row_cascades_its_edges.py create mode 100644 tests_oxpull/test_workflows_hazard_discard_promise.py create mode 100644 tests_oxpull/test_workflows_hazard_django_ox_stats_ox_health_and_the_prometheus.py create mode 100644 tests_oxpull/test_workflows_hazard_every_reconciler_tick_row_locks_every_dead.py create mode 100644 tests_oxpull/test_workflows_hazard_following_w003_s_hint_verbatim_stops_workers.py create mode 100644 tests_oxpull/test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost.py create mode 100644 tests_oxpull/test_workflows_hazard_lost_straggler.py create mode 100644 tests_oxpull/test_workflows_hazard_mariadb_after_a_server_upgrade_from_10_6.py create mode 100644 tests_oxpull/test_workflows_hazard_mixed_fleet_migration.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_a_gated_claim_whose_scan_spans_an.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_edge_columns_created_on_a_different.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_node_name_uniqueness_follows_the_column.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_release_dependents_and_the_cancellation.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_repeatable_read_inside_a_caller_s.py create mode 100644 tests_oxpull/test_workflows_hazard_mysql_repeatable_read_the_cancellation_s.py create mode 100644 tests_oxpull/test_workflows_hazard_nodes_the_reconciler_cancels_emit_no_task.py create mode 100644 tests_oxpull/test_workflows_hazard_on_postgresql_a_worker_class_subclass_of.py create mode 100644 tests_oxpull/test_workflows_hazard_ox_prune_include_failed_deletes_a_workflow.py create mode 100644 tests_oxpull/test_workflows_hazard_postgresql_repeatable_read_concurrent_add.py create mode 100644 tests_oxpull/test_workflows_hazard_prune_membership.py create mode 100644 tests_oxpull/test_workflows_hazard_prune_ungates.py create mode 100644 tests_oxpull/test_workflows_hazard_reconcile_schedule.py create mode 100644 tests_oxpull/test_workflows_hazard_retrying_a_failed_node_after_one_reconciler.py create mode 100644 tests_oxpull/test_workflows_hazard_rolling_back_migrate_oxpull_0003_or_downgrade.py create mode 100644 tests_oxpull/test_workflows_hazard_seal_lock.py create mode 100644 tests_oxpull/test_workflows_hazard_the_cancellation_transaction_s_lock_order.py create mode 100644 tests_oxpull/test_workflows_hazard_the_reconciler_s_per_workflow_except_exception.py create mode 100644 tests_oxpull/test_workflows_hazard_the_reconciler_s_release_sweep_reads_every.py create mode 100644 tests_oxpull/test_workflows_hazard_the_stored_cancelled_count_counts_every.py create mode 100644 tests_oxpull/test_workflows_hazard_under_use_tz_false_on_sqlite_or_mysql_every.py create mode 100644 tests_oxpull/test_workflows_hazard_unique_nodes.py create mode 100644 tests_oxpull/test_workflows_hazard_unique_nodes_let_two_workflows_build_a_dependency.py create mode 100644 tests_oxpull/test_workflows_hazard_unknown_status.py create mode 100644 tests_oxpull/test_workflows_hazard_with_defer_blocked_nodes_releasing_a_node.py create mode 100644 tests_oxpull/test_workflows_one_key_statistics.py create mode 100644 tests_oxpull/test_workflows_outcome_reconnect.py create mode 100644 tests_oxpull/test_workflows_outcomes.py create mode 100644 tests_oxpull/test_workflows_reconciler.py create mode 100644 tests_oxpull/test_workflows_statistics.py create mode 100644 tests_oxpull/test_workflows_waiting.py create mode 100644 tests_oxpull/workers.py create mode 100644 tests_wheel/__init__.py create mode 100644 tests_wheel/conftest.py create mode 100644 tests_wheel/test_wheel_transitions.py create mode 100644 tests_wheel/wheelkit.py delete mode 100755 tools/check-policy-pair create mode 100644 tools/ci/changes.py create mode 100644 tools/ci/shard.py create mode 100644 tools/ci/shards.json diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 83fc9dd..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -custom: ["https://oxpull.com/"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e111822..72aa8f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,12 +3,52 @@ name: CI on: push: branches: [main] + # A release candidate runs the hazard suite on the whole feature matrix. + tags: ["rc*"] pull_request: + schedule: + # Nightly: the hazard tests run in full on every push to main and every + # night, and on a pull request only when `changes` finds a path they pin. + - cron: "17 3 * * *" + workflow_dispatch: permissions: contents: read jobs: + # Decides what the oxpull feature suite (tests_oxpull/) needs on this run. + # There is no `paths:` filter on the workflow: a filtered-out workflow + # reports nothing, and a required check that never reports either blocks the + # pull request for good or can be bypassed by filtering. Every job below + # stays in the run, and ci-ok at the bottom always reports. + # + # The rules are in tools/ci/changes.py, where tests/test_ci_changes.py pins + # them. They fail towards running more: a path no rule knows runs the + # feature suite, and a diff that cannot be computed turns everything on. + changes: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + feature: ${{ steps.classify.outputs.feature }} + hazard: ${{ steps.classify.outputs.hazard }} + full: ${{ steps.classify.outputs.full }} + sqlite_cells: ${{ steps.classify.outputs.sqlite_cells }} + postgres_cells: ${{ steps.classify.outputs.postgres_cells }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.12" + - id: classify + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: python tools/ci/changes.py + test-sqlite: runs-on: ubuntu-latest timeout-minutes: 30 @@ -41,6 +81,11 @@ jobs: # A run that hangs is cancelled at the job's 30-minute cap with # nothing in the log to say where it sat. SIGABRT at 25 minutes # makes faulthandler dump every thread's stack to the log first. + # + # This leg, like the other base-suite legs, runs tests/ only (testpaths). + # tests_oxpull/ is the oxpull feature suite, which needs Django 6.0 and + # runs in the feature-* jobs. On 5.2 only tests/test_oxpull_gate.py + # touches oxpull, to prove it refuses to load. - run: timeout -s ABRT 1500 python -m pytest -q --cov test-postgres: @@ -305,6 +350,121 @@ jobs: - run: python -c "import MySQLdb, sys; assert 'pymysql' not in sys.modules" - run: timeout -s ABRT 1500 python -m pytest -q tests/test_stored_read.py + # The oxpull feature suite (tests_oxpull/), Django 6.0 and later. Six shards + # of whole files (tools/ci/shards.json, fitted to measured durations). The + # hazard tests (marker `hazard`, every file with "hazard" in its name) are + # left out unless `changes` says this run needs them; a shard made only of + # hazard files then passes with nothing to run. The 5.2 legs do not appear + # here: oxpull refuses to load on 5.2, and testpaths keeps them out. + feature-sqlite: + needs: changes + if: needs.changes.outputs.feature == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.changes.outputs.sqlite_cells) }} + shard: [shard1, shard2, shard3, shard4, shard5, shard6] + name: feature sqlite py${{ matrix.cell.python }} dj${{ matrix.cell.django }} ${{ matrix.shard }} + env: + PYTHONFAULTHANDLER: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: ${{ matrix.cell.python }} + - run: python -m pip install --upgrade pip + - run: python -m pip install --group dev -e . "Django~=${{ matrix.cell.django }}.0" + # SIGABRT with thread stacks instead of a silent cancellation; see + # the sqlite job. shard.py replaces itself with pytest so the signal + # reaches pytest. + - env: + SHARD: ${{ matrix.shard }} + HAZARD: ${{ needs.changes.outputs.hazard }} + run: timeout -s ABRT 1500 python tools/ci/shard.py run "$SHARD" tests_oxpull.settings --hazard "$HAZARD" + + feature-postgres: + needs: changes + if: needs.changes.outputs.feature == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.changes.outputs.postgres_cells) }} + shard: [shard1, shard2, shard3, shard4, shard5, shard6] + name: feature ${{ matrix.cell.label }} py${{ matrix.cell.python }} dj${{ matrix.cell.django }} ${{ matrix.shard }} + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: ox + ports: + - 54329:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + PYTHONFAULTHANDLER: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: ${{ matrix.cell.python }} + - run: python -m pip install --upgrade pip + - run: python -m pip install --group dev -e . "Django~=${{ matrix.cell.django }}.0" + - run: python -c "import psycopg_pool" + - env: + SHARD: ${{ matrix.shard }} + HAZARD: ${{ needs.changes.outputs.hazard }} + SETTINGS: ${{ matrix.cell.settings }} + run: timeout -s ABRT 1500 python tools/ci/shard.py run "$SHARD" "$SETTINGS" --hazard "$HAZARD" + + # MySQL: the whole non-hazard suite in one job (about five minutes), then, + # when hazard tests are due, the files named for MySQL. Those are the hazard + # files that only MySQL can fail, and no shard picks them out. + feature-mysql: + needs: changes + if: needs.changes.outputs.feature == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + name: feature mysql py3.14 dj6.1 + services: + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: ox + MYSQL_DATABASE: oxtest + ports: + - 33069:3306 + options: >- + --health-cmd "mysqladmin ping -h 127.0.0.1 -uroot -pox" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + DJANGO_SETTINGS_MODULE: tests_oxpull.settings_mysql + PYTHONFAULTHANDLER: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: "3.14" + - run: python -m pip install --upgrade pip + - run: python -m pip install --group dev -e . "Django~=6.1.0" PyMySQL + - run: timeout -s ABRT 1500 python -m pytest -q -p no:cacheprovider --ds=tests_oxpull.settings_mysql tests_oxpull -m "not hazard" + - if: needs.changes.outputs.hazard == 'true' + run: timeout -s ABRT 1500 python -m pytest -q -p no:cacheprovider --ds=tests_oxpull.settings_mysql tests_oxpull/test_*mysql*.py + # A >= install resolves to the newest release and cannot test a floor. This # leg pins the floor with == so a regression at the bottom of the supported # range fails here. @@ -477,8 +637,9 @@ jobs: - run: python -m pip install --group dev -e . - run: ruff check . - run: ruff format --check . - # A model change without its migration must fail before it ships. - - run: DJANGO_SETTINGS_MODULE=tests.settings python -m django makemigrations django_ox --check --dry-run + # A model change without its migration must fail before it ships. The + # settings add oxpull, which tests.settings leaves out for Django 5.2. + - run: DJANGO_SETTINGS_MODULE=tests.settings_typecheck python -m django makemigrations django_ox oxpull --check --dry-run types: runs-on: ubuntu-latest @@ -564,10 +725,16 @@ jobs: # job instead of every matrix leg means the protection rule survives # matrix changes: add a Python or database to the grid and the rule # still reads "ci-ok", with the new legs pulled in through needs. + # + # It always reports. The jobs that always run must succeed. A job that + # `changes` decided this run does not need may be skipped, and only then: + # a needed job that is skipped, cancelled, failed or missing from `needs` + # fails ci-ok, and so does a job in `needs` that this list does not + # classify, so adding a job means deciding which kind it is. ci-ok: name: ci-ok if: always() - needs: [test-sqlite, test-postgres, test-postgres-naive, test-postgres-pooled, test-backport-floor, test-pool-floor, wagtail-smoke, test-mysql, test-postgres-psycopg2, test-mysql-mysqlclient, lint, types, docs, packaging, zizmor] + needs: [changes, test-sqlite, test-postgres, test-postgres-naive, test-postgres-pooled, test-backport-floor, test-pool-floor, wagtail-smoke, test-mysql, test-postgres-psycopg2, test-mysql-mysqlclient, feature-sqlite, feature-postgres, feature-mysql, lint, types, docs, packaging, zizmor] runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -575,8 +742,36 @@ jobs: env: NEEDS_JSON: ${{ toJSON(needs) }} run: | - set -euo pipefail - printf '%s\n' "$NEEDS_JSON" - [ "${{ contains(needs.*.result, 'failure') }}" = "false" ] - [ "${{ contains(needs.*.result, 'cancelled') }}" = "false" ] - [ "${{ contains(needs.*.result, 'skipped') }}" = "false" ] + python3 - <<'PY' + import json, os, sys + + needs = json.loads(os.environ["NEEDS_JSON"]) + print(json.dumps({k: v["result"] for k, v in needs.items()}, indent=1)) + always = [ + "test-sqlite", "test-postgres", "test-postgres-naive", + "test-postgres-pooled", "test-backport-floor", "test-pool-floor", + "wagtail-smoke", "test-mysql", "test-postgres-psycopg2", + "test-mysql-mysqlclient", "lint", "types", "docs", "packaging", + "zizmor", + ] + if needs["changes"]["result"] != "success": + sys.exit("the changes job did not succeed: nothing can be trusted") + feature = needs["changes"]["outputs"]["feature"] == "true" + conditional = ["feature-sqlite", "feature-postgres", "feature-mysql"] + bad = [] + for job in always: + if needs[job]["result"] != "success": + bad.append(f"{job}: {needs[job]['result']}") + for job in conditional: + result = needs[job]["result"] + if feature and result != "success": + bad.append(f"{job}: {result} (needed by this change)") + if not feature and result not in ("skipped", "success"): + bad.append(f"{job}: {result} (not needed, but it ran and did not pass)") + unclassified = set(needs) - set(always) - set(conditional) - {"changes"} + if unclassified: + bad.append(f"unclassified jobs in needs: {sorted(unclassified)}") + if bad: + sys.exit("\n".join(bad)) + print("ok") + PY diff --git a/AGENTS.md b/AGENTS.md index e873352..c7fb291 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,35 +1,58 @@ # Working in this repository -**If you are adding django-ox to an application, this is the wrong file.** -Read https://oxpull.com/django-ox/agents/ instead. Nothing below applies to -using the package, and the install command in the next section is a -contributor's editable install, not the one you want. - -Facts for anyone, human or tool, making changes to django-ox itself. +**If you are adding django-ox to an application, read the application guide.** +Use https://oxpull.com/django-ox/agents/ instead. The instructions below are +for changing django-ox itself, and the editable install is for contributors. ## Layout -- `src/django_ox/`: the package. `backend.py` (enqueue, result API), - `worker.py` (claim, execute, lease, reaper, schedule dispatch), - `management/commands/` (`ox_worker`, `ox_health`, `ox_prune`), - `supervisor.py` (the `--processes` supervisor), - `heartbeat.py` (local heartbeat-file updates and checks), - `tasks.py` (`PolicyTask`), `testing.py` (public policy-aware test backends), - `schedules.py` and `cron.py` (recurring tasks), `stats.py` (queue metrics), - `metrics.py`, `views.py` and `urls.py` (the Prometheus endpoint), - `actions.py` (retry, discard), `bulk.py` (`enqueue_many`), `admin.py` - (the admin page, imported only when `django.contrib.admin` is installed). -- `tests/`: pytest suite. `tests/settings.py` is SQLite; `settings_postgres.py` - and `settings_mysql.py` switch the database through `DJANGO_SETTINGS_MODULE`. -- `docs/`: MkDocs site. `docs/llms-full.txt` is generated; do not edit it by - hand. -- `tools/`: `check_release.py` (version and metadata gate), - `build_llms_full.py` (regenerates `docs/llms-full.txt`). -- `benchmarks/`: benchmark harness and published results. Not shipped. +django-ox is one distribution. It contains `django_ox` and the optional +`oxpull` app, which requires Django 6.0 or later. + +- `src/django_ox/`: the base app. `backend.py` implements enqueue and result + APIs; `worker.py` handles claims, execution, leases, reaping and schedule + dispatch. `supervisor.py` implements `--processes`, and `heartbeat.py` + handles local heartbeat-file updates and checks. +- Management commands include `ox_worker`, `ox_health`, `ox_prune`, `ox_import_beat_schedules` and the optional app's `oxpull_prune`. +- Other `django_ox` modules include `tasks.py` (`PolicyTask`), `testing.py` + (policy-aware test backends), `schedules.py` and `cron.py` (recurring + tasks), `stats.py` (queue metrics), and `metrics.py`, `views.py` and + `urls.py` (the Prometheus endpoint). `actions.py` implements retry and + discard; `bulk.py` implements `enqueue_many`. `admin.py` is imported only + when `django.contrib.admin` is installed. +- `src/oxpull/`: the optional app. `backend.py` implements `OxpullBackend` + and unique enqueue; `worker.py` implements `OxpullWorker`, rate-limit + admission and the workflow claim gate. Feature modules include + `unique.py`, `batches.py`, `workflows.py` and `rate_limits.py`. + Models, exceptions and migrations live in `models.py`, `exceptions.py` + and `migrations/`. The app adds the `oxpull_prune` command. + `_django_gate.py` and `_requires_django_6.py` enforce the Django version + requirement. Workflows are provisional. +- `src/oxpull/routing.py` enforces the shared database requirement. `OxpullBackend.check()` implements 14 of the app's check ids; `src/oxpull/checks.py` implements the database checks `oxpull.E009` and `oxpull.W007`. +- `tests/`: the base app's pytest suite and the default pytest target. + `tests/settings.py` uses SQLite; `settings_postgres.py` and + `settings_mysql.py` select other databases. +- `tests_oxpull/`: the optional app's suite, requiring Django 6.0 or later. + Use its own settings through `--ds`. Settings cover SQLite, PostgreSQL, + pooled PostgreSQL and MySQL. Hazard tests carry the `hazard` marker. + `fixtures/PROVENANCE.md` documents the immutable historical source + archive loaded by `_historical.py`. +- `tests_wheel/`: opt-in installed-wheel transition tests, selected with + `-m wheel`. +- `docs/`: the MkDocs site. `docs/llms-full.txt` is generated; do not edit + it by hand. +- `tools/check_release.py` checks release metadata. `pyproject.toml`, `django_ox.__version__` and the newest release entry must agree. `oxpull.__version__` is imported from `django_ox`; the gate refuses a separate literal. +- `tools/build_llms_full.py`: regenerates `docs/llms-full.txt`. +- `tools/ci/changes.py`: selects feature jobs from a diff, defaulting to + more coverage when uncertain. `tools/ci/shard.py` and + `tools/ci/shards.json` divide `tests_oxpull/` into six measured shards. +- `benchmarks/`: the benchmark harness and published results. Not shipped. ## Contributor setup -``` +Use Django 6.0 or later when working on the optional app. + +```sh python -m venv .venv .venv/bin/pip install --group dev -e . .venv/bin/pip install -r docs/requirements.txt @@ -37,9 +60,9 @@ python -m venv .venv ## Gates -Every one of these runs in CI on push and pull request, and all must pass: +Run these local checks: -``` +```sh .venv/bin/python -m pytest -q --cov .venv/bin/ruff check . .venv/bin/ruff format --check . @@ -48,34 +71,91 @@ Every one of these runs in CI on push and pull request, and all must pass: .venv/bin/python tools/check_release.py ``` -PostgreSQL and MySQL legs, run locally when a change touches the claim path: +The pytest command above runs `tests/` only. Strict type checking covers +both apps. Run the optional app's suite separately on Django 6.0 or later: + +```sh +.venv/bin/python -m pytest tests_oxpull --ds=tests_oxpull.settings +``` + +Do not use `tests.*` settings for `tests_oxpull/`. Add `-m "not hazard"` +to exclude hazard tests during a focused run. Run hazard tests when +changing worker, claim, transaction, model, migration or workflow behaviour. + +Check for missing migrations for both apps: + +```sh +DJANGO_SETTINGS_MODULE=tests.settings_typecheck .venv/bin/python -m django makemigrations --check --dry-run django_ox oxpull +``` + +Run installed-wheel transition tests when changing packaging or upgrade +behaviour: +```sh +.venv/bin/python -m pytest tests_wheel -m wheel ``` + +CI configures six shards for the `oxpull` feature suite. The base suite is in `tests/`; the optional app's suite is in `tests_oxpull/`. Run the suites affected by a change, including both for shared runtime behavior. + +### Database checks + +Run the relevant suites on PostgreSQL and MySQL when changing database +behaviour, especially claims. + +```sh docker run -d --name ox-pg -e POSTGRES_PASSWORD=ox -p 54329:5432 postgres:16 DJANGO_SETTINGS_MODULE=tests.settings_postgres .venv/bin/python -m pytest -q +.venv/bin/python -m pytest tests_oxpull --ds=tests_oxpull.settings_postgres ``` -``` +Use `--ds=tests_oxpull.settings_postgres_pooled` to exercise the pooled +PostgreSQL configuration. + +```sh .venv/bin/pip install PyMySQL docker run -d --name ox-mysql -e MYSQL_ROOT_PASSWORD=ox \ -e MYSQL_DATABASE=oxtest -p 33069:3306 mysql:8 DJANGO_SETTINGS_MODULE=tests.settings_mysql .venv/bin/python -m pytest -q +.venv/bin/python -m pytest tests_oxpull --ds=tests_oxpull.settings_mysql ``` ## Rules that are not obvious from the code -- Attempts are consumed at claim time, execution is at-least-once, and enqueue - is transactional. Changes to the claim, retry or dispatch paths keep those - invariants; the module docstrings in `worker.py` and `backend.py` state them. +- Attempts are consumed at claim time, execution is at-least-once, and + enqueue is transactional. Preserve these invariants when changing + claim, retry or dispatch paths. Read the module docstrings in + `src/django_ox/worker.py` and `src/django_ox/backend.py`. +- Uniqueness deduplicates pending tasks at enqueue time. It does not + prevent two executions of one key from overlapping. +- `oxpull` creates only its own tables and does not alter `django_ox` + tables, columns, indexes, constraints or migrations. Workflows do write + `django_ox` task rows. Preserve the workflow claim gate and the task-row + lease-epoch checks for cancellation and release. The workflow + `generation` counter is reserved and remains 0; it is not a lease epoch. + Read the module docstrings in `src/oxpull/backend.py`, + `src/oxpull/worker.py` and `src/oxpull/workflows.py`. +- `OPTIONS["WORKFLOWS"]` is a boolean. Set it to `True` to enable + workflows, which are created at runtime rather than from definitions. +- Keep `oxpull` and `django_ox` tables on the same database. + `OxpullBackend.__init__` raises `ImproperlyConfigured` at startup if + routing splits them. `oxpull.E004` reports a backend constructed before + a router change; it is not the initial startup guard. - Behaviour changes need a test that fails without the change. -- User-visible changes get a line in `CHANGELOG.md` under an Unreleased heading. - `tools/check_release.py` requires the version in `pyproject.toml`, - `django_ox.__version__` and the newest changelog entry to agree. +- User-visible changes get a line in `CHANGELOG.md` under an Unreleased + heading. `pyproject.toml`, `django_ox.__version__` and the newest + release entry must agree. + `tools/check_release.py` checks release consistency. - After editing anything under `docs/` or the `nav` in `mkdocs.yml`, run - `python tools/build_llms_full.py`; `tests/test_llms_full.py` fails otherwise. -- `context7.json` `rules` are facts about the shipped code. Add one only when - it is true in `src/` and stated in the docs. -- Comments explain why, not what. No commented-out code and no leftover work - markers; ruff enforces both. -- The docs site publishes by hand with `mkdocs gh-deploy`, never on merge, and - a PyPI release cannot be edited afterwards. + `python tools/build_llms_full.py`; `tests/test_llms_full.py` checks the + generated output. +- Document each `oxpull` event in `docs/monitoring.md`. Keep the + monitoring-docs scanner's coverage aligned with the documented events. +- Keep the README and `oxpull.backend` module docstring explicit about + the schema boundary and workflow task-row writes. The workflow hazard + suite checks both. +- `context7.json` rules describe shipped code. Add a rule only when it is + true in `src/` and stated in the docs. +- Comments explain why, not what. No commented-out code or leftover work + markers. +- The docs site publishes by hand with `mkdocs gh-deploy`, never on merge. + A PyPI release cannot be edited afterwards. diff --git a/CHANGELOG.md b/CHANGELOG.md index a44d2a0..1c04726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 **Upgrading:** No database migration is required. +### Upgrading earlier `oxpull` installs + +If you installed `oxpull` separately, stop the affected processes and back up the database. Uninstall the earlier `oxpull` distribution before upgrading django-ox, then run migrations and restart. + +Do not uninstall an earlier `oxpull` distribution after installing django-ox 1.9.0 as a standalone cleanup step: its uninstall record can delete files now supplied by django-ox. The module-free 1.9.0 metapackage does not have that behavior. + +If the distributions were installed in the wrong order, remove any remaining earlier `oxpull` distribution and then force-reinstall django-ox. See [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs) for the complete sequence and repair commands. + +### Changed + +- The `oxpull` distribution on PyPI is now a metapackage at version 1.9.0. + It supplies no Python module and depends on `django-ox>=1.9.0`. + The django-ox distribution supplies the `oxpull` module. + ### Added +- Include the optional `oxpull` app in the django-ox wheel and source distribution under the BSD-3-Clause licence. It provides: + - Unique tasks, which deduplicate pending tasks at enqueue time. + - Batches. + - Rate limiting. + - Provisional workflows. +- Include the app's five migrations and `py.typed` marker. No separate `oxpull` installation is needed to use the app. + - Log `worker_connection_hint` once at INFO during `Worker.run()` startup on PostgreSQL without pooling and on MySQL when `CONN_MAX_AGE = 0`. The `django_ox` record includes `worker_id`, `database` and `vendor`. @@ -27,6 +48,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Compatibility +- django-ox continues to support Django 5.2 through the `django-tasks` backport. +- The optional `oxpull` app requires Django 6.0+. Enabling the app or importing its runtime entry points on Django 5.2 raises `ImproperlyConfigured`. +- Projects that do not enable the optional app do not load it. +- Existing settings, import paths, task data, the app label, table names, the five `oxpull` migrations and persisted task paths are unchanged. +- `oxpull.__version__` now equals `django_ox.__version__`. +- The app's display name is now "Oxpull". +- The `oxpull` app now enforces its existing Django 6.0+ requirement at load time, raising `ImproperlyConfigured` on Django 5.2; django-ox itself still installs there. The earlier separate `oxpull` 1.8.0 package already declared Django>=6.0 in its metadata. Apart from this load-time check, the app's runtime behavior is unchanged from `oxpull` 1.8.0. One configuration-error message has also been reworded. + - Default connection lifecycle, transaction, delivery and schema semantics are unchanged. Persistence and pooling remain opt-in. Workers with and without persistence shared a queue in tests using @@ -573,8 +602,8 @@ This release preserves the caller's transaction when `run_once()` times out and **Upgrading to 1.5.0:** No database migration is required. Replace every worker to apply schedule isolation, dead-connection outcome recovery and the timeout-watchdog fix throughout the fleet. Schedule isolation adds no system -check. When using Oxpull, deploy django-ox and Oxpull only as the tested -exact-pinned release pair. +check. For this historical release, installations using `oxpull` required +the matching tested versions of both distributions, with exact version pins. If you used `ox_import_beat_schedules`, read the [Security entry](#security) before generating or applying output. Upgrade and regenerate saved output. @@ -859,8 +888,8 @@ connections outside the pool, in addition to `max_size`. Task timeouts (`TASK_TI another connection outside the pool. Each worker process can hold up to `max_size + 1` server connections, or `max_size + 2` with task timeouts. -No migration. For `oxpull` installations, use `oxpull==1.4.0`, -which pins `django-ox==1.4.0`. +No migration. For this historical release, installations using `oxpull` +required `oxpull` 1.4.0, whose dependency required django-ox 1.4.0 exactly. ### PostgreSQL pool correction @@ -1020,13 +1049,12 @@ No migration. Upgrade is optional. - Update the benchmarks page with measurements against django-tasks-db 0.13.0 and recovery after worker death. - Update the comparison page and correct the huey entry. -- Document how to order Oxpull Pro. ## [1.3.0] - 2026-09-17 **One migration ships with this release.** `0008_waiting` adds a status choice and runs no SQL. django-ox never puts a task into the new status by -itself. An install that doesn't use workflows in Oxpull Pro upgrades and +itself. An install that doesn't use workflows in `oxpull` upgrades and rolls back as it always has. A 1.2 process doesn't know the new status. Its admin shows a waiting task's @@ -1040,12 +1068,10 @@ one. row whose status is `WAITING`. No worker claims it, the reaper doesn't reap it, `ox_prune` doesn't delete it, and retry skips it. django-ox writes that status onto no row of its own, and releases no row that holds it. Something -built on django-ox does both, and on this release that is workflows in Oxpull -Pro. +built on django-ox does both, and in this release that is workflows in +`oxpull`. -Turning workflows on, and turning them off again, is Oxpull Pro configuration, -and Oxpull's own documentation has those steps. This section is the django-ox -half. See [Pro](https://oxpull.com/django-ox/pro/). +Before turning workflows off, follow the documented shutdown procedure so that WAITING nodes are not left without a worker that can release them. See [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs) and the [workflow documentation](https://oxpull.com/django-ox/features/workflows/). `migrate django_ox 0007` refuses while any task is WAITING on the database it migrates. A version from before `0008_waiting` can't read a waiting task and @@ -2085,9 +2111,8 @@ appears on the package page. - Packaging metadata now carries a `Documentation` URL, so the documentation site is linked directly from the package page. - README now carries release and CI status badges, a link to the - documentation site, and a scope statement: what the core covers, what is - deliberately outside it, and which features belong to the commercial - tier. + documentation site, and a scope statement distinguishing the core + features from those provided by `oxpull`. ## [0.1.0] - 2026-08-16 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 646344f..8b7ad02 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -37,7 +37,7 @@ This Code of Conduct applies within all community spaces, and also applies when ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at support@oxpull.com. All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at support@oxpull.com. All community leaders are obligated to respect the privacy and security of the reporter of any incident. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c242ca..50d8bd4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,21 +11,49 @@ python -m venv .venv ## Running the tests -SQLite (default): +The default pytest command runs `tests/` only, using SQLite: -``` +```sh .venv/bin/python -m pytest ``` -PostgreSQL 16, which exercises the single-statement `SKIP LOCKED` claim -path: +To run that suite on PostgreSQL 16, including the single-statement +`SKIP LOCKED` claim path: -``` +```sh docker run -d --name ox-pg -e POSTGRES_PASSWORD=ox -p 54329:5432 postgres:16 DJANGO_SETTINGS_MODULE=tests.settings_postgres .venv/bin/python -m pytest ``` -Run both before opening a PR. CI tests every supported Python and Django version on SQLite and PostgreSQL, and the oldest and newest corners on MySQL 8. Separate psycopg2 and mysqlclient jobs are configured to test only the stored-row reader. +Run both before opening a pull request. + +The optional `oxpull` app has a separate suite that requires Django 6.0 or +later and its own settings: + +```sh +.venv/bin/python -m pytest tests_oxpull --ds=tests_oxpull.settings +``` + +For other database configurations, use `--ds=tests_oxpull.settings_postgres`, +`--ds=tests_oxpull.settings_postgres_pooled` or +`--ds=tests_oxpull.settings_mysql`. Do not run this suite with `tests.*` +settings. Add `-m "not hazard"` to exclude tests marked `hazard`; run the +hazard tests when changing worker, claim, transaction, model, migration or +workflow behaviour. Workflows are provisional. + +Installed-wheel tests are opt-in. They build wheels and virtual environments: + +```sh +.venv/bin/python -m pytest tests_wheel -m wheel +``` + +CI configures six shards for the `oxpull` feature suite. Keep feature tests in `tests_oxpull/` and base tests in `tests/`. Changes to shared runtime code must be checked against both suites; a passing base suite alone does not cover the optional app. + +Strict type checking covers both `src/django_ox/` and `src/oxpull/`: + +```sh +.venv/bin/mypy --strict src/ +``` ## Style @@ -66,6 +94,8 @@ upload, so anything wrong in the metadata stays wrong until the next version. ## Pull requests +Contributions are licensed under the project's BSD-3-Clause licence. + - One change per PR, with tests. Behavior changes need a test that fails without the fix. - Anything touching the claim, retry, or dispatch paths must keep the diff --git a/README.md b/README.md index a21f407..7289688 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Stop running Redis to send an email. django-ox runs Django's Tasks framework in your existing database, where a dead worker doesn't mean lost tasks. Run a worker process, not a separate broker. -Supports Django 5.2 LTS through the `django-tasks` backport, and Django 6.0 / 6.1 through core `django.tasks`. Requires Python 3.12+. +Supports Django 5.2 LTS through the `django-tasks` backport, and Django 6.0 / 6.1 through core `django.tasks`. Requires Python 3.12+. The optional `oxpull` app provides task-coordination features and requires Django 6.0+. ## Install @@ -60,10 +60,20 @@ In the [published benchmarks](https://oxpull.com/django-ox/benchmarks/), 2,000 o CI covers Python 3.12 to 3.14, Django 5.2 to 6.1, PostgreSQL, MySQL and SQLite. Open source under the BSD 3-Clause licence. -Need to coordinate work across tasks? [Oxpull Pro](https://oxpull.com/) adds batches, unique tasks, rate limiting and workflows. +django-ox 1.9.0 includes the optional `oxpull` app for [unique tasks, batches, rate limiting and workflows](https://oxpull.com/django-ox/features/). The app requires Django 6.0+. Projects that do not enable it can continue using django-ox on Django 5.2. Documentation: +## Task coordination + +### Enable the optional app + +Add `"oxpull"` to `INSTALLED_APPS`, run migrations, and use `"oxpull.backend.OxpullBackend"` for the task aliases that need its features. No separate package installation is needed. + +Unique tasks deduplicate pending tasks at enqueue time. Batches also need a reconciler schedule. Rate limits need `OPTIONS["RATE_LIMITS"]` and `"oxpull.worker.OxpullWorker"` as the alias's `OPTIONS["WORKER_CLASS"]`. Workflows need `OPTIONS["WORKFLOWS"] = True`, `OxpullWorker` and a reconciler schedule. Workflows are created at runtime, not declared as workflow definitions in settings. + +See the [feature setup](https://oxpull.com/django-ox/features/) and, for an existing installation, [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs). + ## One fewer service to run A broker-based task queue adds a second datastore to your deployment. Redis or @@ -93,19 +103,23 @@ limit, keeping the full traceback of every attempt. ## Measured [The benchmarks page](https://oxpull.com/django-ox/benchmarks/) compares -django-ox with `django-tasks-db`, another database backend for the Tasks -framework, on one machine with both arms on Django core `django.tasks` and -PostgreSQL 16: backlog drain at two queue depths with matched worker -processes, enqueue latency, bulk enqueue, worker death under repeated SIGKILL -with restarts, and exception retry. Every figure there is a median over five -interleaved runs with its range, computed from the raw JSON files committed -beside the page, and every run is reported. +django-ox 1.3.0 with `django-tasks-db` 0.13.0, another database backend +for the Tasks framework, in runs dated 2026-09-19. Both use Django core +`django.tasks`, PostgreSQL 16 in Docker on an Apple M1 Max, no-op task +bodies and package defaults except `LOCK_TIMEOUT` of 15 seconds for the +kill test. It covers backlog drain at two queue depths with one and four +workers per backend, bulk enqueue of 10,000 rows in one transaction, +worker death under repeated SIGKILL with restarts, and exception retry. +The drain comparison uses five interleaved runs per backend per cell; +the kill test uses five trials with 20 worker kills each. The bulk +enqueue result includes its run range. Raw JSON files are downloadable +from the repository. ## How it compares The four backends a Django team is most likely to shortlist. Every cell about -another project comes from that project's own documentation or issue tracker, -each carrying a link and the date it was read on the +another project comes from that project's own documentation, source code or +issue tracker, each carrying a link and the date it was read on the [Choosing a task backend](https://oxpull.com/django-ox/choosing/) page. | | django-ox | django-tasks-db | Celery | huey | @@ -116,6 +130,10 @@ each carrying a link and the date it was read on the | Worker killed mid-task | **Retried.** The lease expires and the task goes back on the queue | **Stuck.** The task stays `RUNNING`, never retried and never failed. Issue #5 has been open since 2024-06-11. | **Lost** when the child process is killed, even with `acks_late` | **Lost.** "will not be retried automatically" | | Retries and backoff | **Exponential by default**, keeping every attempt's traceback. Per-task budget and backoff on Django 6.1 or Django 5.2 with django-tasks 0.12+ | **None** | Yes | Yes | | Recurring schedules | **Cron or a fixed interval, and no scheduler process.** Editable in the Django admin, limited to the tasks your code exposes | **None** | `celery beat`, a separate process you must run exactly one of | Yes | +| Unique tasks / deduplicated enqueue | **Yes.** `@unique` returns the existing result while a match is READY or RUNNING. All four features below require the optional `oxpull` app and `OxpullBackend` on Django 6.0+ | **Business logic.** Issue #164 closed as a duplicate; maintainer: "handled by your business logic" | **No general built-in.** The cookbook's custom lock prevents concurrent runs, not duplicate enqueues | **Not enqueue dedup.** huey's own `lock_task()` prevents concurrent runs | +| Batch completion callback | **Yes.** `on_complete` is enqueued once all member rows are settled or gone, whatever the outcome; `on_success` requires all to succeed. Callback execution is at least once | **None.** "Task Groups" discussion #58 open since 2024-06-20, with 0 comments | **Success callback; failure errback.** `chord` needs a supported result backend; a failed member produces `ChordError`, and an errback attached to the callback runs instead | **Includes failures.** huey's own `chord()` includes permanent failures; a member lost to abrupt shutdown prevents the callback | +| Rate limits shared across workers | **Yes.** Database-backed start windows shared by `OxpullWorker` instances, with bounded overshoot; not concurrency limits | Not claimed | **Per-worker only.** `Task.rate_limit` is not a shared budget | **Yes.** huey's own `rate_limit()` uses a counter in shared storage, checked when the task runs | +| Dependency workflows | **Yes, provisional.** `depends_on` lets a node run only after every upstream succeeds | Not claimed | **Yes.** `chain` / `link` run the next task after success | **Yes.** huey's own pipelines with `.then()` skip the remaining success pipeline after a failure | The full version has six more backends, a footnote and a date on every cell, and a [section on when django-ox is the wrong choice](https://oxpull.com/django-ox/choosing/#when-not-to-use-django-ox). @@ -361,11 +379,12 @@ path, so admin access does not become permission to run anything. See ## Scope -The core is finite on purpose: a durable queue, a worker, recurring -schedules, monitoring, and nothing else to operate. Interrupting one chosen -running task on demand is outside the current scope. Attempt deadlines can -be set with `TASK_TIMEOUT`, per queue with `TASK_TIMEOUTS`, or per task -with `timeout` on Django 6.1 or Django 5.2 with django-tasks 0.12+. +django-ox provides a durable queue, a worker, recurring schedules and +monitoring. The optional `oxpull` app adds task-coordination features. +Interrupting one chosen running task on demand is outside the current scope. +Attempt deadlines can be set with `TASK_TIMEOUT`, per queue with +`TASK_TIMEOUTS`, or per task with `timeout` on Django 6.1 or Django 5.2 +with django-tasks 0.12+. See [Task timeouts](https://oxpull.com/django-ox/production/#task-timeouts) for enforcement and its limits. @@ -381,15 +400,17 @@ at another alias works there and nothing warns, so leave it unset unless you mean it. See [Read replicas](https://oxpull.com/django-ox/configuration/#read-replicas). -Batches, unique tasks, rate limiting and workflows are in -[Oxpull Pro](https://oxpull.com/django-ox/pro/), a paid add-on; has the details. Metrics stay in this -package: `django_ox.stats` and `ox_health` are free and stay free. +The optional `oxpull` app creates its own tables. It does not change any django-ox table, column, index, constraint or migration. + +Workflows do write task rows. They insert blocked nodes as WAITING tasks. Cancellation sets a READY or WAITING task to DISCARDED by a compare-and-set on the task row's status and lease epoch. Release moves a WAITING task to READY and updates `run_after`; `run_after` is never NULL. Revival of a DISCARDED node is not supported. + +`django_ox.stats` and `ox_health` remain available without enabling `oxpull`. ## Stability -What counts as public API, the versioning and deprecation policy, -and the supported Python and Django versions are documented in -[the stability policy](https://oxpull.com/django-ox/stability/). +`OxpullBackend`, its unique-task and rate-limit options, `OxpullWorker`, the unique-task decorator and the documented batches API are stable. Workflow APIs, the `WORKFLOWS` option and `workflow_*` log events are provisional. The [stability policy](https://oxpull.com/django-ox/stability/) lists the public surfaces in each tier. + +What counts as public API, the versioning and deprecation policy, and the guarantees attached to each tier are covered by [the stability policy](https://oxpull.com/django-ox/stability/). ## License diff --git a/SECURITY.md b/SECURITY.md index cdfe30f..22bb246 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -92,18 +92,20 @@ statement `SELECT 1` plus Django's backend-specific No migration uses `RunSQL`. No query is assembled from row data or user input by string formatting. -## Supported versions +## Version scope -| Version | Supported | -| ------- | --------- | -| 1.8.x | yes | -| older | no | +| Release line | Security maintenance | +| ------------ | -------------------- | +| 1.9.x | Current | +| Earlier | Not maintained | ## Reporting a vulnerability -Report vulnerabilities privately via GitHub security advisories: -[Report a vulnerability](https://github.com/oxpull/django-ox/security/advisories/new). -Do not open a public issue for anything security-sensitive. +Report vulnerabilities privately through +[GitHub private vulnerability reporting](https://github.com/oxpull/django-ox/security/advisories/new) +or by email to support@oxpull.com. Do not open a public issue for +security-sensitive information. -You will get an acknowledgement within three business days. Please include a -reproduction or a clear description of the affected code path. +Include the django-ox and Django versions, whether the optional `oxpull` +app is enabled, and a reproduction or clear description of the affected +code path. Omit credentials and unrelated personal data. diff --git a/context7.json b/context7.json index 9f71c4f..03ca151 100644 --- a/context7.json +++ b/context7.json @@ -72,9 +72,9 @@ "Size heartbeat freshness for startup, poll interval, reap/dispatch, one claim and margin; include pool timeout (default 30 s). There is no overall poll-pass deadline. A database hang can trigger fleet-wide liveness restart storms.", "ox_prune --older-than 7d deletes old SUCCESSFUL and DISCARDED rows. FAILED and LOST stay unless --include-failed. READY, RUNNING and WAITING are never deleted.", "django_ox.actions.retry(result_id) requeues FAILED or LOST for one more attempt. discard(result_id) closes READY, WAITING, FAILED or LOST without running it. Both return bool; neither touches RUNNING tasks.", - "OxTask.Status.WAITING is never written by django-ox itself. Workers do not claim it, ox_prune does not delete it and retry skips it. It reads as READY through django.tasks. Oxpull Pro workflows write and release it.", + "Workflows in the optional oxpull app write and release OxTask.Status.WAITING. Workers do not claim it, ox_prune does not delete it and retry skips it. It reads as READY through django.tasks. Workflows are provisional.", "ox_worker, ox_prune and database-mode ox_health take --database, defaulting to the OxTask write alias; overrides are not checked against routing. Keep all django-ox tables together. ox_health --heartbeat-file refuses --database.", - "Batches, unique tasks, rate limiting and workflows belong to Oxpull Pro, a paid Django 6.0+ add-on at https://oxpull.com/. Free create_schedules creates stored schedules. stats and ox_health are free.", + "oxpull is included and requires Django 6.0+. Add the app, migrate, and use OxpullBackend. Follow https://oxpull.com/django-ox/features/ for worker, option and reconciler requirements.", "Stored schedule start_time is its activation boundary. starting_deadline_seconds drops overly late ticks and logs schedule_tick_dropped with lateness; the default is no deadline.", "Alert on schedule_dispatch_error and schedule_dispatch_failed; inspect failures and suppressed, not just log counts. manage.py check cannot establish database acceptance. ox_worker --batch exit 0 does not prove every schedule enqueued.", "SIGTERM/SIGINT request drain then exit 0; a second signal forces exit 130. A main loop hung in a database call cannot begin draining until it returns. If the process cannot act on signals, SIGKILL is needed.", diff --git a/docs/agents.md b/docs/agents.md index 8c6800d..c48a1d2 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -1,22 +1,15 @@ # For AI assistants -Exact steps for setting up django-ox in an existing Django project, the -facts to get right while doing it, and how to prove it works. - -**django-ox 1.8.0, released 2026-10-04:** create many stored schedules in -one all-or-nothing call, use the rebuilt django-celery-beat database importer, -and keep other schedules and queued tasks running when a stored row is invalid. -See [Upgrade to 1.8.0](#upgrade-to-180). -**18 releases in 49 days; public API stable since 1.0.0 (September 2026).** -Explore the [test suite, contributors and project history](#compatibility-and-project-history). - Run Django background tasks in the database your project already uses: transactional enqueue, retries, dead-worker recovery, recurring schedules and monitoring, without a separate broker. +Exact steps for setting up django-ox in an existing Django project, the +facts to get right while doing it, and how to prove it works. + ## Set up django-ox in this project -Requires Python 3.12+ and Django 5.2+. Check before installing: +Requires Python 3.12+ and Django 5.2+. The optional `oxpull` app requires Django 6.0+. Check before installing: ``` python -c "import django, sys; print(django.__version__, sys.version.split()[0])" @@ -111,6 +104,16 @@ TASKS = { } ``` +## Enable the optional app + +The `oxpull` app is included in django-ox and requires Django 6.0+. Add `"oxpull"` to `INSTALLED_APPS`, run migrations, and set the relevant alias's `BACKEND` to `"oxpull.backend.OxpullBackend"`. + +Unique tasks use `@unique` or `OPTIONS["UNIQUE_TASKS"]`. Batches require a reconciler schedule. + +For rate limits or workflows, set that alias's `OPTIONS["WORKER_CLASS"]` to `"oxpull.worker.OxpullWorker"`. Rate limits also need `OPTIONS["RATE_LIMITS"]`. Workflows also need `OPTIONS["WORKFLOWS"] = True` and a reconciler schedule. + +Follow the [feature setup](features/index.md) and the relevant feature page. Do not treat the backend or worker setting alone as complete configuration. + ## Upgrade to 1.8.0 Create many stored schedules in one all-or-nothing call, or bring schedules @@ -325,10 +328,10 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv WAITING, FAILED or LOST task without running it. Neither touches a RUNNING task. With `django.contrib.admin` installed, the task table appears in the admin with the same two actions. -- `OxTask.Status.WAITING` is a status django-ox never writes itself. - Workers never claim a waiting task, `ox_prune` never deletes it, and - `retry` skips it. It reads as READY through `django.tasks`. Workflows in - Oxpull Pro write and release it. +- `OxTask.Status.WAITING` is used by workflows in the optional `oxpull` + app. Workers never claim a waiting task, `ox_prune` never deletes it, + and `retry` skips it. It reads as READY through `django.tasks`. + Workflows write and release this status. - A particular running task cannot be interrupted on demand. Each attempt can have a task, queue or backend execution timeout. Every table lives on the database your router sends `OxTask` to. @@ -337,11 +340,12 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv validate `PolicyTask` fields on supported Django versions. Neither retries, calls backoff callbacks or enforces timeouts. Immediate runs even if the enclosing transaction later rolls back, and rejects - `run_after`. For queued execution, keep `OxBackend` and use the public, - provisional `django_ox.testing.run_tasks()` helper. It runs due attempts - through the configured worker class, including retries and backoff. - Pass `backend=` for the worker class you need, including the Pro alias - for workflows and rate limits. Rows are selected by queue, regardless + `run_after`. For queued execution, keep `OxBackend` or `OxpullBackend` + and use the public, provisional `django_ox.testing.run_tasks()` helper. + It runs due attempts through the configured worker class, including + retries and backoff. For workflows and rate limits, pass `backend=` + with the alias whose `OPTIONS["WORKER_CLASS"]` is + `"oxpull.worker.OxpullWorker"`. Rows are selected by queue, regardless of which backend enqueued them. `TestCase` uses savepoints and commit callback emulation. Use `TransactionTestCase` to test worker autocommit behaviour. Exit caller callback-capture blocks before draining tasks @@ -350,10 +354,13 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv Test timeout enforcement and worker infrastructure against a real worker. See [Run queued tasks in tests](patterns.md#run-queued-tasks-in-tests) for transaction and callback differences. -- Batches, unique tasks, rate limiting and workflows are in - [Oxpull Pro](pro.md), a paid add-on requiring Django 6.0+. - `create_schedules` creates stored schedules in the free package. - `django_ox.stats` and `ox_health` are also in django-ox. +- [Unique tasks, batches, rate limiting and workflows](features/index.md) + are included in django-ox through the optional `oxpull` app and require + Django 6.0+. `OxpullBackend`, `OxpullWorker` and the documented + unique-task, batch and rate-limit entry points are stable. + Workflows are provisional. +- `create_schedules` creates stored schedules. It, `django_ox.stats` and + `ox_health` do not require the optional app. ## How to verify it works @@ -397,21 +404,13 @@ TaskResultStatus.SUCCESSFUL `python manage.py ox_health --worker-timeout 60` now exits 0 and reports a recent `last_claim_age`. Stop the worker with Ctrl-C; it drains and exits 0. -## Compatibility and project history +## Compatibility -django-ox supports Python 3.12-3.14 and Django 5.2 LTS through 6.1. -For Django 5.2, install the backport extra. -PostgreSQL, MySQL 8 and SQLite are in CI. The licence is BSD 3-Clause. +django-ox 1.9.0 supports Django 5.2 LTS through the `django-tasks` backport and Django 6.0 / 6.1 through core `django.tasks`. It requires Python 3.12+. CI covers Python 3.12-3.14, with PostgreSQL, MySQL 8 and SQLite. -As of 2026-10-04, django-ox has shipped **18 releases in 49 days**. -Its **public API has been stable since 1.0.0 (September 2026)**. +The optional `oxpull` app is included in the distribution and requires Django 6.0+. Projects that do not enable it can continue using django-ox on Django 5.2. -- **All 36 checks passed on release PR #124**, including test-suite legs - on PostgreSQL, MySQL and SQLite, plus Wagtail smoke checks. -- **3,761 test cases in the 1.8.0 suite.** -- **22 merged pull requests from 13 outside contributors.** -- Featured in PyCoder's Weekly #752 (2026-09-15). - Listed in awesome-django under Task Queues. +For existing `oxpull` deployments, follow [Earlier `oxpull` installs](migrating.md#earlier-oxpull-installs). Machine-readable copies: [llms.txt](llms.txt) contains facts and links; [llms-full.txt](llms-full.txt) contains every page of this site in one file. diff --git a/docs/choosing.md b/docs/choosing.md index ae55180..46e8ad8 100644 --- a/docs/choosing.md +++ b/docs/choosing.md @@ -22,6 +22,10 @@ fewer service. The last section says where django-ox is not the right fit. | Transactional enqueue | Yes. Enqueue is one INSERT on your default database; a task written inside `atomic()` commits or rolls back with the rows beside it | Not documented [^tdb-readme] | Not documented [^huey-guide] | No. Django's own docs name a background task as the case for `on_commit()` [^dj-oncommit] | Not documented [^q2-brokers] | No. Enqueue is a broker send [^dq-guide] | Not documented [^proc-django] | | Retries and backoff | Exponential backoff by default; per-task `max_attempts` and `backoff` on Django 6.1 or Django 5.2 with django-tasks 0.12+. Backend defaults: `MAX_ATTEMPTS`, `BACKOFF_INITIAL`, `BACKOFF_MAX`. Every attempt's traceback kept. See [Per-task policy](configuration.md#per-task-policy) | No retry option in the README or the worker flags [^tdb-readme] [^tdb-worker] | `retries`, `retry_delay`, `retry_backoff` [^huey-guide] | `autoretry_for`, `retry_backoff`, `retry_backoff_max` (default 600 s), `retry_jitter` [^celery-tasks] | `max_attempts` (default 0, meaning unlimited) and a `retry` interval; backoff not documented [^q2-configure] | Exponential backoff; `max_retries` default 20, `min_backoff` 15 s, `max_backoff` 7 days [^dq-guide] | Retry strategy per task [^proc-index] | | Recurring schedules | Cron or a fixed interval in settings, or rows created and edited in the Django admin, limited to the tasks the code exposes; every worker dispatches, no scheduler process. See [Schedules in the database](stored-schedules.md) | None. Issue #259 open since 2026-08-10 [^tasks-259] | `periodic_task(crontab(...))` [^huey-guide] | `celery beat`, a separate process; "ensure only a single scheduler is running" [^celery-beat]. With `django-celery-beat` the schedules are rows that "can be managed from the Django Admin interface", and the field naming the task has no list to choose from [^beat-admin] | `Schedule` model, editable in admin; cron via croniter [^q2-schedules] | None built in; APScheduler recommended [^dq-cookbook] | `@app.periodic` [^proc-index] | +| Unique tasks / deduplicated enqueue | `@unique` or `UNIQUE_TASKS` returns the existing `TaskResult` while a match is READY or RUNNING; not an execution-exclusivity guarantee. All four features below require the optional `oxpull` app and `OxpullBackend` on Django 6.0+ | Issue #164 closed as a duplicate of discussion #143; maintainer: "handled by your business logic" [^tdb-164] | huey's own `lock_task()` prevents concurrent runs, not duplicate enqueues [^huey-stable] | No general built-in enqueue deduplication. The cookbook uses a custom execution lock; `worker_deduplicate_successful_tasks` handles successful redeliveries, and SQS FIFO has a broker-specific deduplication mechanism [^celery-cookbook] [^celery-dedup-d3] [^celery-sqs-d3] | Not documented [^q2-tasks-d3] | Not documented [^dq-ref] | `queueing_lock` permits one job in `todo` per chosen lock string, but multiple jobs in `doing`; a duplicate defer raises `AlreadyEnqueued` [^proc-qlock] | +| Batch completion callback | `on_complete` is enqueued once every member's row is settled or gone, whatever the outcome; `on_success` requires all to succeed. Callbacks run at least once. Requires a once-a-minute reconciler schedule | "Task Groups" discussion #58, open since 2024-06-20, with 0 comments [^tasks-58] | huey's own `chord()` runs after members finish or fail permanently, passing failures as `huey.Error`; a member lost to abrupt consumer shutdown prevents the callback [^huey-stable] | `chord` runs its callback only if every member succeeds; a failed member produces `ChordError`, and an errback attached to the callback runs instead. Requires a result backend other than RPC and member results must not be ignored [^celery-canvas] | No group completion hook documented; groups label results for polling, while `hook` is per task [^q2-group] [^q2-tasks-d3] | `group.add_completion_callback()` requires optional `GroupCallbacks` middleware; the completion barrier is signalled only on success, so the callback requires every member to succeed [^dq-ref] | A contributor: "there is nothing out of the box"; issue #1219 closed in favour of #1198 [^proc-1219] | +| Rate limits shared across workers | Fixed start windows coordinated through one database row per limit name; requires `OxpullWorker`. Bounded overshoot is possible. Throttled tasks stay READY without spending an attempt; this is not a concurrency cap | Not documented in the README, worker flags or Django Tasks API [^tdb-readme-d3] [^tdb-worker-d3] [^dj-ref-tasks] | huey's own `rate_limit()` uses a fixed-window storage counter keyed by huey name and limit name; workers using the same shared storage use that counter. Checked inside the running task, with throttled calls rescheduled [^huey-stable] [^huey-ratelimit-src] | `Task.rate_limit` is per worker instance, not global; the docs advise restricting to a given queue for global limiting [^celery-ratelimit] | Not documented; pull request #350 proposes a per-cluster throttle and remains open [^q2-configure-d3] [^q2-350] | `WindowRateLimiter` and `BucketRateLimiter` use Redis or Memcached for shared limits, called from actor code [^dq-ref] [^dq-cookbook-d3] | Not documented; issue #1289 open since 2025-01-14 [^proc-1289] | +| Dependency workflows | Provisional. `depends_on` forms a DAG; a node runs only after every upstream succeeds. Requires `WORKFLOWS: True`, `OxpullWorker` and a reconciler schedule | Not documented in the Tasks API; discussion #58 proposes task groups and follow-up work [^dj-ref-tasks] [^tasks-58] | huey's own pipelines with `.then()` run tasks sequentially; a failure skips the remaining success pipeline [^huey-stable] | `chain` / `link` call the next task after success; `chord` provides fan-in [^celery-canvas] | Chains are not success-gated: later tasks are also enqueued after a failure. `save_task()` has no success check before enqueueing the next chain link. Issue #327, open since 2026-05-05 [^q2-327-d3] [^q2-monitor-d3] | `pipeline` passes results to the next actor; the next step is enqueued only after success [^dq-ref] | No built-in workflows; issue #1198 open since 2024-09-18. Execution locks order jobs whether the preceding job succeeds or not [^proc-1198] [^proc-locks-d3] | | Priorities | -100 to 100 | Yes [^tdb-backend] | Yes; on Redis needs 5.0+ and `PriorityRedisHuey` [^huey-guide] | 0 to 255 on RabbitMQ and Redis [^celery-calling] | Not documented [^q2-configure] | Per actor; lower number runs first [^dq-guide] | Yes [^proc-index] | | Time limit on a running task | Configured through backend `TASK_TIMEOUT`, per-queue `TASK_TIMEOUTS`, or a task's declared `timeout` on Django 6.1 or Django 5.2 with django-tasks 0.12+. `TaskTimeout` is raised inside the task at the deadline, followed by a worker recycle if the thread does not stop. The recycle is the whole enforcement on an interpreter with no facility for raising an exception in another thread, and for a thread a coverage tool or debugger is watching. See [Production](production.md#task-timeouts) | Not documented in the README or the worker flags [^tdb-readme] [^tdb-worker] | `timeout` per task or per call; "a `TaskTimeout` is raised and returned to the caller via the result handle" [^huey-guide] | `soft_time_limit` raises `SoftTimeLimitExceeded` inside the task; `time_limit` terminates the process running it, which is replaced. "Time limits don't currently work on platforms that don't support the `SIGUSR1` signal" [^celery-workers] | `timeout`, default `None`: "the number of seconds a worker is allowed to spend on a task before it's terminated" [^q2-configure] | `time_limit` per actor, default 10 minutes, raises `TimeLimitExceeded`. "Time limits are best-effort. They cannot cancel system calls or any function that doesn't currently hold the GIL under CPython" [^dq-guide] | Not documented on the page checked [^proc-index] | | Worker dies mid-task | Lease renewed every `LOCK_TIMEOUT / 3`; a task whose lease goes stale for `LOCK_TIMEOUT` is put back to READY, or marked LOST when attempts are spent. See [Production](production.md#the-reaper) | Issue #5, open since 2024-06-11: the task 'remains marked as "PROCESSING", and thus is never picked up for re-processing nor marked as completed / failed' [^tdb-5] | "tasks that are mid-execution are lost and will not be retried automatically" [^huey-guide] | `acks_late` re-delivers; the worker still acknowledges "if the child process executing the task is terminated" [^celery-tasks] | Issue #327, open since 2026-05-05: worker death not reported to the monitor, `MAX_ATTEMPTS` ignored [^q2-327] | Not stated on the pages checked [^dq-guide] [^dq-cookbook] | Heartbeat every 10 s; jobs stay in `doing` until a `retry_stalled_jobs` periodic task you define picks them up [^proc-stalled] | @@ -37,21 +41,25 @@ function", open since 2020-12-19 with 98 upvotes [^celery-6552]. steady-queue, dj-queue and django-database-task also keep the queue in the database and implement the `django.tasks` backend API. The same rules apply: -every cell about another project comes from its README or its PyPI page, with -the link and the date in the footnotes, and a cell says "not stated" when -those pages do not say. +every cell about another project comes from its README, its PyPI page or its +own issue tracker, with the link and the date in the footnotes, and a cell +says "not stated" when those pages do not say. | | django-ox | steady-queue | dj-queue | django-database-task | | --- | --- | --- | --- | --- | -| Latest release on PyPI | See the [changelog](changelog.md) | 0.2.1, 2026-09-06 [^sq-pypi] | 0.14.1, 2026-09-03 [^djq-pypi] | 0.5.0, 2026-09-05 [^ddt-pypi] | +| Latest release on PyPI | See the [changelog](changelog.md) | 0.2.1, 2026-09-06 [^sq-pypi] | 0.14.2, 2026-09-26 [^djq-pypi] | 0.6.0, 2026-09-30 [^ddt-pypi] | | `django.tasks` backend | Yes, native | Yes, native [^sq-readme] | Yes, native [^djq-readme] | Yes, native [^ddt-readme] | | Django versions | 5.2 LTS with the `django-tasks` backport, 6.0 and 6.1 | 6.0 and later, below 7 (`django>=6.0,<7`) [^sq-pypi] | 6.0 and 6.1 (`django>=6.0.0,<6.2`) [^djq-pypi] | 6.0 and later (`Django>=6.0`) [^ddt-pypi] | | Databases | PostgreSQL, MySQL 8 and SQLite, all run in CI; MariaDB 10.6+ takes the same claim path as MySQL and is not in CI | MySQL, PostgreSQL or SQLite; `FOR UPDATE SKIP LOCKED` on MySQL 8+ and PostgreSQL 9.5+ [^sq-readme] | PostgreSQL first-class, with optional `LISTEN/NOTIFY`; MySQL and MariaDB supported; SQLite "supported with limits", polling only and no `SKIP LOCKED` [^djq-readme] | PostgreSQL 14+, MySQL 8.0.11+, MariaDB 10.6+, SQLite (no row-level locking; "development or single-worker deployments"), Oracle 19c+ ("not tested with this package") [^ddt-readme] | | Transactional enqueue | Yes. Enqueue is one INSERT on your default database; a task written inside `atomic()` commits or rolls back with the rows beside it | The README says a shared database gives transactional integrity and recommends against relying on it: a separate database for the queue, and `transaction.on_commit` for tasks that depend on data [^sq-readme] | "`enqueue()` writes immediately and returns a real persisted task result ID"; for a task that depends on rows in the current transaction the README provides `enqueue_on_commit()` or `transaction.on_commit()` [^djq-readme] | Not stated as a feature. The PostgreSQL broker section says `pg_notify()` runs inside the same transaction as the INSERT and is delivered on commit [^ddt-readme] | | Retries and backoff | Exponential backoff by default; per-task `max_attempts` and `backoff` on Django 6.1 or Django 5.2 with django-tasks 0.12+. Backend defaults: `MAX_ATTEMPTS`, `BACKOFF_INITIAL`, `BACKOFF_MAX`. Every attempt's traceback kept. See [Per-task policy](configuration.md#per-task-policy) | Not stated in the README or the configuration page; failed tasks are retried from the Django admin [^sq-readme] [^sq-config] | None built in: "`dj_queue` does not add an automatic retry policy or backoff engine". Failed jobs are retried from the admin or with `retry_failed_job()` [^djq-readme] | No automatic retry stated. A FAILED task is re-run from the admin action "Retry failed tasks" or with `run_task_by_id(..., allow_retry=True)` [^ddt-readme] | | Recurring schedules | Cron or a fixed interval in settings, or rows edited in the Django admin; every worker dispatches, no scheduler process | Cron-style, with the `@recurring` decorator in code; a scheduler process under the supervisor [^sq-readme] | Static recurring tasks in `OPTIONS["recurring"]` and dynamic recurring tasks managed at runtime; a scheduler under the supervisor [^djq-readme] | None built in. The worker is run from cron, a systemd timer, JP1, Hinemos or Cloud Scheduler [^ddt-readme] | +| Unique tasks / deduplicated enqueue | `@unique` or `UNIQUE_TASKS` returns the existing `TaskResult` while a match is READY or RUNNING; not an execution-exclusivity guarantee. All four features below require the optional `oxpull` app and `OxpullBackend` on Django 6.0+ | Issue #24, open since 2026-02-10; owner: "For now ... I don't think this will get added to the package", but is "open to creating a `contrib` subpackage that could include this" [^sq-24] | `@concurrency(..., on_conflict="discard")` provides singleton-style work through a concurrency semaphore with a TTL; the discarded enqueue's return value is not stated [^djq-readme-d3] | Not stated; `SELECT FOR UPDATE SKIP LOCKED` prevents duplicate claims, not duplicate enqueues [^ddt-readme-d3] | +| Batch completion callback | `on_complete` is enqueued once every member's row is settled or gone, whatever the outcome; `on_success` requires all to succeed. Callbacks run at least once. Requires a once-a-minute reconciler schedule | Not stated [^sq-readme-d3] | Not stated [^djq-readme-d3] | Not stated [^ddt-readme-d3] | +| Rate limits shared across workers | Fixed start windows coordinated through one database row per limit name; requires `OxpullWorker`. Bounded overshoot is possible. Throttled tasks stay READY without spending an attempt; this is not a concurrency cap | Not stated; the README's concurrency controls limit simultaneous execution, not start rate [^sq-readme-d3] | Not stated; shared semaphores provide concurrency limits, not rate limits [^djq-readme-d3] | Not provided by the package; the README points to Google Cloud Tasks per-queue rate settings for `CloudTasksDatabaseBackend` [^ddt-readme-d3] | +| Dependency workflows | Provisional. `depends_on` forms a DAG; a node runs only after every upstream succeeds. Requires `WORKFLOWS: True`, `OxpullWorker` and a reconciler schedule | Not stated [^sq-readme-d3] | Not stated [^djq-readme-d3] | Not stated [^ddt-readme-d3] | | Priorities | -100 to 100 | -100 to 100 [^sq-readme] | `priority` on the task call; higher claimed first within a queue [^djq-readme] | -100 to 100 [^ddt-readme] | -| Concurrency controls | None in django-ox, and none in [Oxpull Pro](pro.md): Pro's rate limiting caps how often a task starts, not how many run at once | "Limit how many instances of a specific task can run simultaneously" [^sq-readme] | `@concurrency` decorator with database-backed limits; semaphores shown in the admin dashboard [^djq-readme] | Not stated [^ddt-readme] | +| Concurrency controls | No per-task concurrency limit. [Rate limiting](features/rate-limiting.md) caps how often a task starts, not how many run at once | "Limit how many instances of a specific task can run simultaneously" [^sq-readme] | `@concurrency` decorator with database-backed limits; semaphores shown in the admin dashboard [^djq-readme] | Not stated [^ddt-readme] | | Worker dies mid-task | Lease renewed every `LOCK_TIMEOUT / 3`; a task whose lease goes stale for `LOCK_TIMEOUT` is put back to READY, or marked LOST when attempts are spent | Marked failed with `ProcessPrunedError` (heartbeat expired) or `ProcessExitError` (killed), to be inspected [^sq-readme] | Kept "as failed work that operators can inspect and retry": `ProcessExitError`, `ProcessPrunedError`, `ProcessMissingError` [^djq-readme] | Requeued by `requeue_stale_database_tasks --older-than`, a separate command run from cron or a systemd timer; `--max-attempts` (default 3) marks the task FAILED instead [^ddt-readme] | | Health and metrics | `ox_health` command, `django_ox.stats`, a Prometheus endpoint (`/ox/metrics`), structured log events, admin retry and discard | Django admin: pause and resume queues, inspect tasks, retry or discard failed ones; operational signals from `steady_queue.signals` [^sq-readme] | Admin dashboard at `/admin/dj_queue/dashboard/`; `dj_queue_health` command; `/dj_queue/metrics` in Prometheus text format with the `prometheus` extra [^djq-readme] | Django admin; structured log fields; opt-in worker exit codes for job schedulers [^ddt-readme] | | Async tasks and results | `async def` tasks run; `get_result()` supported | "Async task enqueueing is not supported"; "Result fetching is not supported" [^sq-readme] | `get_result()` documented; async task functions not stated [^djq-readme] | "Supports async task functions"; `get_result()` documented [^ddt-readme] | @@ -73,9 +81,10 @@ those pages do not say. [benchmarks](benchmarks.md) page gives measured numbers with the method. If your workload is above them, a broker-based queue is the right tool, and the cost is the second datastore. -- **Chains, groups and chords.** Not in django-ox. Batches and workflows - are in [Oxpull Pro](pro.md), a paid add-on; chains are on the Pro - roadmap, undated. +- **Celery-style chains, groups and chords.** django-ox does not implement + these APIs. It provides [batches](features/batches.md) for grouped work + and provisional [workflows](features/workflows.md) for dependency DAGs. + Check their semantics before replacing an existing composition. - **CPU-bound tasks in one process.** Worker concurrency is a thread pool. Run `ox_worker --processes N --concurrency 1` for N interpreters, or pick a queue with a process pool. @@ -92,9 +101,15 @@ of date, open an issue with the link that shows it. [^tdb-44]: https://github.com/RealOrangeOne/django-tasks-db/issues/44, open, checked 2026-08-23. [^tdb-repo]: https://github.com/RealOrangeOne/django-tasks-db, licence field, checked 2026-08-23. [^tasks-259]: https://github.com/RealOrangeOne/django-tasks/issues/259, open, checked 2026-08-23. +[^tdb-164]: https://github.com/RealOrangeOne/django-tasks/issues/164, closed as a duplicate of discussion #143; maintainer assigns deduplication to business logic, checked 2026-10-11. +[^tasks-58]: https://github.com/RealOrangeOne/django-tasks/discussions/58, "Task Groups", open since 2024-06-20 with 0 comments, checked 2026-10-11. +[^tdb-readme-d3]: https://github.com/RealOrangeOne/django-tasks-db, README at `8f79c7f`; no rate-limit option documented, checked 2026-10-11. +[^tdb-worker-d3]: https://github.com/RealOrangeOne/django-tasks-db/blob/master/django_tasks_db/management/commands/db_worker.py, `add_arguments`; no rate-limit flag documented, checked 2026-10-11. [^huey-guide]: https://huey.readthedocs.io/en/latest/guide.html, checked 2026-08-23. [^huey-contrib]: https://huey.readthedocs.io/en/latest/contrib.html, checked 2026-09-19. [^huey-repo]: https://github.com/coleifer/huey, licence field, checked 2026-08-23. +[^huey-stable]: https://huey.readthedocs.io/en/stable/guide.html, huey 3.4.0; huey's own task locks, chords, fixed-window rate limits and pipelines, checked 2026-10-11. +[^huey-ratelimit-src]: https://github.com/coleifer/huey/blob/3.4.0/huey/api.py, `class RateLimit`; counter key includes huey name and limit name and uses `storage.incr()`, checked 2026-10-11. [^celery-tasks]: https://docs.celeryq.dev/en/stable/userguide/tasks.html, checked 2026-08-23. [^celery-brokers]: https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html, checked 2026-08-23. [^celery-workers]: https://docs.celeryq.dev/en/stable/userguide/workers.html, Time Limits, checked 2026-08-23. @@ -105,8 +120,14 @@ of date, open an issue with the link that shows it. [^celery-6552]: https://github.com/celery/celery/issues/6552, open, checked 2026-08-23. [^celery-10062]: https://github.com/celery/celery/issues/10062, closed, checked 2026-08-23. [^celery-license]: https://github.com/celery/celery/blob/main/LICENSE, checked 2026-08-23. +[^celery-cookbook]: https://docs.celeryq.dev/en/stable/tutorials/task-cookbook.html, Celery 5.6.3; custom cache lock for execution exclusivity, checked 2026-10-11. +[^celery-dedup-d3]: https://docs.celeryq.dev/en/stable/userguide/configuration.html, Celery 5.6.3; `worker_deduplicate_successful_tasks` applies to successful redeliveries with the same task identifier and late acknowledgment, checked 2026-10-11. +[^celery-sqs-d3]: https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/sqs.html, Celery 5.6.3; SQS FIFO message properties including `MessageDeduplicationId`, checked 2026-10-11. +[^celery-canvas]: https://docs.celeryq.dev/en/stable/userguide/canvas.html, Celery 5.6.3; chord result-backend requirements and failure behavior, including an errback attached to the chord callback running instead on failure; chains and links, checked 2026-10-11. +[^celery-ratelimit]: https://docs.celeryq.dev/en/stable/userguide/tasks.html#Task.rate_limit, Celery 5.6.3; per-worker rate limits and queue restriction advice, checked 2026-10-11. [^flower]: https://flower.readthedocs.io/en/latest/features.html, checked 2026-08-23. [^dj-oncommit]: https://docs.djangoproject.com/en/6.0/topics/db/transactions/#performing-actions-after-commit, checked 2026-08-23. +[^dj-ref-tasks]: https://docs.djangoproject.com/en/6.0/ref/tasks/, Django 6.0 Tasks API; no rate-limit or dependency option, checked 2026-10-11. [^q2-brokers]: https://django-q2.readthedocs.io/en/master/brokers.html, checked 2026-08-23. [^q2-configure]: https://django-q2.readthedocs.io/en/master/configure.html, checked 2026-08-23. [^q2-schedules]: https://django-q2.readthedocs.io/en/master/schedules.html, checked 2026-08-23. @@ -114,18 +135,35 @@ of date, open an issue with the link that shows it. [^q2-315]: https://github.com/django-q2/django-q2/pull/315, open, checked 2026-08-23. [^q2-327]: https://github.com/django-q2/django-q2/issues/327, open, checked 2026-08-23. [^q2-pyproject]: https://github.com/django-q2/django-q2/blob/master/pyproject.toml, checked 2026-08-23. +[^q2-tasks-d3]: https://django-q2.readthedocs.io/en/master/tasks.html, Django Q2 1.11.1; `async_task()` options and per-task `hook`, checked 2026-10-11. +[^q2-group]: https://django-q2.readthedocs.io/en/master/group.html, result grouping and polling; no group completion hook documented, checked 2026-10-11. +[^q2-configure-d3]: https://django-q2.readthedocs.io/en/master/configure.html, Django Q2 1.11.1; no shared rate-limit option documented, checked 2026-10-11. +[^q2-350]: https://github.com/django-q2/django-q2/pull/350, proposed per-cluster throttle, open since 2026-09-16, checked 2026-10-11. +[^q2-327-d3]: https://github.com/django-q2/django-q2/issues/327, open since 2026-05-05; Bug 2 reports chains continuing after failure, checked 2026-10-11. +[^q2-monitor-d3]: https://github.com/django-q2/django-q2/blob/v1.11.1/django_q/monitor.py, `save_task()` has no success check before enqueueing the next chain link, checked 2026-10-11. [^dq-guide]: https://dramatiq.io/guide.html, checked 2026-08-23. [^dq-cookbook]: https://dramatiq.io/cookbook.html, checked 2026-08-23. [^dq-prom]: https://github.com/Bogdanp/dramatiq/blob/master/dramatiq/middleware/__init__.py, `default_middleware`, and `middleware/prometheus.py`, checked 2026-08-23. [^dq-repo]: https://github.com/Bogdanp/dramatiq, licence field, checked 2026-08-23. +[^dq-ref]: https://dramatiq.io/reference.html, Dramatiq 2.2.0; composition and rate-limiter APIs; success gating also checked in `dramatiq/middleware/group_callbacks.py` and `dramatiq/middleware/pipelines.py` at `8534ffa`, checked 2026-10-11. +[^dq-cookbook-d3]: https://dramatiq.io/cookbook.html, rate limiters used inside actor code with a shared backend, checked 2026-10-11. [^proc-index]: https://procrastinate.readthedocs.io/en/stable/, checked 2026-08-23. [^proc-django]: https://procrastinate.readthedocs.io/en/stable/howto/django/basic_usage.html, checked 2026-08-23. [^proc-stalled]: https://procrastinate.readthedocs.io/en/stable/howto/production/retry_stalled_jobs.html, checked 2026-08-23. [^proc-repo]: https://github.com/procrastinate-org/procrastinate, licence field, checked 2026-08-23. +[^proc-qlock]: https://procrastinate.readthedocs.io/en/stable/howto/advanced/queueing_locks.html, `queueing_lock`, `todo` versus `doing`, and `AlreadyEnqueued`, checked 2026-10-11. +[^proc-1219]: https://github.com/procrastinate-org/procrastinate/issues/1219, batch and workflow request closed in favour of #1198; a contributor, not a maintainer, says "there is nothing out of the box", checked 2026-10-11. +[^proc-1289]: https://github.com/procrastinate-org/procrastinate/issues/1289, rate-limit request, open since 2025-01-14, checked 2026-10-11. +[^proc-1198]: https://github.com/procrastinate-org/procrastinate/issues/1198, dependency workflow request, open since 2024-09-18, checked 2026-10-11. +[^proc-locks-d3]: https://procrastinate.readthedocs.io/en/stable/howto/advanced/locks.html, execution locks order jobs whether the preceding job succeeds or not, checked 2026-10-11. [^sq-readme]: https://github.com/knifecake/steady-queue README, checked 2026-09-19. [^sq-config]: https://steady-queue.readthedocs.io/en/latest/configuration.html, checked 2026-09-19. -[^sq-pypi]: https://pypi.org/project/steady-queue/ 0.2.1, uploaded 2026-09-06; `Requires-Dist: django<7,>=6.0`; licence MIT. Checked 2026-09-19. +[^sq-pypi]: https://pypi.org/project/steady-queue/ 0.2.1, uploaded 2026-09-06; `Requires-Dist: django<7,>=6.0`; licence MIT. Checked 2026-10-11. +[^sq-24]: https://github.com/knifecake/steady-queue/issues/24, enqueue uniqueness request, open since 2026-02-10; owner: "For now ... I don't think this will get added to the package", but is "open to creating a `contrib` subpackage that could include this", checked 2026-10-11. +[^sq-readme-d3]: https://github.com/knifecake/steady-queue, README at `db46170`; concurrency controls and documented feature scope, checked 2026-10-11. [^djq-readme]: https://github.com/coriocactus/dj_queue README, checked 2026-09-19. -[^djq-pypi]: https://pypi.org/project/dj-queue/ 0.14.1, uploaded 2026-09-03; `Requires-Dist: django<6.2,>=6.0.0`; licence MIT. Checked 2026-09-19. +[^djq-pypi]: https://pypi.org/project/dj-queue/ 0.14.2, uploaded 2026-09-26; `Requires-Dist: django<6.2,>=6.0.0`; licence MIT. Checked 2026-10-11. +[^djq-readme-d3]: https://github.com/coriocactus/dj_queue, README at `086ce2a`; singleton-style concurrency controls and documented feature scope, checked 2026-10-11. [^ddt-readme]: https://github.com/tokibito/django-database-task README, checked 2026-09-19. -[^ddt-pypi]: https://pypi.org/project/django-database-task/ 0.5.0, uploaded 2026-09-05; `Requires-Dist: Django>=6.0`; licence MIT. Checked 2026-09-19. +[^ddt-pypi]: https://pypi.org/project/django-database-task/ 0.6.0, uploaded 2026-09-30; `Requires-Dist: Django>=6.0`; licence MIT. Checked 2026-10-11. +[^ddt-readme-d3]: https://github.com/tokibito/django-database-task, README at `9f4db93`; claim exclusivity, Google Cloud Tasks rate settings and documented feature scope, checked 2026-10-11. diff --git a/docs/configuration.md b/docs/configuration.md index 71817c9..11e865a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -506,10 +506,64 @@ Check semantics, probe examples, and guidance on which check fits which alert are on the [Monitoring](monitoring.md#health-checks-ox_health) page. +## `oxpull` options + +The optional `oxpull` app requires Django 6.0+. Add it to `INSTALLED_APPS`, run migrations, and use `oxpull.backend.OxpullBackend` for each alias that needs its features. + +These settings belong in the alias's `OPTIONS`: + +| Key | Default | Meaning | +| --- | --- | --- | +| `UNIQUE_TASKS` | `[]` | A list or tuple of dotted task-path strings. Each entry has the same effect as decorating that task with `@unique`. Uniqueness is scoped to the backend alias. | +| `RATE_LIMITS` | `{}` | A dictionary of named admission limits. Each name identifies a shared window across every backend that configures it. See the validation rules below. | +| `RATE_REFRESH_INTERVAL` | `0.25` | Seconds between a worker's refresh reads of the rate-limit window rows. Accepts a finite, non-negative `int` or `float`, but not a boolean. | +| `WORKFLOWS` | `False` | Enables workflows. Accepts only `True` or `False`. This option is provisional. | + +A `RATE_LIMITS` name must be non-empty and at most 128 characters. Two names that differ only by case are refused. + +Each limit's value is a dictionary containing only `limit`, `per`, `tasks` and `queues`: + +- `limit` is a whole number of admissions per window, at least 0. A boolean is not accepted. +- `per` is a whole number of seconds, at least 1, or a `timedelta` of at least one second. Sub-second periods are refused. +- `tasks` is a list of dotted task-path strings. +- `queues` is a list of queue names accepted by the backend. +- At least one of `tasks` and `queues` is required. + +Rate limits and workflows require `OPTIONS["WORKER_CLASS"] = "oxpull.worker.OxpullWorker"`. Rate limits also require `RATE_LIMITS`; workflows also require `WORKFLOWS = True`. + +Batches and workflows need their respective reconciler schedules. Each reconciler must have a working scheduling path and an accepted queue. Creating a batch without its reconciler schedule is refused; creating or sealing a workflow without its reconciler schedule is also refused. + +The `django_ox` and `oxpull` tables must share one database. A split router raises `ImproperlyConfigured` when the backend is constructed. + +See [Optional features](features/index.md) for feature-specific setup. + ## System checks `manage.py check` validates the setup: +### `oxpull` checks + +The optional app provides these 16 checks: + +- **`oxpull.E001`**: `oxpull` is not in `INSTALLED_APPS`. Add the app and run migrations. +- **`oxpull.E002`**: A configured unique-task entry cannot be imported or is not a `Task`. Correct the dotted task path. +- **`oxpull.E003`**: The backend alias is absent from `TASKS`. Construct the backend through the `TASKS` setting. +- **`oxpull.E004`**: The router splits `django_ox` and `oxpull` after backend construction. Route their tables to one database. A split present at construction raises `ImproperlyConfigured` before this check can run. +- **`oxpull.E005`**: `RATE_LIMITS` or `RATE_REFRESH_INTERVAL` is invalid, or a configured rate-limit task entry is not a `Task`. Correct the option or task path. +- **`oxpull.E006`**: Rate limits are configured without a worker that applies them, or `WORKER_CLASS` cannot be resolved. Configure a resolving worker class that applies the rate-limit gate, normally `oxpull.worker.OxpullWorker`. +- **`oxpull.E007`**: `WORKFLOWS` is not a boolean, or workflows are enabled without a resolving worker class that applies the workflow gate. Use `True` or `False` and configure `oxpull.worker.OxpullWorker` when enabling workflows. +- **`oxpull.E008`**: A reconciler is exposed through `SCHEDULABLE_TASKS`, but no backend reading schedule rows accepts its queue and no `SCHEDULES` entry names it. Add a `SCHEDULES` entry for the reconciler, or read schedule rows on a django-ox backend whose `QUEUES` include its queue. +- **`oxpull.E009`**: On MySQL, the relevant table collations do not match. Correct the collation mismatch reported by the database check. +- **`oxpull.W001`**: A scheduled task is also unique. It can coalesce with a pending manual enqueue, and the tick may record against that task. Remove `@unique` or the `UNIQUE_TASKS` entry from the scheduled task if that behavior is not intended. Schedules already deduplicate once per tick. +- **`oxpull.W002`**: The batch reconciler has no configured schedule. Configure its schedule before creating batches. +- **`oxpull.W003`**: Workflows are enabled without a workflow reconciler schedule. Configure the schedule; `workflows.create()` and `Workflow.seal()` refuse until one exists. +- **`oxpull.W004`**: Backends configure different admission limits under the same limit name. Align their definitions. +- **`oxpull.W005`**: Backends configure different periods under the same limit name. Align their definitions. +- **`oxpull.W006`**: Rate-limit names differ only by case or accents. Use names that remain distinct under the database's comparison rules. +- **`oxpull.W007`**: WAITING workflow nodes exist, but no backend in `TASKS` declares `OPTIONS["WORKFLOWS"]`. Restore the workflow configuration or follow the documented shutdown procedure. + +### django_ox checks + - `django_ox.E001`: `django_ox` is missing from `INSTALLED_APPS`. The app itself registers these checks with Django, so a project that also lacks any other import of `django.tasks` gets no `E001` from `manage.py check`; diff --git a/docs/features/batches.md b/docs/features/batches.md new file mode 100644 index 0000000..4f288f0 --- /dev/null +++ b/docs/features/batches.md @@ -0,0 +1,263 @@ +# Batches + +A batch groups tasks and fires a callback once every member's row is settled or gone. Completion describes the recorded task rows, not proof that every member has stopped executing. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +Batches derive completion from task rows rather than signals. +Batch state is derived by querying member task rows, never accumulated +from `task_finished`: a counter decremented by that signal hangs forever +whenever a worker dies mid-task, which is exactly what batches exist to +survive. The reconciler asks the rows instead, so it cannot miss a task +however it died. Completion is a compare-and-set on the batch status, +with the callbacks enqueued inside that same transaction. The rowcount +picks a single winner among racing reconcilers, so the callbacks are +enqueued once however many reconcilers race. They then run like any other +task, which is at least once: a callback that is retried runs again. A +crash mid-pass either leaves the batch sealed for the next tick or has +already committed the completion and its callbacks together. +These guarantees require the [database-routing constraints](unique-tasks.md#database-routing-constraints). + +`OxpullBatchMember.task_id` is a bare UUID, not a ForeignKey, so task pruning +cannot cascade through membership or be blocked by a membership foreign-key constraint. The completion query uses the ORM rather than +SQL text, so `django_ox` table and column names come from model +metadata rather than hard-coded names. + +## Behaviour + +A batch groups tasks and fires a callback once every member's row is +settled (SUCCESSFUL, FAILED, LOST or DISCARDED) or gone; a row in any +other status, including one unrecognised by django-ox 1.9.0, holds it open: + +```python +from oxpull import batches + + +batch = batches.create("nightly-invoices", on_complete=summarize_run) +batch.add(send_invoice, 42) +batch.add(send_invoice, 43) +batch.seal() + +batch.progress() # BatchProgress(total=2, pending=1, succeeded=1, failed=0) +batches.get(batch.id) +``` + +Completion is detected by a reconciler task, which runs once a minute through +django-ox's schedule dispatch (`OPTIONS["SCHEDULES"]`). Batches need this schedule; without it a batch never +completes: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, + }, + } +} +``` + +The schedule may sit on any django-ox backend. Without one anywhere, +`manage.py check` reports `oxpull.W002`, and `create()` and `seal()` +refuse with `ImproperlyConfigured` naming the block above, so a batch +cannot be left sealed with nothing to complete it. The refusal reads +the settings of the process that calls, so the `SCHEDULES` block has to +be present in every process that calls `create()` or `seal()`, web +processes included, not only in the one that runs the scheduler. A +deployment whose web settings omit it, because only the worker's carry +the schedules, refuses every `create()` from the web; +declaring the same block there changes nothing about which process +runs the reconciler. `seal()` on a batch that is already SEALED or +COMPLETE raises `BatchSealed` whatever the settings say. + +Semantics, all deliberate: + +- **Sealing freezes membership.** `add()` enqueues the task and writes the + membership row in one transaction, so the two commit or roll back + together. `seal()` snapshots the member count; after that, `add()` + raises `BatchSealed`. There is no un-seal and no reopen, and a batch + that is never sealed never completes. On MySQL, an `add()` that loses + an InnoDB deadlock reissues its own transaction, up to five times. + Called inside your `atomic()` block it cannot: InnoDB has rolled your + transaction back too, the error propagates, and it is your transaction + to retry. A reissue enqueues again, so `task_enqueued` fires once per + attempt, the first time for a row the rollback then discarded; a + receiver deferred with `transaction.on_commit` sees only the row that + committed, which is the rule for that signal in any case. +- **`on_complete` fires once every member's row is settled (SUCCESSFUL, + FAILED, LOST or DISCARDED) or gone, whatever the outcome; a row in any + other status holds the batch open. `on_success` fires in addition, and + only when every member's outcome is known and none of them failed.** + Finished is read off the row, not off the code; the bullet on what + complete means says what that leaves open. There is no failed state + for the batch itself: completion is completion, and `failed_count` + plus the absent `on_success` is how a bad run reads. +- **Callbacks are ordinary tasks.** They are enqueued in the same + transaction that completes the batch, then retry and fail like any + other task. The batch stores their ids and does not supervise them. + A callback must be a `@task` on a django-ox backend, and is stored as + its dotted path: `create()` refuses a callback that carries `using()` + overrides, because only the path is kept and the overrides would be + dropped in silence. Set them on the decorator instead. A callback + cannot be a unique task either: two batches finishing with the same + callback and arguments would share one task, and one batch's callback + would never run. A stored callback that became unique after + `create()` is refused at completion instead, and its batch stays + SEALED, logged at each tick, until the callback is made non-unique + again; existing batches do not complete before then. +- **A retry is not a finish.** A member with attempts remaining goes back + to READY, and a batch never completes past a member that will run + again. +- **A member that outlives `LOCK_TIMEOUT` reads by its row, not by what + it did.** When the worker running a member stops renewing its lease, + the reaper takes the task back. With attempts remaining the row returns + to READY and its lease epoch moves, which drops the old attempt's + result. With no attempts left the row is recorded as LOST, which says + the attempt went unobserved and nothing about its outcome, so the batch + counts that member as unattributed rather than failed: `on_complete` + fires, `on_success` is withheld, and `failed_count` is unchanged. + Renewal keeps the lease only while it reaches the database on time. + With Django's PostgreSQL pool, renewal needs a connection: the worker + normally opens one outside the pool with a bounded deadline, with a + short pooled fallback if needed. Sizing `LOCK_TIMEOUT` alone does not + prevent a reclaim; a `task_reclaimed` warning in the worker log is + the symptom. +- **Complete is a reading of the rows, not proof that every member has + stopped running.** The reconciler completes a batch at the tick where + it finds every member's row settled or gone. A member recorded LOST + may still be executing: its worker stopped renewing the lease, the + code kept going, and django-ox lets that execution write SUCCESSFUL + or FAILED onto its own row afterwards. The worker's bounded + recycle produces this on purpose when a member outlives `LOCK_TIMEOUT` + on its last attempt. So `on_complete` is not a barrier against a + member that is still running, and teardown in `on_complete` has to + tolerate one. + `on_success` fires only when every member's row is SUCCESSFUL, which + is the one outcome a running member cannot still have. Two things + follow. Member outcomes and the batch's totals are not final at + COMPLETE: a straggler can turn a LOST member SUCCESSFUL after + `failed_count` was written and the callback enqueued, and `progress()` + shows the new count. And an operator retry that commits during the + completing transaction can leave a READY member on a COMPLETE batch + whose callback is already enqueued: the reconciler re-reads the counts + under the batch row's lock, but `django_ox.actions.retry` writes the + member's task row and takes no batch lock, so the two writes cannot be + ordered. +- **A batch sealed with no members completes at the first tick**, and + both callbacks fire. +- **The reconciler runs on the backend whose `SCHEDULES` name it.** + django-ox rebinds a scheduled task to that backend at load. Its queue + name comes from the default task backend's `QUEUES`: `"default"` when + that list has it, otherwise the first entry. A queue the scheduling + backend does not accept is refused at load with `ImproperlyConfigured` + naming the schedule. Set `queue_name` in the schedule to put it + somewhere specific. +- **Latency is one tick.** Cron granularity is one minute, so a callback + lands up to about a minute after the last member finishes. Completion + is decided by reading the member task rows, which is what makes it + impossible to miss a member whose worker died without a word. +- **Pruned members read as finished, but not as successful.** By + default `ox_prune` deletes SUCCESSFUL and DISCARDED rows that finished + more than `--older-than` ago (7 days), and FAILED and LOST rows too + under `--include-failed`. A membership row whose task is gone counts + as finished, the sealed `total_count` keeps totals honest, and + `BatchProgress.unattributed` reports how many are no longer + attributable to success or failure. Those members are why `on_success` + needs every outcome to be known: a pruned row could have been the one + that failed, so the batch completes and `on_complete` fires while + `on_success` is withheld. That includes the default prune. A batch + that stays open longer than `--older-than`, because it is wide, its + members are throttled, or the reconciler was down, loses its members' + SUCCESSFUL rows and completes without `on_success`. Set `--older-than` + above the longest batch you run. `BatchProgress.pruned` is deprecated + and will be removed in 2.0. It returns the same number as + `unattributed` and emits a `DeprecationWarning` when read. +- **A discarded member is finished, and the run is not clean.** + `django_ox.actions.discard` closes a task an operator does not want + run. That member is terminal, so it never holds the batch open, but it + is cancelled work rather than work that was done. It counts as + unattributed, not as failed, and `on_success` is withheld. + `on_complete` still fires, so cleanup still runs. +- **Retrying a member reopens the wait, not the membership.** + `django_ox.actions.retry` puts a FAILED or LOST member back on the + queue. The batch stays SEALED until that attempt settles, and no + callback fires in the meantime. Retrying a member of a batch that has + already completed does not reopen it, and its callbacks have already + fired. + +## Batch API + +Start with `oxpull.batches.create()`. It returns a `Batch` with an `id` and the methods `add()`, `seal()` and `progress()`. Use `oxpull.batches.get(batch.id)` to retrieve the batch. + +`Batch.add()` raises `oxpull.exceptions.BatchMemberExists` if the task is already a member of that batch. Adding to a sealed batch raises `oxpull.exceptions.BatchSealed`. + +`Batch.progress()` returns `BatchProgress(total, pending, succeeded, failed)`, read from the member task rows when called. Before sealing, `total` is the live member count. After sealing, it is the sealed snapshot. + +The result also exposes: + +- `unattributed`: `max(total - pending - succeeded - failed, 0)`. This covers members whose task rows `ox_prune` deleted, LOST members and DISCARDED members. +- `finished`: `total - pending`. +- `pruned`: a deprecated alias of `unattributed` that emits `DeprecationWarning`. Use `unattributed` in new code. + +These batch entry points, result fields and exceptions are stable. + +## How wide a batch can be + +Measured to **1,000,000 members** in a single batch, on SQLite, +PostgreSQL 16 and MySQL 8, with every count checked against the task rows +rather than against what the API reports about itself. The completion +query is a subquery, not an expanded list of ids, so it binds one +parameter whatever the width, on every database and at every width +measured, and cannot run into SQLite's variable limit or PostgreSQL's +parameter ceiling. + +At a million members the costs separate sharply by database: + +| | SQLite | PostgreSQL 16 | MySQL 8 | +| --- | --- | --- | --- | +| seal, inside a caller's transaction | 34 ms | 0.7 s | **28 s** | +| reconcile | 4.9 s | 0.4 s | 7.3 s | +| finish all members | 4.8 s | 9.1 s | 27.6 s | + +The seal row is the cost of a locking count: a row lock per member +wherever the database offers one, and a million InnoDB locks in one +transaction is where that shows (0.24 s at 100,000 members, 28 s at a +million). `seal()` now takes that count only when it is called inside a +caller's transaction, where the caller's snapshot could predate a member +committed since. Called outside a transaction it counts with a single +`COUNT(*)`: on a million-member batch, timing the two statements alone +with fresh statistics, the locking read took 12 s on MySQL 8 and 0.55 s +on PostgreSQL 16, the `COUNT(*)` 0.7 s and 0.03 s, and the whole `seal()` +call outside a transaction 0.74 s and 0.03 s. SQLite counts either +way, which is why it is the cheapest of the three at exactly the point +the other two are dearest. If you seal batches that wide on MySQL from +inside a transaction, budget for it, split them, or seal outside the +transaction. + +Measured on an arm64 macOS host, with PostgreSQL 16 and MySQL 8 in +Docker on localhost and SQLite on a local file, one process at a time +and each width against an empty database. The individual-batch figures were measured on the earlier separate `oxpull` package: its 0.3.1 release on SQLite and PostgreSQL, and its unpublished 0.4.0 build on MySQL, with the same batch code. The fifty-batch sweep and the task-table figures below it are from the 2026-08-19 runs on pre-release code of that separate `oxpull` package labelled 0.1.0. These are historical measurements, not a new benchmark of the combined distribution. + +Reconciler cost tracks the member count: from 100,000 members to a +million, the pass that completes a batch grows 7.5x on PostgreSQL, 14x on +SQLite and 13x on MySQL. The slowest reconciler pass measured at 100,000 +members is the completing one: 0.56 s on MySQL, 0.34 s on SQLite and +55 ms on PostgreSQL, against a one-minute tick. Fifty sealed batches of +5,000 members each cost 667 ms per pass on PostgreSQL and 319 ms on +SQLite while their members are still pending, which is what every tick +costs until they finish. + +One thing to know if your task table is large. On PostgreSQL the planner +hashes the batch's members and scans the task table, so a reconciler pass +costs roughly in proportion to that table rather than to the batch: about +2.3 ms against 5,000 task rows and about 38 ms against a million, per +sealed batch. Two levers, both yours: prune finished tasks with +`ox_prune`, which you want anyway, or lower `random_page_cost`. At 1.1 +the planner picks a nested loop unaided and the same work took 6.4 ms +instead of 36 ms. SQLite probes the primary key per member, so the same +table growth costs it 3x rather than 16.6x. diff --git a/docs/features/index.md b/docs/features/index.md new file mode 100644 index 0000000..bdd3f41 --- /dev/null +++ b/docs/features/index.md @@ -0,0 +1,64 @@ +# Optional features + +django-ox 1.9.0 includes the optional `oxpull` app. It adds unique tasks, batches, rate limiting and workflows without a separate package installation. + +The app requires Django 6.0+. django-ox itself continues to support Django 5.2 through the `django-tasks` backport. + +## Setup + +1. Add `"oxpull"` to `INSTALLED_APPS` and run migrations. +2. Set the relevant task alias's `BACKEND` to `"oxpull.backend.OxpullBackend"`. +3. Keep the `django_ox` and `oxpull` tables on the same database. A router that splits them is refused at backend construction with `ImproperlyConfigured`. The check consults routers by model, without hints. Hint-dependent routing that sends task rows elsewhere is unsupported and may escape the startup check. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). +4. Configure the features you use: + - Unique tasks use `@unique` or `OPTIONS["UNIQUE_TASKS"]`. + - Batches need a reconciler schedule. + - Rate limits need `OPTIONS["RATE_LIMITS"]` and `OPTIONS["WORKER_CLASS"] = "oxpull.worker.OxpullWorker"`. + - Workflows need `OPTIONS["WORKFLOWS"] = True`, `OxpullWorker` and a reconciler schedule. + +Apply the relevant backend and worker settings in every process that enqueues or claims those tasks. Follow the feature page for the complete configuration, including the reconciler's queue and schedule. + +Already have a separately installed `oxpull` distribution? Follow [Earlier `oxpull` installs](../migrating.md#earlier-oxpull-installs) before upgrading. + +## Unique tasks + +[Unique tasks](unique-tasks.md) deduplicate pending tasks at enqueue time. While a task with the same key is READY or RUNNING, another enqueue returns the pending result. + +This is not a guarantee that two executions of the same key cannot overlap. A LOST task releases the key even if its original execution is still running. + +## Batches + +[Batches](batches.md) collect tasks and arrange a callback after the sealed batch settles. Create a batch, add its members, then seal it. A reconciler schedule is required. + +`Batch.add()` raises `oxpull.exceptions.BatchMemberExists` if the task is already a member of that batch. + +## Rate limiting + +[Rate limiting](rate-limiting.md) controls admissions through `OxpullWorker`. Configure limits on task paths, queues or both. + +A limit name identifies one shared window across the backends that configure it. A backend setting alone does not make a plain django-ox worker enforce that limit. + +## Workflows + +[Workflows](workflows.md) coordinate tasks with dependencies. A workflow is created at runtime. Enable `OPTIONS["WORKFLOWS"]`, use `OxpullWorker`, and configure the reconciler schedule before creating or sealing workflows. + +Workflow APIs, settings and log events are provisional. + +## Database and task rows + +The app creates its own `oxpull_*` tables. It does not change any django-ox table, column, index, constraint or migration. + +Workflows do write task rows. They insert blocked nodes as WAITING tasks. Cancellation moves a READY or WAITING task to DISCARDED by a compare-and-set on the task row's status and lease epoch. Release moves a WAITING task to READY and updates `run_after`; `run_after` is never NULL. DISCARDED nodes are not revived. + +Use `ox_prune` for task rows and `oxpull_prune` for the optional app's records. + +## Signals + +Tasks enqueued through `OxpullBackend` use that backend as the signal sender. Receivers restricted to the base backend's sender must account for this when an alias changes backend. This applies to every task sent through `OxpullBackend`, not only unique tasks. + +## Stability + +The backend, its unique-task and rate-limit options, the worker, the unique-task decorator and the documented batches API are stable. The `oxpull_*` schema, check ids, `oxpull_prune` and non-workflow log events are also stable. + +Everything in the documented workflow surface is provisional, including the `WORKFLOWS` option and every `workflow_*` event. Workflow internals are not public. + +See the [stability policy](../stability.md) for the complete lists and versioning rules. diff --git a/docs/features/rate-limiting.md b/docs/features/rate-limiting.md new file mode 100644 index 0000000..0f8f1fe --- /dev/null +++ b/docs/features/rate-limiting.md @@ -0,0 +1,257 @@ +# Rate limiting + +Rate limiting controls task claims through `OxpullWorker`. Its guarantee depends on the workers applying the limit and recording claims; the bound and its assumptions are described below. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +Rate limiting coordinates workers through database state. Every decision is a +compare-and-set resolved by a rowcount, never a check-then-act: the +window roll is an UPDATE conditioned on the window id, so exactly one +worker among any number opens a window and the losers report zero rows +and keep the view they read. The increment that counts an admission is +deliberately unconditional, because by the time it runs the attempt is +already spent and a statement that could refuse would burn it. What stops +a worker taking a second token is not that statement but the next +refresh, which every worker is required to make before its next limited +claim. + +The window's clock is the database's, on every database and under any +`USE_TZ` setting. `window_started_at` is written with `Now()` and is only +ever compared against `Now()` in the same statement family, so there is +no second writer for it to agree with and no per-host clock in the +decision. A worker's own clock decides only when to look, which can waste +a statement but can never admit a task. A database clock that steps +backwards delays the next roll by the size of the step and under-admits; +one that steps forwards rolls the window early and releases its capacity +ahead of schedule, by at most one window per step. Neither compounds: the +window re-anchors to database time on the next roll. + +## Behaviour + +A rate limit caps how often a task starts. Give it a name, a capacity, +and the tasks or queues it covers: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", "sms", "reports"], + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": { + # Both tasks draw on one budget of 100 per minute. + "stripe": { + "limit": 100, + "per": 60, + "tasks": [ + "myapp.billing.charge_card", + "myapp.billing.refund_card", + ], + }, + "reports": { + "limit": 10, + "per": 3600, + "tasks": ["myapp.reports.build_monthly"], + }, + # A whole queue, whatever runs on it. + "sms": {"limit": 500, "per": 60, "queues": ["sms"]}, + }, + }, + } +} +``` + +A limit is declared in settings. A worker imports no task modules, so a +limit written next to its task would be enforced only when something else +happened to import that module first. Declared here, it is visible to +every worker and to `manage.py check`. + +`RATE_LIMITS`, `UNIQUE_TASKS` and `@unique` take effect only with +`oxpull.backend.OxpullBackend`. On an alias still using +`django_ox.backend.OxBackend`, nothing reads these settings: +`manage.py check` is clean, and nothing is throttled or deduplicated. + +For aliases using `OxpullBackend`, `WORKER_CLASS` is required: it makes +`ox_worker` run the worker that applies the limits. Configuring limits +without it is a hard system check error (`oxpull.E006`). `per` is whole +seconds, or a `timedelta` of at least one second. A limit that covers +nothing, names a queue the backend does not accept, or lists a `tasks` +path that is not a task is `oxpull.E005`. These checks run only for +aliases whose `BACKEND` is `oxpull.backend.OxpullBackend`. + +## Deployment guarantees + +Rate limits are applied in `OxpullWorker.claim_one()`, not enforced by a database constraint. The base `Worker` claims tasks without applying the limit or recording a count. + +Deploy django-ox 1.9.0 and use `OxpullWorker` in every process that claims limited tasks before relying on the guarantee. `manage.py check` reads only its own process's settings and cannot verify which worker other processes run. + +## The guarantee + +> For a limit of N admissions per period P, with W `ox_worker` processes +> claiming from the queues that carry the limit's tasks, the number of +> task attempts started in any one window is at most N + W - 1 + U. +> U is the number of admissions whose count did not land: one for each +> count that raised, and one for each count that the window rolled out +> from under five times in a row. Each one is logged at warning as +> `rate_limit_uncounted`; alert on that event, and U is the number of +> times it fired in the window. Add one for each worker process that +> dies between claiming a limited task and recording it. With a single +> worker process, no such death and every count landed, the limiter is +> exact: at most N per window. +> +> A window is one generation of the limit's shared counter row: it opens +> when a worker rolls the row and lasts P. It is not every interval of +> length P. Windows are contiguous, so an arbitrary interval of length P +> that spans a boundary can carry up to 2(N + W - 1 + U). The limiter +> therefore bounds a long-run average of (N + W - 1 + U)/P and an +> instantaneous burst of 2(N + W - 1 + U). +> +> A count that cannot be written is logged and the attempt runs. Three +> admissions in a row that the counter did not record close the limit +> for that worker process, whether each increment raised or ran and +> matched no row. A closed limit admits nothing until a write lands, +> which the next refresh probes for. The run and the closed state are +> per process, and only an increment that lands ends the run, so a +> counter that fails on every other admission closes after three of +> them rather than admitting through the fault. + +The W - 1 is structural. The budget has to be read before the claim, +because the claim UPDATE spends a retry attempt and a lease epoch and +neither can be given back; between reading "this limit has room" and the +claim landing, another worker can take the last token. Closing that gap +would mean burning attempts on tasks that never ran. + +W is yours to know, not ours. One `ox_worker` process is one claimer, and +under `--processes N` each child is a claimer. Size against the bound +rather than against the capacity. For a ceiling of T attempts in one +window, configure **`limit = T - (W - 1)`**. For a ceiling of T across +any interval of one period, including one that straddles a boundary, +halve it: **`limit = T // 2 - (W - 1)`**. Both assume U is zero, which +holds while `rate_limit_uncounted` is not firing; every time it fires, +the window it fired in can carry one attempt more than the ceiling. +Scaling workers up loosens a limit that is left alone, and each worker +that dies mid-claim costs one more admission. + +Semantics, all deliberate: + +- **It limits attempt starts, not completions and not concurrency.** Ten + per minute on a task that runs for ten minutes will have a hundred + attempts running at once. "At most K running" is a concurrency limit, + which is a different mechanism. +- **Retries count.** A reaped or failed attempt that goes back to READY + consumes another admission when it is claimed again. That is correct + for what limits usually protect: a retry is another call to the third + party. +- **A throttled task stays READY.** It is not claimed, not locked, and it + spends no attempt and no lease epoch. It is indistinguishable from any + other queued task, and nothing in the reaper, the lease or the timeouts + interacts with it. +- **A throttled task does not hold up anything else.** The exclusion is + applied inside the claim's candidate select on all three claim paths, + ahead of its ordering and its limit, so a worker whose limit is + exhausted still claims other tasks and other queues on the same pass. +- **Windows are fixed and anchored at first use**, so a P-length interval + straddling a boundary can carry twice the capacity. If what you are + protecting counts per fixed minute, size N at half your true ceiling or + accept the doubling. +- **A task may belong to several limits.** It is admitted only when every + one of them has room, and admitting it increments every one. +- **`limit: 0` blocks the limit entirely.** It is a supported pause + switch, and it holds unique keys open, holds batches open, and drives + the ready-age metric up for as long as it is set. +- **Capacity is settings, not data.** A change takes effect for a worker + when that worker restarts. During a rolling change of `limit`, + admissions are bounded by the larger of the two capacities. During a + rolling change of `per`, the bound is set by the shorter period until + the last worker on it is gone: the window row carries no period, so a + worker on the shorter one rolls the shared window as soon as its own + period allows, and every worker then counts against the fresh window. + Change `per` in its own deploy, and expect the shorter period to hold + until that deploy finishes. +- **A limit's name spans every backend.** Its window row is keyed on the + name alone, so two backends that use the same name share a window and + count each other's admissions. Give a limit meant for one backend a name + only that backend uses. A name with a different `per` on two backends is + warning `oxpull.W005`, and that limit is not enforced as configured. + Each backend rolls the shared window at its own period. Each roll + restarts the count, so a backend on a longer period can admit more than + its limit in that period. A different `limit` with the same `per` is + warning `oxpull.W004`. Each backend then stops when the shared count + reaches its own `limit`. That shared budget can be deliberate. +- **A limit's name is matched by the database's collation.** The name is + the primary key of the limit's window row, so on MySQL's case-folding + collations renaming `Stripe` to `stripe` is not a rename: the worker + finds the row it already has and carries on counting on it. On + PostgreSQL and SQLite the spelling is the identity, and a renamed limit + starts a fresh window. Within one backend, names that differ only by case + are refused at `check` as `oxpull.E005`. Anything else the database folds + (`Café` and `cafe` on MySQL) is refused at the worker's first refresh, + before it claims, naming both. Across backends, names that differ only by + case or accents are warning `oxpull.W006`. MySQL's default collation + counts them on one window, and PostgreSQL and SQLite count them on two. On + one backend, `oxpull.W006` also covers names that differ only by accents, + or by case and accents, such as `stripe` and `stripé`. On MySQL's default + collation, the worker refuses such a pair too. When `check` compares + names, it drops accents from Latin and Greek letters only. It does not + catch other folds MySQL's default collation makes, such as `ø` into `o`. + Spell a name the same way on every backend. +- **Renaming a limited task detaches its queued rows.** The limit matches + on the task path stored in the row, so rows enqueued under the old path + claim unthrottled. List both paths under the limit for one release. +- **A counter that cannot be written closes its limit.** One failed count + is logged and the claimed task is returned for execution: the claim is + already committed by then, and losing a count is better than losing + the task. The same holds for any database failure after the claim, the + re-read after a window rolled under the count included, and for that + re-read finding the limit's row gone: the loss is logged as + `rate_limit_uncounted` and the claimed task is returned for execution. + Before the claim a failure ends the pass with nothing spent, and a + missing row propagates as `ImproperlyConfigured`. Three unrecorded + admissions in a row under one limit stop it admitting altogether, + because a counter that never advances is not a limit, and every + refresh would read room that is not there. An increment that raised + and one that ran but matched no row both count; only an increment that + lands ends the run. The refresh probes the row and the limit reopens + when a write lands, without a worker restart. + +## Claim cost + +Nothing when it is unused. With no limits configured, and no backend on the +database declaring `WORKFLOWS`, the claim emits django-ox's own SQL, +statement for statement. The suite asserts that by capturing both. +See [workflow claim cost](workflows.md#claim-cost) for the work a `WORKFLOWS` backend adds to every claim on that database. +With limits configured, a task that no limit covers costs one +refresh SELECT per cooldown interval +(`RATE_REFRESH_INTERVAL`, 0.25 s by default), shared by every claim in +it. A limited task costs two extra statements: that refresh and one +increment. A window roll costs a third. One roll lands per window per +limit across the whole fleet; a worker that reads the expired window +before the winner's write commits spends one more UPDATE, which matches +no row. + +The state is one row per limit name in `oxpull_oxpullratewindow`, +forever. The table does not grow with traffic and needs no pruning. + +## Throttled work reads as backlog, because it is + +`django_ox.stats.oldest_ready_age` and the Prometheus ready-age gauge +climb for as long as work is throttled. A queue-age alert tuned before +rate limiting was introduced will fire. That is the limiter working, not +a stuck queue: alert on the age of queues that carry no limited tasks, or +raise the threshold above the longest period you configured. + +Two interactions matter when you combine features. A throttled `@unique` +task keeps its key, so throttling extends the dedup window; that is what +"unique while pending" means, and releasing the key would let duplicates +stack up behind the throttle. And a batch does not complete past a +throttled member, so a limit sets the floor on how long the batch runs. +On one worker, a 1,000-member batch under a limit of 10 per minute fills +100 windows, and the hundredth opens 99 minutes after the first. The +batch completes once its last member finishes, plus one reconciler tick. +Size batches against limits. + +## Database aliases and pools + +Rate-limit window reads and writes use the alias returned by `oxpull_db_alias()`. See [worker database aliases and pools](workflows.md#worker-database-aliases-and-pools) for the recording alias's pool requirements and the limits of the worker's startup warning. + +Use the same database alias for task rows and rate-limit windows: `OxpullWorker` reads the window on that alias. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). diff --git a/docs/features/unique-tasks.md b/docs/features/unique-tasks.md new file mode 100644 index 0000000..f71c3f9 --- /dev/null +++ b/docs/features/unique-tasks.md @@ -0,0 +1,236 @@ +# Unique tasks + +Unique tasks deduplicate enqueues while a matching task is pending. The key is held while the task is READY or RUNNING; this is not a guarantee that two executions of the same key cannot overlap. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +## Behaviour + +A task marked unique is deduplicated while a matching task is pending: + +```python +from django.tasks import task +from oxpull.unique import unique + + +@unique +@task +def send_invoice(order_id): ... + + +send_invoice.enqueue(42) # enqueues +send_invoice.enqueue(42) # returns the pending task's TaskResult; no new row +``` + +`enqueue_many()` raises `InvalidTask` for a unique task and names it. +django-ox writes that call as one bulk INSERT, which never reaches the +deduplicating path, so no uniqueness lock would be taken and the rows +would not be deduplicated. Refusing the call keeps the guarantee honest. +Enqueue unique tasks one call at a time; plain tasks keep the bulk path. + +The lock is an enqueue-time gate. `django_ox.actions.retry` is an +operator instruction to run one named row, not an enqueue, so it does not +pass that gate. Retrying a unique task that a later `enqueue()` has +already superseded therefore runs that attempt beside the newer one, and +the key carries two READY rows until both settle. `oxpull` honours the +retry an operator asked for rather than discarding it. + +Tasks can also be made unique from settings instead of the decorator: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", ...], + "OPTIONS": { + "UNIQUE_TASKS": ["myapp.tasks.send_invoice"], + }, + } +} +``` + +Semantics, all deliberate: + +- **Window: pending-uniqueness.** The key is held while the matching task + is READY or RUNNING. It releases as soon as + the task leaves those two statuses (SUCCESSFUL, FAILED, LOST or + DISCARDED), or when its row is pruned. The key follows the row's + status, not the code: a row the reaper recorded LOST releases the key + while the execution it belonged to may still be running, and django-ox + lets that execution write SUCCESSFUL or FAILED onto its own row later. + A RUNNING task whose worker has died still holds the key, so a duplicate + enqueue returns that task's result and queues nothing. The reaper takes + the task back after its lease expires. With attempts left, it puts the + task back to READY, and the key stays held. On the task's last attempt, + the reaper records it LOST instead. That frees the key, and the result the + duplicate got back reads FAILED after `refresh()`. + Uniqueness deduplicates pending rows. It does not stop two executions + of one key overlapping, and a task that must never overlap itself + needs its own guard. +- **A retry is not a duplicate.** A failed attempt with attempts remaining + returns to READY, which is still inside the window; the key stays held + through backoff, and at-least-once execution is unchanged. +- **Duplicates return the existing `TaskResult`** rather than raising, and + `task_enqueued` is not sent for a suppressed duplicate. The result + reflects the stored row (its priority, queue and run_after), not any + `using()` overrides passed on the duplicate call. +- **Key derivation:** SHA-256 over `(backend_alias, task_path, queue_name, + args, kwargs)` after `django.utils.json.normalize_json`, the same + normalisation the task row stores. Uniqueness is scoped per backend + alias. Dict order is irrelevant; tuples equal lists; `1` and `1.0` + differ; int dict keys match their string form, exactly as JSON stores + them. `queue_name` is in the key; `priority` and `run_after` are not. + There is no signature binding: `add(1, 2)` and `add(1, b=2)` are + different keys. +- **Enqueue cost:** a unique enqueue is one transaction containing three + statements (lock INSERT under a savepoint, task INSERT, lock UPDATE) + instead of one. Non-unique tasks take the parent code path untouched. + +Signals: `task_enqueued` fires inside the enqueue transaction, before +commit, unchanged from django-ox. Receivers that trigger external +effects, including anything that wakes workers out of band, must defer +with `transaction.on_commit`: a pre-commit receiver cannot see the row +from another connection, and for unique tasks a suppressed duplicate +emits no signal at all. + +Scheduling a unique task raises system check `oxpull.W001`: schedules +already dedup once per tick, and a unique scheduled task coalesces with +pending manual enqueues of the same key. + +## Deployment guarantees + +Uniqueness is enforced by the lock table's primary-key constraint. `OxpullBackend.enqueue()` writes the lock row and task row in one transaction and handles a competing insert's `IntegrityError`, rather than checking for the key first. + +The constraint covers enqueues through `OxpullBackend`. An enqueue through `django_ox.backend.OxBackend` writes no lock row, so it can create a duplicate beside a task whose key is locked. Later unique enqueues return the task associated with the lock; they do not remove those extra rows. + +Deploy django-ox 1.9.0 and configure `OxpullBackend` in every process that enqueues unique tasks, including web processes and cron jobs, before relying on this guarantee. `manage.py check` reads only the settings of the process running it; it cannot verify the rest of the fleet. + +Before enabling uniqueness over existing traffic, drain or reconcile pending tasks enqueued without lock rows. A READY or RUNNING task without a lock row is not deduplicated against. + +The dedup point is a unique constraint on `OxpullUniqueLock.key` plus an +`IntegrityError` loser path, the same mechanism used by `django_ox` +for schedule ticks. Check-then-insert is never used. The lock row and the +task row are written in one transaction, so enqueue keeps the parent +queue's durability guarantee (a `transaction.atomic()` opened on the +database these tables live on covers the enqueue too, and a rollback +discards task and lock along with every other row you wrote there). +These guarantees require the [database-routing constraints](#database-routing-constraints). + +Lock release never depends on signals. The django-ox reaper settles an +abandoned task by writing LOST through a queryset update that emits no +signal, so any signal-driven release would strand keys exactly when a +worker dies. Instead, staleness is derived from the task row at the next +enqueue of the same key: when the holder has left READY and RUNNING, or +its row is gone (pruned), the lock is released and the enqueue proceeds. +A holder that is still READY or RUNNING is read without a lock: there +is nothing to release, and a row lock taken inside a caller's +transaction (`ATOMIC_REQUESTS`, any `atomic()` block, or a connection +with autocommit turned off) would live until that transaction commits +and hold off the holder's own lease renewal and outcome write for as +long as the request ran. On MySQL at REPEATABLE +READ that unlocked read, inside a caller's transaction, answers from the +transaction's snapshot, so a holder the snapshot calls live is read once +more on a connection of its own, which sees the committed status and +takes no lock; at READ COMMITTED, the level Django sets for MySQL unless +told otherwise, the plain read is current and no connection is opened. + +At PostgreSQL's REPEATABLE READ or SERIALIZABLE, that read answers from +the snapshot. So the enqueue asks for a `FOR SHARE` lock on the holder's row +without waiting, inside a savepoint it rolls back at once. PostgreSQL +grants the lock when the row hasn't changed since the snapshot began, and +the enqueue returns the holder. If another transaction is writing the row +and hasn't committed, PostgreSQL refuses at once. The enqueue then returns +the last committed row, as a read at READ COMMITTED would. If the row has +changed, the enqueue reads the holder and the key's lock once more on a +new connection, in a READ COMMITTED transaction of its own, and closes it. +A task that's still pending and still holds the key is returned from that +read. The caller's transaction keeps its older snapshot. The level is read +in the same statement as the holder. + +That read's connection waits at most 2 s to connect to each address the +host name resolves to. The read doesn't set that limit when the project +sets `connect_timeout` in `OPTIONS` or in `PGCONNECT_TIMEOUT`, or names a +connection service. Once connected, the read waits at most 100 ms for a +lock and runs for at most 2 s. It sets both with `SET LOCAL`, so a pooled +server session doesn't keep them. Nothing limits the host name lookup, or +a wait in a connection pooler's queue before the read reaches PostgreSQL. +The caller's transaction holds ACCESS SHARE locks on the task and lock +tables, from its own reads of them. A request such as a migration's +`ALTER TABLE` then waits for that transaction to end. The read queues +behind that request while the caller waits on the read, and PostgreSQL +can't see that cycle. Without those limits, the read would wait until that +request gave up. If the read stops, or its connection fails or times out, +the enqueue raises `UniqueEnqueueRace`. + +Apart from that check, which keeps nothing, only the release reads the +holder with a locking read (`SELECT ... FOR +UPDATE` on PostgreSQL and MySQL), so that it acts on the committed +status rather than the enclosing transaction's snapshot and waits for an +operator retry still in flight; a retry that has just put the holder +back to READY keeps its lock instead of losing it to a stale read. +Inside a caller's transaction that lock, on a settled row, rides to the +caller's commit, and a retry of that row, a straggler's outcome write +onto a LOST row and `ox_prune` of the row wait behind it. The DELETE +itself carries the holder's status in its WHERE clause, which is the +whole of the guard on SQLite, where no row lock exists and a statement +is atomic against the single writer. `oxpull_prune` clears stale locks +for keys that are never enqueued again; it is hygiene, not correctness. + +`task_id` on the lock is a bare UUID, not a ForeignKey, so `ox_prune` +deleting finished task rows cannot cascade through the lock or be blocked by a foreign-key constraint; a dangling id reads +as "finished". The `oxpull` migration creates only tables belonging to the +optional `oxpull` app and does not alter `django_ox` tables. + +Portability note: MySQL has no partial unique indexes, so the lock-table +design is the one mechanism that is identical on SQLite, PostgreSQL and +MySQL, rather than a partial index over task status on the databases that +support one. The lock table keeps unique-task coordination within the +`oxpull` app's schema without adding an index to a `django_ox` table. + +Known corner: inside a transaction at MySQL's REPEATABLE READ, or at +PostgreSQL's REPEATABLE READ or SERIALIZABLE, a unique enqueue can raise +`oxpull.exceptions.UniqueEnqueueRace` (a +`django.tasks.exceptions.TaskException`) where READ COMMITTED would +return a result. It never queues a duplicate instead, and it creates +nothing. On PostgreSQL that happens when the task holding the key +finished or was deleted after the transaction's snapshot began, or when a +settled holder was retried after that. It also happens there when the +read on a new connection stops or fails. On both databases it happens when +another enqueue took the key after the snapshot began, because the +transaction can't see the new lock. The message names the cause it +detects. Retry in a new, short transaction at READ COMMITTED, or outside a +transaction. READ COMMITTED on PostgreSQL or MySQL, and SQLite, don't hit +this. + +Known limit: on MySQL at SERIALIZABLE, a duplicate enqueued inside a +transaction locks the holder's row until that transaction ends. Until +then the holder's lease renewal, its outcome write and the reaper wait. +They fail with error 1205 once `innodb_lock_wait_timeout` passes, 50 +seconds by default. A worker won't claim a READY holder either. Keep +those transactions short, or enqueue unique tasks outside them. + +A second corner, on every database: the enqueue joins the transaction the +caller opened on this database, so a slow request holds its key for as +long as it runs, and a duplicate enqueued meanwhile waits on it for as +long as the database allows. SQLite waits for the connection's +`OPTIONS["timeout"]` (5 s unless set), MySQL for +`innodb_lock_wait_timeout` (50 s unless set), PostgreSQL for as long as +the holder runs unless `lock_timeout` is set. Past that the duplicate +raises `UniqueEnqueueRace`, chained to the database's own error, rather +than a raw `OperationalError`. Nothing was written, and the message says +whether to retry the enqueue or, inside an open transaction, the whole +transaction. What that transaction is left as depends on the server. +MySQL's default rolls back the statement, so the enqueue rolls back to its +savepoint and the transaction is still usable; with +`innodb_rollback_on_timeout=ON` the server rolls the whole transaction +back, nothing done in it survives, and the message says so. SQLite in WAL +mode has a third shape: a transaction that read before it enqueued is +refused at once when another connection has committed since +(`SQLITE_BUSY_SNAPSHOT`), without waiting out the timeout; the message +names that too. + + + +Constraint: the `oxpull` and `django_ox` tables must live on the same database. The lock INSERT and its task INSERT share a transaction, the membership INSERT and its task INSERT share a transaction, and the completion query joins the two tables. None of these operations crosses a database alias. The rate window is neither co-written nor joined with task rows; it belongs on the same alias because `OxpullWorker` reads and writes it there, regardless of a router naming another database for it. At construction, the backend checks all seven `oxpull` models against the active router and refuses a split with `ImproperlyConfigured`. The batch API re-checks this placement on every call. Every read on these paths goes to the write database, so read-replica routing cannot serve a stale liveness answer, complete a batch early, or read capacity from a lagged window. + +The check consults routers by model, without hints. `django_ox` saves each task row through the same router with the row as a hint, so a router that routes by model puts the task row on that database too. Hint-dependent routing that sends task rows elsewhere, such as one database per backend, is unsupported and may escape the startup check. With that routing, the lock or membership row would commit apart from its task, and an `add()` reissued after a deadlock would enqueue again on the other database. diff --git a/docs/features/workflows.md b/docs/features/workflows.md new file mode 100644 index 0000000..26f7e3a --- /dev/null +++ b/docs/features/workflows.md @@ -0,0 +1,368 @@ +# Workflows + +Workflows connect tasks through dependencies, keeping dependent nodes waiting until they can be released. Workflows are provisional in django-ox 1.9.0. The sections below describe release, completion, retention, and safe shutdown. + +See the [feature overview](index.md) for setup, requirements, Django 5.2 behaviour, and the stability policy. + +## Behaviour + +A workflow is a set of tasks with dependencies. A node runs only after every +node it depends on has succeeded. + +```python +from oxpull import workflows + +flow = workflows.create("nightly", on_complete=summarize) +flow.add("extract", extract) +flow.add("transform-a", transform, "a", depends_on="extract") +flow.add("transform-b", transform, "b", depends_on="extract") +flow.add("load", load, depends_on=["transform-a", "transform-b"]) +flow.seal() + +flow.progress() # WorkflowProgress(total=4, pending=4, blocked=3, ...) +``` + +A node's backend must declare `WORKFLOWS` and run `OxpullWorker`. The +reconciler must be on a schedule. `add()` refuses a node whose backend does +not declare `WORKFLOWS` or runs another worker class. `create()` and `seal()` +refuse when no schedule runs the reconciler. +Keep workflow tables and their task rows on the same database. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "WORKFLOWS": True, + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "SCHEDULES": { + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, + }, + }, + } +} +``` + +`depends_on` names nodes already added to the workflow, so a cycle cannot be +built. A unique task cannot be a node. To make a task unique that workflows +already use, drain its pending nodes first. + +## Workflow API + +The documented workflow API is provisional: + +- `oxpull.workflows.create()` +- `oxpull.workflows.get()` +- `oxpull.workflows.reconcile` +- `oxpull.workflows.retained_rows()` +- `Workflow.add()` +- `Workflow.seal()` +- `Workflow.progress()` +- `WorkflowProgress` + +The workflow exceptions are also provisional: + +- `oxpull.exceptions.WorkflowSealed` +- `oxpull.exceptions.WorkflowNodeExists` +- `oxpull.exceptions.UnknownWorkflowNode` + +They distinguish a sealed workflow, a duplicate node and an unknown node. The `WORKFLOWS` option and all `workflow_*` log events have the same provisional status. Workflow implementation helpers are not public API. + +## Waiting and release + +- A node with no dependencies is enqueued READY. +- A node with dependencies is inserted in django-ox's WAITING status. No + worker claims, reaps or prunes a WAITING row. +- An edge is marked satisfied only on proof that its upstream succeeded. The + proof is the upstream row, or its recorded outcome once the row is pruned. + SUCCESSFUL is final, so the mark stays true after the upstream row is pruned. +- A node moves from WAITING to READY only when every edge into it is marked. + The move is a compare-and-set on the row's status and lease epoch. +- The worker that runs a node records its outcome and marks the edges out of + it in the transaction that writes the task's result. After that commits, + it releases the nodes those edges held. +- A `run_after` you gave a node stays on its row. The release keeps the + later of now and that time, so a released node shows a `run_after`. +- The reconciler releases what a worker missed. When release owns its + transaction, it makes two attempts per chunk of node IDs. If both attempts + lose a lock conflict, the nodes wait for the reconciler. + + SQLite is supported. Django uses DEFERRED transactions by default. + `OPTIONS["transaction_mode"]` can change that mode. + + Under DEFERRED transactions, a read-to-write lock conflict can return + `SQLITE_BUSY` immediately, without waiting for the busy handler. + After both release attempts lose a lock conflict, release logs + `workflow_release_deferred` at WARNING level. The reconciler retries + the deferred nodes. + + Reconciler cadence affects workflow latency. The documented schedule + runs once a minute. Keep the reconciler scheduled and monitor + `workflow_release_deferred`. + +## The claim still checks + +The claim refuses a READY node while an edge into it is not marked and its +upstream row is missing or not SUCCESSFUL. A missing upstream row is never +taken as success. This covers READY rows no release made, such as a django-ox +retry or an UPDATE by hand. What it costs is below. + +## Failure, cancellation and completion + +- A node does not run while one of its upstreams is failed, lost or + discarded. In a sealed workflow the reconciler discards that node. +- An upstream whose row is gone counts as dead only when its outcome was + recorded as dead. With nothing recorded, the nodes downstream of it wait. +- A sealed workflow completes once no node is pending. WAITING, READY and + RUNNING nodes are pending, as is any status unrecognised by django-ox 1.9.0. +- Completion counts each node by its recorded outcome. A success or a failure + still counts after `ox_prune` deletes the row. A node whose row went before + anything recorded it counts as unattributed. +- The reconciler records any outcome a worker did not record, and completes + the workflow on a later tick. +- `on_complete` and `on_success` are enqueued in the transaction that + completes the workflow. `on_success` is enqueued only when nothing failed + and nothing is unattributed. +- A django-ox retry of a node after its workflow completed runs that node. + The nodes downstream of it stay discarded, and the workflow stays COMPLETE. + +## The reconciler's backstop + +Each tick pages through open and sealed workflows, one visit each in turn. A +visit marks edges whose upstream succeeded, records outcomes, and releases +waiting nodes. After five seconds, or a quarter of what its task timeout +leaves, a tick starts no further visit or page. Its first page always runs, +and a page that has started runs to its end. The retries of the queue below +come first and are not counted. + +On PostgreSQL and MySQL each write waits at most one second for a lock. On +PostgreSQL each of its statements runs at most two seconds. On MySQL that +bound applies to SELECT statements only. The page reads have no time bound. A +row the backstop cannot lock goes to a queue that later ticks retry first. The +next pass reads it again too, so the backstop never gives up on a row. + +For SQLite release deferrals and their effect on workflow latency, +see [Waiting and release](#waiting-and-release). + +## Retention + +`oxpull_prune` keeps batch members. For workflows it follows these rules. + +- It considers a workflow only once it is COMPLETE and completed longer ago + than `--older-than`. +- It stamps the workflow as retired, then deletes nothing while any node is + pending, such as a node retried after completion. +- It deletes a node, with the edges into it, only when the node's outcome is + recorded and its task row is SUCCESSFUL, DISCARDED or gone. +- A FAILED or LOST node stays, with the edges into it, while its row exists. + A retry of it is still checked. It goes once `ox_prune --include-failed` + deletes the row. +- A node with no recorded outcome is never deleted. +- The workflow row stays. +- A later run passes over a retired workflow whose node rows are all gone. It + also passes over one that keeps only recorded FAILED or LOST nodes and nodes + with no recorded outcome. It sends statements for one workflow only while + it has something to do there. +- A run still reads the row of every workflow that completed before the + cutoff, and the task row of every node it keeps. Its count of kept rows can + read the whole node table. Those reads grow with the tables. +- It prints how many node and edge rows it keeps. `workflows.retained_rows()` + returns the same counts for monitoring. + +## Claim cost + +Nothing where no backend on the database declares `WORKFLOWS`. Workflows +then add no conjunct to a claim and no statement to a worker. The suite +asserts that against the base worker's statements. + +With `WORKFLOWS` on a worker's own backend, its claim probes the edge +table's `oxpull_wf_edge_down_idx` once for every candidate row it judges. +The table is `oxpull_oxpullworkflowedge` and the index is on +`(downstream_task_id, satisfied)`. Every candidate, not only the nodes: a +task that is in no workflow pays the probe, and the probe finds nothing. +That is what the option being the switch costs. A workflow is created at +run time. A cached "there are no workflows" would be a window in which a +node runs out of order. Closing that window with a query per claim pass +would cost more than the probe it replaced. + +The reach is the database, not the backend. Wherever any backend on the +database declares `WORKFLOWS`, every `OxpullWorker` there records node +outcomes, because a claim is scoped to queues and not to backends. For a +worker whose own backend does not declare it, that is one column more on the +statement returning the claimed row. Nothing else, for a task that is no +node. Such a worker also looks for the node table every 30 seconds until it +is there, then stops: a cost per worker, not per claim. + +The cost is per candidate row, not per blocked node. A blocked node sits in +WAITING, which no claim reads, so a wide fan-in waiting on one root does not +make the claims around it dearer. Measured on PostgreSQL with +`EXPLAIN (ANALYZE)` at 50 and at 450 blocked nodes, and pinned in the suite. + +The probe stays a probe. On SQLite and MySQL the claim names the index. On +PostgreSQL the migration sets the column's `n_distinct`, so the planner's +estimate stays small. Statistics gathered while a wide fan-in fills the edge +table do not turn the check into a scan. A plan read on stale statistics is +not comparable to one read on fresh ones, so the suite runs `ANALYZE` before +it measures. + +The state is the edge table. It grows with the nodes you add, and +`oxpull_prune` deletes a node's edges with the node, under Retention above. + +## Transactions + +- A call owns its transaction when the connection is in autocommit and no + atomic block is open. `add()` then runs its transaction again after a + deadlock or a serialization failure. `seal()` and completion do so after a + serialization failure. The cancellation, the record repair and + `oxpull_prune`'s stamp, deletes and scan row cleanup do so after either. +- Inside your transaction nothing runs again. The error reaches you, and you + run your transaction again. +- To add to or seal several workflows in one transaction, take them in + ascending id order. Two transactions that take the same workflows in + opposite orders can deadlock. +- In that transaction, do not take a locking read or an UPDATE of an existing + task row. + +Use `django_ox.testing.run_tasks(backend="flows")` to run tasks with the workflow backend. Replace `"flows"` with the alias configured for your workflow backend. `run_tasks()` uses that alias's configured worker class. An `OxpullBackend` alias alone does not make the test worker enforce rate limits or release workflow nodes. + +## Not supported + +- Claiming or running workflow nodes with `run_once()` or `execute()` inside + a caller's atomic block. Dependents are released by a commit callback of + the outcome write. Use `run_tasks()` for tests instead. +- Deleting workflow, node or edge rows through the ORM or SQL. +- Mixing releases. Every process using django-ox on the database must run + django-ox 1.9.0 before any backend declares `WORKFLOWS`. An older release + without WAITING support cannot read that status. + +## Turning workflows off + +Two different things go by that name, and only one of them stays available. + +**Stop using workflows.** Drain the waiting nodes, then take `WORKFLOWS` out +of `OPTIONS` and the reconciler out of `SCHEDULES`. The tables stay, holding +finished work and nothing running. This is available whenever you want it, +and django-ox's own rollback needs nothing more. + +**Drop the tables.** `migrate oxpull` back past +`0005_workflow_edge_upstream_idx` or `0004_workflows` drops the tables that +hold every node, edge and record. Both refuse while any workflow row exists, +and no command deletes a workflow row: `oxpull_prune` deletes nodes and edges +and keeps the row it hangs them on. So this is supported only before the +first workflow is created. After that the row is there for good. + +The refusal catches a mistake. It does not make a rollback safe while workers +run. + +## Draining the waiting nodes + +Take these steps in order, on every database that holds django-ox's tables. + +1. Stop creating workflows. Stop every call to `workflows.create()` and + `add()`, then wait for the requests, jobs and transactions already inside + one to end. +2. End every open workflow. A node leaves WAITING when it is released or + discarded. Let the work finish, or close the branches you do not want. + `django_ox.actions.discard` closes one node's task row, WAITING + included, and on its next tick the reconciler discards every node below + it. No call ends a whole workflow at once. +3. Keep the workers and the reconciler running while step 2 runs. Only an + `OxpullWorker` on a backend that declares `WORKFLOWS` releases a waiting + node, and only the reconciler discards a branch. Take the option away + before the rows are gone and nothing is left that can move them. +4. Stop every process that can write a waiting node, and keep it stopped. + That is anything that calls `add()`. A process already running keeps the + settings it started with, so changing settings does not stop one. +5. On every alias, not only the one that holds `oxpull`'s tables, run + `OxTask.objects.using(alias).filter(status="WAITING").count()`. Every + count must be 0. A count that will not fall means something is still + adding nodes. Find it, stop it, and go back to step 2. +6. Take `WORKFLOWS` out of `OPTIONS` and the reconciler out of `SCHEDULES`, + in the settings every process starts with, and restart. + +Once no backend declares `WORKFLOWS`, `manage.py check --database ` +reports `oxpull.W007` for any workflow node still WAITING. Name the alias +that holds `oxpull`'s tables. A bare `manage.py check` reports nothing +here, because a check that reads a database runs only against an alias you +name. `ox_health` and `ox_prune` pass their own, so both report it without +the flag. W007 means step 5 was not finished. The hint says to declare +`WORKFLOWS` again, which is how you get back a database that can drain: +declare it, finish step 2, then take it away again. The check reads one +database, so step 5 is what covers the rest. + +On MySQL, `oxpull.E009` flags a task-id column on another collation than the +task table's, and a workflow worker refuses to start. + +## Deployment order + +**Turn it on in this order.** A node with dependencies is inserted in django-ox's WAITING +status. Older django-ox releases without WAITING support +cannot read or release such a row. Every process on the database must run django-ox 1.9.0 +before the first waiting node is written. For each database that holds the tables: + +1. Upgrade every process using django-ox against that database to django-ox 1.9.0. + That means workers, web and ASGI processes, enqueue-only services, the reconciler + and cron jobs. +2. Run `migrate django_ox` and `migrate oxpull` on that database. +3. Check your deploy inventory for processes still running an older release. A process cannot + see which django-ox version another process runs. +4. Add the workflow settings described above and restart. A running process keeps the settings + it started with, so nothing changes until it restarts. + +For this upgrade, install django-ox 1.9.0 on every enqueuing and claiming process before enabling workflows. This is a consistent-fleet upgrade requirement, not a claim that releases 1.3.0 through 1.8.0 lack WAITING support. WAITING was introduced in django-ox 1.3.0. + +Leave step 4 until steps 1 to 3 are done everywhere. Nothing refuses `WORKFLOWS` on the grounds that a process elsewhere is too old, because no process can see the others. + +Going back to a django-ox from before WAITING is django-ox's own procedure. Its migration is +the one that refuses while a waiting row exists. Its changelog carries those steps, under +**Rolling back to 1.2 after workflows have run**: +[django-ox changelog](../changelog.md). + +## Worker database aliases and pools + +For the worker's general connection budget, pool sizing, and renewal fallback requirements, see [database connections and PostgreSQL pooling](../production.md#database-connections-and-postgresql-pooling). The additional alias rules below apply to workflow outcome recording and rate-limit windows. + +`oxpull` reads and writes rate-limit windows and records workflow-node outcomes +(node records, edge marks, and release) on the alias returned by +`oxpull_db_alias()`, which must also hold the django_ox tables. When it differs +from `--database`, it needs its own pool and server budget. The worker's +startup warning does not check that alias or available server slots. + +### Renewal and watchdog connections + +The watchdog's own stuck-attempt READY retries and FAILED records for rows +that are not workflow nodes use its private connection on `--database`. Its +workflow-node final FAILED records also use that connection when the +recording alias matches `--database`. Among watchdog writes, only a workflow +node's final FAILED record on a separate alias uses that alias's ordinary +pooled connection and needs a pooled spare. Task-thread +workflow-node writes (SUCCESSFUL, edge marks, release) and poll-loop +rate-window I/O use the recording alias's ordinary pool; ordinary task +outcome writes use the `--database` pool; neither uses the watchdog's +private connection. The separate-alias recording path has not been tested. + +The watchdog uses at most one acquisition sequence per pass: the group of +stuck attempts it records together. This grouping is unrelated to task batches. + +Nothing needs setting for renewal or the watchdog. To bound task and poll-loop +connects, which the private-path deadlines do not cover, set +`OPTIONS["connect_timeout"]` on the alias. A positive value below 5 seconds +also shortens private deadlines when it is below their existing budget. + +See [pool sizing, connection handling, and deadline limits](../production.md#database-connections-and-postgresql-pooling) +and [monitoring events](../monitoring.md). +On pooled PostgreSQL, connection-acquisition failures no longer use +`lease_renew_failed`; update alerts. Statement failures still use it. + +After a PostgreSQL restart, tasks that were running at that moment can run again. + +### Rolling upgrades + +For workers on django-ox 0.2.0 through 1.3.1, which are affected by the renewal defect, +set pools to at least +`concurrency + 2`, or `concurrency + 3` with task timeouts, before rollout, +or disable their Django pool. Keep that budget until every affected worker has +stopped. The same mitigation applies while an affected worker remains deployed. diff --git a/docs/index.md b/docs/index.md index 26a8ce1..4339125 100644 --- a/docs/index.md +++ b/docs/index.md @@ -99,10 +99,15 @@ coming back. The mechanics, and the one case to know about, are in - [Monitoring](monitoring.md): a queue-stats API, an `ox_health` command for fleet alerting and optional local heartbeat probes, a Prometheus endpoint, structured log events, and an admin page with retry and discard. +- [Unique tasks](features/unique-tasks.md), [batches](features/batches.md), + [rate limiting](features/rate-limiting.md) and + [workflows](features/workflows.md), through the optional `oxpull` app + included in django-ox. These features require Django 6.0+. + Workflows are provisional. See [setup and API status](features/index.md). ## Install -Requires Python 3.12+ and Django 5.2+. Django 6.0 and later ship the Tasks +Requires Python 3.12+ and Django 5.2+. The optional `oxpull` app requires Django 6.0+. Django 6.0 and later ship the Tasks framework in core. On Django 5.2 LTS it comes from the `django-tasks` backport, so install the `backport` extra there. **Your own imports differ with it**: on Django 6.0+ you write `from django.tasks import task`, and on @@ -191,9 +196,10 @@ That is the whole integration. Next steps: ## Scope -The core is finite on purpose: a durable queue, a worker, recurring -schedules, and monitoring, with nothing extra to operate. Design -decisions to know before you commit: +django-ox provides a durable queue, a worker, recurring schedules and +monitoring in your existing database. Its optional `oxpull` app adds +coordination features without a separate broker. Design decisions to know +before you commit: - A queued task can be discarded before a worker claims it, and a failed one retried, from the admin or with `django_ox.actions`. Attempt @@ -203,16 +209,19 @@ decisions to know before you commit: interrupted on demand. See [Task timeouts](production.md#task-timeouts) for enforcement and its limits. - Every django-ox table lives on one database, the one your router sends -`OxTask` to. A queue on a different database from the rows it refers to gives -up the transactional enqueue. + `OxTask` to. A queue on a different database from the rows it refers to gives + up the transactional enqueue. - Worker concurrency is a thread pool, which fits I/O-bound tasks. For CPU-bound work, run `--processes N --concurrency 1`, which is N worker processes under one supervisor. See [Production](production.md#threads-and-processes). -Batches, unique tasks, rate limiting and workflows are in [Oxpull Pro](pro.md), a paid add-on; has the details. -Metrics stay in this package: `django_ox.stats` and `ox_health` are free and -stay free. +[Unique tasks, batches, rate limiting and workflows](features/index.md) +are included in django-ox through the optional `oxpull` app and require +Django 6.0+. Workflows are provisional. + +Metrics and health checks are included: `django_ox.stats` and `ox_health` +do not require the optional app. ## License diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 8187f1d..34d99f6 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -106,10 +106,15 @@ coming back. The mechanics, and the one case to know about, are in - [Monitoring](monitoring.md): a queue-stats API, an `ox_health` command for fleet alerting and optional local heartbeat probes, a Prometheus endpoint, structured log events, and an admin page with retry and discard. +- [Unique tasks](features/unique-tasks.md), [batches](features/batches.md), + [rate limiting](features/rate-limiting.md) and + [workflows](features/workflows.md), through the optional `oxpull` app + included in django-ox. These features require Django 6.0+. + Workflows are provisional. See [setup and API status](features/index.md). ## Install -Requires Python 3.12+ and Django 5.2+. Django 6.0 and later ship the Tasks +Requires Python 3.12+ and Django 5.2+. The optional `oxpull` app requires Django 6.0+. Django 6.0 and later ship the Tasks framework in core. On Django 5.2 LTS it comes from the `django-tasks` backport, so install the `backport` extra there. **Your own imports differ with it**: on Django 6.0+ you write `from django.tasks import task`, and on @@ -198,9 +203,10 @@ That is the whole integration. Next steps: ## Scope -The core is finite on purpose: a durable queue, a worker, recurring -schedules, and monitoring, with nothing extra to operate. Design -decisions to know before you commit: +django-ox provides a durable queue, a worker, recurring schedules and +monitoring in your existing database. Its optional `oxpull` app adds +coordination features without a separate broker. Design decisions to know +before you commit: - A queued task can be discarded before a worker claims it, and a failed one retried, from the admin or with `django_ox.actions`. Attempt @@ -210,16 +216,19 @@ decisions to know before you commit: interrupted on demand. See [Task timeouts](production.md#task-timeouts) for enforcement and its limits. - Every django-ox table lives on one database, the one your router sends -`OxTask` to. A queue on a different database from the rows it refers to gives -up the transactional enqueue. + `OxTask` to. A queue on a different database from the rows it refers to gives + up the transactional enqueue. - Worker concurrency is a thread pool, which fits I/O-bound tasks. For CPU-bound work, run `--processes N --concurrency 1`, which is N worker processes under one supervisor. See [Production](production.md#threads-and-processes). -Batches, unique tasks, rate limiting and workflows are in [Oxpull Pro](pro.md), a paid add-on; has the details. -Metrics stay in this package: `django_ox.stats` and `ox_health` are free and -stay free. +[Unique tasks, batches, rate limiting and workflows](features/index.md) +are included in django-ox through the optional `oxpull` app and require +Django 6.0+. Workflows are provisional. + +Metrics and health checks are included: `django_ox.stats` and `ox_health` +do not require the optional app. ## License @@ -1598,10 +1607,64 @@ Check semantics, probe examples, and guidance on which check fits which alert are on the [Monitoring](monitoring.md#health-checks-ox_health) page. +## `oxpull` options + +The optional `oxpull` app requires Django 6.0+. Add it to `INSTALLED_APPS`, run migrations, and use `oxpull.backend.OxpullBackend` for each alias that needs its features. + +These settings belong in the alias's `OPTIONS`: + +| Key | Default | Meaning | +| --- | --- | --- | +| `UNIQUE_TASKS` | `[]` | A list or tuple of dotted task-path strings. Each entry has the same effect as decorating that task with `@unique`. Uniqueness is scoped to the backend alias. | +| `RATE_LIMITS` | `{}` | A dictionary of named admission limits. Each name identifies a shared window across every backend that configures it. See the validation rules below. | +| `RATE_REFRESH_INTERVAL` | `0.25` | Seconds between a worker's refresh reads of the rate-limit window rows. Accepts a finite, non-negative `int` or `float`, but not a boolean. | +| `WORKFLOWS` | `False` | Enables workflows. Accepts only `True` or `False`. This option is provisional. | + +A `RATE_LIMITS` name must be non-empty and at most 128 characters. Two names that differ only by case are refused. + +Each limit's value is a dictionary containing only `limit`, `per`, `tasks` and `queues`: + +- `limit` is a whole number of admissions per window, at least 0. A boolean is not accepted. +- `per` is a whole number of seconds, at least 1, or a `timedelta` of at least one second. Sub-second periods are refused. +- `tasks` is a list of dotted task-path strings. +- `queues` is a list of queue names accepted by the backend. +- At least one of `tasks` and `queues` is required. + +Rate limits and workflows require `OPTIONS["WORKER_CLASS"] = "oxpull.worker.OxpullWorker"`. Rate limits also require `RATE_LIMITS`; workflows also require `WORKFLOWS = True`. + +Batches and workflows need their respective reconciler schedules. Each reconciler must have a working scheduling path and an accepted queue. Creating a batch without its reconciler schedule is refused; creating or sealing a workflow without its reconciler schedule is also refused. + +The `django_ox` and `oxpull` tables must share one database. A split router raises `ImproperlyConfigured` when the backend is constructed. + +See [Optional features](features/index.md) for feature-specific setup. + ## System checks `manage.py check` validates the setup: +### `oxpull` checks + +The optional app provides these 16 checks: + +- **`oxpull.E001`**: `oxpull` is not in `INSTALLED_APPS`. Add the app and run migrations. +- **`oxpull.E002`**: A configured unique-task entry cannot be imported or is not a `Task`. Correct the dotted task path. +- **`oxpull.E003`**: The backend alias is absent from `TASKS`. Construct the backend through the `TASKS` setting. +- **`oxpull.E004`**: The router splits `django_ox` and `oxpull` after backend construction. Route their tables to one database. A split present at construction raises `ImproperlyConfigured` before this check can run. +- **`oxpull.E005`**: `RATE_LIMITS` or `RATE_REFRESH_INTERVAL` is invalid, or a configured rate-limit task entry is not a `Task`. Correct the option or task path. +- **`oxpull.E006`**: Rate limits are configured without a worker that applies them, or `WORKER_CLASS` cannot be resolved. Configure a resolving worker class that applies the rate-limit gate, normally `oxpull.worker.OxpullWorker`. +- **`oxpull.E007`**: `WORKFLOWS` is not a boolean, or workflows are enabled without a resolving worker class that applies the workflow gate. Use `True` or `False` and configure `oxpull.worker.OxpullWorker` when enabling workflows. +- **`oxpull.E008`**: A reconciler is exposed through `SCHEDULABLE_TASKS`, but no backend reading schedule rows accepts its queue and no `SCHEDULES` entry names it. Add a `SCHEDULES` entry for the reconciler, or read schedule rows on a django-ox backend whose `QUEUES` include its queue. +- **`oxpull.E009`**: On MySQL, the relevant table collations do not match. Correct the collation mismatch reported by the database check. +- **`oxpull.W001`**: A scheduled task is also unique. It can coalesce with a pending manual enqueue, and the tick may record against that task. Remove `@unique` or the `UNIQUE_TASKS` entry from the scheduled task if that behavior is not intended. Schedules already deduplicate once per tick. +- **`oxpull.W002`**: The batch reconciler has no configured schedule. Configure its schedule before creating batches. +- **`oxpull.W003`**: Workflows are enabled without a workflow reconciler schedule. Configure the schedule; `workflows.create()` and `Workflow.seal()` refuse until one exists. +- **`oxpull.W004`**: Backends configure different admission limits under the same limit name. Align their definitions. +- **`oxpull.W005`**: Backends configure different periods under the same limit name. Align their definitions. +- **`oxpull.W006`**: Rate-limit names differ only by case or accents. Use names that remain distinct under the database's comparison rules. +- **`oxpull.W007`**: WAITING workflow nodes exist, but no backend in `TASKS` declares `OPTIONS["WORKFLOWS"]`. Restore the workflow configuration or follow the documented shutdown procedure. + +### django_ox checks + - `django_ox.E001`: `django_ox` is missing from `INSTALLED_APPS`. The app itself registers these checks with Django, so a project that also lacks any other import of `django.tasks` gets no `E001` from `manage.py check`; @@ -2780,14 +2843,16 @@ Worth knowing before you turn it on: due-tick and deadline rules; only its reporting is rate-limited. - **Database read failures are not bad rows.** Database errors during full schedule or shared dispatch reads, including connection, missing-table and lock errors, stop the pass. They are not attributed to a row. Server-side errors propagate unchanged. In autocommit, recognized value-local failures may be isolated by diagnostic re-reads while the connection remains usable. - Inside a transaction, diagnostic re-reads are restricted: PostgreSQL requires a usable transaction; SQLite permits only its recognized UTF-8 decoding refusal; MySQL permits none. Unwind through the transaction or savepoint boundary before further queries where required. Reads create no savepoints. `update_schedule` and `delete_schedule` have their own atomic blocks; source reads and `ox_prune` do not. Unlike 1.8.0, a source read inside your transaction can leave a server error for you to roll back. + Inside a transaction, diagnostic re-reads are restricted: PostgreSQL requires a usable transaction; SQLite permits only its recognized UTF-8 decoding refusal; MySQL permits none. Unwind through the transaction or savepoint boundary before further queries where required. Reads create no savepoints. `update_schedule` and `delete_schedule` have their own atomic blocks; source reads and `ox_prune` do not. Unlike 1.8.0, a source read inside your transaction can leave a server error for you to roll back. + - **An abandoned pass stops further traversal.** An escaping `django.db.Error`, other unhandled exception, failed rollback or unusable connection reports `schedule_dispatch_failed`. Dispatch retries on a later pass; committed ticks remain committed. Marker reads and boundary heals retain their own events. - An unreadable marker reports `schedule_source_unavailable` once, then at most once a minute; full reads continue at each reconcile interval. A database error on the marker read reports every pass while workers use the last schedules read. Failed boundary healing reports `schedule_boundary_heal_failed`, first with a traceback, then counted summaries at most once a minute per row per worker. Healing retries each pass; the affected schedule cannot dispatch meanwhile. + An unreadable marker reports `schedule_source_unavailable` once, then at most once a minute; full reads continue at each reconcile interval. A database error on the marker read reports every pass while workers use the last schedules read. Failed boundary healing reports `schedule_boundary_heal_failed`, first with a traceback, then counted summaries at most once a minute per row per worker. Healing retries each pass; the affected schedule cannot dispatch meanwhile. + + An unreadable newest tick within the pass's bound skips only its schedule with `schedule_tick_unreadable`. A settings schedule can also be blocked by its unreadable earliest tick. History is not treated as absent. The pass completes, so this skip does not hold a batch open. Follow the recovery procedure under [Configuration](configuration.md#ox_prune). - An unreadable newest tick within the pass's bound skips only its schedule with `schedule_tick_unreadable`. A settings schedule can also be blocked by its unreadable earliest tick. History is not treated as absent. The pass completes, so this skip does not hold a batch open. Follow the recovery procedure under [Configuration](configuration.md#ox_prune). + Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. - Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. - **`manage.py check` cannot see rows.** Checks run before `migrate`, so a bad schedule in the database is a log line, not a start-up error. Settings-declared schedules still fail fast for errors their checks can @@ -2843,6 +2908,1219 @@ dispatch leaves no tick row. --- +# Source: https://oxpull.com/django-ox/features/ + +# Optional features + +django-ox 1.9.0 includes the optional `oxpull` app. It adds unique tasks, batches, rate limiting and workflows without a separate package installation. + +The app requires Django 6.0+. django-ox itself continues to support Django 5.2 through the `django-tasks` backport. + +## Setup + +1. Add `"oxpull"` to `INSTALLED_APPS` and run migrations. +2. Set the relevant task alias's `BACKEND` to `"oxpull.backend.OxpullBackend"`. +3. Keep the `django_ox` and `oxpull` tables on the same database. A router that splits them is refused at backend construction with `ImproperlyConfigured`. The check consults routers by model, without hints. Hint-dependent routing that sends task rows elsewhere is unsupported and may escape the startup check. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). +4. Configure the features you use: + - Unique tasks use `@unique` or `OPTIONS["UNIQUE_TASKS"]`. + - Batches need a reconciler schedule. + - Rate limits need `OPTIONS["RATE_LIMITS"]` and `OPTIONS["WORKER_CLASS"] = "oxpull.worker.OxpullWorker"`. + - Workflows need `OPTIONS["WORKFLOWS"] = True`, `OxpullWorker` and a reconciler schedule. + +Apply the relevant backend and worker settings in every process that enqueues or claims those tasks. Follow the feature page for the complete configuration, including the reconciler's queue and schedule. + +Already have a separately installed `oxpull` distribution? Follow [Earlier `oxpull` installs](../migrating.md#earlier-oxpull-installs) before upgrading. + +## Unique tasks + +[Unique tasks](unique-tasks.md) deduplicate pending tasks at enqueue time. While a task with the same key is READY or RUNNING, another enqueue returns the pending result. + +This is not a guarantee that two executions of the same key cannot overlap. A LOST task releases the key even if its original execution is still running. + +## Batches + +[Batches](batches.md) collect tasks and arrange a callback after the sealed batch settles. Create a batch, add its members, then seal it. A reconciler schedule is required. + +`Batch.add()` raises `oxpull.exceptions.BatchMemberExists` if the task is already a member of that batch. + +## Rate limiting + +[Rate limiting](rate-limiting.md) controls admissions through `OxpullWorker`. Configure limits on task paths, queues or both. + +A limit name identifies one shared window across the backends that configure it. A backend setting alone does not make a plain django-ox worker enforce that limit. + +## Workflows + +[Workflows](workflows.md) coordinate tasks with dependencies. A workflow is created at runtime. Enable `OPTIONS["WORKFLOWS"]`, use `OxpullWorker`, and configure the reconciler schedule before creating or sealing workflows. + +Workflow APIs, settings and log events are provisional. + +## Database and task rows + +The app creates its own `oxpull_*` tables. It does not change any django-ox table, column, index, constraint or migration. + +Workflows do write task rows. They insert blocked nodes as WAITING tasks. Cancellation moves a READY or WAITING task to DISCARDED by a compare-and-set on the task row's status and lease epoch. Release moves a WAITING task to READY and updates `run_after`; `run_after` is never NULL. DISCARDED nodes are not revived. + +Use `ox_prune` for task rows and `oxpull_prune` for the optional app's records. + +## Signals + +Tasks enqueued through `OxpullBackend` use that backend as the signal sender. Receivers restricted to the base backend's sender must account for this when an alias changes backend. This applies to every task sent through `OxpullBackend`, not only unique tasks. + +## Stability + +The backend, its unique-task and rate-limit options, the worker, the unique-task decorator and the documented batches API are stable. The `oxpull_*` schema, check ids, `oxpull_prune` and non-workflow log events are also stable. + +Everything in the documented workflow surface is provisional, including the `WORKFLOWS` option and every `workflow_*` event. Workflow internals are not public. + +See the [stability policy](../stability.md) for the complete lists and versioning rules. + +--- + +# Source: https://oxpull.com/django-ox/features/unique-tasks/ + +# Unique tasks + +Unique tasks deduplicate enqueues while a matching task is pending. The key is held while the task is READY or RUNNING; this is not a guarantee that two executions of the same key cannot overlap. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +## Behaviour + +A task marked unique is deduplicated while a matching task is pending: + +```python +from django.tasks import task +from oxpull.unique import unique + + +@unique +@task +def send_invoice(order_id): ... + + +send_invoice.enqueue(42) # enqueues +send_invoice.enqueue(42) # returns the pending task's TaskResult; no new row +``` + +`enqueue_many()` raises `InvalidTask` for a unique task and names it. +django-ox writes that call as one bulk INSERT, which never reaches the +deduplicating path, so no uniqueness lock would be taken and the rows +would not be deduplicated. Refusing the call keeps the guarantee honest. +Enqueue unique tasks one call at a time; plain tasks keep the bulk path. + +The lock is an enqueue-time gate. `django_ox.actions.retry` is an +operator instruction to run one named row, not an enqueue, so it does not +pass that gate. Retrying a unique task that a later `enqueue()` has +already superseded therefore runs that attempt beside the newer one, and +the key carries two READY rows until both settle. `oxpull` honours the +retry an operator asked for rather than discarding it. + +Tasks can also be made unique from settings instead of the decorator: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", ...], + "OPTIONS": { + "UNIQUE_TASKS": ["myapp.tasks.send_invoice"], + }, + } +} +``` + +Semantics, all deliberate: + +- **Window: pending-uniqueness.** The key is held while the matching task + is READY or RUNNING. It releases as soon as + the task leaves those two statuses (SUCCESSFUL, FAILED, LOST or + DISCARDED), or when its row is pruned. The key follows the row's + status, not the code: a row the reaper recorded LOST releases the key + while the execution it belonged to may still be running, and django-ox + lets that execution write SUCCESSFUL or FAILED onto its own row later. + A RUNNING task whose worker has died still holds the key, so a duplicate + enqueue returns that task's result and queues nothing. The reaper takes + the task back after its lease expires. With attempts left, it puts the + task back to READY, and the key stays held. On the task's last attempt, + the reaper records it LOST instead. That frees the key, and the result the + duplicate got back reads FAILED after `refresh()`. + Uniqueness deduplicates pending rows. It does not stop two executions + of one key overlapping, and a task that must never overlap itself + needs its own guard. +- **A retry is not a duplicate.** A failed attempt with attempts remaining + returns to READY, which is still inside the window; the key stays held + through backoff, and at-least-once execution is unchanged. +- **Duplicates return the existing `TaskResult`** rather than raising, and + `task_enqueued` is not sent for a suppressed duplicate. The result + reflects the stored row (its priority, queue and run_after), not any + `using()` overrides passed on the duplicate call. +- **Key derivation:** SHA-256 over `(backend_alias, task_path, queue_name, + args, kwargs)` after `django.utils.json.normalize_json`, the same + normalisation the task row stores. Uniqueness is scoped per backend + alias. Dict order is irrelevant; tuples equal lists; `1` and `1.0` + differ; int dict keys match their string form, exactly as JSON stores + them. `queue_name` is in the key; `priority` and `run_after` are not. + There is no signature binding: `add(1, 2)` and `add(1, b=2)` are + different keys. +- **Enqueue cost:** a unique enqueue is one transaction containing three + statements (lock INSERT under a savepoint, task INSERT, lock UPDATE) + instead of one. Non-unique tasks take the parent code path untouched. + +Signals: `task_enqueued` fires inside the enqueue transaction, before +commit, unchanged from django-ox. Receivers that trigger external +effects, including anything that wakes workers out of band, must defer +with `transaction.on_commit`: a pre-commit receiver cannot see the row +from another connection, and for unique tasks a suppressed duplicate +emits no signal at all. + +Scheduling a unique task raises system check `oxpull.W001`: schedules +already dedup once per tick, and a unique scheduled task coalesces with +pending manual enqueues of the same key. + +## Deployment guarantees + +Uniqueness is enforced by the lock table's primary-key constraint. `OxpullBackend.enqueue()` writes the lock row and task row in one transaction and handles a competing insert's `IntegrityError`, rather than checking for the key first. + +The constraint covers enqueues through `OxpullBackend`. An enqueue through `django_ox.backend.OxBackend` writes no lock row, so it can create a duplicate beside a task whose key is locked. Later unique enqueues return the task associated with the lock; they do not remove those extra rows. + +Deploy django-ox 1.9.0 and configure `OxpullBackend` in every process that enqueues unique tasks, including web processes and cron jobs, before relying on this guarantee. `manage.py check` reads only the settings of the process running it; it cannot verify the rest of the fleet. + +Before enabling uniqueness over existing traffic, drain or reconcile pending tasks enqueued without lock rows. A READY or RUNNING task without a lock row is not deduplicated against. + +The dedup point is a unique constraint on `OxpullUniqueLock.key` plus an +`IntegrityError` loser path, the same mechanism used by `django_ox` +for schedule ticks. Check-then-insert is never used. The lock row and the +task row are written in one transaction, so enqueue keeps the parent +queue's durability guarantee (a `transaction.atomic()` opened on the +database these tables live on covers the enqueue too, and a rollback +discards task and lock along with every other row you wrote there). +These guarantees require the [database-routing constraints](#database-routing-constraints). + +Lock release never depends on signals. The django-ox reaper settles an +abandoned task by writing LOST through a queryset update that emits no +signal, so any signal-driven release would strand keys exactly when a +worker dies. Instead, staleness is derived from the task row at the next +enqueue of the same key: when the holder has left READY and RUNNING, or +its row is gone (pruned), the lock is released and the enqueue proceeds. +A holder that is still READY or RUNNING is read without a lock: there +is nothing to release, and a row lock taken inside a caller's +transaction (`ATOMIC_REQUESTS`, any `atomic()` block, or a connection +with autocommit turned off) would live until that transaction commits +and hold off the holder's own lease renewal and outcome write for as +long as the request ran. On MySQL at REPEATABLE +READ that unlocked read, inside a caller's transaction, answers from the +transaction's snapshot, so a holder the snapshot calls live is read once +more on a connection of its own, which sees the committed status and +takes no lock; at READ COMMITTED, the level Django sets for MySQL unless +told otherwise, the plain read is current and no connection is opened. + +At PostgreSQL's REPEATABLE READ or SERIALIZABLE, that read answers from +the snapshot. So the enqueue asks for a `FOR SHARE` lock on the holder's row +without waiting, inside a savepoint it rolls back at once. PostgreSQL +grants the lock when the row hasn't changed since the snapshot began, and +the enqueue returns the holder. If another transaction is writing the row +and hasn't committed, PostgreSQL refuses at once. The enqueue then returns +the last committed row, as a read at READ COMMITTED would. If the row has +changed, the enqueue reads the holder and the key's lock once more on a +new connection, in a READ COMMITTED transaction of its own, and closes it. +A task that's still pending and still holds the key is returned from that +read. The caller's transaction keeps its older snapshot. The level is read +in the same statement as the holder. + +That read's connection waits at most 2 s to connect to each address the +host name resolves to. The read doesn't set that limit when the project +sets `connect_timeout` in `OPTIONS` or in `PGCONNECT_TIMEOUT`, or names a +connection service. Once connected, the read waits at most 100 ms for a +lock and runs for at most 2 s. It sets both with `SET LOCAL`, so a pooled +server session doesn't keep them. Nothing limits the host name lookup, or +a wait in a connection pooler's queue before the read reaches PostgreSQL. +The caller's transaction holds ACCESS SHARE locks on the task and lock +tables, from its own reads of them. A request such as a migration's +`ALTER TABLE` then waits for that transaction to end. The read queues +behind that request while the caller waits on the read, and PostgreSQL +can't see that cycle. Without those limits, the read would wait until that +request gave up. If the read stops, or its connection fails or times out, +the enqueue raises `UniqueEnqueueRace`. + +Apart from that check, which keeps nothing, only the release reads the +holder with a locking read (`SELECT ... FOR +UPDATE` on PostgreSQL and MySQL), so that it acts on the committed +status rather than the enclosing transaction's snapshot and waits for an +operator retry still in flight; a retry that has just put the holder +back to READY keeps its lock instead of losing it to a stale read. +Inside a caller's transaction that lock, on a settled row, rides to the +caller's commit, and a retry of that row, a straggler's outcome write +onto a LOST row and `ox_prune` of the row wait behind it. The DELETE +itself carries the holder's status in its WHERE clause, which is the +whole of the guard on SQLite, where no row lock exists and a statement +is atomic against the single writer. `oxpull_prune` clears stale locks +for keys that are never enqueued again; it is hygiene, not correctness. + +`task_id` on the lock is a bare UUID, not a ForeignKey, so `ox_prune` +deleting finished task rows cannot cascade through the lock or be blocked by a foreign-key constraint; a dangling id reads +as "finished". The `oxpull` migration creates only tables belonging to the +optional `oxpull` app and does not alter `django_ox` tables. + +Portability note: MySQL has no partial unique indexes, so the lock-table +design is the one mechanism that is identical on SQLite, PostgreSQL and +MySQL, rather than a partial index over task status on the databases that +support one. The lock table keeps unique-task coordination within the +`oxpull` app's schema without adding an index to a `django_ox` table. + +Known corner: inside a transaction at MySQL's REPEATABLE READ, or at +PostgreSQL's REPEATABLE READ or SERIALIZABLE, a unique enqueue can raise +`oxpull.exceptions.UniqueEnqueueRace` (a +`django.tasks.exceptions.TaskException`) where READ COMMITTED would +return a result. It never queues a duplicate instead, and it creates +nothing. On PostgreSQL that happens when the task holding the key +finished or was deleted after the transaction's snapshot began, or when a +settled holder was retried after that. It also happens there when the +read on a new connection stops or fails. On both databases it happens when +another enqueue took the key after the snapshot began, because the +transaction can't see the new lock. The message names the cause it +detects. Retry in a new, short transaction at READ COMMITTED, or outside a +transaction. READ COMMITTED on PostgreSQL or MySQL, and SQLite, don't hit +this. + +Known limit: on MySQL at SERIALIZABLE, a duplicate enqueued inside a +transaction locks the holder's row until that transaction ends. Until +then the holder's lease renewal, its outcome write and the reaper wait. +They fail with error 1205 once `innodb_lock_wait_timeout` passes, 50 +seconds by default. A worker won't claim a READY holder either. Keep +those transactions short, or enqueue unique tasks outside them. + +A second corner, on every database: the enqueue joins the transaction the +caller opened on this database, so a slow request holds its key for as +long as it runs, and a duplicate enqueued meanwhile waits on it for as +long as the database allows. SQLite waits for the connection's +`OPTIONS["timeout"]` (5 s unless set), MySQL for +`innodb_lock_wait_timeout` (50 s unless set), PostgreSQL for as long as +the holder runs unless `lock_timeout` is set. Past that the duplicate +raises `UniqueEnqueueRace`, chained to the database's own error, rather +than a raw `OperationalError`. Nothing was written, and the message says +whether to retry the enqueue or, inside an open transaction, the whole +transaction. What that transaction is left as depends on the server. +MySQL's default rolls back the statement, so the enqueue rolls back to its +savepoint and the transaction is still usable; with +`innodb_rollback_on_timeout=ON` the server rolls the whole transaction +back, nothing done in it survives, and the message says so. SQLite in WAL +mode has a third shape: a transaction that read before it enqueued is +refused at once when another connection has committed since +(`SQLITE_BUSY_SNAPSHOT`), without waiting out the timeout; the message +names that too. + + + +Constraint: the `oxpull` and `django_ox` tables must live on the same database. The lock INSERT and its task INSERT share a transaction, the membership INSERT and its task INSERT share a transaction, and the completion query joins the two tables. None of these operations crosses a database alias. The rate window is neither co-written nor joined with task rows; it belongs on the same alias because `OxpullWorker` reads and writes it there, regardless of a router naming another database for it. At construction, the backend checks all seven `oxpull` models against the active router and refuses a split with `ImproperlyConfigured`. The batch API re-checks this placement on every call. Every read on these paths goes to the write database, so read-replica routing cannot serve a stale liveness answer, complete a batch early, or read capacity from a lagged window. + +The check consults routers by model, without hints. `django_ox` saves each task row through the same router with the row as a hint, so a router that routes by model puts the task row on that database too. Hint-dependent routing that sends task rows elsewhere, such as one database per backend, is unsupported and may escape the startup check. With that routing, the lock or membership row would commit apart from its task, and an `add()` reissued after a deadlock would enqueue again on the other database. + +--- + +# Source: https://oxpull.com/django-ox/features/batches/ + +# Batches + +A batch groups tasks and fires a callback once every member's row is settled or gone. Completion describes the recorded task rows, not proof that every member has stopped executing. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +Batches derive completion from task rows rather than signals. +Batch state is derived by querying member task rows, never accumulated +from `task_finished`: a counter decremented by that signal hangs forever +whenever a worker dies mid-task, which is exactly what batches exist to +survive. The reconciler asks the rows instead, so it cannot miss a task +however it died. Completion is a compare-and-set on the batch status, +with the callbacks enqueued inside that same transaction. The rowcount +picks a single winner among racing reconcilers, so the callbacks are +enqueued once however many reconcilers race. They then run like any other +task, which is at least once: a callback that is retried runs again. A +crash mid-pass either leaves the batch sealed for the next tick or has +already committed the completion and its callbacks together. +These guarantees require the [database-routing constraints](unique-tasks.md#database-routing-constraints). + +`OxpullBatchMember.task_id` is a bare UUID, not a ForeignKey, so task pruning +cannot cascade through membership or be blocked by a membership foreign-key constraint. The completion query uses the ORM rather than +SQL text, so `django_ox` table and column names come from model +metadata rather than hard-coded names. + +## Behaviour + +A batch groups tasks and fires a callback once every member's row is +settled (SUCCESSFUL, FAILED, LOST or DISCARDED) or gone; a row in any +other status, including one unrecognised by django-ox 1.9.0, holds it open: + +```python +from oxpull import batches + + +batch = batches.create("nightly-invoices", on_complete=summarize_run) +batch.add(send_invoice, 42) +batch.add(send_invoice, 43) +batch.seal() + +batch.progress() # BatchProgress(total=2, pending=1, succeeded=1, failed=0) +batches.get(batch.id) +``` + +Completion is detected by a reconciler task, which runs once a minute through +django-ox's schedule dispatch (`OPTIONS["SCHEDULES"]`). Batches need this schedule; without it a batch never +completes: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, + }, + } +} +``` + +The schedule may sit on any django-ox backend. Without one anywhere, +`manage.py check` reports `oxpull.W002`, and `create()` and `seal()` +refuse with `ImproperlyConfigured` naming the block above, so a batch +cannot be left sealed with nothing to complete it. The refusal reads +the settings of the process that calls, so the `SCHEDULES` block has to +be present in every process that calls `create()` or `seal()`, web +processes included, not only in the one that runs the scheduler. A +deployment whose web settings omit it, because only the worker's carry +the schedules, refuses every `create()` from the web; +declaring the same block there changes nothing about which process +runs the reconciler. `seal()` on a batch that is already SEALED or +COMPLETE raises `BatchSealed` whatever the settings say. + +Semantics, all deliberate: + +- **Sealing freezes membership.** `add()` enqueues the task and writes the + membership row in one transaction, so the two commit or roll back + together. `seal()` snapshots the member count; after that, `add()` + raises `BatchSealed`. There is no un-seal and no reopen, and a batch + that is never sealed never completes. On MySQL, an `add()` that loses + an InnoDB deadlock reissues its own transaction, up to five times. + Called inside your `atomic()` block it cannot: InnoDB has rolled your + transaction back too, the error propagates, and it is your transaction + to retry. A reissue enqueues again, so `task_enqueued` fires once per + attempt, the first time for a row the rollback then discarded; a + receiver deferred with `transaction.on_commit` sees only the row that + committed, which is the rule for that signal in any case. +- **`on_complete` fires once every member's row is settled (SUCCESSFUL, + FAILED, LOST or DISCARDED) or gone, whatever the outcome; a row in any + other status holds the batch open. `on_success` fires in addition, and + only when every member's outcome is known and none of them failed.** + Finished is read off the row, not off the code; the bullet on what + complete means says what that leaves open. There is no failed state + for the batch itself: completion is completion, and `failed_count` + plus the absent `on_success` is how a bad run reads. +- **Callbacks are ordinary tasks.** They are enqueued in the same + transaction that completes the batch, then retry and fail like any + other task. The batch stores their ids and does not supervise them. + A callback must be a `@task` on a django-ox backend, and is stored as + its dotted path: `create()` refuses a callback that carries `using()` + overrides, because only the path is kept and the overrides would be + dropped in silence. Set them on the decorator instead. A callback + cannot be a unique task either: two batches finishing with the same + callback and arguments would share one task, and one batch's callback + would never run. A stored callback that became unique after + `create()` is refused at completion instead, and its batch stays + SEALED, logged at each tick, until the callback is made non-unique + again; existing batches do not complete before then. +- **A retry is not a finish.** A member with attempts remaining goes back + to READY, and a batch never completes past a member that will run + again. +- **A member that outlives `LOCK_TIMEOUT` reads by its row, not by what + it did.** When the worker running a member stops renewing its lease, + the reaper takes the task back. With attempts remaining the row returns + to READY and its lease epoch moves, which drops the old attempt's + result. With no attempts left the row is recorded as LOST, which says + the attempt went unobserved and nothing about its outcome, so the batch + counts that member as unattributed rather than failed: `on_complete` + fires, `on_success` is withheld, and `failed_count` is unchanged. + Renewal keeps the lease only while it reaches the database on time. + With Django's PostgreSQL pool, renewal needs a connection: the worker + normally opens one outside the pool with a bounded deadline, with a + short pooled fallback if needed. Sizing `LOCK_TIMEOUT` alone does not + prevent a reclaim; a `task_reclaimed` warning in the worker log is + the symptom. +- **Complete is a reading of the rows, not proof that every member has + stopped running.** The reconciler completes a batch at the tick where + it finds every member's row settled or gone. A member recorded LOST + may still be executing: its worker stopped renewing the lease, the + code kept going, and django-ox lets that execution write SUCCESSFUL + or FAILED onto its own row afterwards. The worker's bounded + recycle produces this on purpose when a member outlives `LOCK_TIMEOUT` + on its last attempt. So `on_complete` is not a barrier against a + member that is still running, and teardown in `on_complete` has to + tolerate one. + `on_success` fires only when every member's row is SUCCESSFUL, which + is the one outcome a running member cannot still have. Two things + follow. Member outcomes and the batch's totals are not final at + COMPLETE: a straggler can turn a LOST member SUCCESSFUL after + `failed_count` was written and the callback enqueued, and `progress()` + shows the new count. And an operator retry that commits during the + completing transaction can leave a READY member on a COMPLETE batch + whose callback is already enqueued: the reconciler re-reads the counts + under the batch row's lock, but `django_ox.actions.retry` writes the + member's task row and takes no batch lock, so the two writes cannot be + ordered. +- **A batch sealed with no members completes at the first tick**, and + both callbacks fire. +- **The reconciler runs on the backend whose `SCHEDULES` name it.** + django-ox rebinds a scheduled task to that backend at load. Its queue + name comes from the default task backend's `QUEUES`: `"default"` when + that list has it, otherwise the first entry. A queue the scheduling + backend does not accept is refused at load with `ImproperlyConfigured` + naming the schedule. Set `queue_name` in the schedule to put it + somewhere specific. +- **Latency is one tick.** Cron granularity is one minute, so a callback + lands up to about a minute after the last member finishes. Completion + is decided by reading the member task rows, which is what makes it + impossible to miss a member whose worker died without a word. +- **Pruned members read as finished, but not as successful.** By + default `ox_prune` deletes SUCCESSFUL and DISCARDED rows that finished + more than `--older-than` ago (7 days), and FAILED and LOST rows too + under `--include-failed`. A membership row whose task is gone counts + as finished, the sealed `total_count` keeps totals honest, and + `BatchProgress.unattributed` reports how many are no longer + attributable to success or failure. Those members are why `on_success` + needs every outcome to be known: a pruned row could have been the one + that failed, so the batch completes and `on_complete` fires while + `on_success` is withheld. That includes the default prune. A batch + that stays open longer than `--older-than`, because it is wide, its + members are throttled, or the reconciler was down, loses its members' + SUCCESSFUL rows and completes without `on_success`. Set `--older-than` + above the longest batch you run. `BatchProgress.pruned` is deprecated + and will be removed in 2.0. It returns the same number as + `unattributed` and emits a `DeprecationWarning` when read. +- **A discarded member is finished, and the run is not clean.** + `django_ox.actions.discard` closes a task an operator does not want + run. That member is terminal, so it never holds the batch open, but it + is cancelled work rather than work that was done. It counts as + unattributed, not as failed, and `on_success` is withheld. + `on_complete` still fires, so cleanup still runs. +- **Retrying a member reopens the wait, not the membership.** + `django_ox.actions.retry` puts a FAILED or LOST member back on the + queue. The batch stays SEALED until that attempt settles, and no + callback fires in the meantime. Retrying a member of a batch that has + already completed does not reopen it, and its callbacks have already + fired. + +## Batch API + +Start with `oxpull.batches.create()`. It returns a `Batch` with an `id` and the methods `add()`, `seal()` and `progress()`. Use `oxpull.batches.get(batch.id)` to retrieve the batch. + +`Batch.add()` raises `oxpull.exceptions.BatchMemberExists` if the task is already a member of that batch. Adding to a sealed batch raises `oxpull.exceptions.BatchSealed`. + +`Batch.progress()` returns `BatchProgress(total, pending, succeeded, failed)`, read from the member task rows when called. Before sealing, `total` is the live member count. After sealing, it is the sealed snapshot. + +The result also exposes: + +- `unattributed`: `max(total - pending - succeeded - failed, 0)`. This covers members whose task rows `ox_prune` deleted, LOST members and DISCARDED members. +- `finished`: `total - pending`. +- `pruned`: a deprecated alias of `unattributed` that emits `DeprecationWarning`. Use `unattributed` in new code. + +These batch entry points, result fields and exceptions are stable. + +## How wide a batch can be + +Measured to **1,000,000 members** in a single batch, on SQLite, +PostgreSQL 16 and MySQL 8, with every count checked against the task rows +rather than against what the API reports about itself. The completion +query is a subquery, not an expanded list of ids, so it binds one +parameter whatever the width, on every database and at every width +measured, and cannot run into SQLite's variable limit or PostgreSQL's +parameter ceiling. + +At a million members the costs separate sharply by database: + +| | SQLite | PostgreSQL 16 | MySQL 8 | +| --- | --- | --- | --- | +| seal, inside a caller's transaction | 34 ms | 0.7 s | **28 s** | +| reconcile | 4.9 s | 0.4 s | 7.3 s | +| finish all members | 4.8 s | 9.1 s | 27.6 s | + +The seal row is the cost of a locking count: a row lock per member +wherever the database offers one, and a million InnoDB locks in one +transaction is where that shows (0.24 s at 100,000 members, 28 s at a +million). `seal()` now takes that count only when it is called inside a +caller's transaction, where the caller's snapshot could predate a member +committed since. Called outside a transaction it counts with a single +`COUNT(*)`: on a million-member batch, timing the two statements alone +with fresh statistics, the locking read took 12 s on MySQL 8 and 0.55 s +on PostgreSQL 16, the `COUNT(*)` 0.7 s and 0.03 s, and the whole `seal()` +call outside a transaction 0.74 s and 0.03 s. SQLite counts either +way, which is why it is the cheapest of the three at exactly the point +the other two are dearest. If you seal batches that wide on MySQL from +inside a transaction, budget for it, split them, or seal outside the +transaction. + +Measured on an arm64 macOS host, with PostgreSQL 16 and MySQL 8 in +Docker on localhost and SQLite on a local file, one process at a time +and each width against an empty database. The individual-batch figures were measured on the earlier separate `oxpull` package: its 0.3.1 release on SQLite and PostgreSQL, and its unpublished 0.4.0 build on MySQL, with the same batch code. The fifty-batch sweep and the task-table figures below it are from the 2026-08-19 runs on pre-release code of that separate `oxpull` package labelled 0.1.0. These are historical measurements, not a new benchmark of the combined distribution. + +Reconciler cost tracks the member count: from 100,000 members to a +million, the pass that completes a batch grows 7.5x on PostgreSQL, 14x on +SQLite and 13x on MySQL. The slowest reconciler pass measured at 100,000 +members is the completing one: 0.56 s on MySQL, 0.34 s on SQLite and +55 ms on PostgreSQL, against a one-minute tick. Fifty sealed batches of +5,000 members each cost 667 ms per pass on PostgreSQL and 319 ms on +SQLite while their members are still pending, which is what every tick +costs until they finish. + +One thing to know if your task table is large. On PostgreSQL the planner +hashes the batch's members and scans the task table, so a reconciler pass +costs roughly in proportion to that table rather than to the batch: about +2.3 ms against 5,000 task rows and about 38 ms against a million, per +sealed batch. Two levers, both yours: prune finished tasks with +`ox_prune`, which you want anyway, or lower `random_page_cost`. At 1.1 +the planner picks a nested loop unaided and the same work took 6.4 ms +instead of 36 ms. SQLite probes the primary key per member, so the same +table growth costs it 3x rather than 16.6x. + +--- + +# Source: https://oxpull.com/django-ox/features/rate-limiting/ + +# Rate limiting + +Rate limiting controls task claims through `OxpullWorker`. Its guarantee depends on the workers applying the limit and recording claims; the bound and its assumptions are described below. + +See the [feature overview](index.md) for setup, requirements, and compatibility. + +Rate limiting coordinates workers through database state. Every decision is a +compare-and-set resolved by a rowcount, never a check-then-act: the +window roll is an UPDATE conditioned on the window id, so exactly one +worker among any number opens a window and the losers report zero rows +and keep the view they read. The increment that counts an admission is +deliberately unconditional, because by the time it runs the attempt is +already spent and a statement that could refuse would burn it. What stops +a worker taking a second token is not that statement but the next +refresh, which every worker is required to make before its next limited +claim. + +The window's clock is the database's, on every database and under any +`USE_TZ` setting. `window_started_at` is written with `Now()` and is only +ever compared against `Now()` in the same statement family, so there is +no second writer for it to agree with and no per-host clock in the +decision. A worker's own clock decides only when to look, which can waste +a statement but can never admit a task. A database clock that steps +backwards delays the next roll by the size of the step and under-admits; +one that steps forwards rolls the window early and releases its capacity +ahead of schedule, by at most one window per step. Neither compounds: the +window re-anchors to database time on the next roll. + +## Behaviour + +A rate limit caps how often a task starts. Give it a name, a capacity, +and the tasks or queues it covers: + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", "sms", "reports"], + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": { + # Both tasks draw on one budget of 100 per minute. + "stripe": { + "limit": 100, + "per": 60, + "tasks": [ + "myapp.billing.charge_card", + "myapp.billing.refund_card", + ], + }, + "reports": { + "limit": 10, + "per": 3600, + "tasks": ["myapp.reports.build_monthly"], + }, + # A whole queue, whatever runs on it. + "sms": {"limit": 500, "per": 60, "queues": ["sms"]}, + }, + }, + } +} +``` + +A limit is declared in settings. A worker imports no task modules, so a +limit written next to its task would be enforced only when something else +happened to import that module first. Declared here, it is visible to +every worker and to `manage.py check`. + +`RATE_LIMITS`, `UNIQUE_TASKS` and `@unique` take effect only with +`oxpull.backend.OxpullBackend`. On an alias still using +`django_ox.backend.OxBackend`, nothing reads these settings: +`manage.py check` is clean, and nothing is throttled or deduplicated. + +For aliases using `OxpullBackend`, `WORKER_CLASS` is required: it makes +`ox_worker` run the worker that applies the limits. Configuring limits +without it is a hard system check error (`oxpull.E006`). `per` is whole +seconds, or a `timedelta` of at least one second. A limit that covers +nothing, names a queue the backend does not accept, or lists a `tasks` +path that is not a task is `oxpull.E005`. These checks run only for +aliases whose `BACKEND` is `oxpull.backend.OxpullBackend`. + +## Deployment guarantees + +Rate limits are applied in `OxpullWorker.claim_one()`, not enforced by a database constraint. The base `Worker` claims tasks without applying the limit or recording a count. + +Deploy django-ox 1.9.0 and use `OxpullWorker` in every process that claims limited tasks before relying on the guarantee. `manage.py check` reads only its own process's settings and cannot verify which worker other processes run. + +## The guarantee + +> For a limit of N admissions per period P, with W `ox_worker` processes +> claiming from the queues that carry the limit's tasks, the number of +> task attempts started in any one window is at most N + W - 1 + U. +> U is the number of admissions whose count did not land: one for each +> count that raised, and one for each count that the window rolled out +> from under five times in a row. Each one is logged at warning as +> `rate_limit_uncounted`; alert on that event, and U is the number of +> times it fired in the window. Add one for each worker process that +> dies between claiming a limited task and recording it. With a single +> worker process, no such death and every count landed, the limiter is +> exact: at most N per window. +> +> A window is one generation of the limit's shared counter row: it opens +> when a worker rolls the row and lasts P. It is not every interval of +> length P. Windows are contiguous, so an arbitrary interval of length P +> that spans a boundary can carry up to 2(N + W - 1 + U). The limiter +> therefore bounds a long-run average of (N + W - 1 + U)/P and an +> instantaneous burst of 2(N + W - 1 + U). +> +> A count that cannot be written is logged and the attempt runs. Three +> admissions in a row that the counter did not record close the limit +> for that worker process, whether each increment raised or ran and +> matched no row. A closed limit admits nothing until a write lands, +> which the next refresh probes for. The run and the closed state are +> per process, and only an increment that lands ends the run, so a +> counter that fails on every other admission closes after three of +> them rather than admitting through the fault. + +The W - 1 is structural. The budget has to be read before the claim, +because the claim UPDATE spends a retry attempt and a lease epoch and +neither can be given back; between reading "this limit has room" and the +claim landing, another worker can take the last token. Closing that gap +would mean burning attempts on tasks that never ran. + +W is yours to know, not ours. One `ox_worker` process is one claimer, and +under `--processes N` each child is a claimer. Size against the bound +rather than against the capacity. For a ceiling of T attempts in one +window, configure **`limit = T - (W - 1)`**. For a ceiling of T across +any interval of one period, including one that straddles a boundary, +halve it: **`limit = T // 2 - (W - 1)`**. Both assume U is zero, which +holds while `rate_limit_uncounted` is not firing; every time it fires, +the window it fired in can carry one attempt more than the ceiling. +Scaling workers up loosens a limit that is left alone, and each worker +that dies mid-claim costs one more admission. + +Semantics, all deliberate: + +- **It limits attempt starts, not completions and not concurrency.** Ten + per minute on a task that runs for ten minutes will have a hundred + attempts running at once. "At most K running" is a concurrency limit, + which is a different mechanism. +- **Retries count.** A reaped or failed attempt that goes back to READY + consumes another admission when it is claimed again. That is correct + for what limits usually protect: a retry is another call to the third + party. +- **A throttled task stays READY.** It is not claimed, not locked, and it + spends no attempt and no lease epoch. It is indistinguishable from any + other queued task, and nothing in the reaper, the lease or the timeouts + interacts with it. +- **A throttled task does not hold up anything else.** The exclusion is + applied inside the claim's candidate select on all three claim paths, + ahead of its ordering and its limit, so a worker whose limit is + exhausted still claims other tasks and other queues on the same pass. +- **Windows are fixed and anchored at first use**, so a P-length interval + straddling a boundary can carry twice the capacity. If what you are + protecting counts per fixed minute, size N at half your true ceiling or + accept the doubling. +- **A task may belong to several limits.** It is admitted only when every + one of them has room, and admitting it increments every one. +- **`limit: 0` blocks the limit entirely.** It is a supported pause + switch, and it holds unique keys open, holds batches open, and drives + the ready-age metric up for as long as it is set. +- **Capacity is settings, not data.** A change takes effect for a worker + when that worker restarts. During a rolling change of `limit`, + admissions are bounded by the larger of the two capacities. During a + rolling change of `per`, the bound is set by the shorter period until + the last worker on it is gone: the window row carries no period, so a + worker on the shorter one rolls the shared window as soon as its own + period allows, and every worker then counts against the fresh window. + Change `per` in its own deploy, and expect the shorter period to hold + until that deploy finishes. +- **A limit's name spans every backend.** Its window row is keyed on the + name alone, so two backends that use the same name share a window and + count each other's admissions. Give a limit meant for one backend a name + only that backend uses. A name with a different `per` on two backends is + warning `oxpull.W005`, and that limit is not enforced as configured. + Each backend rolls the shared window at its own period. Each roll + restarts the count, so a backend on a longer period can admit more than + its limit in that period. A different `limit` with the same `per` is + warning `oxpull.W004`. Each backend then stops when the shared count + reaches its own `limit`. That shared budget can be deliberate. +- **A limit's name is matched by the database's collation.** The name is + the primary key of the limit's window row, so on MySQL's case-folding + collations renaming `Stripe` to `stripe` is not a rename: the worker + finds the row it already has and carries on counting on it. On + PostgreSQL and SQLite the spelling is the identity, and a renamed limit + starts a fresh window. Within one backend, names that differ only by case + are refused at `check` as `oxpull.E005`. Anything else the database folds + (`Café` and `cafe` on MySQL) is refused at the worker's first refresh, + before it claims, naming both. Across backends, names that differ only by + case or accents are warning `oxpull.W006`. MySQL's default collation + counts them on one window, and PostgreSQL and SQLite count them on two. On + one backend, `oxpull.W006` also covers names that differ only by accents, + or by case and accents, such as `stripe` and `stripé`. On MySQL's default + collation, the worker refuses such a pair too. When `check` compares + names, it drops accents from Latin and Greek letters only. It does not + catch other folds MySQL's default collation makes, such as `ø` into `o`. + Spell a name the same way on every backend. +- **Renaming a limited task detaches its queued rows.** The limit matches + on the task path stored in the row, so rows enqueued under the old path + claim unthrottled. List both paths under the limit for one release. +- **A counter that cannot be written closes its limit.** One failed count + is logged and the claimed task is returned for execution: the claim is + already committed by then, and losing a count is better than losing + the task. The same holds for any database failure after the claim, the + re-read after a window rolled under the count included, and for that + re-read finding the limit's row gone: the loss is logged as + `rate_limit_uncounted` and the claimed task is returned for execution. + Before the claim a failure ends the pass with nothing spent, and a + missing row propagates as `ImproperlyConfigured`. Three unrecorded + admissions in a row under one limit stop it admitting altogether, + because a counter that never advances is not a limit, and every + refresh would read room that is not there. An increment that raised + and one that ran but matched no row both count; only an increment that + lands ends the run. The refresh probes the row and the limit reopens + when a write lands, without a worker restart. + +## Claim cost + +Nothing when it is unused. With no limits configured, and no backend on the +database declaring `WORKFLOWS`, the claim emits django-ox's own SQL, +statement for statement. The suite asserts that by capturing both. +See [workflow claim cost](workflows.md#claim-cost) for the work a `WORKFLOWS` backend adds to every claim on that database. +With limits configured, a task that no limit covers costs one +refresh SELECT per cooldown interval +(`RATE_REFRESH_INTERVAL`, 0.25 s by default), shared by every claim in +it. A limited task costs two extra statements: that refresh and one +increment. A window roll costs a third. One roll lands per window per +limit across the whole fleet; a worker that reads the expired window +before the winner's write commits spends one more UPDATE, which matches +no row. + +The state is one row per limit name in `oxpull_oxpullratewindow`, +forever. The table does not grow with traffic and needs no pruning. + +## Throttled work reads as backlog, because it is + +`django_ox.stats.oldest_ready_age` and the Prometheus ready-age gauge +climb for as long as work is throttled. A queue-age alert tuned before +rate limiting was introduced will fire. That is the limiter working, not +a stuck queue: alert on the age of queues that carry no limited tasks, or +raise the threshold above the longest period you configured. + +Two interactions matter when you combine features. A throttled `@unique` +task keeps its key, so throttling extends the dedup window; that is what +"unique while pending" means, and releasing the key would let duplicates +stack up behind the throttle. And a batch does not complete past a +throttled member, so a limit sets the floor on how long the batch runs. +On one worker, a 1,000-member batch under a limit of 10 per minute fills +100 windows, and the hundredth opens 99 minutes after the first. The +batch completes once its last member finishes, plus one reconciler tick. +Size batches against limits. + +## Database aliases and pools + +Rate-limit window reads and writes use the alias returned by `oxpull_db_alias()`. See [worker database aliases and pools](workflows.md#worker-database-aliases-and-pools) for the recording alias's pool requirements and the limits of the worker's startup warning. + +Use the same database alias for task rows and rate-limit windows: `OxpullWorker` reads the window on that alias. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). + +--- + +# Source: https://oxpull.com/django-ox/features/workflows/ + +# Workflows + +Workflows connect tasks through dependencies, keeping dependent nodes waiting until they can be released. Workflows are provisional in django-ox 1.9.0. The sections below describe release, completion, retention, and safe shutdown. + +See the [feature overview](index.md) for setup, requirements, Django 5.2 behaviour, and the stability policy. + +## Behaviour + +A workflow is a set of tasks with dependencies. A node runs only after every +node it depends on has succeeded. + +```python +from oxpull import workflows + +flow = workflows.create("nightly", on_complete=summarize) +flow.add("extract", extract) +flow.add("transform-a", transform, "a", depends_on="extract") +flow.add("transform-b", transform, "b", depends_on="extract") +flow.add("load", load, depends_on=["transform-a", "transform-b"]) +flow.seal() + +flow.progress() # WorkflowProgress(total=4, pending=4, blocked=3, ...) +``` + +A node's backend must declare `WORKFLOWS` and run `OxpullWorker`. The +reconciler must be on a schedule. `add()` refuses a node whose backend does +not declare `WORKFLOWS` or runs another worker class. `create()` and `seal()` +refuse when no schedule runs the reconciler. +Keep workflow tables and their task rows on the same database. See the [database-routing constraints](unique-tasks.md#database-routing-constraints). + +```python +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "WORKFLOWS": True, + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "SCHEDULES": { + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, + }, + }, + } +} +``` + +`depends_on` names nodes already added to the workflow, so a cycle cannot be +built. A unique task cannot be a node. To make a task unique that workflows +already use, drain its pending nodes first. + +## Workflow API + +The documented workflow API is provisional: + +- `oxpull.workflows.create()` +- `oxpull.workflows.get()` +- `oxpull.workflows.reconcile` +- `oxpull.workflows.retained_rows()` +- `Workflow.add()` +- `Workflow.seal()` +- `Workflow.progress()` +- `WorkflowProgress` + +The workflow exceptions are also provisional: + +- `oxpull.exceptions.WorkflowSealed` +- `oxpull.exceptions.WorkflowNodeExists` +- `oxpull.exceptions.UnknownWorkflowNode` + +They distinguish a sealed workflow, a duplicate node and an unknown node. The `WORKFLOWS` option and all `workflow_*` log events have the same provisional status. Workflow implementation helpers are not public API. + +## Waiting and release + +- A node with no dependencies is enqueued READY. +- A node with dependencies is inserted in django-ox's WAITING status. No + worker claims, reaps or prunes a WAITING row. +- An edge is marked satisfied only on proof that its upstream succeeded. The + proof is the upstream row, or its recorded outcome once the row is pruned. + SUCCESSFUL is final, so the mark stays true after the upstream row is pruned. +- A node moves from WAITING to READY only when every edge into it is marked. + The move is a compare-and-set on the row's status and lease epoch. +- The worker that runs a node records its outcome and marks the edges out of + it in the transaction that writes the task's result. After that commits, + it releases the nodes those edges held. +- A `run_after` you gave a node stays on its row. The release keeps the + later of now and that time, so a released node shows a `run_after`. +- The reconciler releases what a worker missed. When release owns its + transaction, it makes two attempts per chunk of node IDs. If both attempts + lose a lock conflict, the nodes wait for the reconciler. + + SQLite is supported. Django uses DEFERRED transactions by default. + `OPTIONS["transaction_mode"]` can change that mode. + + Under DEFERRED transactions, a read-to-write lock conflict can return + `SQLITE_BUSY` immediately, without waiting for the busy handler. + After both release attempts lose a lock conflict, release logs + `workflow_release_deferred` at WARNING level. The reconciler retries + the deferred nodes. + + Reconciler cadence affects workflow latency. The documented schedule + runs once a minute. Keep the reconciler scheduled and monitor + `workflow_release_deferred`. + +## The claim still checks + +The claim refuses a READY node while an edge into it is not marked and its +upstream row is missing or not SUCCESSFUL. A missing upstream row is never +taken as success. This covers READY rows no release made, such as a django-ox +retry or an UPDATE by hand. What it costs is below. + +## Failure, cancellation and completion + +- A node does not run while one of its upstreams is failed, lost or + discarded. In a sealed workflow the reconciler discards that node. +- An upstream whose row is gone counts as dead only when its outcome was + recorded as dead. With nothing recorded, the nodes downstream of it wait. +- A sealed workflow completes once no node is pending. WAITING, READY and + RUNNING nodes are pending, as is any status unrecognised by django-ox 1.9.0. +- Completion counts each node by its recorded outcome. A success or a failure + still counts after `ox_prune` deletes the row. A node whose row went before + anything recorded it counts as unattributed. +- The reconciler records any outcome a worker did not record, and completes + the workflow on a later tick. +- `on_complete` and `on_success` are enqueued in the transaction that + completes the workflow. `on_success` is enqueued only when nothing failed + and nothing is unattributed. +- A django-ox retry of a node after its workflow completed runs that node. + The nodes downstream of it stay discarded, and the workflow stays COMPLETE. + +## The reconciler's backstop + +Each tick pages through open and sealed workflows, one visit each in turn. A +visit marks edges whose upstream succeeded, records outcomes, and releases +waiting nodes. After five seconds, or a quarter of what its task timeout +leaves, a tick starts no further visit or page. Its first page always runs, +and a page that has started runs to its end. The retries of the queue below +come first and are not counted. + +On PostgreSQL and MySQL each write waits at most one second for a lock. On +PostgreSQL each of its statements runs at most two seconds. On MySQL that +bound applies to SELECT statements only. The page reads have no time bound. A +row the backstop cannot lock goes to a queue that later ticks retry first. The +next pass reads it again too, so the backstop never gives up on a row. + +For SQLite release deferrals and their effect on workflow latency, +see [Waiting and release](#waiting-and-release). + +## Retention + +`oxpull_prune` keeps batch members. For workflows it follows these rules. + +- It considers a workflow only once it is COMPLETE and completed longer ago + than `--older-than`. +- It stamps the workflow as retired, then deletes nothing while any node is + pending, such as a node retried after completion. +- It deletes a node, with the edges into it, only when the node's outcome is + recorded and its task row is SUCCESSFUL, DISCARDED or gone. +- A FAILED or LOST node stays, with the edges into it, while its row exists. + A retry of it is still checked. It goes once `ox_prune --include-failed` + deletes the row. +- A node with no recorded outcome is never deleted. +- The workflow row stays. +- A later run passes over a retired workflow whose node rows are all gone. It + also passes over one that keeps only recorded FAILED or LOST nodes and nodes + with no recorded outcome. It sends statements for one workflow only while + it has something to do there. +- A run still reads the row of every workflow that completed before the + cutoff, and the task row of every node it keeps. Its count of kept rows can + read the whole node table. Those reads grow with the tables. +- It prints how many node and edge rows it keeps. `workflows.retained_rows()` + returns the same counts for monitoring. + +## Claim cost + +Nothing where no backend on the database declares `WORKFLOWS`. Workflows +then add no conjunct to a claim and no statement to a worker. The suite +asserts that against the base worker's statements. + +With `WORKFLOWS` on a worker's own backend, its claim probes the edge +table's `oxpull_wf_edge_down_idx` once for every candidate row it judges. +The table is `oxpull_oxpullworkflowedge` and the index is on +`(downstream_task_id, satisfied)`. Every candidate, not only the nodes: a +task that is in no workflow pays the probe, and the probe finds nothing. +That is what the option being the switch costs. A workflow is created at +run time. A cached "there are no workflows" would be a window in which a +node runs out of order. Closing that window with a query per claim pass +would cost more than the probe it replaced. + +The reach is the database, not the backend. Wherever any backend on the +database declares `WORKFLOWS`, every `OxpullWorker` there records node +outcomes, because a claim is scoped to queues and not to backends. For a +worker whose own backend does not declare it, that is one column more on the +statement returning the claimed row. Nothing else, for a task that is no +node. Such a worker also looks for the node table every 30 seconds until it +is there, then stops: a cost per worker, not per claim. + +The cost is per candidate row, not per blocked node. A blocked node sits in +WAITING, which no claim reads, so a wide fan-in waiting on one root does not +make the claims around it dearer. Measured on PostgreSQL with +`EXPLAIN (ANALYZE)` at 50 and at 450 blocked nodes, and pinned in the suite. + +The probe stays a probe. On SQLite and MySQL the claim names the index. On +PostgreSQL the migration sets the column's `n_distinct`, so the planner's +estimate stays small. Statistics gathered while a wide fan-in fills the edge +table do not turn the check into a scan. A plan read on stale statistics is +not comparable to one read on fresh ones, so the suite runs `ANALYZE` before +it measures. + +The state is the edge table. It grows with the nodes you add, and +`oxpull_prune` deletes a node's edges with the node, under Retention above. + +## Transactions + +- A call owns its transaction when the connection is in autocommit and no + atomic block is open. `add()` then runs its transaction again after a + deadlock or a serialization failure. `seal()` and completion do so after a + serialization failure. The cancellation, the record repair and + `oxpull_prune`'s stamp, deletes and scan row cleanup do so after either. +- Inside your transaction nothing runs again. The error reaches you, and you + run your transaction again. +- To add to or seal several workflows in one transaction, take them in + ascending id order. Two transactions that take the same workflows in + opposite orders can deadlock. +- In that transaction, do not take a locking read or an UPDATE of an existing + task row. + +Use `django_ox.testing.run_tasks(backend="flows")` to run tasks with the workflow backend. Replace `"flows"` with the alias configured for your workflow backend. `run_tasks()` uses that alias's configured worker class. An `OxpullBackend` alias alone does not make the test worker enforce rate limits or release workflow nodes. + +## Not supported + +- Claiming or running workflow nodes with `run_once()` or `execute()` inside + a caller's atomic block. Dependents are released by a commit callback of + the outcome write. Use `run_tasks()` for tests instead. +- Deleting workflow, node or edge rows through the ORM or SQL. +- Mixing releases. Every process using django-ox on the database must run + django-ox 1.9.0 before any backend declares `WORKFLOWS`. An older release + without WAITING support cannot read that status. + +## Turning workflows off + +Two different things go by that name, and only one of them stays available. + +**Stop using workflows.** Drain the waiting nodes, then take `WORKFLOWS` out +of `OPTIONS` and the reconciler out of `SCHEDULES`. The tables stay, holding +finished work and nothing running. This is available whenever you want it, +and django-ox's own rollback needs nothing more. + +**Drop the tables.** `migrate oxpull` back past +`0005_workflow_edge_upstream_idx` or `0004_workflows` drops the tables that +hold every node, edge and record. Both refuse while any workflow row exists, +and no command deletes a workflow row: `oxpull_prune` deletes nodes and edges +and keeps the row it hangs them on. So this is supported only before the +first workflow is created. After that the row is there for good. + +The refusal catches a mistake. It does not make a rollback safe while workers +run. + +## Draining the waiting nodes + +Take these steps in order, on every database that holds django-ox's tables. + +1. Stop creating workflows. Stop every call to `workflows.create()` and + `add()`, then wait for the requests, jobs and transactions already inside + one to end. +2. End every open workflow. A node leaves WAITING when it is released or + discarded. Let the work finish, or close the branches you do not want. + `django_ox.actions.discard` closes one node's task row, WAITING + included, and on its next tick the reconciler discards every node below + it. No call ends a whole workflow at once. +3. Keep the workers and the reconciler running while step 2 runs. Only an + `OxpullWorker` on a backend that declares `WORKFLOWS` releases a waiting + node, and only the reconciler discards a branch. Take the option away + before the rows are gone and nothing is left that can move them. +4. Stop every process that can write a waiting node, and keep it stopped. + That is anything that calls `add()`. A process already running keeps the + settings it started with, so changing settings does not stop one. +5. On every alias, not only the one that holds `oxpull`'s tables, run + `OxTask.objects.using(alias).filter(status="WAITING").count()`. Every + count must be 0. A count that will not fall means something is still + adding nodes. Find it, stop it, and go back to step 2. +6. Take `WORKFLOWS` out of `OPTIONS` and the reconciler out of `SCHEDULES`, + in the settings every process starts with, and restart. + +Once no backend declares `WORKFLOWS`, `manage.py check --database ` +reports `oxpull.W007` for any workflow node still WAITING. Name the alias +that holds `oxpull`'s tables. A bare `manage.py check` reports nothing +here, because a check that reads a database runs only against an alias you +name. `ox_health` and `ox_prune` pass their own, so both report it without +the flag. W007 means step 5 was not finished. The hint says to declare +`WORKFLOWS` again, which is how you get back a database that can drain: +declare it, finish step 2, then take it away again. The check reads one +database, so step 5 is what covers the rest. + +On MySQL, `oxpull.E009` flags a task-id column on another collation than the +task table's, and a workflow worker refuses to start. + +## Deployment order + +**Turn it on in this order.** A node with dependencies is inserted in django-ox's WAITING +status. Older django-ox releases without WAITING support +cannot read or release such a row. Every process on the database must run django-ox 1.9.0 +before the first waiting node is written. For each database that holds the tables: + +1. Upgrade every process using django-ox against that database to django-ox 1.9.0. + That means workers, web and ASGI processes, enqueue-only services, the reconciler + and cron jobs. +2. Run `migrate django_ox` and `migrate oxpull` on that database. +3. Check your deploy inventory for processes still running an older release. A process cannot + see which django-ox version another process runs. +4. Add the workflow settings described above and restart. A running process keeps the settings + it started with, so nothing changes until it restarts. + +For this upgrade, install django-ox 1.9.0 on every enqueuing and claiming process before enabling workflows. This is a consistent-fleet upgrade requirement, not a claim that releases 1.3.0 through 1.8.0 lack WAITING support. WAITING was introduced in django-ox 1.3.0. + +Leave step 4 until steps 1 to 3 are done everywhere. Nothing refuses `WORKFLOWS` on the grounds that a process elsewhere is too old, because no process can see the others. + +Going back to a django-ox from before WAITING is django-ox's own procedure. Its migration is +the one that refuses while a waiting row exists. Its changelog carries those steps, under +**Rolling back to 1.2 after workflows have run**: +[django-ox changelog](../changelog.md). + +## Worker database aliases and pools + +For the worker's general connection budget, pool sizing, and renewal fallback requirements, see [database connections and PostgreSQL pooling](../production.md#database-connections-and-postgresql-pooling). The additional alias rules below apply to workflow outcome recording and rate-limit windows. + +`oxpull` reads and writes rate-limit windows and records workflow-node outcomes +(node records, edge marks, and release) on the alias returned by +`oxpull_db_alias()`, which must also hold the django_ox tables. When it differs +from `--database`, it needs its own pool and server budget. The worker's +startup warning does not check that alias or available server slots. + +### Renewal and watchdog connections + +The watchdog's own stuck-attempt READY retries and FAILED records for rows +that are not workflow nodes use its private connection on `--database`. Its +workflow-node final FAILED records also use that connection when the +recording alias matches `--database`. Among watchdog writes, only a workflow +node's final FAILED record on a separate alias uses that alias's ordinary +pooled connection and needs a pooled spare. Task-thread +workflow-node writes (SUCCESSFUL, edge marks, release) and poll-loop +rate-window I/O use the recording alias's ordinary pool; ordinary task +outcome writes use the `--database` pool; neither uses the watchdog's +private connection. The separate-alias recording path has not been tested. + +The watchdog uses at most one acquisition sequence per pass: the group of +stuck attempts it records together. This grouping is unrelated to task batches. + +Nothing needs setting for renewal or the watchdog. To bound task and poll-loop +connects, which the private-path deadlines do not cover, set +`OPTIONS["connect_timeout"]` on the alias. A positive value below 5 seconds +also shortens private deadlines when it is below their existing budget. + +See [pool sizing, connection handling, and deadline limits](../production.md#database-connections-and-postgresql-pooling) +and [monitoring events](../monitoring.md). +On pooled PostgreSQL, connection-acquisition failures no longer use +`lease_renew_failed`; update alerts. Statement failures still use it. + +After a PostgreSQL restart, tasks that were running at that moment can run again. + +### Rolling upgrades + +For workers on django-ox 0.2.0 through 1.3.1, which are affected by the renewal defect, +set pools to at least +`concurrency + 2`, or `concurrency + 3` with task timeouts, before rollout, +or disable their Django pool. Keep that budget until every affected worker has +stopped. The same mitigation applies while an affected worker remains deployed. + +--- + # Source: https://oxpull.com/django-ox/patterns/ # Common patterns @@ -3056,9 +4334,13 @@ and inside a `transaction.atomic()` opened on the database that holds `OxTask` it commits or rolls back with the rest of the work you write there, as `enqueue()` does. -This is a bulk insert and nothing more. Grouping the -tasks, reading their progress as one number and firing a callback when the -last one settles is a [batch](pro.md), in Oxpull Pro. +This is a bulk insert and nothing more. To group tasks, read their +progress as one number and fire a callback when the last one settles, +use a [batch](features/batches.md). Batches start with `oxpull.batches.create()`; the returned batch has `add()`, `seal()` and `progress()`. They use the optional `oxpull` app and require a reconciler schedule. See [Batches](features/batches.md) for setup and guarantees. + +`enqueue_many()` does not accept unique tasks: it raises `InvalidTask` +and names the task. See [Unique tasks](features/unique-tasks.md) for +deduplicated enqueue. ## Keep slow work off the fast queue @@ -3180,7 +4462,7 @@ Test timeout enforcement and worker infrastructure against a real worker. run_tasks(*, backend="default", queues=None, max_tasks=None, raise_failures=False) -> list[TaskResult] ``` -All arguments are keyword-only. `backend` must name an `OxBackend` alias. It selects the worker class and its default queues, including a configured `WORKER_CLASS`. Rows are claimed by queue, regardless of which backend enqueued them. Aliases sharing queues share work. For Pro workflows and rate limits, pass the Pro alias as `backend`. +All arguments are keyword-only. `backend` must name an `OxBackend` or `OxpullBackend` alias. It selects the worker class and its default queues, including a configured `WORKER_CLASS`. Rows are claimed by queue, regardless of which backend enqueued them. Aliases sharing queues share work. For workflows and rate limits, pass the alias whose `OPTIONS["WORKER_CLASS"]` is `"oxpull.worker.OxpullWorker"`. `queues=None` uses the worker's queues. An empty list does the same. Empty backend `QUEUES` means every queue. An explicit list restricts claiming to those queues. @@ -3249,7 +4531,7 @@ class DelayedTaskTests(TestCase): self.assertEqual([result.return_value for result in results], [5]) ``` -For retries, advance time between calls by at least the configured backoff delay. Freezing a later time does not drain every retry. Each new retry is scheduled relative to that frozen instant. A zero backoff can retry within one call. Pro rate-limit windows use the database clock, which this patch does not advance. +For retries, advance time between calls by at least the configured backoff delay. Freezing a later time does not drain every retry. Each new retry is scheduled relative to that frozen instant. A zero backoff can retry within one call. Rate-limit windows use the database clock, which this patch does not advance. ### Transactions and callbacks @@ -3281,12 +4563,17 @@ Use a real worker to test timeout enforcement and worker infrastructure. `run_ta Timeouts are inert. `deadline()` and `remaining()` return `None`. A configured timeout produces `run_tasks_timeout_inert` once per task path per call. A hanging body or callback hangs the test. -## Not in the core +## Optional coordination features + +[Unique tasks](features/unique-tasks.md), [batches](features/batches.md), +[rate limiting](features/rate-limiting.md) and +[workflows](features/workflows.md) are included in django-ox through the +optional `oxpull` app. They require Django 6.0+. +`OxpullBackend`, `OxpullWorker` and the documented unique-task, batch and +rate-limit entry points are stable. Workflows are provisional. +See the [feature overview](features/index.md) for setup. -Batches, unique or deduplicated tasks, rate limiting and workflows are in -[Oxpull Pro](pro.md), a paid add-on. Metrics are in the -free tier: `django_ox.stats` and `manage.py ox_health` ship in the core. Chains -are on the Pro roadmap, undated. +`django_ox.stats` and `manage.py ox_health` do not require the optional app. --- @@ -3433,10 +4720,12 @@ Here the enqueue is an `INSERT` on the database that holds `OxTask`, your default one. Open the transaction there and the task commits or rolls back with the row it belongs to, so there is nothing to wrap. -Queues, priorities and `run_after` map directly. Celery's chains, -groups and chords, and routing across multiple brokers, are outside the -package; workflows are in [Oxpull Pro](pro.md), a paid add-on, and chains -are on its roadmap, undated. +Queues, priorities and `run_after` map directly. django-ox does not +implement Celery's chain, group or chord APIs, or routing across multiple +brokers. [Batches](features/batches.md) provide grouped work, and +provisional [workflows](features/workflows.md) provide dependency DAGs. +Both require Django 6.0+ and the optional `oxpull` app. Review their +semantics rather than treating them as drop-in Celery replacements. ## From huey @@ -3502,6 +4791,78 @@ the admin today, see [Schedules in the database](stored-schedules.md). | `retries`, `retry_delay` | Backend defaults in `MAX_ATTEMPTS` and the backoff options, or per-task `max_attempts` and `backoff`. django-ox counts total claims, including the first. Per-task declarations require Django 6.1, or Django 5.2 with django-tasks 0.12+. | | `huey.immediate` in tests | Use `django_ox.testing.ImmediateBackend` to run once at enqueue, or `DummyBackend` to record enqueues. Both accept policy declarations without enforcing retries, backoff or timeouts. For queued execution, keep `OxBackend` and use the public, provisional [`run_tasks()`](patterns.md#run-queued-tasks-in-tests) helper. It runs due attempts with retry and backoff policy. Test timeout enforcement against a real worker. | +## Earlier `oxpull` installs + +The optional `oxpull` app is included in django-ox 1.9.0 and later. You no longer need a separately installed `oxpull` distribution. + +The PyPI `oxpull` 1.9.0 distribution is a compatibility metapackage. It depends on django-ox 1.9.0 or later and contains no Python module. The importable package comes from django-ox. + +The former private package index is retired. Remove it and its credentials from installer configuration. + +### Upgrade an existing installation + +The order matters. + +1. Stop every affected worker and other process that enqueues or claims tasks. + +2. Back up the database. + +3. Uninstall the earlier `oxpull` distribution before upgrading django-ox. + + ```console + python -m pip uninstall oxpull + ``` + +4. Remove the former private index and its credentials from your pip configuration, requirements and deployment secrets. + +5. Upgrade django-ox. + + ```console + python -m pip install -U django-ox + ``` + +6. Run migrations and check the installation. + + ```console + python manage.py migrate + python -m pip check + ``` + + For the verified oxpull 1.8.0 schema, the migration run reports "No migrations to apply." Earlier schemas may have migrations to apply. + +7. Restart the affected processes with the updated installation and existing application settings. + +Keep the existing `INSTALLED_APPS` and `TASKS` configuration. Import paths, task data, the `oxpull` app label, table names, the five migrations and persisted task paths are unchanged. + +The optional app requires Django 6.0+. django-ox without that app continues to support Django 5.2. + +### Repair an overlapping installation + +If you ran `pip install -U oxpull` or `pip install -U django-ox` before removing an earlier `oxpull` distribution, inspect and repair the installation before restarting processes. + +If the earlier distribution is still installed, remove it and then restore django-ox's files: + +```console +python -m pip uninstall oxpull +python -m pip install --force-reinstall --no-deps django-ox +python -m pip check +``` + +Do both steps even if imports still worked before the uninstall. The old distribution can leave stale metadata that makes `pip check` fail, and uninstalling it can remove files now supplied by django-ox. + +If you already uninstalled the earlier distribution, restore the files with: + +```console +python -m pip install --force-reinstall --no-deps django-ox +python -m pip check +``` + +The same repair applies if uninstalling the old 0.0.1 stub removed `oxpull/__init__.py`, leaving a namespace import without `oxpull.__version__`. + +These package-file failures do not alter the database data. Run migrations and complete the normal checks before restarting. + +This shared-file warning concerns earlier distributions that installed an `oxpull` module. Uninstalling the module-free 1.9.0 metapackage does not remove django-ox's module files. + ## Switching over The two systems use different tables. Neither reads the other's rows. So if you @@ -3607,9 +4968,9 @@ Inspect tracebacks and use Retry or Discard on failed tasks. Edit recurring sche Both provide a `django.tasks` backend. django-ox automatically requeues a dead worker's tasks. huey's guide says mid-execution tasks are lost and aren't retried automatically. -### Do retries and recovery require Pro? +### Do retries and recovery need the optional app? -No. The free package includes both, plus schedules and Django admin integration. Health checks and metrics are also included. +No. Retries, dead-worker recovery, schedules, Django admin integration, health checks and metrics work without the optional `oxpull` app. ### How do I make recovery safe? @@ -3619,11 +4980,11 @@ Write idempotent tasks. A repeated execution must not duplicate effects such as Python 3.12+ and Django 5.2 LTS, 6.0 or 6.1. Django 5.2 uses the `django-tasks` backport. PostgreSQL, MySQL 8 and SQLite run in CI. -### What does Pro add? +### What does the optional app add? -Batches, unique tasks, shared rate limits and workflows, with maintainer email support. +[Unique tasks](features/unique-tasks.md), [batches](features/batches.md), [shared rate limits](features/rate-limiting.md) and [workflows](features/workflows.md). The `oxpull` app is included in django-ox and requires Django 6.0+. -Oxpull Pro costs $399 per year, per company, excluding VAT where it applies. Every environment is included. Pro requires Django 6.0+. +`OxpullBackend`, `OxpullWorker` and the documented unique-task, batch and rate-limit entry points are stable. Workflows are provisional. See the [feature overview](features/index.md) for setup and API status. [**Get started**](background-tasks.md) @@ -3753,11 +5114,11 @@ Keep schedules in settings, or let your team pause and edit them as [rows in Dja Schedules need at least one running worker and a usable database. A schedule-scoped failure does not stop later entries if rollback succeeds and the same connection remains usable. After an outage, only the most recent missed tick is enqueued. -## Deduplicate a repeated webhook with Oxpull Pro +## Deduplicate a repeated webhook A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue rather than making the queue do the same work again. -With django-ox alone, these calls create separate tasks: +Without a unique-task declaration, these calls create separate tasks: ```python from django.tasks import task @@ -3768,20 +5129,20 @@ def sync_invoice(invoice_id): ... sync_invoice.enqueue(1001) -sync_invoice.enqueue(1001) # django-ox: a second row, a second run +sync_invoice.enqueue(1001) # A second row, a second run. ``` -Oxpull Pro's unique tasks deduplicate inside the task's own transaction. While a task with the same key is READY or RUNNING, enqueueing it again returns the pending task's result without writing another row. +[Unique tasks](features/unique-tasks.md) deduplicate inside the task's own transaction. While a task with the same key is READY or RUNNING, enqueueing it again returns the pending task's result without writing another row. Once the task settles, the same call can enqueue again. This prevents duplicate pending work; it does not make external effects exactly-once. Tasks still need to be safe to repeat. -Pro also lets you track a batch through completion, control task starts with shared rate limits, and connect dependent tasks into workflows. You get email support from the maintainer and priority feature requests. +Unique tasks are included in django-ox through the optional `oxpull` app and require Django 6.0+. Follow the [setup overview](features/index.md), then the [unique-task declaration guide](features/unique-tasks.md). -[Order Oxpull Pro](https://oxpull.com/) for $399 per year, per company, excluding VAT where it applies. Submit the form; your invoice arrives within two business days. Once paid, private package index credentials arrive by email. Installation stays `pip install`. Pro requires Django 6.0+. +The same app provides [batches](features/batches.md) to track grouped work, [rate limiting](features/rate-limiting.md) to control task starts, and provisional [workflows](features/workflows.md) to connect dependent tasks. ## When to choose another tool -Choose another backend if you need to revoke a chosen running task, use chains, groups or chords, or exceed the throughput one database can serve. Celery supports revocation and those composition patterns. +Choose another backend if you need to revoke a chosen running task, require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md), not those Celery APIs. Transactional enqueue requires the task and model rows to share a database. A separate queue database cannot give you that shared commit. @@ -3817,6 +5178,10 @@ fewer service. The last section says where django-ox is not the right fit. | Transactional enqueue | Yes. Enqueue is one INSERT on your default database; a task written inside `atomic()` commits or rolls back with the rows beside it | Not documented [^tdb-readme] | Not documented [^huey-guide] | No. Django's own docs name a background task as the case for `on_commit()` [^dj-oncommit] | Not documented [^q2-brokers] | No. Enqueue is a broker send [^dq-guide] | Not documented [^proc-django] | | Retries and backoff | Exponential backoff by default; per-task `max_attempts` and `backoff` on Django 6.1 or Django 5.2 with django-tasks 0.12+. Backend defaults: `MAX_ATTEMPTS`, `BACKOFF_INITIAL`, `BACKOFF_MAX`. Every attempt's traceback kept. See [Per-task policy](configuration.md#per-task-policy) | No retry option in the README or the worker flags [^tdb-readme] [^tdb-worker] | `retries`, `retry_delay`, `retry_backoff` [^huey-guide] | `autoretry_for`, `retry_backoff`, `retry_backoff_max` (default 600 s), `retry_jitter` [^celery-tasks] | `max_attempts` (default 0, meaning unlimited) and a `retry` interval; backoff not documented [^q2-configure] | Exponential backoff; `max_retries` default 20, `min_backoff` 15 s, `max_backoff` 7 days [^dq-guide] | Retry strategy per task [^proc-index] | | Recurring schedules | Cron or a fixed interval in settings, or rows created and edited in the Django admin, limited to the tasks the code exposes; every worker dispatches, no scheduler process. See [Schedules in the database](stored-schedules.md) | None. Issue #259 open since 2026-08-10 [^tasks-259] | `periodic_task(crontab(...))` [^huey-guide] | `celery beat`, a separate process; "ensure only a single scheduler is running" [^celery-beat]. With `django-celery-beat` the schedules are rows that "can be managed from the Django Admin interface", and the field naming the task has no list to choose from [^beat-admin] | `Schedule` model, editable in admin; cron via croniter [^q2-schedules] | None built in; APScheduler recommended [^dq-cookbook] | `@app.periodic` [^proc-index] | +| Unique tasks / deduplicated enqueue | `@unique` or `UNIQUE_TASKS` returns the existing `TaskResult` while a match is READY or RUNNING; not an execution-exclusivity guarantee. All four features below require the optional `oxpull` app and `OxpullBackend` on Django 6.0+ | Issue #164 closed as a duplicate of discussion #143; maintainer: "handled by your business logic" [^tdb-164] | huey's own `lock_task()` prevents concurrent runs, not duplicate enqueues [^huey-stable] | No general built-in enqueue deduplication. The cookbook uses a custom execution lock; `worker_deduplicate_successful_tasks` handles successful redeliveries, and SQS FIFO has a broker-specific deduplication mechanism [^celery-cookbook] [^celery-dedup-d3] [^celery-sqs-d3] | Not documented [^q2-tasks-d3] | Not documented [^dq-ref] | `queueing_lock` permits one job in `todo` per chosen lock string, but multiple jobs in `doing`; a duplicate defer raises `AlreadyEnqueued` [^proc-qlock] | +| Batch completion callback | `on_complete` is enqueued once every member's row is settled or gone, whatever the outcome; `on_success` requires all to succeed. Callbacks run at least once. Requires a once-a-minute reconciler schedule | "Task Groups" discussion #58, open since 2024-06-20, with 0 comments [^tasks-58] | huey's own `chord()` runs after members finish or fail permanently, passing failures as `huey.Error`; a member lost to abrupt consumer shutdown prevents the callback [^huey-stable] | `chord` runs its callback only if every member succeeds; a failed member produces `ChordError`, and an errback attached to the callback runs instead. Requires a result backend other than RPC and member results must not be ignored [^celery-canvas] | No group completion hook documented; groups label results for polling, while `hook` is per task [^q2-group] [^q2-tasks-d3] | `group.add_completion_callback()` requires optional `GroupCallbacks` middleware; the completion barrier is signalled only on success, so the callback requires every member to succeed [^dq-ref] | A contributor: "there is nothing out of the box"; issue #1219 closed in favour of #1198 [^proc-1219] | +| Rate limits shared across workers | Fixed start windows coordinated through one database row per limit name; requires `OxpullWorker`. Bounded overshoot is possible. Throttled tasks stay READY without spending an attempt; this is not a concurrency cap | Not documented in the README, worker flags or Django Tasks API [^tdb-readme-d3] [^tdb-worker-d3] [^dj-ref-tasks] | huey's own `rate_limit()` uses a fixed-window storage counter keyed by huey name and limit name; workers using the same shared storage use that counter. Checked inside the running task, with throttled calls rescheduled [^huey-stable] [^huey-ratelimit-src] | `Task.rate_limit` is per worker instance, not global; the docs advise restricting to a given queue for global limiting [^celery-ratelimit] | Not documented; pull request #350 proposes a per-cluster throttle and remains open [^q2-configure-d3] [^q2-350] | `WindowRateLimiter` and `BucketRateLimiter` use Redis or Memcached for shared limits, called from actor code [^dq-ref] [^dq-cookbook-d3] | Not documented; issue #1289 open since 2025-01-14 [^proc-1289] | +| Dependency workflows | Provisional. `depends_on` forms a DAG; a node runs only after every upstream succeeds. Requires `WORKFLOWS: True`, `OxpullWorker` and a reconciler schedule | Not documented in the Tasks API; discussion #58 proposes task groups and follow-up work [^dj-ref-tasks] [^tasks-58] | huey's own pipelines with `.then()` run tasks sequentially; a failure skips the remaining success pipeline [^huey-stable] | `chain` / `link` call the next task after success; `chord` provides fan-in [^celery-canvas] | Chains are not success-gated: later tasks are also enqueued after a failure. `save_task()` has no success check before enqueueing the next chain link. Issue #327, open since 2026-05-05 [^q2-327-d3] [^q2-monitor-d3] | `pipeline` passes results to the next actor; the next step is enqueued only after success [^dq-ref] | No built-in workflows; issue #1198 open since 2024-09-18. Execution locks order jobs whether the preceding job succeeds or not [^proc-1198] [^proc-locks-d3] | | Priorities | -100 to 100 | Yes [^tdb-backend] | Yes; on Redis needs 5.0+ and `PriorityRedisHuey` [^huey-guide] | 0 to 255 on RabbitMQ and Redis [^celery-calling] | Not documented [^q2-configure] | Per actor; lower number runs first [^dq-guide] | Yes [^proc-index] | | Time limit on a running task | Configured through backend `TASK_TIMEOUT`, per-queue `TASK_TIMEOUTS`, or a task's declared `timeout` on Django 6.1 or Django 5.2 with django-tasks 0.12+. `TaskTimeout` is raised inside the task at the deadline, followed by a worker recycle if the thread does not stop. The recycle is the whole enforcement on an interpreter with no facility for raising an exception in another thread, and for a thread a coverage tool or debugger is watching. See [Production](production.md#task-timeouts) | Not documented in the README or the worker flags [^tdb-readme] [^tdb-worker] | `timeout` per task or per call; "a `TaskTimeout` is raised and returned to the caller via the result handle" [^huey-guide] | `soft_time_limit` raises `SoftTimeLimitExceeded` inside the task; `time_limit` terminates the process running it, which is replaced. "Time limits don't currently work on platforms that don't support the `SIGUSR1` signal" [^celery-workers] | `timeout`, default `None`: "the number of seconds a worker is allowed to spend on a task before it's terminated" [^q2-configure] | `time_limit` per actor, default 10 minutes, raises `TimeLimitExceeded`. "Time limits are best-effort. They cannot cancel system calls or any function that doesn't currently hold the GIL under CPython" [^dq-guide] | Not documented on the page checked [^proc-index] | | Worker dies mid-task | Lease renewed every `LOCK_TIMEOUT / 3`; a task whose lease goes stale for `LOCK_TIMEOUT` is put back to READY, or marked LOST when attempts are spent. See [Production](production.md#the-reaper) | Issue #5, open since 2024-06-11: the task 'remains marked as "PROCESSING", and thus is never picked up for re-processing nor marked as completed / failed' [^tdb-5] | "tasks that are mid-execution are lost and will not be retried automatically" [^huey-guide] | `acks_late` re-delivers; the worker still acknowledges "if the child process executing the task is terminated" [^celery-tasks] | Issue #327, open since 2026-05-05: worker death not reported to the monitor, `MAX_ATTEMPTS` ignored [^q2-327] | Not stated on the pages checked [^dq-guide] [^dq-cookbook] | Heartbeat every 10 s; jobs stay in `doing` until a `retry_stalled_jobs` periodic task you define picks them up [^proc-stalled] | @@ -3832,21 +5197,25 @@ function", open since 2020-12-19 with 98 upvotes [^celery-6552]. steady-queue, dj-queue and django-database-task also keep the queue in the database and implement the `django.tasks` backend API. The same rules apply: -every cell about another project comes from its README or its PyPI page, with -the link and the date in the footnotes, and a cell says "not stated" when -those pages do not say. +every cell about another project comes from its README, its PyPI page or its +own issue tracker, with the link and the date in the footnotes, and a cell +says "not stated" when those pages do not say. | | django-ox | steady-queue | dj-queue | django-database-task | | --- | --- | --- | --- | --- | -| Latest release on PyPI | See the [changelog](changelog.md) | 0.2.1, 2026-09-06 [^sq-pypi] | 0.14.1, 2026-09-03 [^djq-pypi] | 0.5.0, 2026-09-05 [^ddt-pypi] | +| Latest release on PyPI | See the [changelog](changelog.md) | 0.2.1, 2026-09-06 [^sq-pypi] | 0.14.2, 2026-09-26 [^djq-pypi] | 0.6.0, 2026-09-30 [^ddt-pypi] | | `django.tasks` backend | Yes, native | Yes, native [^sq-readme] | Yes, native [^djq-readme] | Yes, native [^ddt-readme] | | Django versions | 5.2 LTS with the `django-tasks` backport, 6.0 and 6.1 | 6.0 and later, below 7 (`django>=6.0,<7`) [^sq-pypi] | 6.0 and 6.1 (`django>=6.0.0,<6.2`) [^djq-pypi] | 6.0 and later (`Django>=6.0`) [^ddt-pypi] | | Databases | PostgreSQL, MySQL 8 and SQLite, all run in CI; MariaDB 10.6+ takes the same claim path as MySQL and is not in CI | MySQL, PostgreSQL or SQLite; `FOR UPDATE SKIP LOCKED` on MySQL 8+ and PostgreSQL 9.5+ [^sq-readme] | PostgreSQL first-class, with optional `LISTEN/NOTIFY`; MySQL and MariaDB supported; SQLite "supported with limits", polling only and no `SKIP LOCKED` [^djq-readme] | PostgreSQL 14+, MySQL 8.0.11+, MariaDB 10.6+, SQLite (no row-level locking; "development or single-worker deployments"), Oracle 19c+ ("not tested with this package") [^ddt-readme] | | Transactional enqueue | Yes. Enqueue is one INSERT on your default database; a task written inside `atomic()` commits or rolls back with the rows beside it | The README says a shared database gives transactional integrity and recommends against relying on it: a separate database for the queue, and `transaction.on_commit` for tasks that depend on data [^sq-readme] | "`enqueue()` writes immediately and returns a real persisted task result ID"; for a task that depends on rows in the current transaction the README provides `enqueue_on_commit()` or `transaction.on_commit()` [^djq-readme] | Not stated as a feature. The PostgreSQL broker section says `pg_notify()` runs inside the same transaction as the INSERT and is delivered on commit [^ddt-readme] | | Retries and backoff | Exponential backoff by default; per-task `max_attempts` and `backoff` on Django 6.1 or Django 5.2 with django-tasks 0.12+. Backend defaults: `MAX_ATTEMPTS`, `BACKOFF_INITIAL`, `BACKOFF_MAX`. Every attempt's traceback kept. See [Per-task policy](configuration.md#per-task-policy) | Not stated in the README or the configuration page; failed tasks are retried from the Django admin [^sq-readme] [^sq-config] | None built in: "`dj_queue` does not add an automatic retry policy or backoff engine". Failed jobs are retried from the admin or with `retry_failed_job()` [^djq-readme] | No automatic retry stated. A FAILED task is re-run from the admin action "Retry failed tasks" or with `run_task_by_id(..., allow_retry=True)` [^ddt-readme] | | Recurring schedules | Cron or a fixed interval in settings, or rows edited in the Django admin; every worker dispatches, no scheduler process | Cron-style, with the `@recurring` decorator in code; a scheduler process under the supervisor [^sq-readme] | Static recurring tasks in `OPTIONS["recurring"]` and dynamic recurring tasks managed at runtime; a scheduler under the supervisor [^djq-readme] | None built in. The worker is run from cron, a systemd timer, JP1, Hinemos or Cloud Scheduler [^ddt-readme] | +| Unique tasks / deduplicated enqueue | `@unique` or `UNIQUE_TASKS` returns the existing `TaskResult` while a match is READY or RUNNING; not an execution-exclusivity guarantee. All four features below require the optional `oxpull` app and `OxpullBackend` on Django 6.0+ | Issue #24, open since 2026-02-10; owner: "For now ... I don't think this will get added to the package", but is "open to creating a `contrib` subpackage that could include this" [^sq-24] | `@concurrency(..., on_conflict="discard")` provides singleton-style work through a concurrency semaphore with a TTL; the discarded enqueue's return value is not stated [^djq-readme-d3] | Not stated; `SELECT FOR UPDATE SKIP LOCKED` prevents duplicate claims, not duplicate enqueues [^ddt-readme-d3] | +| Batch completion callback | `on_complete` is enqueued once every member's row is settled or gone, whatever the outcome; `on_success` requires all to succeed. Callbacks run at least once. Requires a once-a-minute reconciler schedule | Not stated [^sq-readme-d3] | Not stated [^djq-readme-d3] | Not stated [^ddt-readme-d3] | +| Rate limits shared across workers | Fixed start windows coordinated through one database row per limit name; requires `OxpullWorker`. Bounded overshoot is possible. Throttled tasks stay READY without spending an attempt; this is not a concurrency cap | Not stated; the README's concurrency controls limit simultaneous execution, not start rate [^sq-readme-d3] | Not stated; shared semaphores provide concurrency limits, not rate limits [^djq-readme-d3] | Not provided by the package; the README points to Google Cloud Tasks per-queue rate settings for `CloudTasksDatabaseBackend` [^ddt-readme-d3] | +| Dependency workflows | Provisional. `depends_on` forms a DAG; a node runs only after every upstream succeeds. Requires `WORKFLOWS: True`, `OxpullWorker` and a reconciler schedule | Not stated [^sq-readme-d3] | Not stated [^djq-readme-d3] | Not stated [^ddt-readme-d3] | | Priorities | -100 to 100 | -100 to 100 [^sq-readme] | `priority` on the task call; higher claimed first within a queue [^djq-readme] | -100 to 100 [^ddt-readme] | -| Concurrency controls | None in django-ox, and none in [Oxpull Pro](pro.md): Pro's rate limiting caps how often a task starts, not how many run at once | "Limit how many instances of a specific task can run simultaneously" [^sq-readme] | `@concurrency` decorator with database-backed limits; semaphores shown in the admin dashboard [^djq-readme] | Not stated [^ddt-readme] | +| Concurrency controls | No per-task concurrency limit. [Rate limiting](features/rate-limiting.md) caps how often a task starts, not how many run at once | "Limit how many instances of a specific task can run simultaneously" [^sq-readme] | `@concurrency` decorator with database-backed limits; semaphores shown in the admin dashboard [^djq-readme] | Not stated [^ddt-readme] | | Worker dies mid-task | Lease renewed every `LOCK_TIMEOUT / 3`; a task whose lease goes stale for `LOCK_TIMEOUT` is put back to READY, or marked LOST when attempts are spent | Marked failed with `ProcessPrunedError` (heartbeat expired) or `ProcessExitError` (killed), to be inspected [^sq-readme] | Kept "as failed work that operators can inspect and retry": `ProcessExitError`, `ProcessPrunedError`, `ProcessMissingError` [^djq-readme] | Requeued by `requeue_stale_database_tasks --older-than`, a separate command run from cron or a systemd timer; `--max-attempts` (default 3) marks the task FAILED instead [^ddt-readme] | | Health and metrics | `ox_health` command, `django_ox.stats`, a Prometheus endpoint (`/ox/metrics`), structured log events, admin retry and discard | Django admin: pause and resume queues, inspect tasks, retry or discard failed ones; operational signals from `steady_queue.signals` [^sq-readme] | Admin dashboard at `/admin/dj_queue/dashboard/`; `dj_queue_health` command; `/dj_queue/metrics` in Prometheus text format with the `prometheus` extra [^djq-readme] | Django admin; structured log fields; opt-in worker exit codes for job schedulers [^ddt-readme] | | Async tasks and results | `async def` tasks run; `get_result()` supported | "Async task enqueueing is not supported"; "Result fetching is not supported" [^sq-readme] | `get_result()` documented; async task functions not stated [^djq-readme] | "Supports async task functions"; `get_result()` documented [^ddt-readme] | @@ -3868,9 +5237,10 @@ those pages do not say. [benchmarks](benchmarks.md) page gives measured numbers with the method. If your workload is above them, a broker-based queue is the right tool, and the cost is the second datastore. -- **Chains, groups and chords.** Not in django-ox. Batches and workflows - are in [Oxpull Pro](pro.md), a paid add-on; chains are on the Pro - roadmap, undated. +- **Celery-style chains, groups and chords.** django-ox does not implement + these APIs. It provides [batches](features/batches.md) for grouped work + and provisional [workflows](features/workflows.md) for dependency DAGs. + Check their semantics before replacing an existing composition. - **CPU-bound tasks in one process.** Worker concurrency is a thread pool. Run `ox_worker --processes N --concurrency 1` for N interpreters, or pick a queue with a process pool. @@ -3887,9 +5257,15 @@ of date, open an issue with the link that shows it. [^tdb-44]: https://github.com/RealOrangeOne/django-tasks-db/issues/44, open, checked 2026-08-23. [^tdb-repo]: https://github.com/RealOrangeOne/django-tasks-db, licence field, checked 2026-08-23. [^tasks-259]: https://github.com/RealOrangeOne/django-tasks/issues/259, open, checked 2026-08-23. +[^tdb-164]: https://github.com/RealOrangeOne/django-tasks/issues/164, closed as a duplicate of discussion #143; maintainer assigns deduplication to business logic, checked 2026-10-11. +[^tasks-58]: https://github.com/RealOrangeOne/django-tasks/discussions/58, "Task Groups", open since 2024-06-20 with 0 comments, checked 2026-10-11. +[^tdb-readme-d3]: https://github.com/RealOrangeOne/django-tasks-db, README at `8f79c7f`; no rate-limit option documented, checked 2026-10-11. +[^tdb-worker-d3]: https://github.com/RealOrangeOne/django-tasks-db/blob/master/django_tasks_db/management/commands/db_worker.py, `add_arguments`; no rate-limit flag documented, checked 2026-10-11. [^huey-guide]: https://huey.readthedocs.io/en/latest/guide.html, checked 2026-08-23. [^huey-contrib]: https://huey.readthedocs.io/en/latest/contrib.html, checked 2026-09-19. [^huey-repo]: https://github.com/coleifer/huey, licence field, checked 2026-08-23. +[^huey-stable]: https://huey.readthedocs.io/en/stable/guide.html, huey 3.4.0; huey's own task locks, chords, fixed-window rate limits and pipelines, checked 2026-10-11. +[^huey-ratelimit-src]: https://github.com/coleifer/huey/blob/3.4.0/huey/api.py, `class RateLimit`; counter key includes huey name and limit name and uses `storage.incr()`, checked 2026-10-11. [^celery-tasks]: https://docs.celeryq.dev/en/stable/userguide/tasks.html, checked 2026-08-23. [^celery-brokers]: https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html, checked 2026-08-23. [^celery-workers]: https://docs.celeryq.dev/en/stable/userguide/workers.html, Time Limits, checked 2026-08-23. @@ -3900,8 +5276,14 @@ of date, open an issue with the link that shows it. [^celery-6552]: https://github.com/celery/celery/issues/6552, open, checked 2026-08-23. [^celery-10062]: https://github.com/celery/celery/issues/10062, closed, checked 2026-08-23. [^celery-license]: https://github.com/celery/celery/blob/main/LICENSE, checked 2026-08-23. +[^celery-cookbook]: https://docs.celeryq.dev/en/stable/tutorials/task-cookbook.html, Celery 5.6.3; custom cache lock for execution exclusivity, checked 2026-10-11. +[^celery-dedup-d3]: https://docs.celeryq.dev/en/stable/userguide/configuration.html, Celery 5.6.3; `worker_deduplicate_successful_tasks` applies to successful redeliveries with the same task identifier and late acknowledgment, checked 2026-10-11. +[^celery-sqs-d3]: https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/sqs.html, Celery 5.6.3; SQS FIFO message properties including `MessageDeduplicationId`, checked 2026-10-11. +[^celery-canvas]: https://docs.celeryq.dev/en/stable/userguide/canvas.html, Celery 5.6.3; chord result-backend requirements and failure behavior, including an errback attached to the chord callback running instead on failure; chains and links, checked 2026-10-11. +[^celery-ratelimit]: https://docs.celeryq.dev/en/stable/userguide/tasks.html#Task.rate_limit, Celery 5.6.3; per-worker rate limits and queue restriction advice, checked 2026-10-11. [^flower]: https://flower.readthedocs.io/en/latest/features.html, checked 2026-08-23. [^dj-oncommit]: https://docs.djangoproject.com/en/6.0/topics/db/transactions/#performing-actions-after-commit, checked 2026-08-23. +[^dj-ref-tasks]: https://docs.djangoproject.com/en/6.0/ref/tasks/, Django 6.0 Tasks API; no rate-limit or dependency option, checked 2026-10-11. [^q2-brokers]: https://django-q2.readthedocs.io/en/master/brokers.html, checked 2026-08-23. [^q2-configure]: https://django-q2.readthedocs.io/en/master/configure.html, checked 2026-08-23. [^q2-schedules]: https://django-q2.readthedocs.io/en/master/schedules.html, checked 2026-08-23. @@ -3909,21 +5291,38 @@ of date, open an issue with the link that shows it. [^q2-315]: https://github.com/django-q2/django-q2/pull/315, open, checked 2026-08-23. [^q2-327]: https://github.com/django-q2/django-q2/issues/327, open, checked 2026-08-23. [^q2-pyproject]: https://github.com/django-q2/django-q2/blob/master/pyproject.toml, checked 2026-08-23. +[^q2-tasks-d3]: https://django-q2.readthedocs.io/en/master/tasks.html, Django Q2 1.11.1; `async_task()` options and per-task `hook`, checked 2026-10-11. +[^q2-group]: https://django-q2.readthedocs.io/en/master/group.html, result grouping and polling; no group completion hook documented, checked 2026-10-11. +[^q2-configure-d3]: https://django-q2.readthedocs.io/en/master/configure.html, Django Q2 1.11.1; no shared rate-limit option documented, checked 2026-10-11. +[^q2-350]: https://github.com/django-q2/django-q2/pull/350, proposed per-cluster throttle, open since 2026-09-16, checked 2026-10-11. +[^q2-327-d3]: https://github.com/django-q2/django-q2/issues/327, open since 2026-05-05; Bug 2 reports chains continuing after failure, checked 2026-10-11. +[^q2-monitor-d3]: https://github.com/django-q2/django-q2/blob/v1.11.1/django_q/monitor.py, `save_task()` has no success check before enqueueing the next chain link, checked 2026-10-11. [^dq-guide]: https://dramatiq.io/guide.html, checked 2026-08-23. [^dq-cookbook]: https://dramatiq.io/cookbook.html, checked 2026-08-23. [^dq-prom]: https://github.com/Bogdanp/dramatiq/blob/master/dramatiq/middleware/__init__.py, `default_middleware`, and `middleware/prometheus.py`, checked 2026-08-23. [^dq-repo]: https://github.com/Bogdanp/dramatiq, licence field, checked 2026-08-23. +[^dq-ref]: https://dramatiq.io/reference.html, Dramatiq 2.2.0; composition and rate-limiter APIs; success gating also checked in `dramatiq/middleware/group_callbacks.py` and `dramatiq/middleware/pipelines.py` at `8534ffa`, checked 2026-10-11. +[^dq-cookbook-d3]: https://dramatiq.io/cookbook.html, rate limiters used inside actor code with a shared backend, checked 2026-10-11. [^proc-index]: https://procrastinate.readthedocs.io/en/stable/, checked 2026-08-23. [^proc-django]: https://procrastinate.readthedocs.io/en/stable/howto/django/basic_usage.html, checked 2026-08-23. [^proc-stalled]: https://procrastinate.readthedocs.io/en/stable/howto/production/retry_stalled_jobs.html, checked 2026-08-23. [^proc-repo]: https://github.com/procrastinate-org/procrastinate, licence field, checked 2026-08-23. +[^proc-qlock]: https://procrastinate.readthedocs.io/en/stable/howto/advanced/queueing_locks.html, `queueing_lock`, `todo` versus `doing`, and `AlreadyEnqueued`, checked 2026-10-11. +[^proc-1219]: https://github.com/procrastinate-org/procrastinate/issues/1219, batch and workflow request closed in favour of #1198; a contributor, not a maintainer, says "there is nothing out of the box", checked 2026-10-11. +[^proc-1289]: https://github.com/procrastinate-org/procrastinate/issues/1289, rate-limit request, open since 2025-01-14, checked 2026-10-11. +[^proc-1198]: https://github.com/procrastinate-org/procrastinate/issues/1198, dependency workflow request, open since 2024-09-18, checked 2026-10-11. +[^proc-locks-d3]: https://procrastinate.readthedocs.io/en/stable/howto/advanced/locks.html, execution locks order jobs whether the preceding job succeeds or not, checked 2026-10-11. [^sq-readme]: https://github.com/knifecake/steady-queue README, checked 2026-09-19. [^sq-config]: https://steady-queue.readthedocs.io/en/latest/configuration.html, checked 2026-09-19. -[^sq-pypi]: https://pypi.org/project/steady-queue/ 0.2.1, uploaded 2026-09-06; `Requires-Dist: django<7,>=6.0`; licence MIT. Checked 2026-09-19. +[^sq-pypi]: https://pypi.org/project/steady-queue/ 0.2.1, uploaded 2026-09-06; `Requires-Dist: django<7,>=6.0`; licence MIT. Checked 2026-10-11. +[^sq-24]: https://github.com/knifecake/steady-queue/issues/24, enqueue uniqueness request, open since 2026-02-10; owner: "For now ... I don't think this will get added to the package", but is "open to creating a `contrib` subpackage that could include this", checked 2026-10-11. +[^sq-readme-d3]: https://github.com/knifecake/steady-queue, README at `db46170`; concurrency controls and documented feature scope, checked 2026-10-11. [^djq-readme]: https://github.com/coriocactus/dj_queue README, checked 2026-09-19. -[^djq-pypi]: https://pypi.org/project/dj-queue/ 0.14.1, uploaded 2026-09-03; `Requires-Dist: django<6.2,>=6.0.0`; licence MIT. Checked 2026-09-19. +[^djq-pypi]: https://pypi.org/project/dj-queue/ 0.14.2, uploaded 2026-09-26; `Requires-Dist: django<6.2,>=6.0.0`; licence MIT. Checked 2026-10-11. +[^djq-readme-d3]: https://github.com/coriocactus/dj_queue, README at `086ce2a`; singleton-style concurrency controls and documented feature scope, checked 2026-10-11. [^ddt-readme]: https://github.com/tokibito/django-database-task README, checked 2026-09-19. -[^ddt-pypi]: https://pypi.org/project/django-database-task/ 0.5.0, uploaded 2026-09-05; `Requires-Dist: Django>=6.0`; licence MIT. Checked 2026-09-19. +[^ddt-pypi]: https://pypi.org/project/django-database-task/ 0.6.0, uploaded 2026-09-30; `Requires-Dist: Django>=6.0`; licence MIT. Checked 2026-10-11. +[^ddt-readme-d3]: https://github.com/tokibito/django-database-task, README at `9f4db93`; claim exclusivity, Google Cloud Tasks rate settings and documented feature scope, checked 2026-10-11. --- @@ -4065,8 +5464,8 @@ batch open; the stop-time look waits up to five seconds for that claim. If the claim is still in flight then, the look gives up with `worker_claim_recovery_failed` and a row that landed is left to the reaper. `--max-tasks` counts only claims -that returned; a claim that raised uses no slot. Oxpull Pro also counts -admission only for a claim that returned, so a released row is counted +that returned; a claim that raised uses no slot. Rate-limit admission is +also counted only for a claim that returned, so a released row is counted when it is claimed again. No database statement is added to the ordinary task path. A recovery look @@ -4283,13 +5682,15 @@ python manage.py ox_worker --batch --concurrency 4 "Nothing to claim" is a point-in-time observation by this worker, not a guarantee that the queue is empty or a workflow is complete. Future `run_after` tasks, backed-off retries, locked tasks and tasks excluded by -claim filters may remain. With [Oxpull Pro](pro.md), rate-limited READY tasks -and WAITING workflow children may remain too. A pass that sees due tasks but -loses them all to other workers isn't empty, and the worker polls again. -Immediately eligible follow-up work committed before a local task finishes -can be picked up on a subsequent pass, unless another stop condition wins. -Batch mode does not wait for later commits, schedule ticks or reconciler -hand-offs; arrange another run or use a long-running worker for that work. +claim filters may remain. With the optional `oxpull` app, +[rate-limited](features/rate-limiting.md) READY tasks and WAITING +[workflow](features/workflows.md) children may remain too. +A pass that sees due tasks but loses them all to other workers isn't empty, +and the worker polls again. Immediately eligible follow-up work committed +before a local task finishes can be picked up on a subsequent pass, unless +another stop condition wins. Batch mode does not wait for later commits, +schedule ticks or reconciler hand-offs; arrange another run or use a +long-running worker for that work. Schedules dispatch only while a worker runs. A batch checks them when it starts and keeps checking while it runs. Each schedule then enqueues only its @@ -4848,13 +6249,11 @@ replacement connections. ### Rolling out per-task policy -No database migration is required. The attempt-budget column already -exists, but old and new workers do not execute the same live policy. +Per-task policy was introduced in django-ox 1.5.0. When upgrading from 1.4.0, use the mixed-fleet guidance below until every worker has been upgraded. -Deploy django-ox 1.5.0 to every producer, worker and other process that -imports task modules before adding declarations. Deploy Oxpull Pro 1.5.0 -everywhere too if it is installed. Only then enable `max_attempts`, -`backoff` and `timeout` declarations. +Per-task policy does not require a migration of either app. Treat it as a mixed-fleet configuration rollout: update every process that enqueues or claims the affected tasks before relying on the new policy. + +Earlier `oxpull` installs have their own [upgrade steps](migrating.md#earlier-oxpull-installs). Those package upgrade steps are separate from enabling per-task policy. | Mixed-fleet case | Behaviour | | --- | --- | @@ -5546,6 +6945,40 @@ your own. Flag reference on the default so tracebacks survive until you have looked at them; add `--include-failed` once that is not needed. +### oxpull_prune + +Run `oxpull_prune` to clean up records owned by the optional app: + +```console +python manage.py oxpull_prune +``` + +It removes: + +- Unique-task lock rows whose tasks are no longer READY or RUNNING, including locks whose tasks were pruned, and locks with a NULL task id. Locks are not aged. Enqueuing the same key also releases a stale lock. +- Eligible workflow node rows and the edges into them, for workflows that have been COMPLETE longer than `--older-than`. +- Backstop scan rows for workflows that are COMPLETE or no longer exist. + +It never deletes workflow rows or batch members. + +For each eligible workflow, pruning first stamps `metadata_retired_at`, once. If a node is still pending, it deletes nothing and tries again on a later run. It repairs unrecorded outcomes, then deletes only nodes whose outcomes are recorded and whose task rows are SUCCESSFUL, DISCARDED or gone. + +Nodes whose task rows are FAILED or LOST remain until `ox_prune --include-failed` removes those task rows. A node with no recorded outcome is never deleted. + +The options are: + +| Option | Default | Meaning | +| --- | --- | --- | +| `--batch-size` | `1000` | Rows per DELETE. Must be at least 1. A workflow statement deletes at most 1,000 rows. | +| `--older-than` | `7d` | Workflow retention age. Accepts values such as `7d`, `24h`, `90m`, `45s` or plain seconds. Does not age unique-task locks. | +| `--dry-run` | Off | Count eligible records without deleting rows or writing retirement stamps. | + +There is no `--database` option. The command uses the database assigned by the routers to both the `django_ox` and `oxpull` tables and refuses a split. If the workflow tables do not exist yet, it skips workflow cleanup and reports that `migrate oxpull` is needed. + +No ordering with `ox_prune` is required. Running `ox_prune --include-failed` first lets more workflow nodes be removed. A lock whose task `ox_prune` deleted is stale and can also be removed. + +The command ends by reporting retained workflow, node and edge counts. See [Monitoring](monitoring.md) for their meaning. + --- # Source: https://oxpull.com/django-ox/monitoring/ @@ -5613,6 +7046,16 @@ functions. (`oldest_ready_age`).** Neither works alone. Depth looks fine while one poisoned task starves the queue. Age looks fine during a flood of fresh work. +### Retained workflow rows + +`oxpull.workflows.retained_rows()` returns `RetainedRows(workflows, nodes, edges)`: three plain counts from the `oxpull` database. + +These counts cover workflows whose retirement has begun, identified by `metadata_retired_at`, and the node and edge rows still retained for them. Retained nodes include those whose task rows are FAILED or LOST and can still move through retry, and those whose outcomes were not recorded before their task rows were deleted. Edges into those nodes are retained too. + +Counts that only grow can indicate task rows that `ox_prune --include-failed` has not removed, or workflows with a node still pending. `oxpull_prune` prints these counts at the end of every run. + +This workflow API and its documented result are provisional. + ## Health checks: ox_health `ox_health` has two modes. With no flags, it checks only that the database @@ -6206,18 +7649,62 @@ The same shape reads `oldest_ready_age()` or `failure_rate()` per queue. ## Log events -The worker logs through the standard library logger named `django_ox`. No -logging dependency, no imposed format. Configure handlers and formatters in -`LOGGING` as usual. The `django_ox.testing` backends also emit -`task_policy_inert`. The `run_tasks()` helper emits worker events, -`run_tasks_timeout_inert` and `run_tasks_callback_failed` as described below. +The `django_ox` worker logs through the standard library logger named +`django_ox`. The `oxpull` worker uses `oxpull`; batch and workflow operations +also use `oxpull.batches` and `oxpull.workflows`. No logging dependency, no +imposed format. Configure handlers and formatters in `LOGGING` as usual. +The `django_ox.testing` backends also emit `task_policy_inert`. The +`run_tasks()` helper emits worker events, `run_tasks_timeout_inert` and +`run_tasks_callback_failed` as described below. Lifecycle events carry an `extra` dictionary with stable keys, so a JSON formatter that serialises record attributes gets consistent fields to index. The message text is not part of the contract. The keys are. +For task lifecycle events, `attempt` is the task's attempt count. For +`batch_complete_serialization_retry`, `workflow_complete_serialization_retry`, +`workflow_seal_serialization_retry` and `workflow_reconcile_conflict_retry`, +it is the transaction attempt number instead. + +The optional app emits 31 distinct log events. Non-workflow events are stable; every `workflow_*` event is provisional. + +An `attempt` value on `batch_add_deadlock_retry`, `batch_add_serialization_retry`, `workflow_add_deadlock_retry` or `workflow_add_serialization_retry` is a transaction attempt number, not a task execution attempt. + +A stopped reconciler sweep is not a completed sweep. Check `visited_count`, `unvisited_count` and `remaining_seconds`. If every tick stops before visiting all sealed objects, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. + | Event | Level | When | | --- | --- | --- | +| `batch_add_deadlock_retry` | WARNING | `Batch.add()` owns its transaction, lost a deadlock and is running the transaction again. Keys: `batch_id`, `task_path`, `attempt` | +| `batch_add_serialization_retry` | WARNING | `Batch.add()` owns its transaction, lost a PostgreSQL serialization failure and is running the transaction again. Keys: `batch_id`, `task_path`, `attempt` | +| `batch_complete_serialization_retry` | INFO | Batch completion encountered a serialization failure and is retrying its transaction. | +| `batch_completed` | INFO | A batch completed, with failed and unattributed member counts. | +| `batch_reconcile_failed` | ERROR | A batch could not be reconciled. Includes a traceback. | +| `batch_reconcile_stopped` | WARNING | A reconciler tick under a task timeout stopped before visiting every sealed batch. A later tick visits the rest. If every tick stops, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. Keys: `visited_count`, `unvisited_count`, `remaining_seconds` | +| `rate_limit_closed` | WARNING | Consecutive failures to count against a rate limit have closed admission under that limit until its counter accepts a write again. | +| `rate_limit_reopened` | WARNING | A rate-limit counter accepts writes again, so admission under that limit has resumed. | +| `rate_limit_row_adopted` | INFO | The database collation treats a configured limit name and a stored window-row name as equal, so the limit uses that row. | +| `rate_limit_uncounted` | WARNING | A task runs without being counted against a rate limit because counting failed or its window repeatedly rolled during counting attempts. | +| `workflow_add_deadlock_retry` | WARNING | `Workflow.add()` owns its transaction, lost a deadlock and is running the transaction again. Keys: `workflow_id`, `task_path`, `attempt` | +| `workflow_add_serialization_retry` | WARNING | `Workflow.add()` owns its transaction, lost a PostgreSQL serialization failure and is running the transaction again. Keys: `workflow_id`, `task_path`, `attempt` | +| `workflow_backstop_skipped` | INFO | The workflow backstop could not lock some rows and will try them again. | +| `workflow_complete_serialization_retry` | INFO | Workflow completion encountered a serialization failure and is retrying its transaction. | +| `workflow_completed` | INFO | A workflow completed, with failed, cancelled and unattributed node counts. | +| `workflow_nodes_cancelled` | INFO | Unreachable workflow nodes were cancelled. | +| `workflow_nodes_released` | DEBUG (worker), INFO (reconciler) | A worker released dependent workflow nodes, or the reconciler released nodes that no worker released. | +| `workflow_outcome_retry` | INFO | Recording a workflow node's outcome encountered a lock conflict and is being retried. | +| `workflow_outcome_unrecorded` | ERROR | A workflow node's outcome could not be recorded after the reported number of tries. The task row is unchanged, and the reaper will reclaim it. Includes a traceback. | +| `workflow_reconcile_conflict_retry` | INFO | A workflow transaction run by the reconciler or by `oxpull_prune` lost a deadlock or serialization failure and is running again. `conflict` is `deadlock` or `serialization_failure`. `workflow_id` is absent only for `oxpull_prune`'s scan-row cleanup. | +| `workflow_reconcile_failed` | ERROR | A workflow could not be reconciled. Includes a traceback. | +| `workflow_reconcile_stopped` | WARNING | A reconciler tick under a task timeout stopped before visiting every sealed workflow. A later tick visits the rest. If every tick stops, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. Keys: `visited_count`, `unvisited_count`, `remaining_seconds` | +| `workflow_records_repaired` | INFO | The reconciler or `oxpull_prune` recorded previously unrecorded node outcomes. `node_count` is the number of node records repaired. | +| `workflow_release_backstop_failed` | ERROR | The workflow release backstop failed. The same tick still cancels nodes and completes workflows. Includes a traceback. | +| `workflow_release_deferred` | WARNING | Workflow node task rows could not be released after repeated lock conflicts. The reconciler will retry them. | +| `workflow_release_failed` | WARNING | A worker could not release a task's dependents, or a workflow node could not be rechecked after `add()`. The reconciler will retry the release. | +| `workflow_release_skip_overflow` | WARNING | A workflow's skip queue is full, so some nodes were not queued. The reconciler's pass over waiting nodes still finds them. | +| `workflow_release_skip_unqueued` | WARNING | Workflow nodes could not be queued for the reconciler to retry. Its pass over waiting nodes still finds them. | +| `workflow_schema_check_failed` | WARNING | The worker could not check the workflow schema before starting. | +| `workflow_seal_serialization_retry` | INFO | Sealing a workflow encountered a serialization failure and is retrying its transaction. | +| `workflow_session_bounds_not_restored` | WARNING | The session's lock-wait and statement bounds could not be restored, so the connection was closed. | | `worker_started` | INFO | The run loop starts. | | `connection_pool_too_small` | WARNING | Once per `Worker.run()`, after `worker_started` and before threads start: the worker alias's effective PostgreSQL pool maximum is below `concurrency + 1`. Each `--processes` child checks separately. The warning does not resize the pool, refuse startup, measure available server slots, or reserve fallback capacity. | | `connection_pool_cannot_drain` | WARNING | Pooled PostgreSQL only. The installed psycopg_pool has no `ConnectionPool.drain()` method, which was added in 3.3.0. Emitted once per `Worker.run()`, after `worker_started` and the `connection_pool_too_small` check, before threads start. Each `--processes` child checks separately. Also emitted at the first sweep of such a pool in a process, for each database alias, unless startup already emitted it in that process. Concurrent first sweeps may emit duplicate warnings. The sweep record is the only notice available to callers of `run_once()` or `run_tasks()`. Idle connections are still tested as in django-ox 1.7.0, preserving restart recovery but also the wait on a connection that stays open without answering. A stop is not read until that test returns. Upgrade psycopg_pool to 3.3.3 or newer. This warning does not refuse startup or change the pool. No traceback. Keys: `event` and `database` always; `worker_id` only on the startup record. | @@ -6313,6 +7800,28 @@ A failed connect while recording a stuck attempt logs | Key | Present on | Meaning | | --- | --- | --- | +| `batch_id` | `batch_complete_serialization_retry`, `batch_completed`, `batch_reconcile_failed` | The batch identifier. | +| `failed_count` | `batch_completed`, `workflow_completed` | The number of failed batch members or workflow nodes. | +| `unattributed_count` | `batch_completed`, `workflow_completed` | The number of batch members or workflow nodes reported as unattributed. | +| `limit` | `rate_limit_closed`, `rate_limit_reopened`, `rate_limit_row_adopted`, `rate_limit_uncounted` | The configured rate-limit name. | +| `failures` | `rate_limit_closed` | The number of consecutive counting failures that caused admission under the limit to close. | +| `stored_as` | `rate_limit_row_adopted` | The name stored on the window row that the database collation treats as equal to the configured limit name. | +| `row_count` | `workflow_backstop_skipped` | The number of rows the workflow backstop could not lock. | +| `table` | `workflow_backstop_skipped` | The table containing the rows the workflow backstop could not lock. | +| `workflow_id` | `workflow_complete_serialization_retry`, `workflow_completed`, `workflow_nodes_cancelled`, `workflow_reconcile_failed`, `workflow_records_repaired`, `workflow_release_skip_overflow`, `workflow_seal_serialization_retry`; conditionally on `workflow_reconcile_conflict_retry` and `workflow_release_skip_unqueued` | The affected workflow. On `workflow_reconcile_conflict_retry`, it is absent only for `oxpull_prune`'s scan-row cleanup. | +| `cancelled_count` | `workflow_completed`, `workflow_nodes_cancelled` | The number of cancelled nodes reported by the event. | +| `released_count` | `workflow_nodes_released` | The number of workflow nodes released. | +| `try_number` | `workflow_outcome_retry` | The try number for recording the workflow node's outcome. | +| `dropped_status` | `workflow_outcome_unrecorded` | The outcome status that could not be recorded. | +| `tries` | `workflow_outcome_unrecorded` | The number of tries made to record the workflow node's outcome. | +| `conflict` | `workflow_reconcile_conflict_retry` | The transaction conflict being retried: `deadlock` or `serialization_failure`. | +| `node_count` | `workflow_records_repaired` | On `workflow_records_repaired`, the number of node records repaired by the reconciler or `oxpull_prune`. | +| `task_count` | `workflow_release_deferred`, `workflow_release_skip_unqueued` | For a deferred release, the upper bound on task rows that could not be released. For a queue failure, the number of nodes that could not be queued for retry. | +| `dropped_count` | `workflow_release_skip_overflow` | The number of nodes not queued because the workflow's skip queue was full. | +| `attempt` | `batch_add_deadlock_retry`, `batch_add_serialization_retry`, `workflow_add_deadlock_retry`, `workflow_add_serialization_retry` | For the four `*_add_*_retry` events, the transaction attempt number, not the task execution attempt. | +| `visited_count` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The number of sealed batches or workflows visited before the reconciler sweep stopped. | +| `unvisited_count` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The number not visited before the sweep stopped. | +| `remaining_seconds` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The remaining task-timeout budget when the reconciler stopped the sweep. | | `event` | all events | The event name from the table above. | | `worker_id` | Worker events except `heartbeat_write_failed`, `schedule_source_unavailable`, `schedule_boundary_healed`, `schedule_boundary_heal_failed`, `schedule_row_skipped` and `schedule_lock_unavailable` for a stored schedule; also `run_tasks_callback_failed`; includes `worker_claim_released` | Unique id of the worker emitting the record. With `--processes`, the slot number is the last part of the id. A Worker used in a process forked after its creation takes a new id in the child, retaining any `-` suffix; the parent keeps its id. An at-fork hook re-identifies the child. If the fork runs no at-fork hook, as with uWSGI's default, a pid check re-identifies it at its first `claim_one()`, claim, recovery look or `run()`; until then, the child's copy can still report the parent's id. Fork before claiming: handing an already-claimed row to a child for execution is unsupported, because the child's new id cannot renew the parent's lease and the reaper can requeue the row while the child runs it. This identity handling does not make arbitrary native forks safe. `ox_worker --processes` starts fresh children rather than forking them. Settings-declared `schedule_lock_unavailable` events carry this key. Records from `run_tasks()` use the drain worker's hostname-pid-random id, created for each call. `task_policy_inert` and `run_tasks_timeout_inert` omit this key. On `worker_claim_released`, identifies the worker that made the unconfirmed claim and released it. | | `worker_class` | `claim_filter_sql_missing` | The Worker subclass's class name. | @@ -6830,6 +8339,53 @@ updates `run_after`, rather than preserving its original value as proposal See [Configuration](configuration.md#per-task-policy) for validation, framework support and precedence. +### Stable: optional `oxpull` app + +The stable public surface is: + +- `oxpull.backend.OxpullBackend`. +- Its `UNIQUE_TASKS`, `RATE_LIMITS` and `RATE_REFRESH_INTERVAL` options. +- `oxpull.worker.OxpullWorker`. +- `oxpull.unique.unique`. +- `oxpull.batches.Batch` and `oxpull.batches.BatchProgress`. +- `oxpull.batches.create`, `oxpull.batches.get` and `oxpull.batches.reconcile`. +- `Batch.add`, `Batch.seal`, `Batch.progress` and `batch.id`. +- `oxpull.exceptions.UniqueEnqueueRace`. +- `oxpull.exceptions.BatchSealed`. +- `oxpull.exceptions.BatchMemberExists`. +- The `oxpull_prune` management command. +- All 16 `oxpull` check ids: `oxpull.E001` through `oxpull.E009`, and `oxpull.W001` through `oxpull.W007`. +- The `oxpull_*` schema. +- The non-workflow `oxpull` log events documented under [Log events](monitoring.md#log-events). + +`BatchProgress.pruned` is a deprecated alias of `unattributed`. Use `unattributed`; the alias emits `DeprecationWarning` and is marked for removal in 2.0. + +### Provisional: workflows + +The documented workflow surface is provisional: + +- `oxpull.workflows.create`. +- `oxpull.workflows.get`. +- `oxpull.workflows.reconcile`. +- `oxpull.workflows.retained_rows`. +- `Workflow.add`, `Workflow.seal` and `Workflow.progress`. +- `WorkflowProgress`. +- `oxpull.exceptions.WorkflowSealed`. +- `oxpull.exceptions.WorkflowNodeExists`. +- `oxpull.exceptions.UnknownWorkflowNode`. +- The `WORKFLOWS` option. +- Every `workflow_*` log event. + +The provisional tier also applies to the documented results returned by these workflow entry points. A return type's use in documentation does not make unrelated implementation helpers public. + +### Not public: workflow internals + +Workflow implementation helpers are not public API. In particular, `node_flag`, `runnable_filter_q` and `record_node_outcome` are internal, even though they appear in the module's `__all__`. + +Internal contracts between `django_ox` and `oxpull` are tested by the feature +suite in the same CI. They are not public API and are not maintained through +a separate dependency pin. + ### Not public Everything else is an implementation detail and may change in any release @@ -7006,19 +8562,24 @@ changelog, and in a security advisory where relevant. ## Supported Python and Django -Each django-ox release is tested against the matrix below in CI, on SQLite -and PostgreSQL 16 across the grid and MySQL 8 on the oldest and newest -corners; these are the supported combinations. +The `oxpull` app requires Django 6.0+. Its SQLite and PostgreSQL suites cover the Django 6.0 / 6.1 grid on `rc*` tags and in nightly runs. Its MySQL 8 suite runs at Python 3.14 / Django 6.1. + +The separate django-ox matrix continues to cover Django 5.2, and its MySQL coverage includes the oldest and newest supported corners. That broader base-app matrix is not a claim that the optional app runs on Django 5.2. | | Django 5.2 LTS | Django 6.0 | Django 6.1 | | --- | --- | --- | --- | -| **Python 3.12** | tested | tested | tested | -| **Python 3.13** | tested | tested | tested | +| **Python 3.12** | `django_ox` tested; `oxpull` not supported | tested | tested | +| **Python 3.13** | `django_ox` tested; `oxpull` not supported | tested | tested | | **Python 3.14** | not supported by Django 5.2 | tested | tested | Django 6.0 and later ship the Tasks framework in core. The Django 5.2 legs -install the `django-tasks` backport and run the whole suite against it, which -is what `django-ox[backport]` pulls in. +install the `django-tasks` backport and run the `django_ox` suite against it, +which is what `django-ox[backport]` pulls in. Those legs do not run the +`oxpull` feature suite. + +`oxpull` is not supported on Django 5.2. It raises `ImproperlyConfigured` +during application startup, before system checks can run. Installing the +backport does not enable `oxpull` on Django 5.2. Support for per-task policy declarations differs from support for the backend itself: @@ -7089,153 +8650,20 @@ part of the tested matrix. --- -# Source: https://oxpull.com/django-ox/pro/ - -# Pro - -django-ox is free and open source (BSD 3-Clause). The durable queue, -transactional enqueue, retries, reaper, graceful drain, priorities, deferred -tasks, recurring tasks and pruning are in the free package. - -**Oxpull Pro** is a paid add-on for four problems that show up once a queue -is carrying real volume. All four are built and tested. has the details. - -Pro requires Django 6.0 or later. It builds on `django.tasks`, which is part -of Django core from 6.0. django-ox itself also runs on Django 5.2 LTS through -the `django-tasks` backport; Pro does not. - -## What Pro adds - -- **Unique tasks.** Deduplicate at enqueue time: while a task with the same - key is READY or RUNNING, enqueueing it again returns the pending task's - result rather than inserting a second row. The lock is written in the - same transaction as the task, so the two commit or roll back together, - and a lock whose task settled or was pruned is released rather than - stranded. - A bulk enqueue is one INSERT that the deduplicating path never sees, so - `enqueue_many()` raises `InvalidTask` for a unique task and names it. -- **Batches.** Enqueue a group, read its progress as a count, and fire a - callback once every member has settled. Completion is computed by querying - the task rows rather than by counting signals, so a worker dying mid-task - cannot strand a batch: the reconciler picks it up on the next tick. - Complete is a reading of the task rows, not proof that every member has - stopped running. - Batches take one setting: `oxpull.batches.reconcile` on a one-minute cron - in `OPTIONS["SCHEDULES"]`, run by the django-ox cron you already have. -- **Rate limiting.** Cap how often a task starts. A named limit of N - admissions per period is declared in `OPTIONS["RATE_LIMITS"]`, and every - worker shares it through one row in the database. A throttled task stays - READY: it is not claimed, so it spends no retry attempt and holds no - lease. Set `OPTIONS["WORKER_CLASS"]` to `oxpull.worker.OxpullWorker`, the - worker that applies the limits. `manage.py check` reports `oxpull.E006` - when a limit is configured without it. - - For a limit of N admissions per period P, with W `ox_worker` processes - claiming from the queues that carry the limit's tasks, the number of - task attempts started in any one window is at most N + W - 1 + U. U is - the number of admissions whose count did not land. Each one logs - `rate_limit_uncounted`, so U is the number of times that event fired. - Add one for each worker process that dies between claiming a limited - task and recording it. With a single worker process, no such death and - every count landed, the limiter is exact: at most N per window. Windows - are contiguous, so an arbitrary interval of length P that spans a - boundary can carry up to 2(N + W - 1 + U). A count that cannot be - written is logged and the attempt runs; three uncounted admissions in a - row on one limit close it until a write lands. -- **Workflows.** Declare a set of tasks with dependencies. A node runs only - after every node it depends on has succeeded. The backend declares - `OPTIONS["WORKFLOWS"]`, sets `OPTIONS["WORKER_CLASS"]` to - `oxpull.worker.OxpullWorker`, and schedules `oxpull.workflows.reconcile`. - Without that schedule, `create()` and `seal()` refuse, and - `manage.py check` reports `oxpull.W003`. A reconciler exposed to - schedule rows on a queue no backend accepts reports `oxpull.E008` - instead. Workflows need Oxpull Pro 1.3.0 or later. - -Each Pro release pins django-ox exactly: `oxpull==1.8.0` pins -`django-ox==1.8.0`. - -Pro runs on the databases the free tier tests in CI: SQLite, -PostgreSQL and MySQL 8. MariaDB 10.6+ takes the same claim path but is not -part of the tested matrix. Batches have been measured to 1,000,000 members -in a single batch on all three, with every count checked against the task -rows rather than against what the API reports about itself. - -Sealing a batch that wide from inside a caller's transaction takes about -34 ms on SQLite, 0.7 s on PostgreSQL and 28 s on MySQL. That path counts -with a row lock per member wherever the database offers one, and a -million InnoDB locks in one transaction is where the MySQL figure comes -from. Called outside a transaction, `seal()` counts with a single -`COUNT(*)` instead. Timing those two statements alone against a -million-member batch with fresh statistics, the locking read took 12 s on -MySQL and 0.55 s on PostgreSQL, and the `COUNT(*)` 0.7 s on MySQL and -0.03 s on PostgreSQL. The whole `seal()` call on the same harness and the -same batch, outside a transaction, took 0.74 s on MySQL and 0.03 s on -PostgreSQL. -Reconciling the same batch is 4.9 s on SQLite, 0.4 s on PostgreSQL and -7.3 s on MySQL, against a reconciler that runs on a one-minute tick. - -Measured on an arm64 macOS host, with SQLite on a local file in WAL mode -at `synchronous=NORMAL`, and stock PostgreSQL 16 and MySQL 8 images in -Docker on localhost, each width against an empty database. - -## What Pro is not - -Chains are on the roadmap, undated. Rate limiting caps how -often a task starts, not how many run at once; concurrency limiting is a -different mechanism and is not in Pro. Metrics stay free: the stats API and -the health command are in the open source package and remain there. - -## Delivery - -Pro installs from a private package index using credentials issued per -company. There is no licence key and no runtime check. A licence check would -put a validation step in the path of code that has to keep running, and it -does nothing for a company that has already paid. Nothing in the package -phones home: no network call to Oxpull, and no telemetry. The credential controls -access to the index rather than to code you have already installed, so if it -lapses, what is deployed keeps running. - -## Pricing - -**$399 per year, per company**, excluding VAT where it applies. One licence -covers a whole organisation and every environment. The term is 12 months and -renews for successive 12-month terms unless you cancel. Cancel by writing to -support@oxpull.com. Cancellation takes effect at the end of the period you -have paid for. The full terms are stated at purchase. - -## Ordering - -Order at [oxpull.com](https://oxpull.com/#order) with your company name, a -contact name, an email address, the billing address and, if you have one, a -VAT ID for the invoice. The invoice goes out within two business days. When it -is paid, the credentials for the private package index arrive by email, and -install is `pip install`. Questions to support@oxpull.com. - -[Order Pro](https://oxpull.com/#order){ .md-button } - ---- - # Source: https://oxpull.com/django-ox/agents/ # For AI assistants -Exact steps for setting up django-ox in an existing Django project, the -facts to get right while doing it, and how to prove it works. - -**django-ox 1.8.0, released 2026-10-04:** create many stored schedules in -one all-or-nothing call, use the rebuilt django-celery-beat database importer, -and keep other schedules and queued tasks running when a stored row is invalid. -See [Upgrade to 1.8.0](#upgrade-to-180). -**18 releases in 49 days; public API stable since 1.0.0 (September 2026).** -Explore the [test suite, contributors and project history](#compatibility-and-project-history). - Run Django background tasks in the database your project already uses: transactional enqueue, retries, dead-worker recovery, recurring schedules and monitoring, without a separate broker. +Exact steps for setting up django-ox in an existing Django project, the +facts to get right while doing it, and how to prove it works. + ## Set up django-ox in this project -Requires Python 3.12+ and Django 5.2+. Check before installing: +Requires Python 3.12+ and Django 5.2+. The optional `oxpull` app requires Django 6.0+. Check before installing: ``` python -c "import django, sys; print(django.__version__, sys.version.split()[0])" @@ -7330,6 +8758,16 @@ TASKS = { } ``` +## Enable the optional app + +The `oxpull` app is included in django-ox and requires Django 6.0+. Add `"oxpull"` to `INSTALLED_APPS`, run migrations, and set the relevant alias's `BACKEND` to `"oxpull.backend.OxpullBackend"`. + +Unique tasks use `@unique` or `OPTIONS["UNIQUE_TASKS"]`. Batches require a reconciler schedule. + +For rate limits or workflows, set that alias's `OPTIONS["WORKER_CLASS"]` to `"oxpull.worker.OxpullWorker"`. Rate limits also need `OPTIONS["RATE_LIMITS"]`. Workflows also need `OPTIONS["WORKFLOWS"] = True` and a reconciler schedule. + +Follow the [feature setup](features/index.md) and the relevant feature page. Do not treat the backend or worker setting alone as complete configuration. + ## Upgrade to 1.8.0 Create many stored schedules in one all-or-nothing call, or bring schedules @@ -7544,10 +8982,10 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv WAITING, FAILED or LOST task without running it. Neither touches a RUNNING task. With `django.contrib.admin` installed, the task table appears in the admin with the same two actions. -- `OxTask.Status.WAITING` is a status django-ox never writes itself. - Workers never claim a waiting task, `ox_prune` never deletes it, and - `retry` skips it. It reads as READY through `django.tasks`. Workflows in - Oxpull Pro write and release it. +- `OxTask.Status.WAITING` is used by workflows in the optional `oxpull` + app. Workers never claim a waiting task, `ox_prune` never deletes it, + and `retry` skips it. It reads as READY through `django.tasks`. + Workflows write and release this status. - A particular running task cannot be interrupted on demand. Each attempt can have a task, queue or backend execution timeout. Every table lives on the database your router sends `OxTask` to. @@ -7556,11 +8994,12 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv validate `PolicyTask` fields on supported Django versions. Neither retries, calls backoff callbacks or enforces timeouts. Immediate runs even if the enclosing transaction later rolls back, and rejects - `run_after`. For queued execution, keep `OxBackend` and use the public, - provisional `django_ox.testing.run_tasks()` helper. It runs due attempts - through the configured worker class, including retries and backoff. - Pass `backend=` for the worker class you need, including the Pro alias - for workflows and rate limits. Rows are selected by queue, regardless + `run_after`. For queued execution, keep `OxBackend` or `OxpullBackend` + and use the public, provisional `django_ox.testing.run_tasks()` helper. + It runs due attempts through the configured worker class, including + retries and backoff. For workflows and rate limits, pass `backend=` + with the alias whose `OPTIONS["WORKER_CLASS"]` is + `"oxpull.worker.OxpullWorker"`. Rows are selected by queue, regardless of which backend enqueued them. `TestCase` uses savepoints and commit callback emulation. Use `TransactionTestCase` to test worker autocommit behaviour. Exit caller callback-capture blocks before draining tasks @@ -7569,10 +9008,13 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv Test timeout enforcement and worker infrastructure against a real worker. See [Run queued tasks in tests](patterns.md#run-queued-tasks-in-tests) for transaction and callback differences. -- Batches, unique tasks, rate limiting and workflows are in - [Oxpull Pro](pro.md), a paid add-on requiring Django 6.0+. - `create_schedules` creates stored schedules in the free package. - `django_ox.stats` and `ox_health` are also in django-ox. +- [Unique tasks, batches, rate limiting and workflows](features/index.md) + are included in django-ox through the optional `oxpull` app and require + Django 6.0+. `OxpullBackend`, `OxpullWorker` and the documented + unique-task, batch and rate-limit entry points are stable. + Workflows are provisional. +- `create_schedules` creates stored schedules. It, `django_ox.stats` and + `ox_health` do not require the optional app. ## How to verify it works @@ -7616,21 +9058,13 @@ TaskResultStatus.SUCCESSFUL `python manage.py ox_health --worker-timeout 60` now exits 0 and reports a recent `last_claim_age`. Stop the worker with Ctrl-C; it drains and exits 0. -## Compatibility and project history +## Compatibility -django-ox supports Python 3.12-3.14 and Django 5.2 LTS through 6.1. -For Django 5.2, install the backport extra. -PostgreSQL, MySQL 8 and SQLite are in CI. The licence is BSD 3-Clause. +django-ox 1.9.0 supports Django 5.2 LTS through the `django-tasks` backport and Django 6.0 / 6.1 through core `django.tasks`. It requires Python 3.12+. CI covers Python 3.12-3.14, with PostgreSQL, MySQL 8 and SQLite. -As of 2026-10-04, django-ox has shipped **18 releases in 49 days**. -Its **public API has been stable since 1.0.0 (September 2026)**. +The optional `oxpull` app is included in the distribution and requires Django 6.0+. Projects that do not enable it can continue using django-ox on Django 5.2. -- **All 36 checks passed on release PR #124**, including test-suite legs - on PostgreSQL, MySQL and SQLite, plus Wagtail smoke checks. -- **3,761 test cases in the 1.8.0 suite.** -- **22 merged pull requests from 13 outside contributors.** -- Featured in PyCoder's Weekly #752 (2026-09-15). - Listed in awesome-django under Task Queues. +For existing `oxpull` deployments, follow [Earlier `oxpull` installs](migrating.md#earlier-oxpull-installs). Machine-readable copies: [llms.txt](llms.txt) contains facts and links; [llms-full.txt](llms-full.txt) contains every page of this site in one file. @@ -7667,8 +9101,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 **Upgrading:** No database migration is required. +### Upgrading earlier `oxpull` installs + +If you installed `oxpull` separately, stop the affected processes and back up the database. Uninstall the earlier `oxpull` distribution before upgrading django-ox, then run migrations and restart. + +Do not uninstall an earlier `oxpull` distribution after installing django-ox 1.9.0 as a standalone cleanup step: its uninstall record can delete files now supplied by django-ox. The module-free 1.9.0 metapackage does not have that behavior. + +If the distributions were installed in the wrong order, remove any remaining earlier `oxpull` distribution and then force-reinstall django-ox. See [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs) for the complete sequence and repair commands. + +### Changed + +- The `oxpull` distribution on PyPI is now a metapackage at version 1.9.0. + It supplies no Python module and depends on `django-ox>=1.9.0`. + The django-ox distribution supplies the `oxpull` module. + ### Added +- Include the optional `oxpull` app in the django-ox wheel and source distribution under the BSD-3-Clause licence. It provides: + - Unique tasks, which deduplicate pending tasks at enqueue time. + - Batches. + - Rate limiting. + - Provisional workflows. +- Include the app's five migrations and `py.typed` marker. No separate `oxpull` installation is needed to use the app. + - Log `worker_connection_hint` once at INFO during `Worker.run()` startup on PostgreSQL without pooling and on MySQL when `CONN_MAX_AGE = 0`. The `django_ox` record includes `worker_id`, `database` and `vendor`. @@ -7685,6 +9140,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Compatibility +- django-ox continues to support Django 5.2 through the `django-tasks` backport. +- The optional `oxpull` app requires Django 6.0+. Enabling the app or importing its runtime entry points on Django 5.2 raises `ImproperlyConfigured`. +- Projects that do not enable the optional app do not load it. +- Existing settings, import paths, task data, the app label, table names, the five `oxpull` migrations and persisted task paths are unchanged. +- `oxpull.__version__` now equals `django_ox.__version__`. +- The app's display name is now "Oxpull". +- The `oxpull` app now enforces its existing Django 6.0+ requirement at load time, raising `ImproperlyConfigured` on Django 5.2; django-ox itself still installs there. The earlier separate `oxpull` 1.8.0 package already declared Django>=6.0 in its metadata. Apart from this load-time check, the app's runtime behavior is unchanged from `oxpull` 1.8.0. One configuration-error message has also been reworded. + - Default connection lifecycle, transaction, delivery and schema semantics are unchanged. Persistence and pooling remain opt-in. Workers with and without persistence shared a queue in tests using @@ -8231,8 +9694,8 @@ This release preserves the caller's transaction when `run_once()` times out and **Upgrading to 1.5.0:** No database migration is required. Replace every worker to apply schedule isolation, dead-connection outcome recovery and the timeout-watchdog fix throughout the fleet. Schedule isolation adds no system -check. When using Oxpull, deploy django-ox and Oxpull only as the tested -exact-pinned release pair. +check. For this historical release, installations using `oxpull` required +the matching tested versions of both distributions, with exact version pins. If you used `ox_import_beat_schedules`, read the [Security entry](#security) before generating or applying output. Upgrade and regenerate saved output. @@ -8517,8 +9980,8 @@ connections outside the pool, in addition to `max_size`. Task timeouts (`TASK_TI another connection outside the pool. Each worker process can hold up to `max_size + 1` server connections, or `max_size + 2` with task timeouts. -No migration. For `oxpull` installations, use `oxpull==1.4.0`, -which pins `django-ox==1.4.0`. +No migration. For this historical release, installations using `oxpull` +required `oxpull` 1.4.0, whose dependency required django-ox 1.4.0 exactly. ### PostgreSQL pool correction @@ -8678,13 +10141,12 @@ No migration. Upgrade is optional. - Update the benchmarks page with measurements against django-tasks-db 0.13.0 and recovery after worker death. - Update the comparison page and correct the huey entry. -- Document how to order Oxpull Pro. ## [1.3.0] - 2026-09-17 **One migration ships with this release.** `0008_waiting` adds a status choice and runs no SQL. django-ox never puts a task into the new status by -itself. An install that doesn't use workflows in Oxpull Pro upgrades and +itself. An install that doesn't use workflows in `oxpull` upgrades and rolls back as it always has. A 1.2 process doesn't know the new status. Its admin shows a waiting task's @@ -8698,12 +10160,10 @@ one. row whose status is `WAITING`. No worker claims it, the reaper doesn't reap it, `ox_prune` doesn't delete it, and retry skips it. django-ox writes that status onto no row of its own, and releases no row that holds it. Something -built on django-ox does both, and on this release that is workflows in Oxpull -Pro. +built on django-ox does both, and in this release that is workflows in +`oxpull`. -Turning workflows on, and turning them off again, is Oxpull Pro configuration, -and Oxpull's own documentation has those steps. This section is the django-ox -half. See [Pro](https://oxpull.com/django-ox/pro/). +Before turning workflows off, follow the documented shutdown procedure so that WAITING nodes are not left without a worker that can release them. See [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs) and the [workflow documentation](https://oxpull.com/django-ox/features/workflows/). `migrate django_ox 0007` refuses while any task is WAITING on the database it migrates. A version from before `0008_waiting` can't read a waiting task and @@ -9743,9 +11203,8 @@ appears on the package page. - Packaging metadata now carries a `Documentation` URL, so the documentation site is linked directly from the package page. - README now carries release and CI status badges, a link to the - documentation site, and a scope statement: what the core covers, what is - deliberately outside it, and which features belong to the commercial - tier. + documentation site, and a scope statement distinguishing the core + features from those provided by `oxpull`. ## [0.1.0] - 2026-08-16 diff --git a/docs/llms.txt b/docs/llms.txt index eadd0a5..2f12316 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -8,7 +8,10 @@ > use Django 6.1 or Django 5.2 with django-tasks 0.12+; Django 6.0 supports > bare tasks. Import schedules from django-celery-beat's database scheduler > tables. Your Django database handles the queue. BSD 3-Clause. -> Package name on PyPI: django-ox. App label: django_ox. +> django-ox 1.9.0 also includes unique tasks, batches, rate limiting and +> provisional workflows through the optional `oxpull` app, on Django 6.0+. +> Package name on PyPI: django-ox. Main app label: django_ox. +> Optional app label: oxpull. **Current release: django-ox 1.8.0, uploaded to PyPI on 2026-10-04.** @@ -286,10 +289,10 @@ READY, WAITING, FAILED or LOST task without running it. Both return a bool, and neither touches a RUNNING task. With `django.contrib.admin` installed the task table is registered in the admin with the same two actions. -- `OxTask.Status.WAITING` is a status django-ox never writes itself. - Workers never claim a waiting task, `ox_prune` never deletes it, and - `retry` skips it. It reads as READY through `django.tasks`. Workflows in - Oxpull Pro write and release it. +- `OxTask.Status.WAITING` is used by workflows in the optional `oxpull` + app. Workers never claim a waiting task, `ox_prune` never deletes it, + and `retry` skips it. It reads as READY through `django.tasks`. + Workflows write and release this status. - Run `migrate` before rolling workers, not from the worker process. - Out of scope in the core: interrupting one chosen running task on demand (every attempt is unbounded by default and can have a configured deadline @@ -321,26 +324,23 @@ between calls for delayed retries. No timeouts, lease renewal, schedules or reconcilers run automatically. Use `TransactionTestCase` to test worker autocommit behaviour and a real worker for infrastructure. -- Batches, unique (deduplicated) tasks, rate limiting and workflows are in - Oxpull Pro, a paid add-on requiring Django 6.0+; - https://oxpull.com/ has the details. - `create_schedules` creates stored schedules in the free package. - `django_ox.stats` and `ox_health` are also in the free package. +- Unique tasks, batches, rate limiting and workflows are included in + django-ox through the optional `oxpull` app and require Django 6.0+. + `OxpullBackend`, `OxpullWorker` and the documented unique-task, batch + and rate-limit entry points are stable. Workflows are provisional. + Setup: https://oxpull.com/django-ox/features/ + `create_schedules` creates stored schedules. It, `django_ox.stats` and + `ox_health` do not require the optional app. -**Compatibility and project history** +**Compatibility** -Supported versions: Python 3.12-3.14 and Django 5.2 LTS through 6.1. +django-ox 1.9.0 supports Python 3.12-3.14 and Django 5.2 LTS through 6.1. For Django 5.2, install the backport extra. PostgreSQL, MySQL 8 and SQLite are in CI. -As of 2026-10-04: -- 18 releases in 49 days. Public API stable since 1.0.0 (September 2026). -- All 36 checks passed on release PR #124, including test-suite legs on - PostgreSQL, MySQL and SQLite, plus Wagtail smoke checks. -- 3,761 test cases in the 1.8.0 suite. -- 22 merged pull requests from 13 outside contributors. -- Featured in PyCoder's Weekly #752 (2026-09-15). -- Listed in awesome-django under Task Queues. +The optional `oxpull` app requires Django 6.0+. +For earlier `oxpull` installations, follow: +https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs Use Context7 library id `/oxpull/django-ox` to look up the documentation. @@ -353,16 +353,20 @@ Use Context7 library id `/oxpull/django-ox` to look up the documentation. - [Recurring tasks](https://oxpull.com/django-ox/recurring-tasks/): schedule keys, cron syntax, many workers sharing a tick and missed ticks. - [Schedules in the database](https://oxpull.com/django-ox/stored-schedules/): admin-editable schedules, schedulable keys, create_schedules and the django-celery-beat database importer. - [Common patterns](https://oxpull.com/django-ox/patterns/): transactional enqueue, retries, deferral, bulk enqueue, queue routing, idempotency and tests. -- [Migrating](https://oxpull.com/django-ox/migrating/): moving from another Django Tasks backend, Celery or huey, and migrating away. +- [Migrating](https://oxpull.com/django-ox/migrating/): moving from another Django Tasks backend, Celery or huey, upgrading earlier `oxpull` installs, and migrating away. - [Why django-ox](https://oxpull.com/django-ox/why-django-ox/): requirements, alternatives, limits and evaluation questions. -- [Use cases](https://oxpull.com/django-ox/use-cases/): recovery, broker-free tasks, transactional enqueue, recurring tasks and a paid Pro case. +- [Use cases](https://oxpull.com/django-ox/use-cases/): recovery, broker-free tasks, transactional enqueue, recurring tasks and deduplicating repeated webhooks. - [Choosing a backend](https://oxpull.com/django-ox/choosing/): sourced backend comparisons and when not to use django-ox. - [Production](https://oxpull.com/django-ox/production/): supervision, containers, shutdown, scaling, leases, databases and pruning. - [Monitoring](https://oxpull.com/django-ox/monitoring/): stats, health probes, Prometheus metrics, log events, retry, discard and admin. - [Benchmarks](https://oxpull.com/django-ox/benchmarks/): throughput and latency measurements with their methods. - [API stability](https://oxpull.com/django-ox/stability/): public API, versioning, deprecation policy and supported versions. -- [Changelog](https://oxpull.com/django-ox/changelog/): django-ox 1.8.0 additions, fixes and security changes, plus earlier releases. -- [Pro](https://oxpull.com/django-ox/pro/): what Oxpull Pro adds, delivery and pricing. +- [Changelog](https://oxpull.com/django-ox/changelog/): django-ox 1.9.0 changes and earlier releases. +- [Features](https://oxpull.com/django-ox/features/): optional `oxpull` app setup, Django 6.0+ requirements and API stability. +- [Unique tasks](https://oxpull.com/django-ox/features/unique-tasks/): deduplicate pending work by key and understand when another enqueue creates a task. +- [Batches](https://oxpull.com/django-ox/features/batches/): group tasks, track progress and run a callback when the batch settles. +- [Rate limiting](https://oxpull.com/django-ox/features/rate-limiting/): shared limits on task starts, worker configuration and the distinction from concurrency limits. +- [Workflows](https://oxpull.com/django-ox/features/workflows/): provisional dependency DAGs, worker configuration and workflow execution. - [For AI assistants](https://oxpull.com/django-ox/agents/): exact setup and verification steps, upgrade guidance and operational facts. ## Optional diff --git a/docs/migrating.md b/docs/migrating.md index 1510494..49f6707 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -139,10 +139,12 @@ Here the enqueue is an `INSERT` on the database that holds `OxTask`, your default one. Open the transaction there and the task commits or rolls back with the row it belongs to, so there is nothing to wrap. -Queues, priorities and `run_after` map directly. Celery's chains, -groups and chords, and routing across multiple brokers, are outside the -package; workflows are in [Oxpull Pro](pro.md), a paid add-on, and chains -are on its roadmap, undated. +Queues, priorities and `run_after` map directly. django-ox does not +implement Celery's chain, group or chord APIs, or routing across multiple +brokers. [Batches](features/batches.md) provide grouped work, and +provisional [workflows](features/workflows.md) provide dependency DAGs. +Both require Django 6.0+ and the optional `oxpull` app. Review their +semantics rather than treating them as drop-in Celery replacements. ## From huey @@ -208,6 +210,78 @@ the admin today, see [Schedules in the database](stored-schedules.md). | `retries`, `retry_delay` | Backend defaults in `MAX_ATTEMPTS` and the backoff options, or per-task `max_attempts` and `backoff`. django-ox counts total claims, including the first. Per-task declarations require Django 6.1, or Django 5.2 with django-tasks 0.12+. | | `huey.immediate` in tests | Use `django_ox.testing.ImmediateBackend` to run once at enqueue, or `DummyBackend` to record enqueues. Both accept policy declarations without enforcing retries, backoff or timeouts. For queued execution, keep `OxBackend` and use the public, provisional [`run_tasks()`](patterns.md#run-queued-tasks-in-tests) helper. It runs due attempts with retry and backoff policy. Test timeout enforcement against a real worker. | +## Earlier `oxpull` installs + +The optional `oxpull` app is included in django-ox 1.9.0 and later. You no longer need a separately installed `oxpull` distribution. + +The PyPI `oxpull` 1.9.0 distribution is a compatibility metapackage. It depends on django-ox 1.9.0 or later and contains no Python module. The importable package comes from django-ox. + +The former private package index is retired. Remove it and its credentials from installer configuration. + +### Upgrade an existing installation + +The order matters. + +1. Stop every affected worker and other process that enqueues or claims tasks. + +2. Back up the database. + +3. Uninstall the earlier `oxpull` distribution before upgrading django-ox. + + ```console + python -m pip uninstall oxpull + ``` + +4. Remove the former private index and its credentials from your pip configuration, requirements and deployment secrets. + +5. Upgrade django-ox. + + ```console + python -m pip install -U django-ox + ``` + +6. Run migrations and check the installation. + + ```console + python manage.py migrate + python -m pip check + ``` + + For the verified oxpull 1.8.0 schema, the migration run reports "No migrations to apply." Earlier schemas may have migrations to apply. + +7. Restart the affected processes with the updated installation and existing application settings. + +Keep the existing `INSTALLED_APPS` and `TASKS` configuration. Import paths, task data, the `oxpull` app label, table names, the five migrations and persisted task paths are unchanged. + +The optional app requires Django 6.0+. django-ox without that app continues to support Django 5.2. + +### Repair an overlapping installation + +If you ran `pip install -U oxpull` or `pip install -U django-ox` before removing an earlier `oxpull` distribution, inspect and repair the installation before restarting processes. + +If the earlier distribution is still installed, remove it and then restore django-ox's files: + +```console +python -m pip uninstall oxpull +python -m pip install --force-reinstall --no-deps django-ox +python -m pip check +``` + +Do both steps even if imports still worked before the uninstall. The old distribution can leave stale metadata that makes `pip check` fail, and uninstalling it can remove files now supplied by django-ox. + +If you already uninstalled the earlier distribution, restore the files with: + +```console +python -m pip install --force-reinstall --no-deps django-ox +python -m pip check +``` + +The same repair applies if uninstalling the old 0.0.1 stub removed `oxpull/__init__.py`, leaving a namespace import without `oxpull.__version__`. + +These package-file failures do not alter the database data. Run migrations and complete the normal checks before restarting. + +This shared-file warning concerns earlier distributions that installed an `oxpull` module. Uninstalling the module-free 1.9.0 metapackage does not remove django-ox's module files. + ## Switching over The two systems use different tables. Neither reads the other's rows. So if you diff --git a/docs/monitoring.md b/docs/monitoring.md index 7a51d3d..7b82f73 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -61,6 +61,16 @@ functions. (`oldest_ready_age`).** Neither works alone. Depth looks fine while one poisoned task starves the queue. Age looks fine during a flood of fresh work. +### Retained workflow rows + +`oxpull.workflows.retained_rows()` returns `RetainedRows(workflows, nodes, edges)`: three plain counts from the `oxpull` database. + +These counts cover workflows whose retirement has begun, identified by `metadata_retired_at`, and the node and edge rows still retained for them. Retained nodes include those whose task rows are FAILED or LOST and can still move through retry, and those whose outcomes were not recorded before their task rows were deleted. Edges into those nodes are retained too. + +Counts that only grow can indicate task rows that `ox_prune --include-failed` has not removed, or workflows with a node still pending. `oxpull_prune` prints these counts at the end of every run. + +This workflow API and its documented result are provisional. + ## Health checks: ox_health `ox_health` has two modes. With no flags, it checks only that the database @@ -654,18 +664,62 @@ The same shape reads `oldest_ready_age()` or `failure_rate()` per queue. ## Log events -The worker logs through the standard library logger named `django_ox`. No -logging dependency, no imposed format. Configure handlers and formatters in -`LOGGING` as usual. The `django_ox.testing` backends also emit -`task_policy_inert`. The `run_tasks()` helper emits worker events, -`run_tasks_timeout_inert` and `run_tasks_callback_failed` as described below. +The `django_ox` worker logs through the standard library logger named +`django_ox`. The `oxpull` worker uses `oxpull`; batch and workflow operations +also use `oxpull.batches` and `oxpull.workflows`. No logging dependency, no +imposed format. Configure handlers and formatters in `LOGGING` as usual. +The `django_ox.testing` backends also emit `task_policy_inert`. The +`run_tasks()` helper emits worker events, `run_tasks_timeout_inert` and +`run_tasks_callback_failed` as described below. Lifecycle events carry an `extra` dictionary with stable keys, so a JSON formatter that serialises record attributes gets consistent fields to index. The message text is not part of the contract. The keys are. +For task lifecycle events, `attempt` is the task's attempt count. For +`batch_complete_serialization_retry`, `workflow_complete_serialization_retry`, +`workflow_seal_serialization_retry` and `workflow_reconcile_conflict_retry`, +it is the transaction attempt number instead. + +The optional app emits 31 distinct log events. Non-workflow events are stable; every `workflow_*` event is provisional. + +An `attempt` value on `batch_add_deadlock_retry`, `batch_add_serialization_retry`, `workflow_add_deadlock_retry` or `workflow_add_serialization_retry` is a transaction attempt number, not a task execution attempt. + +A stopped reconciler sweep is not a completed sweep. Check `visited_count`, `unvisited_count` and `remaining_seconds`. If every tick stops before visiting all sealed objects, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. + | Event | Level | When | | --- | --- | --- | +| `batch_add_deadlock_retry` | WARNING | `Batch.add()` owns its transaction, lost a deadlock and is running the transaction again. Keys: `batch_id`, `task_path`, `attempt` | +| `batch_add_serialization_retry` | WARNING | `Batch.add()` owns its transaction, lost a PostgreSQL serialization failure and is running the transaction again. Keys: `batch_id`, `task_path`, `attempt` | +| `batch_complete_serialization_retry` | INFO | Batch completion encountered a serialization failure and is retrying its transaction. | +| `batch_completed` | INFO | A batch completed, with failed and unattributed member counts. | +| `batch_reconcile_failed` | ERROR | A batch could not be reconciled. Includes a traceback. | +| `batch_reconcile_stopped` | WARNING | A reconciler tick under a task timeout stopped before visiting every sealed batch. A later tick visits the rest. If every tick stops, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. Keys: `visited_count`, `unvisited_count`, `remaining_seconds` | +| `rate_limit_closed` | WARNING | Consecutive failures to count against a rate limit have closed admission under that limit until its counter accepts a write again. | +| `rate_limit_reopened` | WARNING | A rate-limit counter accepts writes again, so admission under that limit has resumed. | +| `rate_limit_row_adopted` | INFO | The database collation treats a configured limit name and a stored window-row name as equal, so the limit uses that row. | +| `rate_limit_uncounted` | WARNING | A task runs without being counted against a rate limit because counting failed or its window repeatedly rolled during counting attempts. | +| `workflow_add_deadlock_retry` | WARNING | `Workflow.add()` owns its transaction, lost a deadlock and is running the transaction again. Keys: `workflow_id`, `task_path`, `attempt` | +| `workflow_add_serialization_retry` | WARNING | `Workflow.add()` owns its transaction, lost a PostgreSQL serialization failure and is running the transaction again. Keys: `workflow_id`, `task_path`, `attempt` | +| `workflow_backstop_skipped` | INFO | The workflow backstop could not lock some rows and will try them again. | +| `workflow_complete_serialization_retry` | INFO | Workflow completion encountered a serialization failure and is retrying its transaction. | +| `workflow_completed` | INFO | A workflow completed, with failed, cancelled and unattributed node counts. | +| `workflow_nodes_cancelled` | INFO | Unreachable workflow nodes were cancelled. | +| `workflow_nodes_released` | DEBUG (worker), INFO (reconciler) | A worker released dependent workflow nodes, or the reconciler released nodes that no worker released. | +| `workflow_outcome_retry` | INFO | Recording a workflow node's outcome encountered a lock conflict and is being retried. | +| `workflow_outcome_unrecorded` | ERROR | A workflow node's outcome could not be recorded after the reported number of tries. The task row is unchanged, and the reaper will reclaim it. Includes a traceback. | +| `workflow_reconcile_conflict_retry` | INFO | A workflow transaction run by the reconciler or by `oxpull_prune` lost a deadlock or serialization failure and is running again. `conflict` is `deadlock` or `serialization_failure`. `workflow_id` is absent only for `oxpull_prune`'s scan-row cleanup. | +| `workflow_reconcile_failed` | ERROR | A workflow could not be reconciled. Includes a traceback. | +| `workflow_reconcile_stopped` | WARNING | A reconciler tick under a task timeout stopped before visiting every sealed workflow. A later tick visits the rest. If every tick stops, give the reconciler's queue a longer `TASK_TIMEOUTS` entry. Keys: `visited_count`, `unvisited_count`, `remaining_seconds` | +| `workflow_records_repaired` | INFO | The reconciler or `oxpull_prune` recorded previously unrecorded node outcomes. `node_count` is the number of node records repaired. | +| `workflow_release_backstop_failed` | ERROR | The workflow release backstop failed. The same tick still cancels nodes and completes workflows. Includes a traceback. | +| `workflow_release_deferred` | WARNING | Workflow node task rows could not be released after repeated lock conflicts. The reconciler will retry them. | +| `workflow_release_failed` | WARNING | A worker could not release a task's dependents, or a workflow node could not be rechecked after `add()`. The reconciler will retry the release. | +| `workflow_release_skip_overflow` | WARNING | A workflow's skip queue is full, so some nodes were not queued. The reconciler's pass over waiting nodes still finds them. | +| `workflow_release_skip_unqueued` | WARNING | Workflow nodes could not be queued for the reconciler to retry. Its pass over waiting nodes still finds them. | +| `workflow_schema_check_failed` | WARNING | The worker could not check the workflow schema before starting. | +| `workflow_seal_serialization_retry` | INFO | Sealing a workflow encountered a serialization failure and is retrying its transaction. | +| `workflow_session_bounds_not_restored` | WARNING | The session's lock-wait and statement bounds could not be restored, so the connection was closed. | | `worker_started` | INFO | The run loop starts. | | `connection_pool_too_small` | WARNING | Once per `Worker.run()`, after `worker_started` and before threads start: the worker alias's effective PostgreSQL pool maximum is below `concurrency + 1`. Each `--processes` child checks separately. The warning does not resize the pool, refuse startup, measure available server slots, or reserve fallback capacity. | | `connection_pool_cannot_drain` | WARNING | Pooled PostgreSQL only. The installed psycopg_pool has no `ConnectionPool.drain()` method, which was added in 3.3.0. Emitted once per `Worker.run()`, after `worker_started` and the `connection_pool_too_small` check, before threads start. Each `--processes` child checks separately. Also emitted at the first sweep of such a pool in a process, for each database alias, unless startup already emitted it in that process. Concurrent first sweeps may emit duplicate warnings. The sweep record is the only notice available to callers of `run_once()` or `run_tasks()`. Idle connections are still tested as in django-ox 1.7.0, preserving restart recovery but also the wait on a connection that stays open without answering. A stop is not read until that test returns. Upgrade psycopg_pool to 3.3.3 or newer. This warning does not refuse startup or change the pool. No traceback. Keys: `event` and `database` always; `worker_id` only on the startup record. | @@ -761,6 +815,28 @@ A failed connect while recording a stuck attempt logs | Key | Present on | Meaning | | --- | --- | --- | +| `batch_id` | `batch_complete_serialization_retry`, `batch_completed`, `batch_reconcile_failed` | The batch identifier. | +| `failed_count` | `batch_completed`, `workflow_completed` | The number of failed batch members or workflow nodes. | +| `unattributed_count` | `batch_completed`, `workflow_completed` | The number of batch members or workflow nodes reported as unattributed. | +| `limit` | `rate_limit_closed`, `rate_limit_reopened`, `rate_limit_row_adopted`, `rate_limit_uncounted` | The configured rate-limit name. | +| `failures` | `rate_limit_closed` | The number of consecutive counting failures that caused admission under the limit to close. | +| `stored_as` | `rate_limit_row_adopted` | The name stored on the window row that the database collation treats as equal to the configured limit name. | +| `row_count` | `workflow_backstop_skipped` | The number of rows the workflow backstop could not lock. | +| `table` | `workflow_backstop_skipped` | The table containing the rows the workflow backstop could not lock. | +| `workflow_id` | `workflow_complete_serialization_retry`, `workflow_completed`, `workflow_nodes_cancelled`, `workflow_reconcile_failed`, `workflow_records_repaired`, `workflow_release_skip_overflow`, `workflow_seal_serialization_retry`; conditionally on `workflow_reconcile_conflict_retry` and `workflow_release_skip_unqueued` | The affected workflow. On `workflow_reconcile_conflict_retry`, it is absent only for `oxpull_prune`'s scan-row cleanup. | +| `cancelled_count` | `workflow_completed`, `workflow_nodes_cancelled` | The number of cancelled nodes reported by the event. | +| `released_count` | `workflow_nodes_released` | The number of workflow nodes released. | +| `try_number` | `workflow_outcome_retry` | The try number for recording the workflow node's outcome. | +| `dropped_status` | `workflow_outcome_unrecorded` | The outcome status that could not be recorded. | +| `tries` | `workflow_outcome_unrecorded` | The number of tries made to record the workflow node's outcome. | +| `conflict` | `workflow_reconcile_conflict_retry` | The transaction conflict being retried: `deadlock` or `serialization_failure`. | +| `node_count` | `workflow_records_repaired` | On `workflow_records_repaired`, the number of node records repaired by the reconciler or `oxpull_prune`. | +| `task_count` | `workflow_release_deferred`, `workflow_release_skip_unqueued` | For a deferred release, the upper bound on task rows that could not be released. For a queue failure, the number of nodes that could not be queued for retry. | +| `dropped_count` | `workflow_release_skip_overflow` | The number of nodes not queued because the workflow's skip queue was full. | +| `attempt` | `batch_add_deadlock_retry`, `batch_add_serialization_retry`, `workflow_add_deadlock_retry`, `workflow_add_serialization_retry` | For the four `*_add_*_retry` events, the transaction attempt number, not the task execution attempt. | +| `visited_count` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The number of sealed batches or workflows visited before the reconciler sweep stopped. | +| `unvisited_count` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The number not visited before the sweep stopped. | +| `remaining_seconds` | `batch_reconcile_stopped`, `workflow_reconcile_stopped` | The remaining task-timeout budget when the reconciler stopped the sweep. | | `event` | all events | The event name from the table above. | | `worker_id` | Worker events except `heartbeat_write_failed`, `schedule_source_unavailable`, `schedule_boundary_healed`, `schedule_boundary_heal_failed`, `schedule_row_skipped` and `schedule_lock_unavailable` for a stored schedule; also `run_tasks_callback_failed`; includes `worker_claim_released` | Unique id of the worker emitting the record. With `--processes`, the slot number is the last part of the id. A Worker used in a process forked after its creation takes a new id in the child, retaining any `-` suffix; the parent keeps its id. An at-fork hook re-identifies the child. If the fork runs no at-fork hook, as with uWSGI's default, a pid check re-identifies it at its first `claim_one()`, claim, recovery look or `run()`; until then, the child's copy can still report the parent's id. Fork before claiming: handing an already-claimed row to a child for execution is unsupported, because the child's new id cannot renew the parent's lease and the reaper can requeue the row while the child runs it. This identity handling does not make arbitrary native forks safe. `ox_worker --processes` starts fresh children rather than forking them. Settings-declared `schedule_lock_unavailable` events carry this key. Records from `run_tasks()` use the drain worker's hostname-pid-random id, created for each call. `task_policy_inert` and `run_tasks_timeout_inert` omit this key. On `worker_claim_released`, identifies the worker that made the unconfirmed claim and released it. | | `worker_class` | `claim_filter_sql_missing` | The Worker subclass's class name. | diff --git a/docs/patterns.md b/docs/patterns.md index 5f1c805..f5b8919 100644 --- a/docs/patterns.md +++ b/docs/patterns.md @@ -209,9 +209,13 @@ and inside a `transaction.atomic()` opened on the database that holds `OxTask` it commits or rolls back with the rest of the work you write there, as `enqueue()` does. -This is a bulk insert and nothing more. Grouping the -tasks, reading their progress as one number and firing a callback when the -last one settles is a [batch](pro.md), in Oxpull Pro. +This is a bulk insert and nothing more. To group tasks, read their +progress as one number and fire a callback when the last one settles, +use a [batch](features/batches.md). Batches start with `oxpull.batches.create()`; the returned batch has `add()`, `seal()` and `progress()`. They use the optional `oxpull` app and require a reconciler schedule. See [Batches](features/batches.md) for setup and guarantees. + +`enqueue_many()` does not accept unique tasks: it raises `InvalidTask` +and names the task. See [Unique tasks](features/unique-tasks.md) for +deduplicated enqueue. ## Keep slow work off the fast queue @@ -333,7 +337,7 @@ Test timeout enforcement and worker infrastructure against a real worker. run_tasks(*, backend="default", queues=None, max_tasks=None, raise_failures=False) -> list[TaskResult] ``` -All arguments are keyword-only. `backend` must name an `OxBackend` alias. It selects the worker class and its default queues, including a configured `WORKER_CLASS`. Rows are claimed by queue, regardless of which backend enqueued them. Aliases sharing queues share work. For Pro workflows and rate limits, pass the Pro alias as `backend`. +All arguments are keyword-only. `backend` must name an `OxBackend` or `OxpullBackend` alias. It selects the worker class and its default queues, including a configured `WORKER_CLASS`. Rows are claimed by queue, regardless of which backend enqueued them. Aliases sharing queues share work. For workflows and rate limits, pass the alias whose `OPTIONS["WORKER_CLASS"]` is `"oxpull.worker.OxpullWorker"`. `queues=None` uses the worker's queues. An empty list does the same. Empty backend `QUEUES` means every queue. An explicit list restricts claiming to those queues. @@ -402,7 +406,7 @@ class DelayedTaskTests(TestCase): self.assertEqual([result.return_value for result in results], [5]) ``` -For retries, advance time between calls by at least the configured backoff delay. Freezing a later time does not drain every retry. Each new retry is scheduled relative to that frozen instant. A zero backoff can retry within one call. Pro rate-limit windows use the database clock, which this patch does not advance. +For retries, advance time between calls by at least the configured backoff delay. Freezing a later time does not drain every retry. Each new retry is scheduled relative to that frozen instant. A zero backoff can retry within one call. Rate-limit windows use the database clock, which this patch does not advance. ### Transactions and callbacks @@ -434,9 +438,14 @@ Use a real worker to test timeout enforcement and worker infrastructure. `run_ta Timeouts are inert. `deadline()` and `remaining()` return `None`. A configured timeout produces `run_tasks_timeout_inert` once per task path per call. A hanging body or callback hangs the test. -## Not in the core +## Optional coordination features + +[Unique tasks](features/unique-tasks.md), [batches](features/batches.md), +[rate limiting](features/rate-limiting.md) and +[workflows](features/workflows.md) are included in django-ox through the +optional `oxpull` app. They require Django 6.0+. +`OxpullBackend`, `OxpullWorker` and the documented unique-task, batch and +rate-limit entry points are stable. Workflows are provisional. +See the [feature overview](features/index.md) for setup. -Batches, unique or deduplicated tasks, rate limiting and workflows are in -[Oxpull Pro](pro.md), a paid add-on. Metrics are in the -free tier: `django_ox.stats` and `manage.py ox_health` ship in the core. Chains -are on the Pro roadmap, undated. +`django_ox.stats` and `manage.py ox_health` do not require the optional app. diff --git a/docs/pro.md b/docs/pro.md deleted file mode 100644 index 9e67239..0000000 --- a/docs/pro.md +++ /dev/null @@ -1,121 +0,0 @@ -# Pro - -django-ox is free and open source (BSD 3-Clause). The durable queue, -transactional enqueue, retries, reaper, graceful drain, priorities, deferred -tasks, recurring tasks and pruning are in the free package. - -**Oxpull Pro** is a paid add-on for four problems that show up once a queue -is carrying real volume. All four are built and tested. has the details. - -Pro requires Django 6.0 or later. It builds on `django.tasks`, which is part -of Django core from 6.0. django-ox itself also runs on Django 5.2 LTS through -the `django-tasks` backport; Pro does not. - -## What Pro adds - -- **Unique tasks.** Deduplicate at enqueue time: while a task with the same - key is READY or RUNNING, enqueueing it again returns the pending task's - result rather than inserting a second row. The lock is written in the - same transaction as the task, so the two commit or roll back together, - and a lock whose task settled or was pruned is released rather than - stranded. - A bulk enqueue is one INSERT that the deduplicating path never sees, so - `enqueue_many()` raises `InvalidTask` for a unique task and names it. -- **Batches.** Enqueue a group, read its progress as a count, and fire a - callback once every member has settled. Completion is computed by querying - the task rows rather than by counting signals, so a worker dying mid-task - cannot strand a batch: the reconciler picks it up on the next tick. - Complete is a reading of the task rows, not proof that every member has - stopped running. - Batches take one setting: `oxpull.batches.reconcile` on a one-minute cron - in `OPTIONS["SCHEDULES"]`, run by the django-ox cron you already have. -- **Rate limiting.** Cap how often a task starts. A named limit of N - admissions per period is declared in `OPTIONS["RATE_LIMITS"]`, and every - worker shares it through one row in the database. A throttled task stays - READY: it is not claimed, so it spends no retry attempt and holds no - lease. Set `OPTIONS["WORKER_CLASS"]` to `oxpull.worker.OxpullWorker`, the - worker that applies the limits. `manage.py check` reports `oxpull.E006` - when a limit is configured without it. - - For a limit of N admissions per period P, with W `ox_worker` processes - claiming from the queues that carry the limit's tasks, the number of - task attempts started in any one window is at most N + W - 1 + U. U is - the number of admissions whose count did not land. Each one logs - `rate_limit_uncounted`, so U is the number of times that event fired. - Add one for each worker process that dies between claiming a limited - task and recording it. With a single worker process, no such death and - every count landed, the limiter is exact: at most N per window. Windows - are contiguous, so an arbitrary interval of length P that spans a - boundary can carry up to 2(N + W - 1 + U). A count that cannot be - written is logged and the attempt runs; three uncounted admissions in a - row on one limit close it until a write lands. -- **Workflows.** Declare a set of tasks with dependencies. A node runs only - after every node it depends on has succeeded. The backend declares - `OPTIONS["WORKFLOWS"]`, sets `OPTIONS["WORKER_CLASS"]` to - `oxpull.worker.OxpullWorker`, and schedules `oxpull.workflows.reconcile`. - Without that schedule, `create()` and `seal()` refuse, and - `manage.py check` reports `oxpull.W003`. A reconciler exposed to - schedule rows on a queue no backend accepts reports `oxpull.E008` - instead. Workflows need Oxpull Pro 1.3.0 or later. - -Each Pro release pins django-ox exactly: `oxpull==1.8.0` pins -`django-ox==1.8.0`. - -Pro runs on the databases the free tier tests in CI: SQLite, -PostgreSQL and MySQL 8. MariaDB 10.6+ takes the same claim path but is not -part of the tested matrix. Batches have been measured to 1,000,000 members -in a single batch on all three, with every count checked against the task -rows rather than against what the API reports about itself. - -Sealing a batch that wide from inside a caller's transaction takes about -34 ms on SQLite, 0.7 s on PostgreSQL and 28 s on MySQL. That path counts -with a row lock per member wherever the database offers one, and a -million InnoDB locks in one transaction is where the MySQL figure comes -from. Called outside a transaction, `seal()` counts with a single -`COUNT(*)` instead. Timing those two statements alone against a -million-member batch with fresh statistics, the locking read took 12 s on -MySQL and 0.55 s on PostgreSQL, and the `COUNT(*)` 0.7 s on MySQL and -0.03 s on PostgreSQL. The whole `seal()` call on the same harness and the -same batch, outside a transaction, took 0.74 s on MySQL and 0.03 s on -PostgreSQL. -Reconciling the same batch is 4.9 s on SQLite, 0.4 s on PostgreSQL and -7.3 s on MySQL, against a reconciler that runs on a one-minute tick. - -Measured on an arm64 macOS host, with SQLite on a local file in WAL mode -at `synchronous=NORMAL`, and stock PostgreSQL 16 and MySQL 8 images in -Docker on localhost, each width against an empty database. - -## What Pro is not - -Chains are on the roadmap, undated. Rate limiting caps how -often a task starts, not how many run at once; concurrency limiting is a -different mechanism and is not in Pro. Metrics stay free: the stats API and -the health command are in the open source package and remain there. - -## Delivery - -Pro installs from a private package index using credentials issued per -company. There is no licence key and no runtime check. A licence check would -put a validation step in the path of code that has to keep running, and it -does nothing for a company that has already paid. Nothing in the package -phones home: no network call to Oxpull, and no telemetry. The credential controls -access to the index rather than to code you have already installed, so if it -lapses, what is deployed keeps running. - -## Pricing - -**$399 per year, per company**, excluding VAT where it applies. One licence -covers a whole organisation and every environment. The term is 12 months and -renews for successive 12-month terms unless you cancel. Cancel by writing to -support@oxpull.com. Cancellation takes effect at the end of the period you -have paid for. The full terms are stated at purchase. - -## Ordering - -Order at [oxpull.com](https://oxpull.com/#order) with your company name, a -contact name, an email address, the billing address and, if you have one, a -VAT ID for the invoice. The invoice goes out within two business days. When it -is paid, the credentials for the private package index arrive by email, and -install is `pip install`. Questions to support@oxpull.com. - -[Order Pro](https://oxpull.com/#order){ .md-button } diff --git a/docs/production.md b/docs/production.md index 6d1aca7..d0743c6 100644 --- a/docs/production.md +++ b/docs/production.md @@ -134,8 +134,8 @@ batch open; the stop-time look waits up to five seconds for that claim. If the claim is still in flight then, the look gives up with `worker_claim_recovery_failed` and a row that landed is left to the reaper. `--max-tasks` counts only claims -that returned; a claim that raised uses no slot. Oxpull Pro also counts -admission only for a claim that returned, so a released row is counted +that returned; a claim that raised uses no slot. Rate-limit admission is +also counted only for a claim that returned, so a released row is counted when it is claimed again. No database statement is added to the ordinary task path. A recovery look @@ -352,13 +352,15 @@ python manage.py ox_worker --batch --concurrency 4 "Nothing to claim" is a point-in-time observation by this worker, not a guarantee that the queue is empty or a workflow is complete. Future `run_after` tasks, backed-off retries, locked tasks and tasks excluded by -claim filters may remain. With [Oxpull Pro](pro.md), rate-limited READY tasks -and WAITING workflow children may remain too. A pass that sees due tasks but -loses them all to other workers isn't empty, and the worker polls again. -Immediately eligible follow-up work committed before a local task finishes -can be picked up on a subsequent pass, unless another stop condition wins. -Batch mode does not wait for later commits, schedule ticks or reconciler -hand-offs; arrange another run or use a long-running worker for that work. +claim filters may remain. With the optional `oxpull` app, +[rate-limited](features/rate-limiting.md) READY tasks and WAITING +[workflow](features/workflows.md) children may remain too. +A pass that sees due tasks but loses them all to other workers isn't empty, +and the worker polls again. Immediately eligible follow-up work committed +before a local task finishes can be picked up on a subsequent pass, unless +another stop condition wins. Batch mode does not wait for later commits, +schedule ticks or reconciler hand-offs; arrange another run or use a +long-running worker for that work. Schedules dispatch only while a worker runs. A batch checks them when it starts and keeps checking while it runs. Each schedule then enqueues only its @@ -917,13 +919,11 @@ replacement connections. ### Rolling out per-task policy -No database migration is required. The attempt-budget column already -exists, but old and new workers do not execute the same live policy. +Per-task policy was introduced in django-ox 1.5.0. When upgrading from 1.4.0, use the mixed-fleet guidance below until every worker has been upgraded. -Deploy django-ox 1.5.0 to every producer, worker and other process that -imports task modules before adding declarations. Deploy Oxpull Pro 1.5.0 -everywhere too if it is installed. Only then enable `max_attempts`, -`backoff` and `timeout` declarations. +Per-task policy does not require a migration of either app. Treat it as a mixed-fleet configuration rollout: update every process that enqueues or claims the affected tasks before relying on the new policy. + +Earlier `oxpull` installs have their own [upgrade steps](migrating.md#earlier-oxpull-installs). Those package upgrade steps are separate from enabling per-task policy. | Mixed-fleet case | Behaviour | | --- | --- | @@ -1614,3 +1614,37 @@ your own. Flag reference on the [Configuration](configuration.md#ox_prune) page. FAILED rows are kept by default so tracebacks survive until you have looked at them; add `--include-failed` once that is not needed. + +### oxpull_prune + +Run `oxpull_prune` to clean up records owned by the optional app: + +```console +python manage.py oxpull_prune +``` + +It removes: + +- Unique-task lock rows whose tasks are no longer READY or RUNNING, including locks whose tasks were pruned, and locks with a NULL task id. Locks are not aged. Enqueuing the same key also releases a stale lock. +- Eligible workflow node rows and the edges into them, for workflows that have been COMPLETE longer than `--older-than`. +- Backstop scan rows for workflows that are COMPLETE or no longer exist. + +It never deletes workflow rows or batch members. + +For each eligible workflow, pruning first stamps `metadata_retired_at`, once. If a node is still pending, it deletes nothing and tries again on a later run. It repairs unrecorded outcomes, then deletes only nodes whose outcomes are recorded and whose task rows are SUCCESSFUL, DISCARDED or gone. + +Nodes whose task rows are FAILED or LOST remain until `ox_prune --include-failed` removes those task rows. A node with no recorded outcome is never deleted. + +The options are: + +| Option | Default | Meaning | +| --- | --- | --- | +| `--batch-size` | `1000` | Rows per DELETE. Must be at least 1. A workflow statement deletes at most 1,000 rows. | +| `--older-than` | `7d` | Workflow retention age. Accepts values such as `7d`, `24h`, `90m`, `45s` or plain seconds. Does not age unique-task locks. | +| `--dry-run` | Off | Count eligible records without deleting rows or writing retirement stamps. | + +There is no `--database` option. The command uses the database assigned by the routers to both the `django_ox` and `oxpull` tables and refuses a split. If the workflow tables do not exist yet, it skips workflow cleanup and reports that `migrate oxpull` is needed. + +No ordering with `ox_prune` is required. Running `ox_prune --include-failed` first lets more workflow nodes be removed. A lock whose task `ox_prune` deleted is stale and can also be removed. + +The command ends by reporting retained workflow, node and edge counts. See [Monitoring](monitoring.md) for their meaning. diff --git a/docs/requirements.txt b/docs/requirements.txt index 28e7aa3..c94d5ed 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,5 @@ # pip install -r docs/requirements.txt && mkdocs build mkdocs>=1.6.1,<2 mkdocs-material>=9.7.7,<10 +# 1.2.3 depends on properdocs, a second documentation engine; 1.2.2 needs only mkdocs. +mkdocs-redirects==1.2.2 diff --git a/docs/stability.md b/docs/stability.md index f14cf9b..340b21f 100644 --- a/docs/stability.md +++ b/docs/stability.md @@ -147,6 +147,53 @@ updates `run_after`, rather than preserving its original value as proposal See [Configuration](configuration.md#per-task-policy) for validation, framework support and precedence. +### Stable: optional `oxpull` app + +The stable public surface is: + +- `oxpull.backend.OxpullBackend`. +- Its `UNIQUE_TASKS`, `RATE_LIMITS` and `RATE_REFRESH_INTERVAL` options. +- `oxpull.worker.OxpullWorker`. +- `oxpull.unique.unique`. +- `oxpull.batches.Batch` and `oxpull.batches.BatchProgress`. +- `oxpull.batches.create`, `oxpull.batches.get` and `oxpull.batches.reconcile`. +- `Batch.add`, `Batch.seal`, `Batch.progress` and `batch.id`. +- `oxpull.exceptions.UniqueEnqueueRace`. +- `oxpull.exceptions.BatchSealed`. +- `oxpull.exceptions.BatchMemberExists`. +- The `oxpull_prune` management command. +- All 16 `oxpull` check ids: `oxpull.E001` through `oxpull.E009`, and `oxpull.W001` through `oxpull.W007`. +- The `oxpull_*` schema. +- The non-workflow `oxpull` log events documented under [Log events](monitoring.md#log-events). + +`BatchProgress.pruned` is a deprecated alias of `unattributed`. Use `unattributed`; the alias emits `DeprecationWarning` and is marked for removal in 2.0. + +### Provisional: workflows + +The documented workflow surface is provisional: + +- `oxpull.workflows.create`. +- `oxpull.workflows.get`. +- `oxpull.workflows.reconcile`. +- `oxpull.workflows.retained_rows`. +- `Workflow.add`, `Workflow.seal` and `Workflow.progress`. +- `WorkflowProgress`. +- `oxpull.exceptions.WorkflowSealed`. +- `oxpull.exceptions.WorkflowNodeExists`. +- `oxpull.exceptions.UnknownWorkflowNode`. +- The `WORKFLOWS` option. +- Every `workflow_*` log event. + +The provisional tier also applies to the documented results returned by these workflow entry points. A return type's use in documentation does not make unrelated implementation helpers public. + +### Not public: workflow internals + +Workflow implementation helpers are not public API. In particular, `node_flag`, `runnable_filter_q` and `record_node_outcome` are internal, even though they appear in the module's `__all__`. + +Internal contracts between `django_ox` and `oxpull` are tested by the feature +suite in the same CI. They are not public API and are not maintained through +a separate dependency pin. + ### Not public Everything else is an implementation detail and may change in any release @@ -323,19 +370,24 @@ changelog, and in a security advisory where relevant. ## Supported Python and Django -Each django-ox release is tested against the matrix below in CI, on SQLite -and PostgreSQL 16 across the grid and MySQL 8 on the oldest and newest -corners; these are the supported combinations. +The `oxpull` app requires Django 6.0+. Its SQLite and PostgreSQL suites cover the Django 6.0 / 6.1 grid on `rc*` tags and in nightly runs. Its MySQL 8 suite runs at Python 3.14 / Django 6.1. + +The separate django-ox matrix continues to cover Django 5.2, and its MySQL coverage includes the oldest and newest supported corners. That broader base-app matrix is not a claim that the optional app runs on Django 5.2. | | Django 5.2 LTS | Django 6.0 | Django 6.1 | | --- | --- | --- | --- | -| **Python 3.12** | tested | tested | tested | -| **Python 3.13** | tested | tested | tested | +| **Python 3.12** | `django_ox` tested; `oxpull` not supported | tested | tested | +| **Python 3.13** | `django_ox` tested; `oxpull` not supported | tested | tested | | **Python 3.14** | not supported by Django 5.2 | tested | tested | Django 6.0 and later ship the Tasks framework in core. The Django 5.2 legs -install the `django-tasks` backport and run the whole suite against it, which -is what `django-ox[backport]` pulls in. +install the `django-tasks` backport and run the `django_ox` suite against it, +which is what `django-ox[backport]` pulls in. Those legs do not run the +`oxpull` feature suite. + +`oxpull` is not supported on Django 5.2. It raises `ImproperlyConfigured` +during application startup, before system checks can run. Installing the +backport does not enable `oxpull` on Django 5.2. Support for per-task policy declarations differs from support for the backend itself: diff --git a/docs/stored-schedules.md b/docs/stored-schedules.md index d521ea9..2612f97 100644 --- a/docs/stored-schedules.md +++ b/docs/stored-schedules.md @@ -709,14 +709,16 @@ Worth knowing before you turn it on: due-tick and deadline rules; only its reporting is rate-limited. - **Database read failures are not bad rows.** Database errors during full schedule or shared dispatch reads, including connection, missing-table and lock errors, stop the pass. They are not attributed to a row. Server-side errors propagate unchanged. In autocommit, recognized value-local failures may be isolated by diagnostic re-reads while the connection remains usable. - Inside a transaction, diagnostic re-reads are restricted: PostgreSQL requires a usable transaction; SQLite permits only its recognized UTF-8 decoding refusal; MySQL permits none. Unwind through the transaction or savepoint boundary before further queries where required. Reads create no savepoints. `update_schedule` and `delete_schedule` have their own atomic blocks; source reads and `ox_prune` do not. Unlike 1.8.0, a source read inside your transaction can leave a server error for you to roll back. + Inside a transaction, diagnostic re-reads are restricted: PostgreSQL requires a usable transaction; SQLite permits only its recognized UTF-8 decoding refusal; MySQL permits none. Unwind through the transaction or savepoint boundary before further queries where required. Reads create no savepoints. `update_schedule` and `delete_schedule` have their own atomic blocks; source reads and `ox_prune` do not. Unlike 1.8.0, a source read inside your transaction can leave a server error for you to roll back. + - **An abandoned pass stops further traversal.** An escaping `django.db.Error`, other unhandled exception, failed rollback or unusable connection reports `schedule_dispatch_failed`. Dispatch retries on a later pass; committed ticks remain committed. Marker reads and boundary heals retain their own events. - An unreadable marker reports `schedule_source_unavailable` once, then at most once a minute; full reads continue at each reconcile interval. A database error on the marker read reports every pass while workers use the last schedules read. Failed boundary healing reports `schedule_boundary_heal_failed`, first with a traceback, then counted summaries at most once a minute per row per worker. Healing retries each pass; the affected schedule cannot dispatch meanwhile. + An unreadable marker reports `schedule_source_unavailable` once, then at most once a minute; full reads continue at each reconcile interval. A database error on the marker read reports every pass while workers use the last schedules read. Failed boundary healing reports `schedule_boundary_heal_failed`, first with a traceback, then counted summaries at most once a minute per row per worker. Healing retries each pass; the affected schedule cannot dispatch meanwhile. + + An unreadable newest tick within the pass's bound skips only its schedule with `schedule_tick_unreadable`. A settings schedule can also be blocked by its unreadable earliest tick. History is not treated as absent. The pass completes, so this skip does not hold a batch open. Follow the recovery procedure under [Configuration](configuration.md#ox_prune). - An unreadable newest tick within the pass's bound skips only its schedule with `schedule_tick_unreadable`. A settings schedule can also be blocked by its unreadable earliest tick. History is not treated as absent. The pass completes, so this skip does not hold a batch open. Follow the recovery procedure under [Configuration](configuration.md#ox_prune). + Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. - Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. - **`manage.py check` cannot see rows.** Checks run before `migrate`, so a bad schedule in the database is a log line, not a start-up error. Settings-declared schedules still fail fast for errors their checks can diff --git a/docs/use-cases.md b/docs/use-cases.md index c3354b1..e99194e 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -112,11 +112,11 @@ Keep schedules in settings, or let your team pause and edit them as [rows in Dja Schedules need at least one running worker and a usable database. A schedule-scoped failure does not stop later entries if rollback succeeds and the same connection remains usable. After an outage, only the most recent missed tick is enqueued. -## Deduplicate a repeated webhook with Oxpull Pro +## Deduplicate a repeated webhook A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue rather than making the queue do the same work again. -With django-ox alone, these calls create separate tasks: +Without a unique-task declaration, these calls create separate tasks: ```python from django.tasks import task @@ -127,20 +127,20 @@ def sync_invoice(invoice_id): ... sync_invoice.enqueue(1001) -sync_invoice.enqueue(1001) # django-ox: a second row, a second run +sync_invoice.enqueue(1001) # A second row, a second run. ``` -Oxpull Pro's unique tasks deduplicate inside the task's own transaction. While a task with the same key is READY or RUNNING, enqueueing it again returns the pending task's result without writing another row. +[Unique tasks](features/unique-tasks.md) deduplicate inside the task's own transaction. While a task with the same key is READY or RUNNING, enqueueing it again returns the pending task's result without writing another row. Once the task settles, the same call can enqueue again. This prevents duplicate pending work; it does not make external effects exactly-once. Tasks still need to be safe to repeat. -Pro also lets you track a batch through completion, control task starts with shared rate limits, and connect dependent tasks into workflows. You get email support from the maintainer and priority feature requests. +Unique tasks are included in django-ox through the optional `oxpull` app and require Django 6.0+. Follow the [setup overview](features/index.md), then the [unique-task declaration guide](features/unique-tasks.md). -[Order Oxpull Pro](https://oxpull.com/) for $399 per year, per company, excluding VAT where it applies. Submit the form; your invoice arrives within two business days. Once paid, private package index credentials arrive by email. Installation stays `pip install`. Pro requires Django 6.0+. +The same app provides [batches](features/batches.md) to track grouped work, [rate limiting](features/rate-limiting.md) to control task starts, and provisional [workflows](features/workflows.md) to connect dependent tasks. ## When to choose another tool -Choose another backend if you need to revoke a chosen running task, use chains, groups or chords, or exceed the throughput one database can serve. Celery supports revocation and those composition patterns. +Choose another backend if you need to revoke a chosen running task, require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md), not those Celery APIs. Transactional enqueue requires the task and model rows to share a database. A separate queue database cannot give you that shared commit. diff --git a/docs/why-django-ox.md b/docs/why-django-ox.md index 1adbabd..8359462 100644 --- a/docs/why-django-ox.md +++ b/docs/why-django-ox.md @@ -47,9 +47,9 @@ Inspect tracebacks and use Retry or Discard on failed tasks. Edit recurring sche Both provide a `django.tasks` backend. django-ox automatically requeues a dead worker's tasks. huey's guide says mid-execution tasks are lost and aren't retried automatically. -### Do retries and recovery require Pro? +### Do retries and recovery need the optional app? -No. The free package includes both, plus schedules and Django admin integration. Health checks and metrics are also included. +No. Retries, dead-worker recovery, schedules, Django admin integration, health checks and metrics work without the optional `oxpull` app. ### How do I make recovery safe? @@ -59,11 +59,11 @@ Write idempotent tasks. A repeated execution must not duplicate effects such as Python 3.12+ and Django 5.2 LTS, 6.0 or 6.1. Django 5.2 uses the `django-tasks` backport. PostgreSQL, MySQL 8 and SQLite run in CI. -### What does Pro add? +### What does the optional app add? -Batches, unique tasks, shared rate limits and workflows, with maintainer email support. +[Unique tasks](features/unique-tasks.md), [batches](features/batches.md), [shared rate limits](features/rate-limiting.md) and [workflows](features/workflows.md). The `oxpull` app is included in django-ox and requires Django 6.0+. -Oxpull Pro costs $399 per year, per company, excluding VAT where it applies. Every environment is included. Pro requires Django 6.0+. +`OxpullBackend`, `OxpullWorker` and the documented unique-task, batch and rate-limit entry points are stable. Workflows are provisional. See the [feature overview](features/index.md) for setup and API status. [**Get started**](background-tasks.md) diff --git a/mkdocs.yml b/mkdocs.yml index d5e9a4d..b1b2a9b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,6 +49,12 @@ markdown_extensions: - pymdownx.snippets: base_path: !relative $config_dir +plugins: + - search + - redirects: + redirect_maps: + pro.md: features/index.md + nav: - Home: index.md - Background tasks, step by step: background-tasks.md @@ -56,6 +62,12 @@ nav: - Configuration: configuration.md - Recurring tasks: recurring-tasks.md - Schedules in the database: stored-schedules.md + - Task coordination: + - Overview: features/index.md + - Unique tasks: features/unique-tasks.md + - Batches: features/batches.md + - Rate limiting: features/rate-limiting.md + - Workflows: features/workflows.md - Common patterns: patterns.md - Migrating: migrating.md - Why django-ox: why-django-ox.md @@ -65,6 +77,5 @@ nav: - Monitoring: monitoring.md - Benchmarks: benchmarks.md - API stability: stability.md - - Pro: pro.md - For AI assistants: agents.md - Changelog: changelog.md diff --git a/pyproject.toml b/pyproject.toml index f860ba0..6f0eafe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ dev = [ [tool.hatch.build.targets.sdist] include = [ "/src/django_ox", + "/src/oxpull", "/LICENSE", "/README.md", "/CHANGELOG.md", @@ -91,7 +92,7 @@ exclude = ["**/__pycache__", "**/*.pyc"] path = "hatch_build.py" [tool.hatch.build.targets.wheel] -packages = ["src/django_ox"] +packages = ["src/django_ox", "src/oxpull"] exclude = ["**/__pycache__", "**/*.pyc"] [tool.pytest.ini_options] @@ -152,6 +153,10 @@ ignore = [ [tool.ruff.lint.per-file-ignores] # Tests assert, use throwaway credentials, and write to temp paths on purpose. "tests/**" = ["S101", "S105", "S106", "S108", "S311"] +# The oxpull suite is held to the narrower exemption it had as its own package. +"tests_oxpull/**" = ["S101", "S105", "S106"] +# The installed-wheel tests run pip and python in virtualenvs they create. +"tests_wheel/**" = ["S101", "S603", "S607", "S608"] # Benchmark and maintenance scripts are operator tools, not shipped code. "tools/**" = ["T20"] @@ -169,10 +174,10 @@ module = ["django_tasks", "django_tasks.*"] ignore_missing_imports = true [tool.django-stubs] -django_settings_module = "tests.settings" +django_settings_module = "tests.settings_typecheck" [tool.coverage.run] -source = ["django_ox"] +source = ["django_ox", "oxpull"] branch = true [tool.coverage.report] diff --git a/src/django_ox/_run_tasks.py b/src/django_ox/_run_tasks.py index fda6b64..84a6f21 100644 --- a/src/django_ox/_run_tasks.py +++ b/src/django_ox/_run_tasks.py @@ -619,7 +619,7 @@ def _run_one( if attempt.moved is not None: raise attempt.moved # The outcome write's own callbacks, which a worker's commit of that - # write would run: Pro releases a workflow node's dependents here. + # write would run: `oxpull` releases a workflow node's dependents here. failed = attempt.commit() if failed is not None: raise failed[1] diff --git a/src/oxpull/__init__.py b/src/oxpull/__init__.py new file mode 100644 index 0000000..7a5bae3 --- /dev/null +++ b/src/oxpull/__init__.py @@ -0,0 +1,5 @@ +# One distribution, one version: oxpull ships inside django-ox, so its version +# is django-ox's. tools/check_release.py refuses a literal here. +from django_ox import __version__ + +__all__ = ["__version__"] diff --git a/src/oxpull/_django_gate.py b/src/oxpull/_django_gate.py new file mode 100644 index 0000000..e3cab36 --- /dev/null +++ b/src/oxpull/_django_gate.py @@ -0,0 +1,38 @@ +""" +The Django floor for oxpull, in one place. + +django-ox runs on Django 5.2 and later. oxpull is built on Django's Tasks +framework (django.tasks), which Django ships from 6.0, so it needs Django 6.0 +or later. The modules that import django.tasks import _requires_django_6 +first, and OxpullConfig.ready() calls require_django_6(), so a project on +5.2 gets this message and not a ModuleNotFoundError from inside a management +command. + +The functions are here and nothing runs on import, because oxpull.checks has +to ask whether Django is supported without being refused itself. The modules +that must refuse import _requires_django_6 instead. +""" + +import django +from django.core.exceptions import ImproperlyConfigured + +MINIMUM = (6, 0) + + +def unsupported_message() -> str: + return ( + f"oxpull needs Django 6.0 or later, and this is Django {django.get_version()}. " + "django-ox itself runs on Django 5.2, but oxpull is built on Django's " + "Tasks framework, which Django ships from 6.0. Upgrade Django to 6.0 or " + "later, or remove 'oxpull' from INSTALLED_APPS and set the task backend " + "to 'django_ox.backend.OxBackend'." + ) + + +def supported() -> bool: + return django.VERSION[:2] >= MINIMUM + + +def require_django_6() -> None: + if not supported(): + raise ImproperlyConfigured(unsupported_message()) diff --git a/src/oxpull/_requires_django_6.py b/src/oxpull/_requires_django_6.py new file mode 100644 index 0000000..d82e643 --- /dev/null +++ b/src/oxpull/_requires_django_6.py @@ -0,0 +1,10 @@ +""" +Importing this module raises ImproperlyConfigured on Django before 6.0. + +Every module that imports django.tasks imports this one first. A failed import +is not cached, so every attempt raises again. +""" + +from ._django_gate import require_django_6 + +require_django_6() diff --git a/src/oxpull/apps.py b/src/oxpull/apps.py new file mode 100644 index 0000000..ae18d62 --- /dev/null +++ b/src/oxpull/apps.py @@ -0,0 +1,16 @@ +from django.apps import AppConfig + + +class OxpullConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "oxpull" + verbose_name = "Oxpull" + + def ready(self) -> None: + # Before anything that imports django.tasks, so Django 5.2 gets a + # message that names the fix. + from ._django_gate import require_django_6 + + require_django_6() + # Registers the checks that read the database. + from . import checks # noqa: F401 diff --git a/src/oxpull/backend.py b/src/oxpull/backend.py new file mode 100644 index 0000000..02ea536 --- /dev/null +++ b/src/oxpull/backend.py @@ -0,0 +1,1831 @@ +""" +Optional unique tasks, batches, rate limiting and workflows for django-ox. + +Use OxpullBackend on each task alias that needs these features, in every +enqueuing and claiming process. Rate limits and workflows also require +oxpull.worker.OxpullWorker through OPTIONS["WORKER_CLASS"]. Workflows require +OPTIONS["WORKFLOWS"] = True and a reconciler schedule; batches require their +reconciler schedule. + +Uniqueness deduplicates pending task rows. It does not prevent two executions +of the same key from overlapping. Tasks sent outside this backend and workers +that do not apply its admission gate bypass oxpull's unique-task lock rows and +rate limits, respectively. + +oxpull creates only its own tables. It does not change any django-ox table, +column, index, constraint or migration. + +Workflows do write task rows. They insert blocked nodes as WAITING tasks. +Cancellation moves a READY or WAITING task to DISCARDED by a compare-and-set +on the task row's status and lease epoch. Release moves a WAITING task to READY +and updates run_after; run_after is never NULL. Revival of a DISCARDED node +is not supported. The workflow generation field is reserved and remains 0; +it is not a workflow lease epoch. + +A workflow is created at runtime, not declared as a definition in settings. +WORKFLOWS accepts only True or False. Signal receivers must account for the +OxpullBackend sender on tasks enqueued through this backend. + +Use ox_prune for task rows and oxpull_prune for oxpull records. +""" + +from __future__ import annotations + +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +import os +import unicodedata +from collections.abc import Iterable, Mapping, Sequence +from typing import Any, cast + +from django.apps import apps +from django.conf import settings +from django.core import checks +from django.core.exceptions import ImproperlyConfigured +from django.db import ( + DatabaseError, + IntegrityError, + OperationalError, + connections, + transaction, +) +from django.db.models import CharField, Exists, OuterRef, Subquery +from django.db.models.expressions import RawSQL +from django.tasks.base import DEFAULT_TASK_QUEUE_NAME, Task, TaskResult +from django.tasks.exceptions import InvalidTask +from django.utils import timezone +from django.utils.module_loading import import_string + +from django_ox.backend import OxBackend + +from .exceptions import UniqueEnqueueRace +from .rate_limits import ( + RateLimit, + rate_limits_from_options, + refresh_interval_from_options, +) +from .routing import oxpull_db_alias +from .unique import is_registered, unique_key, unique_tasks_from_options + +# Bound on the duplicate-path retry loop. A pass either returns the live +# holder's result, deletes one stale lock generation, or ends in a +# transaction the server rolled back and is reissued: an InnoDB deadlock +# victim on MySQL, a serialization failure on PostgreSQL. Reaching the +# bound takes that many finished generations or lost transactions during +# a single enqueue call. +MAX_UNIQUE_ATTEMPTS = 5 + +# MySQL's ER_LOCK_DEADLOCK. The insert race here can genuinely deadlock +# under InnoDB: a losing INSERT's duplicate check leaves a shared lock +# on the conflicting index record, and two losers re-inserting after a +# stale lock's DELETE each wait on the other's shared lock. InnoDB +# resolves the cycle by rolling back one victim's entire transaction, +# and its manual's instruction for 1213 is to reissue that transaction. +ER_LOCK_DEADLOCK = 1213 + +# The three ways a database stops waiting for the holder of a key. The +# lock INSERT on the loser's side blocks on the winner's uncommitted row, +# and the enqueue joins the caller's transaction, so a slow request holds +# the key for as long as it runs. SQLite gives up after the connection's +# busy timeout (OPTIONS["timeout"], 5 s unless set) with SQLITE_BUSY; +# every extended busy code shares that primary code, SQLITE_BUSY_SNAPSHOT +# among them, which a WAL-mode read-to-write upgrade raises at once +# rather than after the timeout. MySQL gives up after +# innodb_lock_wait_timeout (50 s unless set) with ER_LOCK_WAIT_TIMEOUT, +# rolling back the statement and not the transaction. PostgreSQL waits +# for as long as the holder runs unless the project sets lock_timeout, +# which ends the wait with lock_not_available. A statement_timeout ends +# it too, as query_canceled (57014); that one is not specific to this +# contention and propagates raw. +SQLITE_BUSY = 5 +# The extended code for the WAL-mode case: a transaction that read before +# it wrote asks to become a writer after another connection committed, +# and SQLite refuses at once rather than waiting out the busy timeout. +SQLITE_BUSY_SNAPSHOT = SQLITE_BUSY | (2 << 8) +ER_LOCK_WAIT_TIMEOUT = 1205 +PG_LOCK_NOT_AVAILABLE = "55P03" +# PostgreSQL's serialization_failure. At REPEATABLE READ and SERIALIZABLE +# a transaction may not lock, update or delete a row that another +# transaction changed after its snapshot began, and the statement fails +# with this code instead. MySQL reports a deadlock under the same SQLSTATE, +# and that case keeps its own path. +PG_SERIALIZATION_FAILURE = "40001" +# The levels at which PostgreSQL answers a plain read inside a transaction +# from the transaction's snapshot, spelled as current_setting() spells them. +PG_SNAPSHOT_LEVELS = frozenset({"repeatable read", "serializable"}) +# The name under which the holder read carries the transaction's level. +ISOLATION_LEVEL_ANNOTATION = "oxpull_isolation_level" +# The name under which the fresh read of a holder carries the task that the +# key's lock names. +LOCK_HOLDER_ANNOTATION = "oxpull_lock_task_id" +# How long that fresh read waits for a lock, and how long its statement may +# run. Any lock wait there is one that can't end while the caller's +# transaction is open (see _read_holder_fresh), so the first is short. The +# second is a backstop for a statement that reads one row by primary key. +FRESH_READ_LOCK_TIMEOUT = "100ms" +FRESH_READ_STATEMENT_TIMEOUT = "2s" +# How long that read's connection waits to connect, in seconds, where the +# project sets no connect timeout of its own. libpq applies it to each address +# the host name resolves to, and treats anything lower as 2. It doesn't cover +# the host name lookup, or a wait in a connection pooler's queue. +FRESH_READ_CONNECT_TIMEOUT = 2 +# What a UniqueEnqueueRace raised inside a snapshot transaction tells the +# caller to do. +RETRY_OUTSIDE_THE_SNAPSHOT = ( + "The enqueue created nothing. Retry it in a new, short transaction at " + "READ COMMITTED, or outside a transaction. A retry inside this transaction " + "reads the same snapshot." +) + + +def _pg_sqlstate(exc: BaseException) -> str | None: + """ + The SQLSTATE of the driver error Django wrapped. psycopg 3 calls it + sqlstate and psycopg2 calls it pgcode. + """ + cause = exc.__cause__ + code = getattr(cause, "sqlstate", None) or getattr(cause, "pgcode", None) + return code if isinstance(code, str) else None + + +def _savepoint(db_alias: str) -> Any: + """ + Open a savepoint on the alias's connection and return its id. Django 6.1 + names that call transaction.savepoint_create() and deprecates + transaction.savepoint(), which Django 7.0 removes. Django 6.0 has only the + old name. + """ + create = getattr(transaction, "savepoint_create", None) + if create is None: + return transaction.savepoint(using=db_alias) + return create(using=db_alias) + + +def _in_callers_transaction(db_alias: str) -> bool: + """ + Whether the enqueue runs inside a transaction that the caller opened and + only the caller ends. That is an atomic block, or a connection with + autocommit turned off, by transaction.set_autocommit(False) or by AUTOCOMMIT + False in the database settings. Either way every statement the enqueue runs + joins that transaction, reads its snapshot at a snapshot level, and is + lost with the caller's work when the server rolls the transaction back. + """ + connection = connections[db_alias] + return bool(connection.in_atomic_block) or not connection.get_autocommit() + + +def _sets_its_own_connect_timeout(options: Mapping[str, Any]) -> bool: + """ + Whether libpq takes a connect timeout from the project rather than from + oxpull. A project sets one in OPTIONS["connect_timeout"] or in the + PGCONNECT_TIMEOUT environment variable, or names a connection service, + whose entry can set one. A value passed with the connection would override + all three. + """ + return ( + "connect_timeout" in options + or "service" in options + or "PGCONNECT_TIMEOUT" in os.environ + or "PGSERVICE" in os.environ + ) + + +# PostgreSQL's serialization_failure. At REPEATABLE READ or SERIALIZABLE, +# a transaction that writes a row another transaction changed after its +# snapshot was taken gets this instead of the newer row, and waiting on +# that writer's lock first does not help: the error arrives when the +# writer commits. The whole transaction is rolled back, and PostgreSQL's +# manual says to retry it from the start. +PG_SERIALIZATION_FAILURE = "40001" + +# PostgreSQL's deadlock_detected. The deadlock detector rolls back one +# transaction of the cycle whole, and the manual says to run it again. +PG_DEADLOCK_DETECTED = "40P01" + + +def is_innodb_deadlock(exc: OperationalError, db_alias: str) -> bool: + if connections[db_alias].vendor != "mysql": + return False + return bool(exc.args and exc.args[0] == ER_LOCK_DEADLOCK) + + +def is_deadlock(exc: OperationalError, db_alias: str) -> bool: + """Whether InnoDB (1213) or PostgreSQL (40P01) rolled this transaction back.""" + if is_innodb_deadlock(exc, db_alias): + return True + if connections[db_alias].vendor != "postgresql": + return False + return getattr(exc.__cause__, "sqlstate", None) == PG_DEADLOCK_DETECTED + + +def is_serialization_failure(exc: OperationalError, db_alias: str) -> bool: + """Whether PostgreSQL rolled this transaction back as a serialization failure.""" + if connections[db_alias].vendor != "postgresql": + return False + return getattr(exc.__cause__, "sqlstate", None) == PG_SERIALIZATION_FAILURE + + +def _is_lock_wait_timeout(exc: OperationalError, db_alias: str) -> bool: + """Whether the database gave up waiting for another transaction's lock.""" + vendor = connections[db_alias].vendor + cause = exc.__cause__ + if vendor == "sqlite": + code = getattr(cause, "sqlite_errorcode", None) + return isinstance(code, int) and code & 0xFF == SQLITE_BUSY + if vendor == "mysql": + return bool(exc.args and exc.args[0] == ER_LOCK_WAIT_TIMEOUT) + if vendor == "postgresql": + return _pg_sqlstate(exc) == PG_LOCK_NOT_AVAILABLE + return False + + +def _is_serialization_failure(exc: OperationalError, db_alias: str) -> bool: + """Whether PostgreSQL refused a row that changed after the snapshot began.""" + if connections[db_alias].vendor != "postgresql": + return False + return _pg_sqlstate(exc) == PG_SERIALIZATION_FAILURE + + +def _unbound_reconciler_error( + task_path: str, schedule_name: str, schedule: Any, consequence: str +) -> checks.CheckMessage: + """oxpull.E008: a reconciler exposed to schedule rows that nothing runs.""" + from .batches import unbound_reconciler_hint, unbound_reconciler_message + + return checks.Error( + unbound_reconciler_message(task_path, schedule, consequence), + hint=unbound_reconciler_hint(task_path, schedule_name, schedule), + id="oxpull.E008", + ) + + +class OxpullBackend(OxBackend): + """ + OxBackend plus unique tasks. + + A task is unique when marked with @oxpull.unique.unique or listed in + this backend's OPTIONS["UNIQUE_TASKS"]. Enqueueing a unique task whose + key is already held by a READY or RUNNING task returns that task's + TaskResult instead of inserting a second row; task_enqueued is not + sent for the suppressed duplicate. Non-unique tasks take the parent + code path untouched. Uniqueness is scoped per backend alias. + + enqueue_many() is the one parent method a unique task cannot use, and + it says so rather than writing duplicates: its bulk INSERT does not + reach the deduplicating path at all. + + Correctness rests on the same mechanism django-ox proves in-tree for + schedule ticks: a unique constraint plus an IntegrityError loser path, + never check-then-insert. The lock row and the task row are written in + one transaction, so they commit or roll back together, preserving the + parent backend's durability guarantee: a transaction.atomic() opened + on the database these tables live on covers the enqueue too, and a + rollback discards task and lock with every other row written there. + + The oxpull tables must live on the same database as the django_ox + tables; the lock INSERT and the task INSERT must share a transaction. + __init__ enforces it against the active router, and every read on the + dedup path is pinned to that database, so a read-replica router + cannot serve a stale liveness answer. + """ + + def __init__(self, alias: str, params: dict[str, Any]) -> None: + super().__init__(alias, params) + # A split router silently breaks lock/task atomicity; fail at + # startup, not at the first lost duplicate. + self._db_alias: str = oxpull_db_alias() + # Shape errors raise here, loudly, at startup. check() validates + # the deeper property (that each path imports to a Task). + self.unique_task_paths: frozenset[str] = unique_tasks_from_options(self.options) + + def _is_unique(self, task: Task[..., Any]) -> bool: + path = task.module_path + return is_registered(path) or path in self.unique_task_paths + + def enqueue[**P, R]( + self, task: Task[P, R], args: list[Any], kwargs: dict[str, Any] + ) -> TaskResult[P, R]: + if not self._is_unique(task): + return super().enqueue(task, args, kwargs) + return self._enqueue_unique(task, args, kwargs) + + def enqueue_many[**P, R]( + self, + task: Task[P, R], + calls: Iterable[tuple[Sequence[Any], Mapping[str, Any]]], + ) -> list[TaskResult[P, R]]: + """ + The parent's bulk insert, refused for a unique task. + + django-ox writes an enqueue_many() call as one bulk INSERT and + never goes through enqueue(), so no uniqueness lock is taken. + The rows land beside a lock that knows nothing about them, every + later enqueue() of that key keeps returning the original task as + though deduplication were working, and the duplicates run. A + guarantee that fails silently is worse than a call that refuses, + so this refuses and names the task. + + Deduplicating in bulk is separate work, not a smaller version of + this: each key needs its lock row and its task row inserted + together, with the losers resolved against whichever holder is + live, and that is not one conditional INSERT on all three + supported databases. Non-unique tasks take the parent path + untouched, bulk INSERT included. + """ + if self._is_unique(task): + raise InvalidTask( + f"Task {task.module_path!r} is unique, and enqueue_many() " + "cannot hold its uniqueness lock: the rows go in as one bulk " + "INSERT that never reaches the deduplicating enqueue path, so " + "the duplicates would be written and run. Call task.enqueue() " + "once per call, or stop marking the task unique." + ) + return super().enqueue_many(task, calls) + + def _enqueue_unique[**P, R]( + self, task: Task[P, R], args: list[Any], kwargs: dict[str, Any] + ) -> TaskResult[P, R]: + from .models import OxpullUniqueLock + + # Deliberate double-validation: on the duplicate path + # super().enqueue() never runs, so this is the only validate_task + # call the request gets; removing it would make InvalidTask + # depend on queue state. + self.validate_task(task) + key = unique_key(self.alias, task.module_path, task.queue_name, args, kwargs) + db_alias = self._db_alias + + # A lost deadlock rolls back the victim's entire transaction, + # not just the statement. When this call owns its transactions, + # a victimised attempt left nothing behind and is reissued in + # the loop below. Inside a caller's transaction the server has + # discarded the caller's work too, and only the caller can + # decide to redo that, so the error propagates instead. A + # connection with autocommit turned off is inside one, as an + # atomic block is. + ambient_atomic = _in_callers_transaction(db_alias) + + last_conflict: DatabaseError | None = None + for _ in range(MAX_UNIQUE_ATTEMPTS): + conflict: IntegrityError | None = None + about_the_lock = False + try: + with transaction.atomic(using=db_alias): + try: + # The lock INSERT gets its own savepoint so that + # only its conflict is caught below; an + # IntegrityError out of super().enqueue() + # propagates raw instead of being misread as a + # duplicate. + with transaction.atomic(using=db_alias): + lock = OxpullUniqueLock.objects.using(db_alias).create( + key=key, + task_path=task.module_path, + queue_name=task.queue_name, + created_at=timezone.now(), + ) + except IntegrityError as exc: + # The primary key on key: another enqueue holds + # it. A conflicting INSERT blocks until the + # holding transaction resolves, within the + # database's lock wait: PostgreSQL waits for as + # long as the holder runs unless lock_timeout is + # set, SQLite for OPTIONS["timeout"] (5 s by + # default), MySQL for innodb_lock_wait_timeout + # (50 s by default). Inside that wait the + # winner has committed by the time IntegrityError + # surfaces. Past it the database raises instead, + # and the OperationalError handler below names + # that condition. + conflict = exc + else: + # Lock first, task second: a losing duplicate + # rolls back before super().enqueue() runs, so + # no task row is written and no task_enqueued + # signal fires for a suppressed duplicate. + result = super().enqueue(task, args, kwargs) + about_the_lock = True + # task_id=None keeps this UPDATE to the row just + # inserted. At PostgreSQL's REPEATABLE READ and + # SERIALIZABLE the INSERT can wait on another + # process's release of the key and go ahead once + # it commits, while the snapshot, taken before the + # wait, still shows the released row. Matched by + # key alone, that row fails the UPDATE with a + # serialization failure (40001). A committed lock + # row always names its task. SERIALIZABLE can + # still refuse the UPDATE for reading the released + # row, and the handler below treats that refusal + # as it treats one met while resolving. + OxpullUniqueLock.objects.using(db_alias).filter( + pk=lock.pk, task_id=None + ).update(task_id=result.id) + return result + last_conflict = conflict + + about_the_lock = True + existing = self._existing_result( + task, key, db_alias, ambient=ambient_atomic + ) + except OperationalError as exc: + if _is_lock_wait_timeout(exc, db_alias): + # The holder outlived the wait. Nothing of this + # enqueue survived: the transaction rolled back when + # this call owns it, the savepoint when it does not, + # so the caller's transaction is still usable. No + # retry here: another wait inside the caller's + # request is what a caller should choose, not have + # chosen for it, and UniqueEnqueueRace already means + # safe to retry. + raise UniqueEnqueueRace( + self._lock_wait_message( + task, exc, db_alias, ambient_atomic=ambient_atomic + ) + ) from exc + if about_the_lock and _is_serialization_failure(exc, db_alias): + # Only a refusal met after the task is written, or + # while resolving the duplicate, is about the key's + # lock or its holder: the locking read of a settled + # holder, the release DELETE, or the lock UPDATE. A + # pending holder's changed row never gets here, because + # _resolve_conflict reads it again instead. + # PostgreSQL's SERIALIZABLE can also fail the INSERTs + # above when it finds a dependency on another + # transaction, and that failure propagates as it would + # from any enqueue. + if ambient_atomic: + # The snapshot is the caller's, and an enqueue + # retried inside that transaction reads it again, + # so only the caller can retry usefully. + raise UniqueEnqueueRace( + self._serialization_message(task, db_alias) + ) from exc + # The transaction the failure rolled back was this + # call's own, as with a deadlock victim below, and + # the next pass takes a new snapshot. + last_conflict = exc + continue + if ambient_atomic or not is_innodb_deadlock(exc, db_alias): + raise + last_conflict = exc + continue + if existing is not None: + return existing + raise UniqueEnqueueRace( + self._race_message(task, db_alias, ambient=ambient_atomic) + ) from last_conflict + + def _lock_wait_message( + self, + task: Task[..., Any], + exc: OperationalError, + db_alias: str, + *, + ambient_atomic: bool, + ) -> str: + """ + Say what happened, per engine, and what is left to retry. + + Three things vary. SQLite in WAL mode refuses a reader's upgrade + to writer at once (SQLITE_BUSY_SNAPSHOT) rather than after the + busy timeout, so that case did not wait for anything. MySQL with + innodb_rollback_on_timeout=ON rolls the whole transaction back on + 1205, not the statement: Django's savepoint rollback then fails + and marks the connection for rollback, which is how it is known + here, and nothing the caller did in that transaction survives. + Everywhere else inside a caller's transaction the enqueue rolled + back to its savepoint and the transaction is still usable. + """ + connection = connections[db_alias] + snapshot = ( + connection.vendor == "sqlite" + and getattr(exc.__cause__, "sqlite_errorcode", None) == SQLITE_BUSY_SNAPSHOT + ) + if snapshot: + message = ( + f"Could not enqueue unique task {task.module_path!r}: this " + "transaction had read before it wrote, another connection " + "committed a write in between, and SQLite in WAL mode refuses " + "to make a writer of a reader whose snapshot is behind " + "(SQLITE_BUSY_SNAPSHOT); it did not wait. Nothing was written." + ) + else: + message = ( + f"Could not enqueue unique task {task.module_path!r}: another " + "enqueue of the same key held it for longer than the database's " + "lock wait, and the database stopped waiting for that transaction " + "to resolve. Nothing was written; the enqueue is safe to retry " + "once the holder has committed or rolled back." + ) + if not ambient_atomic: + return message + if connection.needs_rollback: + return message + ( + " This enqueue ran inside an open transaction, and the server " + "rolled that whole transaction back (MySQL with " + "innodb_rollback_on_timeout=ON): nothing done in it survives, " + "the atomic block rolls back on exit, and the whole transaction " + "is the thing to retry." + ) + message += ( + " This enqueue ran inside an open transaction. The enqueue rolled " + "back to its savepoint and the transaction is still usable, but " + "retry the whole transaction rather than the enqueue alone." + ) + level = self._snapshot_level(db_alias) + if level is None: + return message + return message + ( + f" At {level} the transaction reads from a snapshot, and a retry " + "inside it cannot see a winner that committed after that snapshot " + "began." + ) + + @staticmethod + def _snapshot_level(db_alias: str) -> str | None: + """ + The engine and level to name when this session's plain reads answer + from a snapshot, or None. A winner that committed after the snapshot + began is invisible to the lock read, so a retry inside the same + transaction ends the same way. Asked only on the way to an error. + """ + connection = connections[db_alias] + if connection.vendor == "mysql": + query = "SELECT @@transaction_isolation" + names = {"REPEATABLE READ": "MySQL's REPEATABLE READ"} + elif connection.vendor == "postgresql": + query = "SHOW transaction_isolation" + names = { + "REPEATABLE READ": "PostgreSQL's REPEATABLE READ", + "SERIALIZABLE": "PostgreSQL's SERIALIZABLE", + } + else: + return None + with connection.cursor() as cursor: + cursor.execute(query) + (level,) = cursor.fetchone() + return names.get(str(level).upper().replace("-", " ")) + + @staticmethod + def _serialization_message(task: Task[..., Any], db_alias: str) -> str: + """ + PostgreSQL refused a row the enqueue locked, updated or deleted, + because it changed after the caller's snapshot began. The level is + asked of the transaction, and only on the way to this error. + """ + connection = connections[db_alias] + at = "" + if connection.vendor == "postgresql" and not connection.needs_rollback: + with connection.cursor() as cursor: + cursor.execute("SHOW transaction_isolation") + (level,) = cursor.fetchone() + at = f" at {str(level).upper()}" + return ( + f"Could not enqueue unique task {task.module_path!r} on database " + f"{db_alias!r}. The enqueue ran inside a transaction{at}, and " + "PostgreSQL refused a row that changed after that transaction's " + "snapshot began (SQLSTATE 40001). The row was the task that held " + f"the key, or the key's lock. {RETRY_OUTSIDE_THE_SNAPSHOT}" + ) + + def _race_message( + self, task: Task[..., Any], db_alias: str, *, ambient: bool + ) -> str: + message = ( + f"Could not enqueue or deduplicate unique task " + f"{task.module_path!r} after {MAX_UNIQUE_ATTEMPTS} attempts." + ) + if not ambient: + return message + # The lock row is read on the caller's connection. Where that read + # answers from a snapshot, a winner that committed after the + # snapshot began is invisible to it, and every pass ends the same. + level = self._snapshot_level(db_alias) + if level is None: + return message + return message + ( + f" The enqueue ran inside a transaction at {level}, which reads " + "from a snapshot. The likely cause is a winner that committed " + "after that snapshot began, which the transaction cannot see. " + "Enqueue outside the transaction, or retry the whole transaction." + ) + + def _existing_result[**P, R]( + self, task: Task[P, R], key: str, db_alias: str, *, ambient: bool + ) -> TaskResult[P, R] | None: + """ + Resolve a lock conflict: return the live holder's result, or + release the stale lock and return None so the caller retries. + `ambient` says whether the caller's transaction is open around + this call, which is where a plain read can answer from a + snapshot older than the conflict. Inside one on PostgreSQL at + REPEATABLE READ or SERIALIZABLE, a pending holder whose row changed + after the snapshot began is read again on a connection of its own, + and the enqueue deduplicates only when that read finds it still + pending and still named by the key's lock. + + Staleness is derived purely from the task row. The reaper writes + LOST with no signal (django_ox worker.reap), the ImportError + terminal path can skip its signal, and ox_prune deletes rows + outright. All of them land here as "no longer READY or RUNNING, + or gone", which is the whole point of deriving instead of + listening. + + The lock row is read first, its task second, and the release at + the end is pinned to the (key, task_id) generation inspected, so + a release only ever removes a generation whose task this call + saw terminal or absent. The previous shape, a semijoin from the + task side with a lock re-read when it came back empty, could + re-read a winner that committed after the empty semijoin and + release its live lock; a CI postgres run caught it returning two + winners for one key. + + Every read is pinned to the write database: a router that sends + reads to a lagged replica must not be able to make a live lock + look stale. + """ + if not connections[db_alias].features.has_select_for_update: + # SQLite: no row locks, so there is nothing for a transaction + # to hold between the read and the DELETE, and no atomic + # block is opened here. When this call owns its transactions + # each statement then runs on its own, which matters because + # a deferred transaction that reads and then writes fails + # with "database is locked" the moment two of them race. + # Inside a caller's atomic() the statements run in that + # transaction regardless: skipping the block ends nothing, + # and the read-to-write upgrade it makes is the one + # SQLITE_BUSY_SNAPSHOT refuses (see _lock_wait_message). In + # both cases the DELETE at the end carries the holder's + # status in its own WHERE clause, which is what makes the + # release atomic against a retry: a statement runs whole + # against SQLite's single writer. + return self._resolve_conflict(task, key, db_alias, ambient=ambient) + with transaction.atomic(using=db_alias): + return self._resolve_conflict(task, key, db_alias, ambient=ambient) + + def _resolve_conflict[**P, R]( + self, task: Task[P, R], key: str, db_alias: str, *, ambient: bool + ) -> TaskResult[P, R] | None: + """ + The body of _existing_result, inside one transaction where the + database has row locks. + + The read that decides a release and the DELETE share a + transaction so that the row lock the read takes is still held + when the DELETE runs. This runs after the enqueue transaction + has ended, so without the atomic each statement would commit on + its own and the locking read would be refused. + """ + from django_ox.models import OxTask + from django_ox.results import task_result_from_db + + from .models import OxpullUniqueLock + + # The statuses in which a holder keeps its key. The read that + # decides a release and the DELETE that performs it test the + # same set. + pending = (OxTask.Status.READY, OxTask.Status.RUNNING) + lock = OxpullUniqueLock.objects.using(db_alias).filter(key=key).first() + if lock is None: + # Released by a concurrent enqueue, or rolled back with the + # transaction that inserted it. Nothing to release; retry. + return None + # A live holder is read without a lock. There is nothing to + # release for it, and a row lock taken here inside a caller's + # transaction (ATOMIC_REQUESTS, any atomic() block) would live + # until that transaction commits: for as long as the request ran, + # the holder's worker could not renew its lease or write its + # outcome and the reaper could not take the row. PostgreSQL waits + # for the request unless lock_timeout is set; MySQL gives up with + # 1205 after innodb_lock_wait_timeout. + # + # Inside a caller's transaction on PostgreSQL the same statement + # reads the transaction's isolation level, so asking costs no + # statement. It is asked of the transaction, not the settings or + # the driver, because a session or the transaction itself can set + # its own. At REPEATABLE READ and SERIALIZABLE the holder read + # answers from the snapshot, and a holder that finished after the + # snapshot began still reads as READY or RUNNING. Returning it + # would hand back a finished task's result and queue nothing. + # _check_holder_row asks PostgreSQL whether the row changed after + # the snapshot began, and keeps no lock. Only when it did are the + # holder and the key's lock read again, once, on a connection of + # their own. At READ COMMITTED the read is current, and when this + # call owns its transaction the snapshot began at the lock read a + # moment ago, so neither runs the check. + level_sql = self._isolation_level_sql(db_alias) if ambient else None + holder = self._holder( + lock.task_id, db_alias, locking=False, level_sql=level_sql + ) + live = holder is not None and holder.status in pending + if live and level_sql is not None: + level = str(getattr(holder, ISOLATION_LEVEL_ANNOTATION)).lower() + if level in PG_SNAPSHOT_LEVELS: + refusal = self._check_holder_row(holder.pk, db_alias) + if refusal is not None: + return self._result_from_a_fresh_read( + task, key, lock.task_id, db_alias, level=level, refusal=refusal + ) + if live and ambient and self._reads_a_snapshot(db_alias): + # Inside a caller's transaction on MySQL REPEATABLE READ that + # plain read answers from the transaction's snapshot, and a + # holder that finished since the snapshot began still reads + # as RUNNING: the enqueue would hand back a finished task's + # result and the new work would never be queued. InnoDB has + # no current read that takes no lock, and a lock here is the + # one ruled out above, so the committed status is read on a + # connection of its own, which sees every commit and holds + # nothing. Only a holder the snapshot calls live pays for + # that connection; a holder it calls settled is re-read FOR + # UPDATE below, a current read on this connection. At READ + # COMMITTED, Django's default for the backend, the plain + # read is current and nothing is opened. + live = self._committed_status(lock.task_id, db_alias) in pending + if live: + # Pending window holds. Retries also land here: a failed + # attempt with attempts remaining goes back to READY, so a + # retrying task keeps its key; retrying is not duplication. + # The result is built from the row alone: the caller's Task + # object may carry different using() overrides, and the row + # is the truth about the task that actually pends. + return cast("TaskResult[P, R]", task_result_from_db(holder)) + if holder is not None and connections[db_alias].features.has_select_for_update: + # The release path, and only here, reads the holder with a + # locking read: the DELETE below is a current write and the + # status it acts on must be current too. A plain SELECT under + # REPEATABLE READ answers from the transaction's snapshot: a + # caller that had read the holder as FAILED, while an operator + # retry committed READY in between, would read FAILED again + # here, delete the live lock and insert a second READY row + # beside the retried one. FOR UPDATE reads the committed row + # and waits for a retry still in flight. PostgreSQL at + # REPEATABLE READ or SERIALIZABLE refuses instead when the row + # changed after the snapshot began (SQLSTATE 40001), and + # _enqueue_unique reports that or runs a new pass. The row lock is + # released with the enclosing atomic: at once when this call + # owns it, at the caller's commit when it does not, and until + # then a retry of that settled row, a straggler's outcome + # write onto a LOST row and ox_prune of the row wait behind + # it. A retry that lands after the release reopens the key + # beside the new holder, the documented operator bypass. + holder = self._holder(lock.task_id, db_alias, locking=True) + if holder is not None and holder.status in pending: + # A retry committed between the two reads, or, inside a + # MySQL REPEATABLE READ transaction, since its snapshot. + # The row is locked now and stays so until the enclosing + # atomic ends; this is the one case a live holder's row + # is held, and it is bounded to that race. + return cast("TaskResult[P, R]", task_result_from_db(holder)) + # Stale or dangling: the task finished, was reaped to LOST, or + # its row was pruned; a null task_id never survives a committed + # enqueue. Release exactly the generation inspected; key is the + # primary key, so (key, task_id) pins the generation and the + # delete is a no-op if a concurrent enqueue already replaced the + # lock. The pin covers a replaced generation only: a + # django_ox.actions.retry moves a FAILED or LOST holder back to + # READY without touching this lock row, so its (key, task_id) is + # unchanged. The DELETE therefore carries the holder's status in + # its own WHERE clause as well, and deletes nothing when the + # holder is READY or RUNNING by then: on SQLite, where no row + # lock spans the read and the DELETE, that is the whole of the + # guard, and the statement is atomic against its single writer. + OxpullUniqueLock.objects.using(db_alias).filter( + key=key, task_id=lock.task_id + ).exclude( + Exists( + OxTask.objects.using(db_alias).filter( + id=OuterRef("task_id"), status__in=pending + ) + ) + ).delete() + return None + + @staticmethod + def _holder( + task_id: Any, db_alias: str, *, locking: bool, level_sql: str | None = None + ) -> Any: + """ + The holder's row, or None. With level_sql the row also carries the + transaction's isolation level as ISOLATION_LEVEL_ANNOTATION, read in + the same statement. + """ + from django_ox.models import OxTask + + if task_id is None: + return None + rows = OxTask.objects.using(db_alias).filter(id=task_id) + if locking: + rows = rows.select_for_update() + if level_sql is not None: + # A fixed expression from _isolation_level_sql, never input. + level = RawSQL(level_sql, (), output_field=CharField()) # noqa: S611 + return rows.annotate(**{ISOLATION_LEVEL_ANNOTATION: level}).first() + return rows.first() + + @staticmethod + def _isolation_level_sql(db_alias: str) -> str | None: + """ + The expression the holder read uses to report the transaction's + isolation level, or None where the read does not need it. Only + PostgreSQL does. MySQL asks its session when a holder reads as live, + as it did in 1.2.0, and SQLite has one level. + """ + if connections[db_alias].vendor != "postgresql": + return None + return "current_setting('transaction_isolation')" + + @classmethod + def _check_holder_row(cls, task_pk: Any, db_alias: str) -> OperationalError | None: + """ + Whether the holder's row changed after this transaction's snapshot + began. Returns PostgreSQL's serialization failure when it did. Returns + None when the row the snapshot holds was still the last committed one + at the check, including while another transaction is changing it. + + The check asks for a FOR SHARE lock on the row without waiting, inside a + savepoint, and rolls the savepoint back at once, granted or not. + That ends the lock before anything else runs, so the holder's lease + renewal, its outcome write, the reaper, a claim and ox_prune never + wait on this transaction. Released instead of rolled back, the lock + would last until the caller commits and hold them all off. A lock + this transaction took before the savepoint is kept. The request has + three answers: + + - Granted: no newer version of the row exists. A row this + transaction wrote or locked itself is granted too. + - lock_not_available (55P03): another transaction is writing or + locking the row and has not committed. This is contention that is + compatible with deduplicating, not a fresh read. The snapshot's + row was the last committed one when the check ran, and the other + transaction may commit a finish a moment later. A read at READ + COMMITTED leaves the same window. It is caught here, or + _enqueue_unique would take it for a lock wait. + - serialization_failure (40001): a change or a delete committed + after the snapshot began. The holder may have finished, or it may + only have renewed its lease, been claimed or been requeued. This + transaction cannot read the new row to tell, so the failure is + returned for a read on another connection to decide. + + Any other error propagates. A share lock, not a key share lock. A + key share lock conflicts only with a delete or a change to a key + column, so it is granted after the worker's outcome write and the + reaper's LOST write, on exactly the stale row this check exists to + catch. + """ + sid = _savepoint(db_alias) + try: + try: + cls._lock_holder_nowait(task_pk, db_alias) + finally: + transaction.savepoint_rollback(sid, using=db_alias) + except OperationalError as exc: + code = _pg_sqlstate(exc) + if code == PG_LOCK_NOT_AVAILABLE: + return None + if code == PG_SERIALIZATION_FAILURE: + return exc + raise + return None + + @staticmethod + def _lock_holder_nowait(task_pk: Any, db_alias: str) -> None: + """ + SELECT ... FOR SHARE NOWAIT on the holder's row, on the caller's + connection. The ORM's select_for_update() cannot ask for a share + lock. Table and column names come from the model and are quoted by + the backend. + """ + from django_ox.models import OxTask + + connection = connections[db_alias] + meta = OxTask._meta + id_field = meta.get_field("id") + # Concrete fields always carry a column; the stubs say Optional. + id_column = cast("str", id_field.column) + quote = connection.ops.quote_name + sql = ( + f"SELECT 1 FROM {quote(meta.db_table)} " # noqa: S608 - model names + f"WHERE {quote(id_column)} = %s FOR SHARE NOWAIT" + ) + with connection.cursor() as cursor: + cursor.execute(sql, [id_field.get_db_prep_value(task_pk, connection)]) + + def _result_from_a_fresh_read[**P, R]( + self, + task: Task[P, R], + key: str, + task_id: Any, + db_alias: str, + *, + level: str, + refusal: OperationalError, + ) -> TaskResult[P, R]: + """ + The pending holder's row changed after the caller's snapshot began. + Decide on the committed rows instead, read once on a connection of + their own. + + When the holder is still READY or RUNNING and the key's lock still + names it, a lease renewal, a claim or a reaper requeue changed the + row. The enqueue deduplicates and returns the result built from that + read. The caller's own transaction keeps reading its snapshot, so a + read of the task inside it can still show the older row. + + Otherwise the holder finished, its row is gone, or the key's lock is + gone or names another task. This transaction cannot release the lock, + because the row it would lock changed too, and it cannot see a newer + holder. UniqueEnqueueRace, chained to PostgreSQL's refusal, names the + database and the level and says where to retry. Nothing was written. + + When the read itself fails, the enqueue can't decide either, and it + raises UniqueEnqueueRace chained to that failure: a refused + connection, the read's lock or statement timeout, or any other + database error. The caller's transaction is untouched by it, and a + retry at READ COMMITTED needs no such read. + """ + from django_ox.models import OxTask + from django_ox.results import task_result_from_db + + opening = ( + f"Could not enqueue unique task {task.module_path!r} on database " + f"{db_alias!r}. The enqueue ran inside a transaction at " + f"{level.upper()}, and the task that held the key changed after that " + "transaction's snapshot began." + ) + try: + holder, lock_task_id = self._read_holder_fresh(task_id, key, db_alias) + except DatabaseError as exc: + raise UniqueEnqueueRace( + f"{opening} {self._fresh_read_failure(exc)} " + f"{RETRY_OUTSIDE_THE_SNAPSHOT}" + ) from exc + if holder is None: + found = "the task's row gone" + elif holder.status not in (OxTask.Status.READY, OxTask.Status.RUNNING): + found = f"the task {holder.status}" + elif lock_task_id is None: + found = "the key's lock gone" + elif lock_task_id != holder.pk: + found = "the key held by another task" + else: + return cast("TaskResult[P, R]", task_result_from_db(holder)) + raise UniqueEnqueueRace( + f"{opening} A fresh read found {found}. This transaction can't see " + f"that change. {RETRY_OUTSIDE_THE_SNAPSHOT}" + ) from refusal + + @staticmethod + def _fresh_read_failure(exc: DatabaseError) -> str: + """ + What a failed fresh read tells the caller. The database's words come + from the driver's primary message, without the statement text psycopg + adds after it, or else from the error's first line. + """ + diag = getattr(exc.__cause__, "diag", None) + detail = getattr(diag, "message_primary", None) + if not isinstance(detail, str) or not detail: + lines = str(exc).strip().splitlines() + detail = lines[0] if lines else type(exc).__name__ + message = f"A fresh read on a new connection failed ({detail})." + if _pg_sqlstate(exc) == PG_LOCK_NOT_AVAILABLE: + message += ( + f" That read waits at most {FRESH_READ_LOCK_TIMEOUT} for a lock. " + "A request for the task or lock table, such as ALTER TABLE, can " + "be waiting for this transaction to end, and the read queues " + "behind it." + ) + return message + + @staticmethod + def _read_holder_fresh(task_id: Any, key: str, db_alias: str) -> tuple[Any, Any]: + """ + The holder's committed row, or None when it is gone, and the task id + the key's lock names, or None when the lock is gone. One read. + + The read runs on a connection created for it and closed before + returning, whatever happens. The connection is never registered with + the connection handler, never kept for a later call and never taken + from OPTIONS["pool"], where a request waiting on its own pool could + stall. The read runs in a transaction of its own at READ COMMITTED, + whatever OPTIONS["isolation_level"] or the server's default says, so + it sees every commit made before it started. It takes no row lock. + The ORM builds the statement and Django's converters read the row, so + the result matches a read on the caller's connection. + + The transaction also bounds the wait. The caller's transaction holds + ACCESS SHARE on the task table and the lock table, from its own reads + of them. A request for ACCESS EXCLUSIVE on either (ALTER TABLE, LOCK + TABLE, TRUNCATE, VACUUM FULL) then waits on the caller, and this read + queues behind that request. The caller waits on this read outside the + database, so PostgreSQL sees no deadlock, and by default nothing ends + any of the three waits. Every worker statement on that table queues + behind the request too. While the caller holds its lock, no such + request can be granted, so a lock wait here ends only if that request + gives up. SET LOCAL ends the wait after FRESH_READ_LOCK_TIMEOUT and + the statement after FRESH_READ_STATEMENT_TIMEOUT, for this transaction + alone. A session-level SET would stay on a server session that a + pooler hands to another client. + + Connecting is bounded too. Where the project sets no connect timeout, + the connection waits FRESH_READ_CONNECT_TIMEOUT seconds for each + address. Nothing here bounds the host name lookup, or a wait in a + connection pooler's queue before the statement reaches PostgreSQL. + """ + from django_ox.models import OxTask + + from .models import OxpullUniqueLock + + fields = [field.attname for field in OxTask._meta.concrete_fields] + named = OxpullUniqueLock.objects.filter(key=key).values("task_id")[:1] + query = ( + OxTask.objects.filter(id=task_id) + .annotate(**{LOCK_HOLDER_ANNOTATION: Subquery(named)}) + .values_list(*fields, LOCK_HOLDER_ANNOTATION) + .query + ) + fresh = connections.create_connection(db_alias) + try: + # Only PostgreSQL reads a holder this way. The stubbed tests that + # drive this read on the other databases skip its bounds. + bounded = fresh.vendor == "postgresql" + options = { + name: value + for name, value in fresh.settings_dict.get("OPTIONS", {}).items() + if name not in ("pool", "isolation_level") + } + if bounded and not _sets_its_own_connect_timeout(options): + options["connect_timeout"] = FRESH_READ_CONNECT_TIMEOUT + fresh.settings_dict = { + **fresh.settings_dict, + "AUTOCOMMIT": True, + "OPTIONS": options, + } + with fresh.cursor() as cursor: + if bounded: + # Fixed text. With no parameters both drivers send the + # three statements in one round trip. + cursor.execute( + "BEGIN ISOLATION LEVEL READ COMMITTED; " + f"SET LOCAL lock_timeout = '{FRESH_READ_LOCK_TIMEOUT}'; " + "SET LOCAL statement_timeout = " + f"'{FRESH_READ_STATEMENT_TIMEOUT}'" + ) + compiler = query.get_compiler(connection=fresh) + rows = list(compiler.results_iter(tuple_expected=True)) + if bounded: + cursor.execute("COMMIT") + finally: + fresh.close() + if not rows: + return None, None + *values, lock_task_id = rows[0] + return OxTask.from_db(db_alias, fields, values), lock_task_id + + @staticmethod + def _reads_a_snapshot(db_alias: str) -> bool: + """ + Whether a plain SELECT on this connection, inside its open + transaction, answers from a read view older than the statement. + + MySQL at REPEATABLE READ, InnoDB's server default, fixes the read + view at the transaction's first consistent read; at READ + COMMITTED, which Django's MySQL backend sets on every connection + unless OPTIONS["isolation_level"] says otherwise, each statement + takes a fresh one. The session is asked rather than the setting + read, because the setting may be None for the server's default + and a session can change its own level. + """ + connection = connections[db_alias] + if connection.vendor != "mysql": + return False + with connection.cursor() as cursor: + cursor.execute("SELECT @@transaction_isolation") + (level,) = cursor.fetchone() + return str(level).upper().replace("_", "-") == "REPEATABLE-READ" + + @staticmethod + def _committed_status(task_id: Any, db_alias: str) -> str | None: + """ + The holder's committed status, read on a connection of its own. + + A connection opened here is outside the caller's transaction, so + its one statement sees the latest commit, and a consistent read + takes no lock. Table and column names come from the model and + are quoted by the backend; the ORM is not used because it routes + every query through the connection handler, and this connection + is deliberately not registered there. It is closed before + returning. None means the row is gone. + """ + from django_ox.models import OxTask + + meta = OxTask._meta + id_field = meta.get_field("id") + # Concrete fields always carry a column; the stubs say Optional. + id_column = cast("str", id_field.column) + status_column = cast("str", meta.get_field("status").column) + fresh = connections.create_connection(db_alias) + try: + quote = fresh.ops.quote_name + sql = ( + f"SELECT {quote(status_column)} " # noqa: S608 - model names + f"FROM {quote(meta.db_table)} " + f"WHERE {quote(id_column)} = %s" + ) + with fresh.cursor() as cursor: + cursor.execute(sql, [id_field.get_db_prep_value(task_id, fresh)]) + row = cursor.fetchone() + finally: + fresh.close() + return None if row is None else str(row[0]) + + def check(self, **kwargs: Any) -> list[checks.CheckMessage]: + if self.alias not in settings.TASKS: + # The parent's schedule checks index settings.TASKS by alias + # and would KeyError on a backend constructed outside TASKS; + # fail with a real message instead. + return [ + checks.Error( + f"Backend alias {self.alias!r} is not present in settings.TASKS.", + hint="check() requires a configured alias; construct " + "the backend through the TASKS setting.", + id="oxpull.E003", + ) + ] + errors = super().check(**kwargs) + if not apps.is_installed("oxpull"): + errors.append( + checks.Error( + "oxpull must be in INSTALLED_APPS to use OxpullBackend.", + hint="Add 'oxpull' to INSTALLED_APPS and run migrations.", + id="oxpull.E001", + ) + ) + errors.extend(self._check_unique_task_paths()) + errors.extend(self._check_router()) + errors.extend(self._check_unique_schedules()) + errors.extend(self._check_rate_limits()) + errors.extend(self._check_limit_names_across_backends()) + errors.extend(self._check_reconcile_schedule()) + errors.extend(self._check_workflows()) + return errors + + @staticmethod + def _first_oxpull_alias() -> str | None: + """ + The first OxpullBackend alias in settings.TASKS, in sort order. + + A check about the whole installation is emitted from this alias + alone, so it is reported once rather than once per alias, and one + id silences it. + """ + from .batches import task_backend_classes + + return min( + ( + alias + for alias, backend_class in task_backend_classes().items() + if issubclass(backend_class, OxpullBackend) + ), + default=None, + ) + + def _check_reconcile_schedule(self) -> list[checks.CheckMessage]: + """ + Warn, once, when no django-ox backend schedules the reconciler. + + Batches complete only through oxpull.batches.reconcile on a + schedule, and nothing else says so before the first batch sits + SEALED forever. The check reads the same settings create() and + seal() refuse on, and it is emitted from the first OxpullBackend + alias in sort order rather than from every alias, so one id + silences it in a project that uses uniqueness or limits and + never a batch. + + A Warning rather than an Error, unlike E006: check() runs with no + database, so whether batches exist is unknowable here, and an + Error would stop migrate and runserver for every installation + that never creates a batch. The silent failure an Error would + guard against is closed by the refusal in create() and seal(). + + One layout is an Error, oxpull.E008, in place of the warning: the + reconciler exposed to schedule rows, and no backend that reads + them able to run it, so django-ox skips every row that names it + and nothing else schedules it. Exposing the reconciler says the + project means to run batches, and django-ox reports the same fault + in its settings channel, a SCHEDULES entry on a queue its backend + refuses, as the Error django_ox.E002. A row is accepted by the + admin and create_schedule and then skipped with a log line, so + this is the first place that can refuse it. + """ + from .batches import ( + RECONCILE_SCHEDULE_NAME, + RECONCILE_TASK_PATH, + reconcile_schedule, + reconcile_schedule_hint, + ) + + if self._first_oxpull_alias() != self.alias: + return [] + schedule = reconcile_schedule(RECONCILE_TASK_PATH) + if schedule.aliases: + return [] + if schedule.unbound: + return [ + _unbound_reconciler_error( + RECONCILE_TASK_PATH, + RECONCILE_SCHEDULE_NAME, + schedule, + "no batch will ever complete", + ) + ] + return [ + checks.Warning( + "No django-ox backend in settings.TASKS schedules " + "oxpull.batches.reconcile. Batches complete only through " + "that schedule; create() and seal() refuse until one exists.", + hint=reconcile_schedule_hint(), + id="oxpull.W002", + ) + ] + + def _check_workflows(self) -> list[checks.CheckMessage]: + """ + Workflows declared without a worker that gates on dependencies. + + The same silent failure rate limits have, with a worse outcome: + the claim would carry no dependency conjunct, so every node of + every workflow runs the moment it is enqueued, in enqueue order, + and the queue looks healthy while the DAG is not a DAG. An Error + for that reason. + + WORKFLOWS is a declared option precisely so this check can exist. + A workflow is created at run time and no check could see one. + """ + from django_ox.worker import worker_class + + from .worker import OxpullWorker + from .workflows import workflows_enabled_from_options + + try: + enabled = workflows_enabled_from_options(self.options) + except ImproperlyConfigured as exc: + return [ + checks.Error( + str(exc), + hint="OPTIONS['WORKFLOWS'] switches the dependency " + "gate in the worker's claim; it is validated here so " + "a bad value cannot reach a running worker.", + id="oxpull.E007", + ), + ] + if not enabled: + return [] + # Whatever this backend's worker is: the warning is emitted from + # one backend only, and an Error here must not swallow it. + reconciler = self._check_workflow_reconciler() + try: + runs = worker_class(self.alias) + except ImproperlyConfigured as exc: + return [checks.Error(str(exc), id="oxpull.E007"), *reconciler] + if not issubclass(runs, OxpullWorker): + return [ + checks.Error( + f"Backend {self.alias!r} declares OPTIONS['WORKFLOWS'], " + f"but its worker is {runs.__module__}.{runs.__qualname__}, " + "which does not gate a claim on its dependencies. Every " + "workflow node would run as soon as it was enqueued, " + "silently.", + hint="Set OPTIONS['WORKER_CLASS'] to 'oxpull.worker.OxpullWorker'.", + id="oxpull.E007", + ), + *reconciler, + ] + return reconciler + + def _check_workflow_reconciler(self) -> list[checks.CheckMessage]: + """ + Workflows declared with nothing scheduled to reconcile them. + + The claim cannot cancel the downstream cone of a node that + settled without succeeding, and it cannot complete a sealed + workflow. Both are the reconciler's, and a workflow backend + without it fails silently in the worst way: every check passes, + the queue looks healthy, unreachable + nodes stay READY in every worker's candidate set forever, and no + callback ever fires. Nothing else in the package notices. + + The reconciler is global: it sweeps every SEALED workflow whatever + backend enqueued it, so scheduling it once on any django-ox backend + is a correct install. The count is the one batches use, for the + workflow reconciler's path, so create(), seal() and this check + answer from one place: both schedule channels count, a SCHEDULES + entry on a backend that is not django-ox does not, and a backend + reading schedule rows counts only when it can run the reconciler. + + The condition is global too, so it is reported once, from the first + backend in sort order that declares WORKFLOWS, as oxpull.W002 is. A + warning per workflow backend, each hint pasted into its own + backend, would define one schedule name on several backends, which + django_ox.E003 refuses. The hint names a backend the reconciler + binds to, so the paste passes django_ox.E002 as well. + + A Warning and not an Error, matching W001 and for a practical + reason: `manage.py check` is run by every management command, + `ox_worker` included, and an Error would refuse to start the very + worker an operator is adding the schedule for. The refusal in + workflows.create() and seal() closes the silent failure. The one + Error is oxpull.E008, for the reason _check_reconcile_schedule + gives. + """ + from .batches import reconcile_schedule + from .workflows import ( + RECONCILE_SCHEDULE_NAME, + RECONCILE_TASK_PATH, + declaring_aliases, + reconcile_schedule_hint, + ) + + declaring = declaring_aliases() + if not declaring or declaring[0] != self.alias: + return [] + schedule = reconcile_schedule(RECONCILE_TASK_PATH) + if schedule.aliases: + return [] + if schedule.unbound: + return [ + _unbound_reconciler_error( + RECONCILE_TASK_PATH, + RECONCILE_SCHEDULE_NAME, + schedule, + "no workflow will ever complete, and nothing will cancel " + "the downstream cone of a node that fails", + ) + ] + subject = ( + f"Backend {declaring[0]!r} declares" + if len(declaring) == 1 + else "Backends " + + ", ".join(repr(alias) for alias in declaring) + + " declare" + ) + return [ + checks.Warning( + f"{subject} OPTIONS['WORKFLOWS'], but no django-ox backend " + f"schedules {RECONCILE_TASK_PATH!r}. Nothing will cancel the " + "downstream cone of a node that fails, and no workflow will " + "ever complete or fire its callbacks. The reconciler is also " + "what releases a waiting node whose release a worker missed, " + "so without it that node waits forever. workflows.create() and " + "seal() refuse until one exists.", + hint=reconcile_schedule_hint(schedule), + id="oxpull.W003", + ) + ] + + def _check_rate_limits(self) -> list[checks.CheckMessage]: + """ + Validate RATE_LIMITS, then insist on a worker that applies it. + + Limits configured without WORKER_CLASS is the feature's only + silent failure: everything starts, nothing errors, and no limit + is ever applied. It is an Error rather than a Warning for that + reason. The check can see it because RATE_LIMITS is the only + place a limit is declared, and settings are readable here. + """ + from django_ox.worker import worker_class + + from .worker import OxpullWorker + + try: + limits = rate_limits_from_options(self.options, self.alias, self.queues) + refresh_interval_from_options(self.options) + except ImproperlyConfigured as exc: + return [ + checks.Error( + str(exc), + hint="Fix the OPTIONS entry the message names. " + "RATE_LIMITS and RATE_REFRESH_INTERVAL are validated " + "here so a bad value cannot reach a running worker.", + id="oxpull.E005", + ) + ] + if not limits: + return [] + errors = self._check_limited_task_paths(limits) + if errors: + return errors + try: + runs = worker_class(self.alias) + except ImproperlyConfigured as exc: + return [checks.Error(str(exc), id="oxpull.E006")] + if not issubclass(runs, OxpullWorker): + return [ + checks.Error( + f"Backend {self.alias!r} configures rate limit(s) " + f"{sorted(limits)}, but its worker is " + f"{runs.__module__}.{runs.__qualname__}, which does not " + "apply them. Every limit would be ignored, silently.", + hint="Set OPTIONS['WORKER_CLASS'] to 'oxpull.worker.OxpullWorker'.", + id="oxpull.E006", + ) + ] + return [] + + def _check_limit_names_across_backends(self) -> list[checks.CheckMessage]: + """ + One limit name configured differently on two OxpullBackend aliases. + + A limit's window row is keyed on its name alone, so every alias + that configures a name counts on one shared row. Different periods + cannot both hold on it. A worker on the shorter period rolls the + window as soon as its own period allows, and the count starts again + from zero. A worker on the longer period that reads the fresh window + admits against that count, so its backend can admit more than its + limit in its own period. A worker that has exhausted its limit + waits out its own window first, which slows the overrun and does + not stop it. No restart or deploy order fixes that, so the limit is + not enforced as configured. It is still a Warning, oxpull.W005, and + not an Error. django-ox's ox_worker runs every system check before + it starts, so an Error here would stop the workers of every backend + at upgrade, not only the two that disagree, and task processing with + them. That is the breaking upgrade oxpull.W002 is a Warning to avoid. + Different capacities over one period make a shared budget in which + each alias stops at its own mark in the shared count. That can be + meant, so it is a Warning too, under an id of its own, oxpull.W004. + + Two spellings of a name that differ only by case or accents are a + Warning of their own, oxpull.W006, whatever each alias configures + under them. MySQL's default collation makes one row of them, so they + share one window as one name does. PostgreSQL and SQLite, and a + binary MySQL collation such as utf8mb4_bin, make two, and there the + spellings already work as two separate limits. utf8mb4_0900_as_cs is + not named with them, because it makes one row of é precomposed and e + followed by a combining acute. Two spellings on one backend are + reported too, because the parser refuses only a case difference + there. On a collation that folds them, that backend's workers refuse + them at their first refresh, and the message says so. Settings do + not show the column's collation, so they are reported on every + database. An Error would stop every backend's workers at upgrade for + oxpull.W005's reason, and it would do so on installations where the + two limits work as their author meant. The message says what MySQL, + PostgreSQL and SQLite each do, not only the database check runs + against. check can run against a different database from the one the + workers use, SQLite in CI and MySQL in production, and a message + fitted to the check's database would then describe the wrong one. + _spelling_key decides which spellings are one. What else a collation + folds only the database can say, and a worker refuses it only within + its own limits. + + Limits are compared as rate_limits_from_options resolves them, so + a per of 60 and one of timedelta(minutes=1) are one period, as + they are to the worker. An alias whose RATE_LIMITS do not parse is + left out, and its own oxpull.E005 reports why. An alias whose limits + parse is compared, even when another of its checks fails. + + Settings only, like every check here. It never reads the database. + Emitted from the first OxpullBackend alias in sort order, as + oxpull.W002 is, so each name is reported once. + """ + from .batches import task_backend_classes + + if self._first_oxpull_alias() != self.alias: + return [] + configured: dict[str, list[tuple[str, RateLimit]]] = {} + for alias, backend_class in sorted(task_backend_classes().items()): + if not issubclass(backend_class, OxpullBackend): + continue + params = settings.TASKS[alias] + queues = params.get("QUEUES", [DEFAULT_TASK_QUEUE_NAME]) + try: + limits = rate_limits_from_options( + params.get("OPTIONS", {}), alias, queues + ) + except ImproperlyConfigured: + continue + for name, limit in limits.items(): + configured.setdefault(name, []).append((alias, limit)) + spellings: dict[str, list[str]] = {} + for name in sorted(configured): + spellings.setdefault(_spelling_key(name), []).append(name) + messages: list[checks.CheckMessage] = [] + for names in spellings.values(): + if len(names) > 1: + messages.append(_one_name_spelled_differently(names, configured)) + for name in names: + messages.extend(_one_name_disagreeing(name, configured[name])) + return messages + + def _check_limited_task_paths( + self, limits: dict[str, RateLimit] + ) -> list[checks.CheckMessage]: + """ + Every path under a limit's `tasks` must import to a Task. + + A limit matches on the task path stored in the row, so a typo + covers nothing and throttles nothing, and the queue looks healthy + while the third party being protected is not. Same reasoning as + the UNIQUE_TASKS check, and the same failure it prevents. + """ + errors: list[checks.CheckMessage] = [] + for limit in limits.values(): + for path in sorted(limit.task_paths): + try: + obj = import_string(path) + except ImportError: + obj = None + if not isinstance(obj, Task): + errors.append( + checks.Error( + f"RATE_LIMITS[{limit.name!r}]['tasks'] entry " + f"{path!r} is not a django.tasks Task. The limit " + "would cover nothing under that name.", + hint="Use the dotted path of a @task-decorated " + "function, as it is stored on the task row.", + id="oxpull.E005", + ) + ) + return errors + + def _check_unique_task_paths(self) -> list[checks.CheckMessage]: + """ + Shape errors raise in __init__; this validates the deeper + property, that every configured path imports to a django.tasks + Task. A typo here would otherwise just silently never dedup. + """ + errors: list[checks.CheckMessage] = [] + for path in sorted(self.unique_task_paths): + try: + obj = import_string(path) + except ImportError: + errors.append( + checks.Error( + f"UNIQUE_TASKS entry {path!r} does not import.", + hint="Use the dotted path of a @task-decorated function.", + id="oxpull.E002", + ) + ) + continue + if not isinstance(obj, Task): + errors.append( + checks.Error( + f"UNIQUE_TASKS entry {path!r} is not a django.tasks Task.", + hint="Point it at the module-level name created " + "by the @task decorator.", + id="oxpull.E002", + ) + ) + return errors + + def _check_router(self) -> list[checks.CheckMessage]: + try: + oxpull_db_alias() + except ImproperlyConfigured as exc: + return [ + checks.Error( + str(exc), + hint="Route django_ox and oxpull models to the same database.", + id="oxpull.E004", + ) + ] + return [] + + def _check_unique_schedules(self) -> list[checks.CheckMessage]: + """ + Warn when a schedule enqueues a unique task: schedules already + dedup once per tick, and a unique scheduled task coalesces with + any pending manual enqueue of the same key, so the tick row may + record against the manual task. + """ + raw = self.options.get("SCHEDULES", {}) + if not isinstance(raw, dict): + return [] # shape reported by the parent's schedule checks + warnings: list[checks.CheckMessage] = [] + for name, config in raw.items(): + path = config.get("task") if isinstance(config, dict) else None + if isinstance(path, str) and ( + is_registered(path) or path in self.unique_task_paths + ): + warnings.append( + checks.Warning( + f"Schedule {name!r} enqueues unique task {path!r}; " + "unique scheduled tasks coalesce with pending " + "manual enqueues, and ticks may record against " + "the manual task.", + hint="Schedules already dedup once per tick; " + "@unique on a scheduled task is usually redundant.", + id="oxpull.W001", + ) + ) + return warnings + + +def _one_name_disagreeing( + name: str, entries: list[tuple[str, RateLimit]] +) -> list[checks.CheckMessage]: + """oxpull.W005 or oxpull.W004 for one name, if its aliases disagree.""" + if len({(limit.limit, limit.per) for _, limit in entries}) < 2: + return [] + described = _in_words( + f"{alias!r} (limit={limit.limit}, per={limit.per})" for alias, limit in entries + ) + if len({limit.per for _, limit in entries}) > 1: + return [ + checks.Warning( + f"Rate limit {name!r} has a different period on backends " + f"{described}, and it is not enforced as configured. A limit's " + "window is keyed on its name alone, so these backends roll one " + "shared window, each at its own period. Each roll restarts the " + "count, so a backend on a longer period can admit more than its " + "limit in that period.", + hint="Use one period for this name on every backend, or give " + "each backend's limit a name of its own.", + id="oxpull.W005", + ) + ] + return [ + checks.Warning( + f"Rate limit {name!r} has a different limit on backends " + f"{described}. A limit's window is keyed on its name alone, so " + "they share one window and count each other's admissions. Each " + "backend stops when the shared count reaches its own limit, so " + "together they admit up to the largest one.", + hint="If a shared budget is what you want, use the same limit for " + "this name on every backend. Otherwise give each backend's limit a " + "name of its own.", + id="oxpull.W004", + ) + ] + + +def _one_name_spelled_differently( + names: list[str], configured: dict[str, list[tuple[str, RateLimit]]] +) -> checks.CheckMessage: + """ + oxpull.W006: spellings of one limit name that differ only by case or + accents. + + The spellings can be on one backend. The parser refuses two that differ + only by case there, but not two that differ by accents. A backend that + configures two spellings does not count on the shared window. On a + collation that folds them, its workers refuse them at their first + refresh, so the message says that for it. + """ + shown = _shown_names(names) + by_alias: dict[str, list[str]] = {} + for name in names: + for alias, _ in configured[name]: + by_alias.setdefault(alias, []).append(name) + if len(by_alias) == 1: + alias, spellings = next(iter(by_alias.items())) + subject = ( + f"Rate limit names {_in_words(shown[name] for name in spellings)} " + f"on backend {alias!r}" + ) + else: + described = _in_words( + f"{alias!r} ({_in_words(shown[name] for name in spellings)})" + for alias, spellings in sorted(by_alias.items()) + ) + subject = f"Rate limit names on backends {described}" + doubled = [alias for alias, found in sorted(by_alias.items()) if len(found) > 1] + single = [alias for alias, found in sorted(by_alias.items()) if len(found) == 1] + on_mysql = ( + "On MySQL's default collation, which ignores case and accents, these " + "spellings are one name" + ) + if not doubled: + on_mysql += ( + ", so the backends share one window and count each other's admissions." + ) + hint = ( + "Use one spelling of this name on every backend. If each backend " + "needs a limit of its own, put the backend's alias in each name." + ) + else: + noun = "backend" if len(doubled) == 1 else "backends" + on_mysql += ( + f", so the workers for {noun} {_in_words(map(repr, doubled))} " + "refuse them before claiming any task." + ) + if len(single) > 1: + on_mysql += ( + f" Backends {_in_words(map(repr, single))} count each other's " + "admissions." + ) + hint = ( + "Keep one of these limits, or give each one a name that differs by " + "more than case or accents." + ) + return checks.Warning( + f"{subject} differ only by case or accents. A limit's window is keyed " + "on its name, and the database's collation decides which spellings " + f"match. {on_mysql} On PostgreSQL and SQLite, and on a binary MySQL " + "collation such as utf8mb4_bin, they are two separate limits.", + hint=hint, + id="oxpull.W006", + ) + + +def _shown_names(names: list[str]) -> dict[str, str]: + """ + Each name as oxpull.W006 prints it. + + Two names that NFKC normalization makes one can print alike, such as é + precomposed and e followed by a combining acute. Those are printed with + each non-ASCII character as its code point, so the reader can tell + which one to change. + """ + forms = [unicodedata.normalize("NFKC", name) for name in names] + return { + name: _code_points(name) if forms.count(form) > 1 else repr(name) + for name, form in zip(names, forms, strict=True) + } + + +def _code_points(name: str) -> str: + """repr(name), with each non-ASCII character escaped as its code point.""" + return "".join( + char + if char.isascii() + else f"\\U{ord(char):08x}" + if ord(char) > 0xFFFF + else f"\\u{ord(char):04x}" + for char in repr(name) + ) + + +# Combining marks that Unicode 3.2 already had and that MySQL's default +# collation weighs as accents only. U+0363 to U+036F are small letters +# written above another, and it weighs those as letters. +_ACCENTS = frozenset( + chr(code) + for code in range(0x300, 0x363) + if unicodedata.ucd_3_2_0.category(chr(code)) == "Mn" +) + + +def _spelling_key(name: str) -> str: + """ + One key for the spellings of a limit name that differ only by case or + accents, for oxpull.W006. + + Two names with one key are meant to be one name to MySQL's default + collation, utf8mb4_0900_ai_ci, and to utf8mb4_unicode_ci. On MySQL 8.4, + pieces of names were compared with their own keys under both + collations, and every one matched. The pieces were each assigned code + point except controls, surrogates and private use, alone and + decomposed. They were also 100 base characters, most of them Latin, + Greek and Cyrillic letters, each followed by each mark from U+0300 to + U+036F. The last were ten vowels, each followed by 300 pairs of marks + from U+0300 to U+0362. Whole names were not compared. A longer name + matches only if MySQL weighs its characters one by one, which these + checks assume. The key is narrower than either collation, so it misses + some pairs they fold. + + Only characters Unicode 3.2 already had are folded. MySQL weighs later + ones, such as Georgian Mtavruli, apart from their lower case. Case is + folded with str.casefold(). Accents are dropped only from Latin and + Greek letters, because MySQL keeps й apart from и. A character is + decomposed only when that drops an accent, because MySQL keeps some + precomposed Tamil letters apart from their decomposition. Letters with + no decomposition that MySQL folds, such as ø into o, are left to the + worker, which asks the database. + """ + key = "" + for char in name: + if char in _ACCENTS: + if not (key and _takes_accents(key[-1])): + key += char + elif unicodedata.ucd_3_2_0.category(char) == "Cn": + key += char + else: + if _takes_accents(unicodedata.normalize("NFD", char)[0]): + folded = _unaccented(_unaccented(char).casefold()) + else: + folded = char.casefold() + if any(unicodedata.ucd_3_2_0.category(c) == "Cn" for c in folded): + folded = char + key += folded + return key + + +def _takes_accents(char: str) -> bool: + """A Latin or Greek letter that Unicode 3.2 already had.""" + return unicodedata.ucd_3_2_0.category(char).startswith("L") and ( + unicodedata.name(char, "").startswith(("LATIN ", "GREEK ")) + ) + + +def _unaccented(text: str) -> str: + return "".join( + char for char in unicodedata.normalize("NFD", text) if char not in _ACCENTS + ) + + +def _in_words(items: Iterable[str]) -> str: + """Items as a sentence lists them: "a", "a and b", "a, b and c".""" + listed = list(items) + if len(listed) < 2: + return "".join(listed) + return f"{', '.join(listed[:-1])} and {listed[-1]}" diff --git a/src/oxpull/batches.py b/src/oxpull/batches.py new file mode 100644 index 0000000..07b7a4a --- /dev/null +++ b/src/oxpull/batches.py @@ -0,0 +1,1633 @@ +""" +Batches: a group of tasks, and callbacks that fire once every member's row +is settled or gone. + + from oxpull import batches + + batch = batches.create("nightly-invoices", on_complete=summarize_run) + batch.add(send_invoice, 42) + batch.add(send_invoice, 43) + batch.seal() + + batch.progress() # BatchProgress(total=2, pending=1, succeeded=1, failed=0) + +The load-bearing decision: batch state is derived by querying member task +rows, never accumulated from task_finished signals. The django-ox reaper +reclaims a dead worker's task by writing FAILED through a plain queryset +update that emits no signal at all, and the attempts-exhausted path can +skip its signal when the task module no longer imports. A batch that +counted signals would therefore hang forever in exactly the scenario +batches exist to survive. Every count here is a query against the task +rows, so it cannot miss a death however it happened. + +Completion is detected by the reconcile() task, which django-ox's schedule +dispatcher runs once a minute. Completion detection is asynchronous: the +callback is enqueued when reconciliation observes that all members have +finished. Worker availability and queue delays can extend that interval. + +A sealed batch is not frozen in time, only frozen in membership. +django_ox.actions.retry puts a FAILED or LOST member back on the queue, +and the reconciler treats that as it treats any pending member: the +batch stays SEALED until the retry settles. Retrying a member of a batch +that has already completed does not reopen it, and its callbacks have +already fired. + + TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, + }, + } + } + +The reconciler Task is built on attribute access, not at import: a Task +is validated against its backend the moment it is constructed, so a +module-level @task would make importing this module fail in any project +whose QUEUES omits "default", and would build the default task backend +as a side effect of the import. `oxpull.batches.reconcile` is therefore +resolved lazily, on the default task backend's "default" queue when it +has one and on its first queue otherwise; the schedule's own `queue_name` +overrides that. The task runs on whichever backend's SCHEDULES name it: +django-ox rebinds a scheduled task to that backend at load, and refuses +a queue it does not accept with ImproperlyConfigured naming the schedule. +""" + +from __future__ import annotations + +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +import json +import logging +import random +import time +import uuid +import warnings +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, cast + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.db import IntegrityError, OperationalError, connections, transaction +from django.db.models import Count, QuerySet +from django.tasks import task +from django.tasks.base import Task, TaskResult +from django.utils import timezone +from django.utils.json import normalize_json +from django.utils.module_loading import import_string + +from django_ox import remaining +from django_ox.exceptions import TaskTimeout +from django_ox.models import OxTask + +from .exceptions import BatchMemberExists, BatchSealed +from .models import OxpullBatch, OxpullBatchMember +from .routing import oxpull_db_alias +from .unique import is_registered + +logger = logging.getLogger(__name__) + +# Bound on the reissue of a transaction this package owns after a lost +# InnoDB deadlock, and the longest first pause between two attempts after +# any lost conflict, in seconds. A pass ends that way only when another +# transaction closed a lock cycle with it, so reaching the bound takes that +# many lost cycles inside one call. Mirrors MAX_UNIQUE_ATTEMPTS. +MAX_ADD_ATTEMPTS = 5 +ADD_RETRY_JITTER = 0.05 + +# The bound after a serialization failure is higher. A deadlock has one +# victim, but a serialization failure reaches every transaction that waited +# on the writer that committed, so the same losers meet again: eight add()s +# held behind one at PostgreSQL REPEATABLE READ needed a fifth attempt twice +# in 60 races. The pause doubles with each attempt, up to 16 times the first. +MAX_SERIALIZATION_ATTEMPTS = 10 +SERIALIZATION_PAUSE_DOUBLINGS = 4 + +if TYPE_CHECKING: + # Built by __getattr__ at the bottom of this module; declared here so + # the public name is visible to type checkers and linters. + reconcile: Task[[], int] + +__all__ = ["Batch", "BatchProgress", "create", "get", "reconcile"] + +# A member is unfinished while its task row is in one of these. WAITING is +# a workflow node held back until its upstreams succeed. No batch member is +# ever written WAITING, and wherever the status appears it has not run. +PENDING_STATUSES = ( + OxTask.Status.READY, + OxTask.Status.RUNNING, + OxTask.Status.WAITING, +) + +# A member is finished when its task row has one of these statuses or has +# been deleted by ox_prune. Use this set rather than PENDING_STATUSES so +# an unrecognised status holds the batch open instead of assigning an +# outcome. The tests assert that the two sets cover every OxTask.Status +# in this checkout, so adding a status requires updating that decision. +SETTLED_STATUSES = ( + OxTask.Status.SUCCESSFUL, + OxTask.Status.FAILED, + OxTask.Status.LOST, + OxTask.Status.DISCARDED, +) + +# A member settled in one of these has no outcome that can be called +# clean, and is not a FAILED row either. LOST says the attempt was never +# observed. DISCARDED says an operator closed the task without running +# it, which is a decision on record but not work that was done. Both are +# held out of the counted set, so both land in the unattributed +# shortfall and both withhold on_success; neither is counted in +# failed_count, which stays a count of members recorded as FAILED. +UNATTRIBUTED_STATUSES = (OxTask.Status.LOST, OxTask.Status.DISCARDED) + +NAME_MAX_LENGTH = 128 + +# The path a schedule names to run the reconciler; the task row stores the +# same string, see the bottom of this module. +RECONCILE_TASK_PATH = "oxpull.batches.reconcile" + +# The schedule name every hint gives the reconciler. +RECONCILE_SCHEDULE_NAME = "oxpull-batch-reconcile" + +RECONCILE_SCHEDULE_HINT = ( + "Add to a django-ox backend's OPTIONS: " + '"SCHEDULES": {"oxpull-batch-reconcile": ' + '{"task": "oxpull.batches.reconcile", "cron": "* * * * *"}}.' +) + +# The same requirement, for a project whose schedules are rows rather than +# settings entries. Telling that project to add a SCHEDULES block reads as +# advice to undo the move it just made, so it is told what it is actually +# missing: the reconciler is not exposed, so no row is allowed to name it. +RECONCILE_SCHEDULABLE_HINT = ( + "Expose the reconciler to the schedule rows: add to a django-ox " + 'backend\'s OPTIONS "SCHEDULABLE_TASKS": {"oxpull-batch-reconcile": ' + '"oxpull.batches.reconcile"}, then add a schedule for that key.' +) + + +def task_backend_classes() -> dict[str, type[Any]]: + """Each settings.TASKS alias with the class its BACKEND names, where it imports.""" + classes: dict[str, type[Any]] = {} + for alias, params in settings.TASKS.items(): + if not isinstance(params, dict): + continue + try: + backend_class = import_string(str(params.get("BACKEND", ""))) + except ImportError: + continue + if isinstance(backend_class, type): + classes[alias] = backend_class + return classes + + +def _schedulable_task_path(config: Any) -> Any: + """The task one SCHEDULABLE_TASKS entry names; a bare path is the shorthand.""" + if isinstance(config, str): + return config + return config.get("task") if isinstance(config, dict) else None + + +def _ox_backend_options() -> dict[str, dict[str, Any]]: + """Each settings.TASKS alias backed by django-ox, with its OPTIONS.""" + from django_ox.backend import OxBackend + + found: dict[str, dict[str, Any]] = {} + for alias, backend_class in task_backend_classes().items(): + if not issubclass(backend_class, OxBackend): + continue + params = settings.TASKS[alias] + options = params.get("OPTIONS", {}) + if isinstance(options, dict): + found[alias] = options + return found + + +def _schedules_reconciler(options: dict[str, Any], task_path: str) -> bool: + """Whether OPTIONS["SCHEDULES"] declares the reconciler at `task_path`.""" + raw = options.get("SCHEDULES", {}) + if not isinstance(raw, dict): + return False + return any( + isinstance(config, dict) and config.get("task") == task_path + for config in raw.values() + ) + + +def _engine_reads_schedule_rows() -> bool: + """ + Whether django-ox's schedule dispatcher can load schedules from rows. + + Batch reconciliation requires row-backed schedule dispatch. Merely + accepting SCHEDULE_SOURCE and SCHEDULABLE_TASKS settings does not + establish that capability: a dispatcher that ignores them leaves a + sealed batch without its reconciler. Check the loader function that + implements SCHEDULE_SOURCE rather than inferring support from settings + or a version number. Both apps ship in one distribution, so this + capability is fixed within a release. + """ + from django_ox import schedules + + return callable(getattr(schedules, "schedule_source_from_options", None)) + + +def _reads_schedule_rows(options: dict[str, Any]) -> bool: + """ + Whether OPTIONS["SCHEDULE_SOURCE"] names a source, so schedules can be rows. + + Only that the option names something. django-ox reports a source it + cannot build as django_ox.E006 and the worker refuses to start on it, + so validating the path a second time here would report one fault + twice and disagree with django-ox about what a source may be: + a source is any class with the method, not only the one we ship. + """ + source = options.get("SCHEDULE_SOURCE") + return isinstance(source, str) and bool(source.strip()) + + +def _exposed_reconcilers( + options: dict[str, Any], task_path: str +) -> list[Task[..., Any]]: + """ + The Tasks OPTIONS["SCHEDULABLE_TASKS"] exposes that run the reconciler. + + Each entry is imported and matched on the Task's module_path, which is + the path its task row carries, rather than on the configured string: + the entry a row names may be the reconciler rebound with using() under + a project's own dotted path, and that is the one a row-reading backend + can run when the public one is on a queue it does not accept. An entry + that does not import is django_ox.E007's to report. + """ + raw = options.get("SCHEDULABLE_TASKS", {}) + if not isinstance(raw, dict): + return [] + found = [] + for config in raw.values(): + path = _schedulable_task_path(config) + if not isinstance(path, str): + continue + try: + obj = import_string(path) + except Exception: # noqa: S112 - django_ox.E007 reports it + continue + if isinstance(obj, Task) and obj.module_path == task_path: + found.append(obj) + return found + + +def _binds(task_obj: Task[..., Any], alias: str, queue_name: str | None = None) -> bool: + """ + Whether a schedule on backend `alias` can build `task_obj`. + + The same rebind django-ox makes: a schedule row, or a SCHEDULES entry, + runs its task on the backend that dispatches it, through Task.using(), + which validates the queue against that backend. A row that fails it is + skipped with a warning at every build, and nothing on the row can fix + it. Every exception counts as a failure, as it does where django-ox + builds the row. + """ + try: + if queue_name is not None: + task_obj.using(backend=alias, queue_name=queue_name) + elif task_obj.backend != alias: + task_obj.using(backend=alias) + except Exception: # django-ox skips a row on any exception + return False + return True + + +@dataclass(frozen=True) +class ReconcilerSchedule: + """ + What settings say about the schedule of one reconciler. + + aliases are the django-ox backends that run it. row_readers are the + django-ox backends that read schedule rows, on an engine that reads + them, and exposed are the Tasks SCHEDULABLE_TASKS lets a row name for + it, whether or not any of those backends can run them. + """ + + aliases: tuple[str, ...] + row_readers: tuple[str, ...] + exposed: tuple[Task[..., Any], ...] + + @property + def unbound(self) -> bool: + """ + Exposed to rows that are read, and still run by nothing. + + No backend that reads the rows accepts the queue the reconciler + is built on, so django-ox skips every row that names it, and no + SCHEDULES entry runs it either. + """ + return not self.aliases and bool(self.row_readers) and bool(self.exposed) + + +def reconcile_schedule(task_path: str = RECONCILE_TASK_PATH) -> ReconcilerSchedule: + """ + Where the reconciler at `task_path` is scheduled; see reconcile_schedule_aliases. + + Batches and workflows both ask here, each for its own reconciler. + """ + options_by_alias = _ox_backend_options() + by_settings = { + alias + for alias, options in options_by_alias.items() + if _schedules_reconciler(options, task_path) + } + row_readers: list[str] = [] + exposed: list[Task[..., Any]] = [] + by_rows: set[str] = set() + if _engine_reads_schedule_rows(): + row_readers = sorted( + alias + for alias, options in options_by_alias.items() + if _reads_schedule_rows(options) + ) + if row_readers: + exposed = [ + task_obj + for options in options_by_alias.values() + for task_obj in _exposed_reconcilers(options, task_path) + ] + by_rows = { + alias + for alias in row_readers + if any(_binds(task_obj, alias) for task_obj in exposed) + } + return ReconcilerSchedule( + aliases=tuple(sorted(by_settings | by_rows)), + row_readers=tuple(row_readers), + exposed=tuple(exposed), + ) + + +def reconcile_schedule_aliases(task_path: str = RECONCILE_TASK_PATH) -> list[str]: + """ + The django-ox backend aliases whose OPTIONS schedule the reconciler. + + Read from settings alone, with no database, so create(), seal() and + the system check answer one question from one place. A schedule on + any django-ox backend counts: schedules_from_options rebinds the task + to the backend it is declared on, so that backend's worker runs it, + and the reconciler resolves the oxpull tables through the router + rather than through the alias it was enqueued on. A backend that is + not django-ox dispatches no schedule, so an entry there counts for + nothing. + + Two channels declare a schedule and both count. An OPTIONS["SCHEDULES"] + entry naming the reconciler is the settings one. From django-ox 1.2.0 a + backend can instead name OPTIONS["SCHEDULE_SOURCE"] and keep its + schedules as admin-editable rows, which settings cannot see; for that + one the question settings can answer is whether a row is allowed to + name the reconciler at all, which is OPTIONS["SCHEDULABLE_TASKS"], and + whether the backend reading the rows can run what it exposes. The + registry is process-wide and a row is not bound to an alias, so the + backend that reads the rows and the backend whose OPTIONS expose the + reconciler need not be the same one. The backend that reads the rows + rebinds the task to itself, though, and skips a row whose task is on a + queue it does not accept, so a reader counts only when an exposed Task + binds to it. + + That second channel counts only where the engine implements it. Both + keys are inert on an engine that does not read them, and accepting + them there would seal batches nothing can complete on exactly the + projects that wrote the keys in good faith, so the installed + django-ox is asked first. + + The honest cost of counting a source is a project that reads rows and + exposes the reconciler but never writes the row: it passes here and + its batches sit SEALED. Reading the schedule table instead would make + the system check -- which runs without a database and before migrate + -- answer differently from create() and seal(), and that disagreement + is what this function exists to prevent. Refusing the configuration + outright is worse than the gap: it fails a project that has the + reconciler running, on the line that creates a batch. + """ + return list(reconcile_schedule(task_path).aliases) + + +def queue_accepted_by(backend: Any, params: Any) -> str: + """ + The queue a reconciler takes on `backend`: "default" when it accepts + that, else the first queue its settings declare. + """ + from django.tasks import DEFAULT_TASK_QUEUE_NAME + + queues = backend.queues + if not queues or DEFAULT_TASK_QUEUE_NAME in queues: + return DEFAULT_TASK_QUEUE_NAME + # The declared order is the operator's own; sorted() is only a + # fallback for a backend built outside settings.TASKS. + configured = params.get("QUEUES") if isinstance(params, dict) else None + return str((configured or sorted(queues))[0]) + + +def schedule_entry_hint( + task_path: str, schedule_name: str, candidates: list[str] +) -> str: + """ + A SCHEDULES entry for the reconciler, and the backend to add it to. + + The first candidate that runs the reconciler as it is built is named + with the plain entry. Failing that, the first one that runs it on a + queue of its own is named with that queue_name in the entry. Failing + both, no backend is named. Pasting the entry where it points passes + django-ox's schedule checks, which a backend the reconciler cannot bind + to does not: a queue it refuses is django_ox.E002, and the same + schedule name on two backends is django_ox.E003. + """ + from django.tasks import task_backends + + entry: dict[str, str] = {"task": task_path, "cron": "* * * * *"} + try: + reconciler = import_string(task_path) + except Exception: + reconciler = None + target: tuple[str, str | None] | None = None + if isinstance(reconciler, Task): + target = next( + ((alias, None) for alias in candidates if _binds(reconciler, alias)), None + ) + if isinstance(reconciler, Task) and target is None: + for alias in candidates: + try: + queue = queue_accepted_by(task_backends[alias], settings.TASKS[alias]) + except Exception: # noqa: S112 - the backend's own checks report it + continue + if _binds(reconciler, alias, queue_name=queue): + target = (alias, queue) + break + if target is None: + where = "a django-ox backend's OPTIONS['SCHEDULES']" + else: + where = f"TASKS[{target[0]!r}]['OPTIONS']['SCHEDULES']" + if target[1] is not None: + entry["queue_name"] = target[1] + return f"Add to {where}: {json.dumps({schedule_name: entry})}" + + +def unbound_reconciler_message( + task_path: str, schedule: ReconcilerSchedule, consequence: str +) -> str: + """Why an exposed reconciler runs nowhere; see ReconcilerSchedule.unbound.""" + queues = ", ".join( + sorted({repr(task_obj.queue_name) for task_obj in schedule.exposed}) + ) + readers = ", ".join(repr(alias) for alias in schedule.row_readers) + return ( + f"SCHEDULABLE_TASKS exposes {task_path} to schedule rows, but it is " + f"built on queue {queues}, and no backend that reads the rows accepts " + f"it ({readers}). django-ox skips every row that names it, and no " + f"SCHEDULES entry names it either, so {consequence}." + ) + + +def unbound_reconciler_hint( + task_path: str, schedule_name: str, schedule: ReconcilerSchedule +) -> str: + """What to do about an exposed reconciler that runs nowhere.""" + queues = ", ".join( + sorted({repr(task_obj.queue_name) for task_obj in schedule.exposed}) + ) + entry = schedule_entry_hint(task_path, schedule_name, list(schedule.row_readers)) + return ( + f"{entry}. Or read the schedule rows on a django-ox backend whose " + f"QUEUES include {queues}." + ) + + +def reconcile_schedule_hint() -> str: + """ + What to do about a missing reconciler schedule, given what is set already. + + A project whose backends read their schedules from rows has already + done what the settings hint asks for, in the other channel; naming it + again reads as advice to undo that. It is told the step it is + actually missing instead. + + On an engine that does not read rows at all, SCHEDULE_SOURCE is not a + move this hint should protect: nothing is reading the rows, exposing + the reconciler would change nothing, and the settings block is the + only remedy that starts one. + """ + if _engine_reads_schedule_rows() and any( + _reads_schedule_rows(options) for options in _ox_backend_options().values() + ): + return RECONCILE_SCHEDULABLE_HINT + return RECONCILE_SCHEDULE_HINT + + +def _require_reconcile_schedule() -> None: + """ + Refuse to create or seal a batch that nothing would ever complete. + + Completion happens only in reconcile(), and only a django-ox schedule + runs it. A project that copies the uniqueness and rate-limit settings + and misses the SCHEDULES block used to get batches that sat SEALED + forever, with no error and no log line: the system check can be + silenced and settings can change under a batch that already exists, + so the refusal lives here as well, on the line that configures the + batch and again on the line that seals it. A reconciler exposed to + schedule rows that no backend reading them can run is refused with + what it lacks, as oxpull.E008 reports it. + """ + schedule = reconcile_schedule(RECONCILE_TASK_PATH) + if schedule.aliases: + return + if schedule.unbound: + raise ImproperlyConfigured( + "Batches need a schedule for oxpull.batches.reconcile that a worker " + "can run. " + + unbound_reconciler_message( + RECONCILE_TASK_PATH, schedule, "a sealed batch would never complete" + ) + + " " + + unbound_reconciler_hint( + RECONCILE_TASK_PATH, RECONCILE_SCHEDULE_NAME, schedule + ) + + " The system check reports the same condition as oxpull.E008." + ) + raise ImproperlyConfigured( + "Batches need a schedule for oxpull.batches.reconcile, and no " + "django-ox backend in settings.TASKS declares one, so a sealed batch " + f"would never complete. {reconcile_schedule_hint()} The system check " + "reports the same condition as oxpull.W002." + ) + + +@dataclass(frozen=True) +class BatchProgress: + """ + Live counts for one batch, computed on demand from member task rows. + + total is the sealed snapshot once the batch is sealed, and the live + member count while it is open. pending, succeeded and failed come + from the task rows that still say so, and they need not add up to + total. The difference is unattributed: members whose task row + ox_prune deleted, members the reaper marked LOST because their + attempt was never observed, and members an operator closed with + django_ox.actions.discard. All three count as finished, since none of + them can still be pending, but none is attributable to success or + failure. + """ + + total: int + pending: int + succeeded: int + failed: int + + @property + def unattributed(self) -> int: + """ + Members that finished without an outcome that reads as clean. + + Three things land here. A member whose task row ox_prune deleted + joins to nothing. A member the reaper marked LOST holds a row + that says only that its attempt went unobserved. A member + django_ox.actions.discard closed holds a row saying an operator + cancelled it before it ran. None of the three is a success, and + none is a recorded failure, which is why on_success stays + withheld while this is above zero. + """ + return max(self.total - (self.pending + self.succeeded + self.failed), 0) + + @property + def pruned(self) -> int: + """ + Deprecated alias of unattributed, to be removed in oxpull 2.0. + + The name was accurate while ox_prune was the only way to lose an + outcome. The reaper's LOST rows and the operator's DISCARDED rows + now land in the same bucket, so it reports more than it says. + Reading it still returns the same number, and emits a + DeprecationWarning. + """ + # The warning lives in the getter rather than as a field or a + # __getattr__ hook, so the dataclass's fields, equality, repr, + # asdict and pickling are untouched. Attribute access reaches the + # getter without a Python frame between, so stacklevel=2 is the + # line that read .pruned. + warnings.warn( + "BatchProgress.pruned is deprecated and will be removed in oxpull 2.0. " + "Use BatchProgress.unattributed, which returns the same number.", + DeprecationWarning, + stacklevel=2, + ) + return self.unattributed + + @property + def finished(self) -> int: + return self.total - self.pending + + +def _resolve_callback( + callback: Task[..., Any] | str | None, + args: list[Any] | tuple[Any, ...] | None, + kwargs: dict[str, Any] | None, + label: str = "Batch", +) -> tuple[str, list[Any], dict[str, Any]]: + """ + Normalise a callback into (dotted path, args, kwargs) for storage. + + Everything about the callback is validated here rather than at + completion: a mistake should fail on the line that configures the + batch, not silently three hours later inside the transaction that + completes it. That covers the path (it must import to a Task on a + django-ox backend), the arguments (list and dict, JSON-serialisable + by the same normalize_json the task row uses) and the shape of the + Task itself, because only the path is stored and per-call using() + overrides would be dropped in silence. + """ + if callback is None: + return "", [], {} + if isinstance(callback, str): + try: + resolved = import_string(callback) + except ImportError as exc: + raise ImproperlyConfigured( + f"{label} callback {callback!r} does not import ({exc})." + ) from exc + else: + resolved = callback + if not isinstance(resolved, Task): + raise ImproperlyConfigured( + f"{label} callback {callback!r} is not a django.tasks Task; " + "pass the module-level name created by the @task decorator." + ) + _require_ox_backend(resolved, role="callbacks", label=label) + _reject_using_overrides(resolved, label=label) + _reject_unique(resolved, label=label) + if args is not None and not isinstance(args, list | tuple): + raise ImproperlyConfigured( + f"{label} callback {resolved.module_path!r} args must be a list " + f"or tuple, not {type(args).__name__}." + ) + if kwargs is not None and not isinstance(kwargs, dict): + raise ImproperlyConfigured( + f"{label} callback {resolved.module_path!r} kwargs must be a " + f"dict, not {type(kwargs).__name__}." + ) + try: + stored_args = cast("list[Any]", normalize_json(list(args or []))) + stored_kwargs = cast("dict[str, Any]", normalize_json(dict(kwargs or {}))) + except (TypeError, ValueError) as exc: + raise ImproperlyConfigured( + f"{label} callback {resolved.module_path!r} arguments are not " + f"JSON-serializable ({exc})." + ) from exc + return resolved.module_path, stored_args, stored_kwargs + + +def _member_task_counts(batch_pk: uuid.UUID, db_alias: str) -> dict[str, int]: + """ + Member task rows grouped by status. + + This is the join the whole design rests on, expressed through the ORM + rather than as SQL text: table and column names come from the + django_ox model's _meta through the query compiler, so nothing here + hardcodes a django-ox table name that a future release could rename. + The join cannot cross database aliases, which is why both sides are + pinned to the one alias oxpull_db_alias() resolves. + + A membership row whose task row is gone joins to nothing and so + appears in no bucket: pruned reads as finished, which is what keeps + long-lived batches completing after ox_prune has been through. + """ + members = ( + OxpullBatchMember.objects.using(db_alias) + .filter(batch_id=batch_pk) + .values("task_id") + ) + rows = ( + OxTask.objects.using(db_alias) + .filter(id__in=members) + .values("status") + .annotate(count=Count("id")) + .order_by() + ) + return {row["status"]: row["count"] for row in rows} + + +def _pending_count(counts: dict[str, int]) -> int: + """ + Members whose task row is present and not settled. + + Counted as everything outside SETTLED_STATUSES rather than everything + inside PENDING_STATUSES: the two agree on every status this package + knows, and differ on one it does not, which must read as pending. A + row that is absent altogether is in no bucket and reads as finished. + Workflow completion and progress read this same function. + """ + return sum( + count for status, count in counts.items() if status not in SETTLED_STATUSES + ) + + +DEADLOCK = "deadlock" +SERIALIZATION_FAILURE = "serialization_failure" + + +def _lost_conflict( + exc: OperationalError, db_alias: str, *, deadlocks: bool +) -> str | None: + """ + DEADLOCK or SERIALIZATION_FAILURE when `exc` is one, else None. + + A deadlock is InnoDB's 1213 or PostgreSQL's 40P01. Both kinds roll back + the whole transaction on the server and leave nothing behind, and both + engines' manuals say to run the transaction again. + """ + from .backend import is_deadlock, is_serialization_failure + + if deadlocks and is_deadlock(exc, db_alias): + return DEADLOCK + if is_serialization_failure(exc, db_alias): + return SERIALIZATION_FAILURE + return None + + +def _reissue[T]( + once: Callable[[], T], + db_alias: str, + *, + deadlocks: bool, + retrying: Callable[[str, int, int], None], +) -> T: + """ + Run `once`, and run it again when the transaction it opens is lost to + a conflict: a PostgreSQL serialization failure, and a deadlock (InnoDB's + 1213 or PostgreSQL's 40P01) where `deadlocks` is set. + + Whether this call owns the transaction is read before `once` opens its + atomic block: inside it the block is always active. The call owns it + when the connection is in autocommit and no atomic block is open. Then + the server rolled the whole of it back, so the body can run again from + the top. Inside a caller's atomic block, or a transaction the caller + opened by turning autocommit off, the server discarded the caller's work + too, and only the caller can redo that, so the error propagates. A + savepoint retry would be unsound either way: the server rolled back the + transaction, not the statement. + + `retrying` is told which conflict, which attempt and the bound, before + the pause. The pause is jittered so that the losers of one conflict do + not run again in lockstep. A deadlock has one victim, so a short pause + and MAX_ADD_ATTEMPTS do. A serialization failure reaches every + transaction that waited on the writer that committed, so its pause + doubles with each attempt and its bound is MAX_SERIALIZATION_ATTEMPTS. + """ + connection = connections[db_alias] + owns_transaction = connection.get_autocommit() and not connection.in_atomic_block + for attempt in range(max(MAX_ADD_ATTEMPTS, MAX_SERIALIZATION_ATTEMPTS)): + try: + return once() + except OperationalError as exc: + conflict = ( + _lost_conflict(exc, db_alias, deadlocks=deadlocks) + if owns_transaction + else None + ) + bound = ( + MAX_SERIALIZATION_ATTEMPTS + if conflict == SERIALIZATION_FAILURE + else MAX_ADD_ATTEMPTS + ) + if conflict is None or attempt + 1 >= bound: + raise + retrying(conflict, attempt + 1, bound) + pause = ADD_RETRY_JITTER + if conflict == SERIALIZATION_FAILURE: + pause *= 2 ** min(attempt, SERIALIZATION_PAUSE_DOUBLINGS) + time.sleep(random.uniform(0, pause)) # noqa: S311 + raise AssertionError("unreachable: the loop returns or raises") + + +def _reissue_add[T]( + once: Callable[[], T], + db_alias: str, + *, + label: str, + owner_pk: uuid.UUID, + task_path: str, + log: logging.Logger, +) -> T: + """ + Run one add() attempt, and reissue it after a lost deadlock or + PostgreSQL serialization failure, when add() owns its transaction. + + Batch.add and Workflow.add both go through here. Every add() to one + row claims it with an UPDATE, so at PostgreSQL REPEATABLE READ two + overlapping add()s are a serialization failure for the one that waited. + """ + noun = label.lower() + + def retrying(conflict: str, attempt: int, bound: int) -> None: + what = "a deadlock" if conflict == DEADLOCK else "a serialization failure" + event = "deadlock" if conflict == DEADLOCK else "serialization" + log.warning( + f"{label} %s: add() of %r lost {what} on attempt %d of %d; " + "reissuing its transaction", + owner_pk, + task_path, + attempt, + bound, + extra={ + "event": f"{noun}_add_{event}_retry", + f"{noun}_id": str(owner_pk), + "task_path": task_path, + "attempt": attempt, + }, + ) + + return _reissue(once, db_alias, deadlocks=True, retrying=retrying) + + +def _claim_open(row: QuerySet[Any], open_status: str) -> str | None: + """ + Take the lock that serialises add() against seal(), and return the + status it sees: OPEN, another status, or None when the row is gone. + + `row` is the batch or workflow row, filtered by primary key. A no-op + UPDATE under the OPEN predicate takes the row lock seal() takes. A + rowcount of one settles it. Zero does not: it is the row no longer + OPEN, or it is a MySQL connection counting changed rows rather than + matched ones, which a no-op UPDATE never changes. Django asks for + CLIENT_FOUND_ROWS on every MySQL connection, and a project that sets + OPTIONS["client_flag"] replaces it. So the status is read under the + row lock: where the UPDATE matched, the lock is already this + transaction's; where it did not, this read takes it now. Either way + the answer is current, not the caller's snapshot, which under + REPEATABLE READ can still say OPEN after a seal() committed. + """ + if row.filter(status=open_status).update(status=open_status): + return open_status + return cast( + "str | None", row.select_for_update().values_list("status", flat=True).first() + ) + + +def _count_current(rows: QuerySet[Any], db_alias: str, *, ambient: bool) -> int: + """ + Count `rows` as of now, for seal(); Batch._count_members says why. + + A plain COUNT when seal() owns its transaction. A locking read, which + is a current read, only inside a caller's transaction, whose snapshot + can predate a row committed since, and only where row locks exist. + """ + if not ambient or not connections[db_alias].features.has_select_for_update: + return rows.count() + return len(rows.select_for_update().values_list("pk", flat=True)) + + +class Batch: + """ + A handle on one OxpullBatch row. Build one with create() or get(). + + The handle holds no derived state: status and progress are read from + the database on each call, because the reconciler that completes a + batch runs in another process. + """ + + def __init__(self, row: OxpullBatch) -> None: + self._row = row + + def __repr__(self) -> str: + return f"" + + @property + def id(self) -> uuid.UUID: + return self._row.pk + + @property + def name(self) -> str: + return self._row.name + + @property + def status(self) -> str: + """The status as stored now, not as it was when this handle was built.""" + db_alias = oxpull_db_alias() + return str( + OxpullBatch.objects.using(db_alias) + .values_list("status", flat=True) + .get(pk=self._row.pk) + ) + + def add[**P, R]( + self, task: Task[P, R], *args: P.args, **kwargs: P.kwargs + ) -> TaskResult[P, R]: + """ + Enqueue a task and record its membership, in one transaction. + + The task row and the membership row commit or roll back together, + so a batch can never contain a member that was never enqueued, nor + enqueue work the batch does not know it is waiting for. + + Raises BatchSealed if the batch is no longer OPEN. + """ + db_alias = oxpull_db_alias() + _require_ox_backend(task, role="members") + return _reissue_add( + lambda: self._add_once(db_alias, task, *args, **kwargs), + db_alias, + label="Batch", + owner_pk=self._row.pk, + task_path=task.module_path, + log=logger, + ) + + def _add_once[**P, R]( + self, db_alias: str, task: Task[P, R], *args: P.args, **kwargs: P.kwargs + ) -> TaskResult[P, R]: + """One attempt of add(): the transaction that add() may reissue.""" + with transaction.atomic(using=db_alias): + # Enqueue first, claim second. A unique member's enqueue + # takes the OxpullUniqueLock row for its key, so doing it + # under the batch row lock would make add() hold batch -> + # unique lock while an ordinary caller that enqueues a unique + # task and then calls add() holds unique lock -> batch: a + # deadlock cycle, observed on PostgreSQL and reported as + # ER_LOCK_DEADLOCK on MySQL. This order is the one callers + # can also follow, and it keeps the batch row locked for two + # statements rather than for a whole enqueue. + result = task.enqueue(*args, **kwargs) + # A no-op UPDATE under the OPEN predicate, taken before the + # membership INSERT, is what serialises add() against seal(). + # It takes the same row lock seal() takes, on every supported + # database, so either seal() waits for this member to commit + # and counts it, or this add() finds the batch already SEALED + # and the whole transaction, enqueue included, rolls back. A + # plain SELECT would not do: under MVCC it can read OPEN from + # a snapshot while seal() holds an uncommitted UPDATE, and + # the member would land after the count that total_count + # froze. + status = _claim_open( + OxpullBatch.objects.using(db_alias).filter(pk=self._row.pk), + OxpullBatch.Status.OPEN, + ) + if status != OxpullBatch.Status.OPEN: + raise self._not_open_error(status) + try: + # Its own savepoint: a duplicate membership must not + # poison a caller's surrounding transaction, the same + # confinement the unique enqueue path uses. + with transaction.atomic(using=db_alias): + OxpullBatchMember.objects.using(db_alias).create( + batch_id=self._row.pk, + task_id=result.id, + created_at=timezone.now(), + ) + except IntegrityError as exc: + raise BatchMemberExists( + f"Task {result.id} is already a member of batch " + f"{self._row.pk}; a deduplicated unique enqueue returned " + "a task this batch already waits on." + ) from exc + return result + + def seal(self) -> None: + """ + Freeze membership and snapshot total_count. Completion is possible + only after this; a batch that is never sealed never completes. + + Raises BatchSealed if the batch was already sealed or completed. + There is no un-seal. Raises ImproperlyConfigured if no django-ox + backend schedules the reconciler, since the batch would then stay + SEALED forever; the status is checked first, so a batch that is + already sealed says so whatever the settings. + """ + db_alias = oxpull_db_alias() + # Read before the block opens; inside it the answer is always yes. + ambient = connections[db_alias].in_atomic_block + with transaction.atomic(using=db_alias): + # The status write comes first and locks the row, so any + # add() already in flight commits before the count below runs + # and any later add() sees SEALED. Counting first would let a + # member land after the snapshot and make total_count a lie. + sealed = ( + OxpullBatch.objects.using(db_alias) + .filter(pk=self._row.pk, status=OxpullBatch.Status.OPEN) + .update(status=OxpullBatch.Status.SEALED, sealed_at=timezone.now()) + ) + if not sealed: + raise self._not_open_error(self._stored_status(db_alias)) + # After the status, so that a batch already SEALED or COMPLETE + # reports that rather than the settings. Inside the block, so + # that a refusal here rolls the status write back and the + # batch stays OPEN. + _require_reconcile_schedule() + OxpullBatch.objects.using(db_alias).filter(pk=self._row.pk).update( + total_count=self._count_members(db_alias, ambient=ambient) + ) + + def _count_members(self, db_alias: str, *, ambient: bool) -> int: + """ + The member count as of now, not as of an older snapshot. + + When seal() owns its transaction the status write above is the + transaction's first statement. It waits on every add() still + holding the batch row, and the count that follows is the + transaction's first consistent read, so even under MySQL's + REPEATABLE READ its read view begins after those add()s + committed; a later add() finds the row SEALED. PostgreSQL at + REPEATABLE READ takes its snapshot at that first statement, and + a status write that waited on an add() then finds a row changed + since the snapshot and raises a serialization failure rather + than proceeding, so nothing is counted stale there either: the + batch stays OPEN and the caller retries seal(). Django's + PostgreSQL default is READ COMMITTED, where the UPDATE re-reads + the row and the COUNT takes a fresh snapshot. A plain COUNT is + current wherever seal() proceeds, and it is one aggregate rather + than a row lock per member pulled into Python, which at a million + members on MySQL was the difference between 28 seconds and a + count. + + Inside a caller's transaction the snapshot can be older than the + lock: under REPEATABLE READ the read view was fixed by the + caller's first consistent read, and a member whose add() + committed after that is invisible to a plain COUNT even though + the status write waited on nothing. A locking read is a current + read on every database that has one, and it also holds the + members it counted for the rest of the caller's transaction. That + is the cost of sealing inside a transaction, and only there. + The locking read blocks no INSERT of a new member; the freeze + comes from the status flip above, which add()'s UPDATE ... WHERE + status = OPEN no longer matches. + SQLite has no row locks and needs none: writers are serialised. + """ + return _count_current( + OxpullBatchMember.objects.using(db_alias).filter(batch_id=self._row.pk), + db_alias, + ambient=ambient, + ) + + def progress(self) -> BatchProgress: + """Counts for this batch, queried from the member task rows now.""" + db_alias = oxpull_db_alias() + row = OxpullBatch.objects.using(db_alias).get(pk=self._row.pk) + if row.total_count is not None: + total = row.total_count + counts = _member_task_counts(row.pk, db_alias) + else: + # An open batch has no snapshot yet, so the live membership + # count is the only honest denominator, and it is a separate + # statement from the status counts. Read it first and let the + # counted members raise it: an add() committing between the + # two then makes this reading lag the batch by a member, + # which is harmless. The other order inflates the + # denominator instead, and total - pending reports members + # that neither finished nor were pruned. + members = ( + OxpullBatchMember.objects.using(db_alias) + .filter(batch_id=row.pk) + .count() + ) + counts = _member_task_counts(row.pk, db_alias) + total = max(members, sum(counts.values())) + return BatchProgress( + total=total, + pending=_pending_count(counts), + succeeded=counts.get(OxTask.Status.SUCCESSFUL, 0), + failed=counts.get(OxTask.Status.FAILED, 0), + ) + + def _stored_status(self, db_alias: str) -> str | None: + return ( + OxpullBatch.objects.using(db_alias) + .values_list("status", flat=True) + .filter(pk=self._row.pk) + .first() + ) + + def _not_open_error(self, status: str | None) -> BatchSealed: + if status is None: + raise OxpullBatch.DoesNotExist(f"Batch {self._row.pk} no longer exists.") + return BatchSealed(f"Batch {self._row.pk} is {status}; membership is frozen.") + + +def _require_ox_backend( + task: Task[..., Any], *, role: str, label: str = "Batch" +) -> None: + """ + Members and callbacks alike are only meaningful as OxTask rows. + + A member enqueued through, say, the immediate backend writes no row + for the completion join to find, and the batch would read it as + already finished. A callback on that backend is worse: the immediate + backend runs the function inline, inside the transaction that + completes the batch, and returns an id that is not a UUID, so + recording it raises and rolls the completion back with the side + effect already done. The next tick then repeats it, forever. Fail on + the add() or the create() instead. + """ + from django_ox.backend import OxBackend + + backend = task.get_backend() + if not isinstance(backend, OxBackend): + raise ImproperlyConfigured( + f"{label} {role} must be enqueued on a django-ox backend; task " + f"{task.module_path!r} uses backend {task.backend!r} " + f"({type(backend).__name__}), whose tasks have no OxTask row." + ) + + +def _reject_using_overrides(callback: Task[..., Any], label: str = "Batch") -> None: + """ + Refuse a callback carrying per-call using() overrides. + + Only the dotted path is stored, so the reconciler re-imports the + module-level Task and enqueues that. A queue, priority or run_after + set with using() on the way in would be dropped in silence, and the + callback would land on a queue nobody is draining. Reject it on the + create() line and say so. + """ + declared = import_string(callback.module_path) + if declared == callback: + return + changed = ( + sorted( + field + for field in ("queue_name", "priority", "run_after", "backend") + if getattr(declared, field) != getattr(callback, field) + ) + if isinstance(declared, Task) + else [] + ) + if not changed: + raise ImproperlyConfigured( + f"{label} callback {callback.module_path!r} does not import " + "back to itself; callbacks are stored as a dotted path, so the " + "module-level name must be this Task." + ) + raise ImproperlyConfigured( + f"{label} callback {callback.module_path!r} carries using() " + f"override(s) {', '.join(changed)}, which cannot be stored: " + "only the dotted path is kept, so the callback would be enqueued " + "with the module-level defaults. Declare the overrides on the " + "@task decorator instead." + ) + + +def _reject_unique(callback: Task[..., Any], label: str = "Batch") -> None: + """ + Refuse a unique task as a callback. + + A unique task coalesces: while one with the same arguments is + pending, enqueueing it again returns the pending task instead of + writing a second row. Two batches completing with the same unique + callback and the same arguments would therefore share one callback + task, and the batch that coalesced would record an id whose run + belongs to the other batch. Its on_complete would never fire, and + nothing would say so. Refused on the create() line, and again at + completion, because a later deploy can make the path unique through + UNIQUE_TASKS or the decorator after the batch row was written. + """ + from .backend import OxpullBackend + + path = callback.module_path + backend = callback.get_backend() + configured = ( + isinstance(backend, OxpullBackend) and path in backend.unique_task_paths + ) + # "Batch" keeps the batch wording exactly; any other label pluralises plainly. + plural = "batches" if label == "Batch" else f"{label.lower()}s" + if is_registered(path) or configured: + raise ImproperlyConfigured( + f"{label} callback {path!r} is a unique task. Two {plural} " + "completing with the same unique callback and arguments would " + "coalesce onto one callback task, and one " + f"{label.lower()}'s callback would never run. Use a task that is " + "not unique." + ) + + +def create( + name: str, + *, + on_complete: Task[..., Any] | str | None = None, + on_complete_args: list[Any] | tuple[Any, ...] | None = None, + on_complete_kwargs: dict[str, Any] | None = None, + on_success: Task[..., Any] | str | None = None, + on_success_args: list[Any] | tuple[Any, ...] | None = None, + on_success_kwargs: dict[str, Any] | None = None, +) -> Batch: + """ + Create an OPEN batch. + + on_complete fires once every member's row is settled (see + SETTLED_STATUSES) or gone, whatever the outcome; any other status + holds the batch open, and a member recorded LOST may still be + executing, so this is not a barrier against a running member. on_success fires in + addition, and only when every member's outcome is known and none of + them FAILED. Each is a Task or its + dotted path, with optional arguments; both are enqueued as ordinary + tasks, retried and failed like any other, and the batch does not + supervise them beyond storing their ids. + + Raises ImproperlyConfigured if no django-ox backend schedules the + reconciler: a batch nothing completes should fail on the line that + creates it. + """ + _require_reconcile_schedule() + if len(name) > NAME_MAX_LENGTH: + # SQLite would store it, MySQL and PostgreSQL would not; reject + # it identically everywhere. + raise ValueError(f"Batch name exceeds {NAME_MAX_LENGTH} characters.") + complete_path, complete_args, complete_kwargs = _resolve_callback( + on_complete, on_complete_args, on_complete_kwargs + ) + success_path, success_args, success_kwargs = _resolve_callback( + on_success, on_success_args, on_success_kwargs + ) + db_alias = oxpull_db_alias() + row = OxpullBatch.objects.using(db_alias).create( + name=name, + status=OxpullBatch.Status.OPEN, + on_complete_path=complete_path, + on_complete_args=complete_args, + on_complete_kwargs=complete_kwargs, + on_success_path=success_path, + on_success_args=success_args, + on_success_kwargs=success_kwargs, + created_at=timezone.now(), + ) + return Batch(row) + + +def get(batch_id: uuid.UUID | str) -> Batch: + """The batch with this id. Raises OxpullBatch.DoesNotExist if there is none.""" + db_alias = oxpull_db_alias() + return Batch(OxpullBatch.objects.using(db_alias).get(pk=batch_id)) + + +# A reconciler tick stops before its next item when less time is left +# before its task timeout than this many times the slowest item it has +# taken so far. +SWEEP_STOP_FACTOR = 2.0 + + +def _timed_out(exc: BaseException) -> bool: + """ + Whether this exception is a TaskTimeout, or was raised while one was + unwinding. + + It follows the chain django-ox's worker follows to record an attempt + as timed out. A timeout delivered inside a database driver rarely + arrives bare: PyMySQL turns it into OperationalError 2013, and + Django's atomic() exit can raise over it, each with the TaskTimeout + as cause or context. + """ + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + if isinstance(current, TaskTimeout): + return True + seen.add(id(current)) + if current.__cause__ is not None: + current = current.__cause__ + elif current.__suppress_context__: + current = None + else: + current = current.__context__ + return False + + +def _sweep( + pks: list[uuid.UUID], kind: str, plural: str, log: logging.Logger +) -> Iterator[uuid.UUID]: + """ + The sealed rows one reconciler tick visits: all of them, in a new + random order, stopping early when the task's time is nearly up. + + The order changes every tick so that a row which always takes the rest + of a tick cannot keep the same rows behind it waiting tick after tick. + It is a shuffle and not a rotation: under a rotation, the row just + after a slow one would be visited only on ticks that start with it. + + The stop reads django-ox's remaining(), so it applies only under a + TASK_TIMEOUT or TASK_TIMEOUTS entry. Before each row, it stops when + less time is left than SWEEP_STOP_FACTOR times the slowest row this + tick has taken, and logs how many rows were not visited. The task then + returns normally. A row that overruns anyway gets the TaskTimeout, + which the caller must let out (_timed_out). + """ + order = list(pks) + random.shuffle(order) + slowest = 0.0 + for index, pk in enumerate(order): + left = remaining() + if left is not None and left < SWEEP_STOP_FACTOR * slowest: + log.warning( + "Stopped reconciling %s %.3fs before the task timeout, with %d " + "of %d not visited; a later tick visits them. If every tick " + "stops, give the reconciler's queue a longer TASK_TIMEOUTS entry.", + plural, + left, + len(order) - index, + len(order), + extra={ + "event": f"{kind}_reconcile_stopped", + "visited_count": index, + "unvisited_count": len(order) - index, + "remaining_seconds": left, + }, + ) + return + started = time.monotonic() + yield pk + slowest = max(slowest, time.monotonic() - started) + + +def _reconcile() -> int: + """ + Complete every SEALED batch whose members have all finished, and + return how many were completed by this pass. + + Run it through django-ox's schedule dispatcher once a minute. Completion + and callback enqueue share a transaction, so a crash leaves either an + incomplete batch or a completed batch with its callbacks enqueued. + A later pass can revisit an incomplete batch. + + One batch cannot starve the others. A batch whose callback no longer + imports is logged and skipped, and stays SEALED until the deploy that + broke it is fixed. Under a task timeout, a tick visits sealed batches + in a new order, stops between batches when the next would probably + not finish in time, and lets a TaskTimeout out when one lands anyway, + so django-ox records that attempt as timed out. + """ + db_alias = oxpull_db_alias() + sealed = ( + OxpullBatch.objects.using(db_alias) + .filter(status=OxpullBatch.Status.SEALED) + .order_by() + .values_list("pk", flat=True) + ) + completed = 0 + for batch_pk in _sweep(list(sealed), "batch", "batches", logger): + try: + if _complete_if_finished(batch_pk, db_alias): + completed += 1 + except Exception as exc: + # The tick ran out of time; this batch did not fail. + if _timed_out(exc): + raise + logger.exception( + "Batch %s could not be reconciled", + batch_pk, + extra={"event": "batch_reconcile_failed", "batch_id": str(batch_pk)}, + ) + return completed + + +class _MemberPending(Exception): + """ + Raised inside the completing transaction to roll it back. + + Not an error anyone sees: it is how a member that became pending + again between the cheap filter and the compare-and-set undoes the + CAS. Django rolls an atomic block back on any exception, which is + what makes this the shortest way out of a write already issued. + `transaction.set_rollback(True)` would also do it, at the cost of + every caller between here and the block having to check a flag and + return early; raising says the same thing in one line and cannot be + forgotten halfway up. + """ + + +def _complete_if_finished(batch_pk: uuid.UUID, db_alias: str) -> bool: + """ + Complete one sealed batch if nothing is pending. True if this call is + the one that completed it. + + The first count is read outside the transaction, as a cheap filter + that keeps a sweep from opening a write transaction per SEALED batch. + It is not the authority. "Nothing is pending" used to be monotone and + is not any more: django_ox.actions.retry and retry_many move a FAILED + or LOST row back to READY, so a count read before the CAS can be + stale by the time the CAS lands. The counts are therefore read again + inside the transaction, after the CAS has taken the batch row's write + lock, and a member that became pending again rolls the whole thing + back. Re-reading after the CAS rather than before it is also what + keeps a sweep from deadlocking on SQLite's read-to-write lock + upgrade: by then this transaction is already a writer. + + Everything after the CAS rides in the same transaction, so a crash + between completing the batch and enqueueing its callbacks rolls both + back and the next tick redoes them. + + At PostgreSQL REPEATABLE READ a reconciler whose CAS waited on another + one completing the same batch gets a serialization failure when that + one commits, not a rowcount of 0. The transaction is reissued, and the + CAS then matches no SEALED row. + """ + if _pending_count(_member_task_counts(batch_pk, db_alias)): + return False + + def retrying(conflict: str, attempt: int, bound: int) -> None: + logger.info( + "Batch %s: completion lost a serialization failure on attempt %d " + "of %d; reissuing its transaction", + batch_pk, + attempt, + bound, + extra={ + "event": "batch_complete_serialization_retry", + "batch_id": str(batch_pk), + "attempt": attempt, + }, + ) + + outcome = _reissue( + lambda: _complete_once(batch_pk, db_alias), + db_alias, + deadlocks=False, + retrying=retrying, + ) + if outcome is None: + return False + failed, unattributed = outcome + logger.info( + "Batch %s complete (%d failed, %d unattributed)", + batch_pk, + failed, + unattributed, + extra={ + "event": "batch_completed", + "batch_id": str(batch_pk), + "failed_count": failed, + "unattributed_count": unattributed, + }, + ) + return True + + +def _complete_once(batch_pk: uuid.UUID, db_alias: str) -> tuple[int, int] | None: + """ + The completing transaction: (failed, unattributed) when this call + completed the batch, None when it did not. + """ + try: + with transaction.atomic(using=db_alias): + # The callbacks are enqueued once because of this rowcount: + # racing reconcilers all attempt the same UPDATE, one matches + # a SEALED row, and the losers' transactions change nothing. + # The same class of mechanism as the schedule-tick unique + # constraint, and for the same reason: never check-then-act. + # Enqueued once is the whole promise; the callback then runs + # like any other task, which is at least once. + # failed_count is not written here: it is written below, + # once the authoritative counts have been read under this + # lock, so a rolled-back completion never leaves one behind. + won = ( + OxpullBatch.objects.using(db_alias) + .filter(pk=batch_pk, status=OxpullBatch.Status.SEALED) + .update( + status=OxpullBatch.Status.COMPLETE, + completed_at=timezone.now(), + ) + ) + if not won: + return None + counts = _member_task_counts(batch_pk, db_alias) + if _pending_count(counts): + # An operator retried a member between the filter above + # and this lock. Undo the completion; the batch stays + # SEALED and the next tick reconsiders it. + raise _MemberPending + failed = counts.get(OxTask.Status.FAILED, 0) + # A LOST row is settled, but it records only that the attempt + # was never observed, and the straggler still holding that + # lease may yet resolve it either way. A DISCARDED row is + # settled too, and says an operator cancelled the work rather + # than that it was done. Both stand exactly where a member + # whose row ox_prune deleted stands, so both are held out of + # the counted set here and fall into the unattributed + # shortfall below. + counted = sum(counts.values()) - sum( + counts.get(status, 0) for status in UNATTRIBUTED_STATUSES + ) + row = OxpullBatch.objects.using(db_alias).get(pk=batch_pk) + # Members whose row ox_prune deleted, plus members the reaper + # marked LOST, plus members an operator discarded. They are + # finished, but none of them is work that was done, and + # failed == 0 is therefore not evidence that the run was + # clean. total_count was frozen at seal and never moves, so + # the shortfall against the counted rows is exactly that set. + callbacks: dict[str, Any] = {"failed_count": failed} + total = row.total_count + if total is None: + total = _sealed_without_a_total( + OxpullBatchMember.objects.using(db_alias).filter(batch_id=batch_pk) + ) + callbacks["total_count"] = total + unattributed = max(total - counted, 0) + if row.on_complete_path: + callbacks["callback_task_id"] = _enqueue_callback( + row.on_complete_path, row.on_complete_args, row.on_complete_kwargs + ) + # An outcome nobody recorded is not a success: + # on_success is the operator's "the run was clean" signal, so + # it fires only when every member's outcome is on record and + # none of them failed. A batch with no members has nothing + # unaccounted for and nothing failed, so it still completes + # vacuously successful. + if failed == 0 and unattributed == 0 and row.on_success_path: + callbacks["success_task_id"] = _enqueue_callback( + row.on_success_path, row.on_success_args, row.on_success_kwargs + ) + OxpullBatch.objects.using(db_alias).filter(pk=batch_pk).update(**callbacks) + except _MemberPending: + return None + return failed, unattributed + + +def _sealed_without_a_total(rows: QuerySet[Any]) -> int: + """ + The total for a SEALED row that seal() did not write: its live count. + + seal() writes the status and total_count in one transaction, so this + is a row sealed some other way, such as an UPDATE issued by hand to + finish an abandoned build. Reading the missing total as zero would + hide every LOST, DISCARDED or pruned row and fire on_success for a + run that was not clean. The live count is the total seal() would have + stored: add() claims only an OPEN row, so nothing joins a sealed one, + and prune removes rows only once the owner is COMPLETE. progress() + reads the same count for a row with no total. + """ + return rows.count() + + +def _enqueue_callback( + path: str, args: list[Any], kwargs: dict[str, Any], label: str = "Batch" +) -> str: + """ + Enqueue one stored callback and return its task id. + + create() has already checked all of this, but the row outlives the + process that wrote it and a later deploy can move the path onto + another backend. Both checks therefore run again here, before the + enqueue: on a backend that executes at enqueue time the side effect + would otherwise fire and only then fail to record, rolling the + completion back and repeating every tick. + """ + callback = import_string(path) + if not isinstance(callback, Task): + raise ImproperlyConfigured( + f"{label} callback {path!r} is not a django.tasks Task." + ) + _require_ox_backend(callback, role="callbacks", label=label) + _reject_unique(callback, label=label) + return str(callback.enqueue(*args, **kwargs).id) + + +# The task row stores f"{func.__module__}.{func.__qualname__}", and both +# the schedule and the worker resolve the reconciler by that path, so the +# function carries its public name however it is spelled above. +_reconcile.__name__ = _reconcile.__qualname__ = "reconcile" + + +def _build_reconcile_task() -> Task[[], int]: + """The batch reconciler, on a queue the default backend accepts.""" + return cast("Task[[], int]", _build_task_on_a_queue(_reconcile)) + + +def _build_task_on_a_queue(func: Any) -> Task[..., Any]: + """ + Wrap a function as a Task, on the queue the backend will accept. + + django.tasks validates a Task against its backend the moment the + Task is constructed, queue name included. A module-level `@task` + here would therefore make `import oxpull.batches` raise InvalidTask + outright in any project whose QUEUES omits "default" -- from a + view, from a tasks module, from anywhere -- with a message naming + neither this package nor this module. Building it on access instead + keeps the import side-effect-free, and picking a queue the backend + already accepts keeps the reconciler schedulable without editing + QUEUES for it. The schedule's own `queue_name` overrides this. + """ + from django.tasks import ( + DEFAULT_TASK_BACKEND_ALIAS, + DEFAULT_TASK_QUEUE_NAME, + task_backends, + ) + + queue_name = queue_accepted_by( + task_backends[DEFAULT_TASK_BACKEND_ALIAS], + settings.TASKS.get(DEFAULT_TASK_BACKEND_ALIAS, {}), + ) + if queue_name == DEFAULT_TASK_QUEUE_NAME: + return task(func) + return task(queue_name=queue_name)(func) + + +def __getattr__(name: str) -> Any: + """Resolve the module's lazily built public names.""" + if name == "reconcile": + return _build_reconcile_task() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/oxpull/checks.py b/src/oxpull/checks.py new file mode 100644 index 0000000..ee69eb8 --- /dev/null +++ b/src/oxpull/checks.py @@ -0,0 +1,176 @@ +""" +System checks that read the database. + +Django runs them for `manage.py check --database `, and `migrate` runs +them against the database it is about to migrate. The other oxpull checks run +from each backend's check() and never touch a database. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, cast + +from django.core import checks +from django.core.exceptions import ImproperlyConfigured +from django.db import connections +from django.db.models import Field + +from django_ox.models import OxTask + + +@dataclass(frozen=True) +class CollationMismatch: + """A workflow task-id column on another collation than OxTask.id.""" + + table: str + column: str + column_type: str + nullable: bool + collation: str + task_table: str + task_column: str + task_charset: str + task_collation: str + + def message(self) -> str: + return ( + f"On MySQL, {self.table}.{self.column} is on collation " + f"{self.collation} and {self.task_table}.{self.task_column} is on " + f"{self.task_collation}. MySQL refuses to compare the two, so every " + "claim on a backend that declares WORKFLOWS fails." + ) + + def hint(self) -> str: + null = "NULL" if self.nullable else "NOT NULL" + return ( + f"Run ALTER TABLE `{self.table}` MODIFY `{self.column}` " + f"{self.column_type} CHARACTER SET {self.task_charset} COLLATE " + f"{self.task_collation} {null} to put the column on the task id's " + "collation." + ) + + +def collation_mismatches(db_alias: str) -> list[CollationMismatch]: + """ + The workflow task-id columns whose collation differs from OxTask.id's. + + MySQL only. PostgreSQL stores a UUID as uuid, which has no collation, and + SQLite never refuses a comparison over one. Migration 0004 puts the + columns on the task id's collation when it creates them. This finds a + split made afterwards, such as django-ox's table converted to a new + default. A table that does not exist yet contributes nothing. + """ + connection = connections[db_alias] + if connection.vendor != "mysql": + return [] + from .models import OxpullWorkflowEdge, OxpullWorkflowNode + + task_table = OxTask._meta.db_table + # Concrete fields always carry a column; the stubs say Optional. + task_column = str(cast("Field[Any, Any]", OxTask._meta.get_field("id")).column) + columns = [ + (model._meta.db_table, str(cast("Field[Any, Any]", field).column)) + for model, field in ( + (model, model._meta.get_field(name)) + for model, name in ( + (OxpullWorkflowEdge, "upstream_task_id"), + (OxpullWorkflowEdge, "downstream_task_id"), + (OxpullWorkflowNode, "task_id"), + (OxpullWorkflowNode, "cancelled_by_task_id"), + ) + ) + ] + tables = sorted({task_table, *(table for table, _ in columns)}) + with connection.cursor() as cursor: + cursor.execute( + "SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE," + " CHARACTER_SET_NAME, COLLATION_NAME FROM information_schema.COLUMNS" + " WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (%s, %s, %s)", + tables, + ) + found = {(row[0], row[1]): row[2:] for row in cursor.fetchall()} + task = found.get((task_table, task_column)) + if task is None or task[3] is None: + return [] + mismatches = [] + for table, column in columns: + row = found.get((table, column)) + if row is None or row[3] is None or row[3] == task[3]: + continue + mismatches.append( + CollationMismatch( + table=table, + column=column, + column_type=str(row[0]), + nullable=row[1] == "YES", + collation=str(row[3]), + task_table=task_table, + task_column=task_column, + task_charset=str(task[2]), + task_collation=str(task[3]), + ) + ) + return mismatches + + +def waiting_node_count(db_alias: str) -> int: + """Workflow nodes whose task row is WAITING, or 0 before the tables exist.""" + from .models import OxpullWorkflowNode + + connection = connections[db_alias] + with connection.cursor() as cursor: + tables = connection.introspection.table_names(cursor) + if OxpullWorkflowNode._meta.db_table not in tables: + return 0 + nodes = OxpullWorkflowNode.objects.using(db_alias).values("task_id") + return ( + OxTask.objects.using(db_alias) + .filter(status=OxTask.Status.WAITING, pk__in=nodes) + .count() + ) + + +@checks.register(checks.Tags.database) +def check_workflow_schema( + app_configs: Any = None, databases: Any = None, **kwargs: Any +) -> list[checks.CheckMessage]: + """oxpull.E009 and oxpull.W007, for the database oxpull's tables live on.""" + if not databases: + return [] + from ._django_gate import supported + + if not supported(): + # .workflows imports django.tasks, which Django 5.2 does not have. + # OxpullConfig.ready() has already refused the app with a message. + return [] + from .routing import oxpull_db_alias + + try: + alias = oxpull_db_alias() + except ImproperlyConfigured: + # oxpull.E004 reports a router that splits the tables. + return [] + if alias not in databases: + return [] + messages: list[checks.CheckMessage] = [ + checks.Error(mismatch.message(), hint=mismatch.hint(), id="oxpull.E009") + for mismatch in collation_mismatches(alias) + ] + from .workflows import declaring_aliases + + if not declaring_aliases(): + waiting = waiting_node_count(alias) + if waiting: + messages.append( + checks.Warning( + f"{waiting} workflow node(s) on database {alias!r} are WAITING, " + "and no backend in settings.TASKS declares " + "OPTIONS['WORKFLOWS']. No worker releases them when their " + "upstreams succeed.", + hint="Declare OPTIONS['WORKFLOWS'] again on the backend these " + "workflows ran on, and schedule oxpull.workflows.reconcile.", + id="oxpull.W007", + ) + ) + return messages diff --git a/src/oxpull/exceptions.py b/src/oxpull/exceptions.py new file mode 100644 index 0000000..7269a8e --- /dev/null +++ b/src/oxpull/exceptions.py @@ -0,0 +1,58 @@ +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +from django.tasks.exceptions import TaskException + + +class UniqueEnqueueRace(TaskException): + """ + A unique enqueue exhausted its retries while racing other enqueuers + over the same key. Safe to retry; never a silent double-enqueue. + """ + + +class BatchSealed(TaskException): + """ + A batch was sealed (or completed) before this call ran: add() after + seal, or a second seal. Membership is frozen at seal so total_count + stays true; there is no un-seal. + """ + + +class BatchMemberExists(TaskException): + """ + The task returned by enqueue() is already a member of this batch. + + Only reachable through unique tasks: a deduplicated enqueue returns + the pending task's id, and the (batch, task_id) constraint refuses to + count one task twice in one batch. + """ + + +class WorkflowSealed(TaskException): + """ + A workflow was sealed (or completed) before this call ran: add() + after seal, or a second seal. Membership is frozen at seal so + total_count stays true; there is no un-seal. + """ + + +class WorkflowNodeExists(TaskException): + """ + A node with this name, or this task, is already in the workflow. + + The name clash is a caller mistake: dependencies are declared by + name, so two nodes sharing one would make depends_on ambiguous. The + task clash is only reachable through unique tasks, where a + deduplicated enqueue returns a task the workflow already waits on. + """ + + +class UnknownWorkflowNode(TaskException): + """ + depends_on named a node this workflow does not have. + + Dependencies may only point at nodes already added, which is also + why a cycle cannot be built: every edge points backwards in + insertion order. + """ diff --git a/src/oxpull/management/__init__.py b/src/oxpull/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/oxpull/management/commands/__init__.py b/src/oxpull/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/oxpull/management/commands/oxpull_prune.py b/src/oxpull/management/commands/oxpull_prune.py new file mode 100644 index 0000000..18d6a0d --- /dev/null +++ b/src/oxpull/management/commands/oxpull_prune.py @@ -0,0 +1,173 @@ +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +from typing import Any + +from django.core.management.base import BaseCommand, CommandError, CommandParser +from django.db import connections +from django.db.models import Q, QuerySet +from django.utils import timezone + +from django_ox.durations import parse_duration +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import ( + OxpullUniqueLock, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, +) +from oxpull.routing import oxpull_db_alias + + +class Command(BaseCommand): + help = ( + "Delete unique-task locks whose task is finished or pruned, and the " + "node and edge rows of workflows that completed longer ago than " + "--older-than and that no retry can move again. Batch members are " + "kept. Stale locks are also released lazily by the next enqueue of the " + "same key. This command clears locks for keys that are never enqueued " + "again." + ) + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument( + "--batch-size", + type=int, + default=1000, + help=( + "Rows deleted per DELETE statement, bounding statement " + "size on large tables (default: %(default)s). A statement " + "deletes at most 1,000 workflow rows." + ), + ) + parser.add_argument( + "--older-than", + default="7d", + help=( + "How long a workflow must have been COMPLETE before its node " + "and edge rows are considered. It takes 7d, 24h, 90m, 45s or a " + "plain number of seconds (default: %(default)s). Locks are not " + "aged. A lock is either held by a live task or it is stale." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help=( + "Report how many rows would be deleted without deleting or " + "stamping anything." + ), + ) + + def handle(self, *args: Any, **options: Any) -> None: + if options["batch_size"] < 1: + raise CommandError("--batch-size must be a positive integer.") + live = OxTask.objects.filter( + status__in=[OxTask.Status.READY, OxTask.Status.RUNNING] + ).values("id") + # The isnull arm is not redundant: SQL's NOT IN treats a NULL + # task_id as unknown, so without it a NULL row would be kept + # forever. A committed lock never carries a NULL task_id (enqueue + # writes it in the same transaction), so the arm only clears + # defensive residue. + prunable = OxpullUniqueLock.objects.filter( + Q(task_id__isnull=True) | ~Q(task_id__in=live) + ) + older_than = options["older_than"] + try: + cutoff = timezone.now() - parse_duration(older_than) + except OverflowError: + # A duration timedelta can hold may still reach back past year 1. + raise CommandError( + f"Invalid duration {older_than!r}; it is out of range." + ) from None + db_alias = oxpull_db_alias() + + if options["dry_run"]: + self.stdout.write(f"Would delete {prunable.count()} stale unique lock(s).") + else: + deleted = self._delete_in_batches(prunable, options["batch_size"]) + self.stdout.write(f"Deleted {deleted} stale unique lock(s).") + + # Batch members are never deleted. A batch's progress() reads them + # for as long as the batch row exists, and a pruned member counts as + # unattributed, as it did in 1.2.0. + if not self._workflow_tables_exist(db_alias): + self.stdout.write( + "Skipped workflows, because their tables do not exist on this " + "database yet. Run manage.py migrate oxpull." + ) + return + retirable = workflows.retirable_workflows(cutoff, db_alias) + if options["dry_run"]: + self._dry_run(retirable, db_alias) + return + nodes = edges = pending = 0 + for workflow_pk in retirable: + retired = workflows.retire_workflow( + workflow_pk, + db_alias, + chunk_size=min(options["batch_size"], workflows.ID_CHUNK_SIZE), + ) + nodes += retired.nodes + edges += retired.edges + pending += retired.pending + self.stdout.write(f"Deleted {nodes} workflow node row(s).") + self.stdout.write(f"Deleted {edges} workflow edge row(s).") + if pending: + self.stdout.write( + f"Skipped {pending} workflow(s) with a node still pending." + ) + scans = workflows.clean_scan_rows( + db_alias, + chunk_size=min(options["batch_size"], workflows.ID_CHUNK_SIZE), + ) + self.stdout.write(f"Deleted {scans} workflow scan row(s).") + self._report_retained() + + def _workflow_tables_exist(self, db_alias: str) -> bool: + with connections[db_alias].cursor() as cursor: + tables = connections[db_alias].introspection.table_names(cursor) + return { + OxpullWorkflow._meta.db_table, + OxpullWorkflowNode._meta.db_table, + OxpullWorkflowEdge._meta.db_table, + }.issubset(tables) + + def _dry_run(self, retirable: list[Any], db_alias: str) -> None: + nodes = edges = 0 + for workflow_pk in retirable: + would = workflows.retirable_rows(workflow_pk, db_alias) + nodes += would.nodes + edges += would.edges + self.stdout.write(f"Would delete {nodes} workflow node row(s).") + self.stdout.write(f"Would delete {edges} workflow edge row(s).") + self._report_retained() + + def _report_retained(self) -> None: + kept = workflows.retained_rows() + self.stdout.write( + f"Kept {kept.nodes} workflow node row(s) and {kept.edges} workflow " + f"edge row(s) of {kept.workflows} retired workflow(s)." + ) + + def _delete_in_batches(self, prunable: QuerySet[Any], batch_size: int) -> int: + # Neither the key nor a task's staleness is monotone. Between + # the SELECT and the DELETE a concurrent enqueue can release a + # stale lock and insert a new live generation under the same pk, + # and django_ox.actions.retry can move a FAILED or LOST row back + # to READY, making a lock the SELECT read as stale live again. + # The DELETE therefore goes through prunable, which re-applies + # the staleness predicate inside the statement, so each batch is + # re-verified against current rows: a stolen key survives, and so + # does the lock on a task that was retried. + # Bounded batches are the same idiom as ox_prune's deletes, + # deliberately duplicated. + deleted = 0 + while True: + batch = list(prunable.values_list("pk", flat=True)[:batch_size]) + if not batch: + break + deleted += prunable.filter(pk__in=batch).delete()[0] + return deleted diff --git a/src/oxpull/migrations/0001_initial.py b/src/oxpull/migrations/0001_initial.py new file mode 100644 index 0000000..42e4ba4 --- /dev/null +++ b/src/oxpull/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# Generated by Django 6.1 on 2026-08-18 11:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="OxpullUniqueLock", + fields=[ + ( + "key", + models.CharField(max_length=64, primary_key=True, serialize=False), + ), + ("task_id", models.UUIDField(blank=True, null=True)), + ("task_path", models.CharField(max_length=255)), + ("queue_name", models.CharField(default="default", max_length=128)), + ("created_at", models.DateTimeField()), + ], + ), + ] diff --git a/src/oxpull/migrations/0002_oxpullbatch_oxpullbatchmember.py b/src/oxpull/migrations/0002_oxpullbatch_oxpullbatchmember.py new file mode 100644 index 0000000..c2c1ddf --- /dev/null +++ b/src/oxpull/migrations/0002_oxpullbatch_oxpullbatchmember.py @@ -0,0 +1,99 @@ +# Generated by Django 6.1 on 2026-08-18 21:19 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("oxpull", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="OxpullBatch", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("name", models.CharField(max_length=128)), + ( + "status", + models.CharField( + choices=[ + ("OPEN", "Open"), + ("SEALED", "Sealed"), + ("COMPLETE", "Complete"), + ], + default="OPEN", + max_length=10, + ), + ), + ("total_count", models.PositiveIntegerField(blank=True, null=True)), + ( + "on_complete_path", + models.CharField(blank=True, default="", max_length=255), + ), + ("on_complete_args", models.JSONField(blank=True, default=list)), + ("on_complete_kwargs", models.JSONField(blank=True, default=dict)), + ( + "on_success_path", + models.CharField(blank=True, default="", max_length=255), + ), + ("on_success_args", models.JSONField(blank=True, default=list)), + ("on_success_kwargs", models.JSONField(blank=True, default=dict)), + ("failed_count", models.PositiveIntegerField(blank=True, null=True)), + ("callback_task_id", models.UUIDField(blank=True, null=True)), + ("success_task_id", models.UUIDField(blank=True, null=True)), + ("created_at", models.DateTimeField()), + ("sealed_at", models.DateTimeField(blank=True, null=True)), + ("completed_at", models.DateTimeField(blank=True, null=True)), + ], + options={ + "indexes": [ + models.Index( + fields=["status", "created_at"], name="oxpull_batch_sweep_idx" + ) + ], + }, + ), + migrations.CreateModel( + name="OxpullBatchMember", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("task_id", models.UUIDField()), + ("created_at", models.DateTimeField()), + ( + "batch", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="members", + to="oxpull.oxpullbatch", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("batch", "task_id"), name="oxpull_batch_member_uniq" + ) + ], + }, + ), + ] diff --git a/src/oxpull/migrations/0003_oxpullratewindow.py b/src/oxpull/migrations/0003_oxpullratewindow.py new file mode 100644 index 0000000..fcf095e --- /dev/null +++ b/src/oxpull/migrations/0003_oxpullratewindow.py @@ -0,0 +1,24 @@ +# Generated by Django 6.1 on 2026-09-01 19:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("oxpull", "0002_oxpullbatch_oxpullbatchmember"), + ] + + operations = [ + migrations.CreateModel( + name="OxpullRateWindow", + fields=[ + ( + "limit_name", + models.CharField(max_length=128, primary_key=True, serialize=False), + ), + ("window_id", models.BigIntegerField(default=0)), + ("window_started_at", models.DateTimeField()), + ("used", models.PositiveIntegerField(default=0)), + ], + ), + ] diff --git a/src/oxpull/migrations/0004_workflows.py b/src/oxpull/migrations/0004_workflows.py new file mode 100644 index 0000000..0991f36 --- /dev/null +++ b/src/oxpull/migrations/0004_workflows.py @@ -0,0 +1,338 @@ +import re +import uuid + +import django.db.models.deletion +from django.db import migrations, models +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.state import StateApps + +from oxpull.migrations._workflow_guard import refuse_while_workflows_exist + +# The workflow columns that hold a django-ox task id and are compared with +# OxTask.id in SQL, as (model, field). +TASK_ID_COLUMNS = ( + ("OxpullWorkflowEdge", "upstream_task_id"), + ("OxpullWorkflowEdge", "downstream_task_id"), + ("OxpullWorkflowNode", "task_id"), + ("OxpullWorkflowNode", "cancelled_by_task_id"), +) + +# A character set or collation name as information_schema spells one. +MYSQL_NAME = re.compile(r"^[A-Za-z0-9_]+$") + + +def copy_task_id_collation( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """ + On MySQL, put the task-id columns on the collation of OxTask.id. + + migrate creates a table on the database's default collation at that + moment. django-ox's table can carry another one, after a dump restored + into a server with a newer default or an ALTER DATABASE. MySQL refuses to + compare two columns of one character set on different collations, and + every gated claim compares these columns with OxTask.id. The tables were + created just above and are empty, so the ALTER rewrites nothing. + + A column with no collation, such as MariaDB's native uuid type, is left + alone. oxpull.E009 reports a mismatch that arises later. + """ + connection = schema_editor.connection + if connection.vendor != "mysql": + return + task_model = apps.get_model("django_ox", "OxTask") + quote = schema_editor.quote_name + with connection.cursor() as cursor: + target = _column(cursor, task_model._meta.db_table, "id", task_model) + if target is None or target[2] is None: + return + charset, collation = target[3], target[2] + if not (MYSQL_NAME.match(charset) and MYSQL_NAME.match(collation)): + return + for model_name, field_name in TASK_ID_COLUMNS: + model = apps.get_model("oxpull", model_name) + table = model._meta.db_table + found = _column(cursor, table, field_name, model) + if found is None or found[2] is None or found[2] == collation: + continue + column_type, nullable = found[0], found[1] + null = "NULL" if nullable == "YES" else "NOT NULL" + column = _column_name(model, field_name) + cursor.execute( + f"ALTER TABLE {quote(table)} MODIFY {quote(column)} {column_type}" + f" CHARACTER SET {charset} COLLATE {collation} {null}" + ) + + +# Distinct downstream ids per edge row, as a fraction, that PostgreSQL's +# planner assumes for the edge table: a node has two edges in, on average. +EDGE_DOWNSTREAM_N_DISTINCT = -0.5 + + +def pin_edge_statistics( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """ + On PostgreSQL, fix the planner's count of distinct downstream ids on the + edge table. + + The claim's dependency check probes the edges into each candidate row by + downstream_task_id. ANALYZE run while one wide fan-in fills the table + counts one downstream id for thousands of edges, and the planner then + reads the whole edge table on every claim instead of probing the index. + A node has a few edges in, so the check is a probe. With n_distinct set, + ANALYZE keeps this value for the column and still gathers the rest. + """ + connection = schema_editor.connection + if connection.vendor != "postgresql": + return + model = apps.get_model("oxpull", "OxpullWorkflowEdge") + quote = schema_editor.quote_name + column = _column_name(model, "downstream_task_id") + with connection.cursor() as cursor: + cursor.execute( + f"ALTER TABLE {quote(model._meta.db_table)} ALTER COLUMN {quote(column)}" + f" SET (n_distinct = {EDGE_DOWNSTREAM_N_DISTINCT})" + ) + + +def _column( + cursor: object, table: str, field_name: str, model: type[models.Model] +) -> tuple[str, str, str | None, str] | None: + """COLUMN_TYPE, IS_NULLABLE, COLLATION_NAME and CHARACTER_SET_NAME.""" + column = _column_name(model, field_name) + cursor.execute( # type: ignore[attr-defined] + "SELECT COLUMN_TYPE, IS_NULLABLE, COLLATION_NAME, CHARACTER_SET_NAME" + " FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE()" + " AND TABLE_NAME = %s AND COLUMN_NAME = %s", + [table, column], + ) + row = cursor.fetchone() # type: ignore[attr-defined] + return None if row is None else (str(row[0]), str(row[1]), row[2], str(row[3])) + + +def _column_name(model: type[models.Model], field_name: str) -> str: + field = model._meta.get_field(field_name) + if not isinstance(field, models.Field): + raise TypeError(f"{model.__name__}.{field_name} is not a concrete field") + return str(field.column) + + +class Migration(migrations.Migration): + dependencies = [ + ("django_ox", "0008_waiting"), + ("oxpull", "0003_oxpullratewindow"), + ] + + operations = [ + migrations.CreateModel( + name="OxpullWorkflowRound", + fields=[ + ( + "id", + models.PositiveSmallIntegerField( + default=1, primary_key=True, serialize=False + ), + ), + ("workflow_id", models.UUIDField(blank=True, null=True)), + ("version", models.BigIntegerField(default=0)), + ], + ), + migrations.CreateModel( + name="OxpullWorkflowScan", + fields=[ + ("workflow_id", models.UUIDField(primary_key=True, serialize=False)), + ( + "phase", + models.CharField( + choices=[ + ("EDGES", "Edges"), + ("NODES", "Nodes"), + ("RELEASE", "Release"), + ], + default="EDGES", + max_length=10, + ), + ), + ("position", models.BigIntegerField(default=0)), + ("edge_hwm", models.BigIntegerField(blank=True, null=True)), + ("node_hwm", models.BigIntegerField(blank=True, null=True)), + ("skipped", models.JSONField(blank=True, default=list)), + ("version", models.BigIntegerField(default=0)), + ], + ), + migrations.CreateModel( + name="OxpullWorkflow", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("name", models.CharField(max_length=128)), + ( + "status", + models.CharField( + choices=[ + ("OPEN", "Open"), + ("SEALED", "Sealed"), + ("COMPLETE", "Complete"), + ], + default="OPEN", + max_length=10, + ), + ), + ("total_count", models.PositiveIntegerField(blank=True, null=True)), + ( + "on_complete_path", + models.CharField(blank=True, default="", max_length=255), + ), + ("on_complete_args", models.JSONField(blank=True, default=list)), + ("on_complete_kwargs", models.JSONField(blank=True, default=dict)), + ( + "on_success_path", + models.CharField(blank=True, default="", max_length=255), + ), + ("on_success_args", models.JSONField(blank=True, default=list)), + ("on_success_kwargs", models.JSONField(blank=True, default=dict)), + ("failed_count", models.PositiveIntegerField(blank=True, null=True)), + ("cancelled_count", models.PositiveIntegerField(blank=True, null=True)), + ("callback_task_id", models.UUIDField(blank=True, null=True)), + ("success_task_id", models.UUIDField(blank=True, null=True)), + ("created_at", models.DateTimeField()), + ("sealed_at", models.DateTimeField(blank=True, null=True)), + ("completed_at", models.DateTimeField(blank=True, null=True)), + ("generation", models.PositiveIntegerField(default=0)), + ("cancel_requested_at", models.DateTimeField(blank=True, null=True)), + ("metadata_retired_at", models.DateTimeField(blank=True, null=True)), + ( + "key_hash", + models.CharField(blank=True, max_length=64, null=True, unique=True), + ), + ("completions", models.JSONField(blank=True, default=dict)), + ("node_count", models.PositiveIntegerField(default=0)), + ], + options={ + "indexes": [ + models.Index( + fields=["status", "created_at"], name="oxpull_wf_sweep_idx" + ) + ], + }, + ), + migrations.CreateModel( + name="OxpullWorkflowEdge", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("upstream_task_id", models.UUIDField()), + ("downstream_task_id", models.UUIDField()), + ("satisfied", models.BooleanField(default=False)), + ("created_at", models.DateTimeField()), + ( + "workflow", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.PROTECT, + related_name="edges", + to="oxpull.oxpullworkflow", + ), + ), + ], + options={ + "indexes": [ + models.Index( + fields=["downstream_task_id", "satisfied"], + name="oxpull_wf_edge_down_idx", + ), + models.Index( + fields=["workflow", "satisfied", "id"], + name="oxpull_wf_edge_open_idx", + ), + ], + "constraints": [ + models.UniqueConstraint( + fields=("workflow", "upstream_task_id", "downstream_task_id"), + name="oxpull_wf_edge_uniq", + ) + ], + }, + ), + migrations.CreateModel( + name="OxpullWorkflowNode", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=128)), + ("name_key", models.CharField(max_length=64)), + ("task_id", models.UUIDField()), + ("created_at", models.DateTimeField()), + ("outcome", models.CharField(blank=True, max_length=10, null=True)), + ("outcome_epoch", models.BigIntegerField(blank=True, null=True)), + ("succeeded", models.BooleanField(default=False)), + ( + "cancel_cause", + models.CharField(blank=True, max_length=16, null=True), + ), + ("cancelled_by_task_id", models.UUIDField(blank=True, null=True)), + ( + "cancelled_generation", + models.PositiveIntegerField(blank=True, null=True), + ), + ( + "workflow", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.PROTECT, + related_name="nodes", + to="oxpull.oxpullworkflow", + ), + ), + ], + options={ + "indexes": [ + models.Index( + fields=["workflow", "succeeded", "id"], + name="oxpull_wf_node_open_idx", + ) + ], + "constraints": [ + models.UniqueConstraint( + fields=("workflow", "name_key"), + name="oxpull_wf_node_name_uniq", + ), + models.UniqueConstraint( + fields=("task_id",), name="oxpull_wf_node_task_uniq" + ), + ], + }, + ), + migrations.RunPython( + copy_task_id_collation, reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + pin_edge_statistics, reverse_code=migrations.RunPython.noop + ), + # Last, so migrating back meets it before any table is dropped. + migrations.RunPython( + migrations.RunPython.noop, reverse_code=refuse_while_workflows_exist + ), + ] diff --git a/src/oxpull/migrations/0005_workflow_edge_upstream_idx.py b/src/oxpull/migrations/0005_workflow_edge_upstream_idx.py new file mode 100644 index 0000000..4839696 --- /dev/null +++ b/src/oxpull/migrations/0005_workflow_edge_upstream_idx.py @@ -0,0 +1,32 @@ +from django.db import migrations, models + +from oxpull.migrations._workflow_guard import refuse_dropping_the_upstream_index + + +class Migration(migrations.Migration): + """ + The index the release reads. + + OxpullWorkflowEdge already carries an index on downstream_task_id -- + the one the claim's gate probes, asking "does anything gate this + candidate". The release asks the opposite question, "what did this + task just unblock", and the unique constraint cannot answer it: its + leading column is workflow_id, so upstream_task_id is not a usable + prefix. Without this the release is a sequential scan of the edge + table on every successful task. + """ + + dependencies = [("oxpull", "0004_workflows")] + + operations = [ + migrations.AddIndex( + model_name="oxpullworkflowedge", + index=models.Index( + fields=["upstream_task_id"], name="oxpull_wf_edge_up_idx" + ), + ), + # Last, so migrating back meets it before the index is dropped. + migrations.RunPython( + migrations.RunPython.noop, reverse_code=refuse_dropping_the_upstream_index + ), + ] diff --git a/src/oxpull/migrations/__init__.py b/src/oxpull/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/oxpull/migrations/_workflow_guard.py b/src/oxpull/migrations/_workflow_guard.py new file mode 100644 index 0000000..6970192 --- /dev/null +++ b/src/oxpull/migrations/_workflow_guard.py @@ -0,0 +1,74 @@ +""" +The refusals that end every workflow migration. Not a migration itself: +Django's migration loader skips a module whose name starts with an underscore. + +Migrating oxpull back unapplies its later migrations first, newest first. Each +workflow migration ends with a refusal as its last operation, so migrating +back meets it before that migration undoes anything. With the refusal in 0004 +alone, 0005 would drop the index the release reads by upstream, and then 0004 +would refuse, leaving running workers without it. + +Each migration's refusal says what unapplying that migration drops, so its +message is true whichever target the command names. +""" + +from django.db.backends.base.schema import BaseDatabaseSchemaEditor +from django.db.migrations.exceptions import IrreversibleError +from django.db.migrations.state import StateApps + +ROLLBACK_RULE = "Rolling back is supported only before the first workflow is created." + + +def _workflow_count(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> int: + workflow_model = apps.get_model("oxpull", "OxpullWorkflow") + rows = workflow_model._default_manager.using(schema_editor.connection.alias) + return int(rows.count()) + + +def refuse_while_workflows_exist( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """ + 0004's refusal: do not unapply 0004_workflows while any workflow row exists. + + Unapplying 0004 drops the tables that hold every node, edge and record, + and a version from before it cannot release a waiting node or finish a + workflow. + + This detects a mistake. It does not make a rollback safe while workers + run: a workflow created after this check passes is dropped with the + tables. Stop every process that can create a workflow before migrating + back. + """ + count = _workflow_count(apps, schema_editor) + if count: + raise IrreversibleError( + f"Cannot migrate oxpull back past 0004_workflows. {count} " + "workflow(s) exist on this database. Migrating back drops the " + "tables that record their nodes and edges, and a version before " + f"0004 cannot run or finish them. {ROLLBACK_RULE}" + ) + + +def refuse_dropping_the_upstream_index( + apps: StateApps, schema_editor: BaseDatabaseSchemaEditor +) -> None: + """ + 0005's refusal: do not unapply 0005_workflow_edge_upstream_idx while any + workflow row exists. + + Unapplying 0005 drops oxpull_wf_edge_up_idx, which a worker reads when a + node succeeds, to mark the edges out of it and release what they held. + Migrating further back meets 0004's refusal. The same caveat holds: this + detects a mistake, and processes that create workflows must be stopped + first. + """ + count = _workflow_count(apps, schema_editor) + if count: + raise IrreversibleError( + "Cannot migrate oxpull back past 0005_workflow_edge_upstream_idx. " + f"{count} workflow(s) exist on this database. Unapplying it drops " + "the index a worker reads when a node succeeds, to release the " + "nodes that wait on it. Migrating back past 0004_workflows would " + f"also drop the workflow tables. {ROLLBACK_RULE}" + ) diff --git a/src/oxpull/models.py b/src/oxpull/models.py new file mode 100644 index 0000000..74fe2a0 --- /dev/null +++ b/src/oxpull/models.py @@ -0,0 +1,468 @@ +import uuid + +from django.db import models + +from .rate_limits import MAX_LIMIT_NAME_LENGTH + + +class OxpullUniqueLock(models.Model): + """ + The uniqueness lock for one pending unique task. + + A row means: a task with this key was enqueued and has not been seen + to finish. The row is written in the same transaction as the OxTask + INSERT, so the lock and the task commit or roll back together. key is + the primary key, so the dedup point is a single unique btree, + maintained once per winner INSERT and once per release DELETE. + + Lock state is derived from the task row, never from signals: the + django-ox reaper writes FAILED without emitting task_finished, so any + release mechanism driven by signals would strand locks exactly when a + worker dies. Instead, a lock whose task is no longer READY or RUNNING + (or whose task row was pruned) is stale, and is released by the next + enqueue of the same key or by the oxpull_prune command. + + task_id is a bare UUID, not a ForeignKey: ox_prune deletes finished + OxTask rows, and a dangling reference must read as "finished" rather + than block pruning or cascade the lock away while it is still the + enqueue-time dedup point. + """ + + # SHA-256 hex digest over (backend_alias, task_path, queue_name, + # args, kwargs). + key = models.CharField(max_length=64, primary_key=True) + task_id = models.UUIDField(null=True, blank=True) + + # Denormalised for operators reading the table; the key alone decides. + # task_path and queue_name stay readable after the task row is pruned, + # when task_id already dangles by design. + task_path = models.CharField(max_length=255) + queue_name = models.CharField(max_length=128, default="default") + + created_at = models.DateTimeField() + + def __str__(self) -> str: + return f"{self.task_path} [{self.key[:12]}]" + + +class OxpullBatch(models.Model): + """ + A named group of tasks and the callbacks that fire once every member's + row is settled (SUCCESSFUL, FAILED, LOST or DISCARDED) or gone. + + Status moves OPEN -> SEALED -> COMPLETE and never back. Membership is + frozen at seal, which is also when total_count is snapshotted; the + reconciler completes a SEALED batch under a compare-and-set on that + status, and the CAS rowcount is what gets the callbacks enqueued + once. They then run like any other task, which is at least once. + + There is no FAILED status. Completion is completion: failed_count + plus the absent on_success callback is how a bad run reads. Collapsing + "done" and "how it went" into one enum is the door the signal-counting + design comes back through. + + Batch state is never accumulated from signals. The django-ox reaper + writes FAILED through a queryset update and emits nothing, so a batch + that counted task_finished would hang exactly when a worker dies, + which is the case batches exist to survive. Every count here is + derived by querying member task rows. + + Callbacks are stored as a dotted task path plus JSON arguments, not + as a pickled Task: the row outlives the process that created it, and + the reconciler that enqueues it is a different worker. + """ + + class Status(models.TextChoices): + OPEN = "OPEN" + SEALED = "SEALED" + COMPLETE = "COMPLETE" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=128) + status = models.CharField( + max_length=10, choices=Status.choices, default=Status.OPEN + ) + + # Member count frozen at seal time. NULL while OPEN, where the live + # member count is the only honest answer; after seal it is also the + # denominator that keeps totals true once ox_prune deletes member + # task rows. + total_count = models.PositiveIntegerField(null=True, blank=True) + + on_complete_path = models.CharField(max_length=255, blank=True, default="") + on_complete_args = models.JSONField(default=list, blank=True) + on_complete_kwargs = models.JSONField(default=dict, blank=True) + + on_success_path = models.CharField(max_length=255, blank=True, default="") + on_success_args = models.JSONField(default=list, blank=True) + on_success_kwargs = models.JSONField(default=dict, blank=True) + + # Written once, in the completing transaction. NULL means "not + # determined yet" rather than "no failures", which a default of 0 + # would claim about every open batch. + failed_count = models.PositiveIntegerField(null=True, blank=True) + + # Bare UUIDs of the enqueued callbacks, for the same reason as + # OxpullBatchMember.task_id: ox_prune deletes finished task rows. + callback_task_id = models.UUIDField(null=True, blank=True) + success_task_id = models.UUIDField(null=True, blank=True) + + created_at = models.DateTimeField() + sealed_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + class Meta: + indexes = [ + # The reconciler's sweep: every SEALED batch, oldest first. + models.Index( + fields=["status", "created_at"], name="oxpull_batch_sweep_idx" + ), + ] + + def __str__(self) -> str: + return f"{self.name} [{self.status}]" + + +class OxpullBatchMember(models.Model): + """ + One task's membership of one batch. + + task_id is a bare UUID, not a ForeignKey: ox_prune deletes finished + OxTask rows, and a foreign key would either block the prune or + cascade the membership away. A membership row whose task row is gone + joins to nothing, which is exactly the wanted reading: pruned means + finished. total_count keeps the denominator honest when that happens. + + The (batch, task_id) constraint refuses to count one task twice in + one batch, reachable only through a deduplicated unique enqueue, and + doubles as the index the completion join reads. + """ + + batch = models.ForeignKey( + OxpullBatch, on_delete=models.CASCADE, related_name="members" + ) + task_id = models.UUIDField() + created_at = models.DateTimeField() + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["batch", "task_id"], name="oxpull_batch_member_uniq" + ), + ] + + def __str__(self) -> str: + return f"{self.batch_id} -> {self.task_id}" + + +class OxpullRateWindow(models.Model): + """ + The current window of one named rate limit. + + One row per limit name, forever: the table counts admissions, not + tasks, so it does not grow with traffic and needs no pruner. Every + statement against it is keyed on limit_name, one name for a write + and the worker's configured names with IN for the refresh's select, + with an optional AND window_id = %s; the primary key answers all of + them on its own. Identity is the column's collation: a name the + collation folds onto an existing row is that row, which is why the + worker asks the database which row a renamed limit owns. + + Capacity is not stored. A limit's N and P live in settings, and a + column holding them would be a second source of truth with no + answer to which one wins halfway through a rolling deploy. + + There is no foreign key to anything, matching OxpullUniqueLock and + OxpullBatchMember: there is nothing to reference. + """ + + # The configured name, whose length rate_limits.py validates against + # this same constant, so a name that reaches a row always fits it. + limit_name = models.CharField(max_length=MAX_LIMIT_NAME_LENGTH, primary_key=True) + + # Monotone per row, never reset and never reused. It names one window, + # and it is the compare-and-set target that makes exactly one worker + # the one that rolls, the role lease_epoch plays on OxTask. + window_id = models.BigIntegerField(default=0) + + # When this window opened, on the database clock, always. Nothing but + # this table writes or reads this column, and it is only ever compared + # against the database's own clock in the same statement, so there is + # no other writer for it to agree with. + window_started_at = models.DateTimeField() + + # Admissions counted in the current window. It may exceed the + # configured capacity by up to one per additional worker process; + # that overshoot is the guarantee's slack, and a value above the + # limit here is expected rather than corrupt. + used = models.PositiveIntegerField(default=0) + + def __str__(self) -> str: + return f"{self.limit_name} [{self.used} in window {self.window_id}]" + + +class OxpullWorkflow(models.Model): + """ + A named DAG of tasks, and the callbacks that fire once every node has + finished. + + The status columns are OxpullBatch's, name for name: OPEN -> SEALED -> + COMPLETE, and total_count snapshotted at seal. The reconciler completes + a SEALED row under a compare-and-set, and that CAS rowcount is what gets + the callbacks enqueued once. They then run like any other task, which is + at least once. + + A node whose upstream has not succeeded is a django-ox row in the + WAITING status, which no worker claims. It moves to READY only once + every edge into it is marked satisfied. The claim still reads the edges, + and refuses a READY row behind any edge that is not marked satisfied and + whose upstream row is absent or not SUCCESSFUL. + + This row is the first lock every workflow operation takes. + """ + + class Status(models.TextChoices): + OPEN = "OPEN" + SEALED = "SEALED" + COMPLETE = "COMPLETE" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=128) + status = models.CharField( + max_length=10, choices=Status.choices, default=Status.OPEN + ) + + # Node count frozen at seal. NULL while OPEN, where the live node + # count is the only honest answer; after seal it is the denominator + # that keeps totals true once ox_prune deletes node task rows. + total_count = models.PositiveIntegerField(null=True, blank=True) + + on_complete_path = models.CharField(max_length=255, blank=True, default="") + on_complete_args = models.JSONField(default=list, blank=True) + on_complete_kwargs = models.JSONField(default=dict, blank=True) + + on_success_path = models.CharField(max_length=255, blank=True, default="") + on_success_args = models.JSONField(default=list, blank=True) + on_success_kwargs = models.JSONField(default=dict, blank=True) + + failed_count = models.PositiveIntegerField(null=True, blank=True) + # Nodes this workflow cancelled because an upstream of theirs settled + # without succeeding. They are DISCARDED rows and therefore also + # unattributed, but an operator reading the table wants to know the + # workflow did it rather than a person. + cancelled_count = models.PositiveIntegerField(null=True, blank=True) + + callback_task_id = models.UUIDField(null=True, blank=True) + success_task_id = models.UUIDField(null=True, blank=True) + + created_at = models.DateTimeField() + sealed_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + # Reserved. Nothing in this release writes generation or + # cancel_requested_at. generation is read as the key completions are + # stored under, and it is 0 on every row. + generation = models.PositiveIntegerField(default=0) + cancel_requested_at = models.DateTimeField(null=True, blank=True) + # When oxpull_prune began retiring this workflow's node and edge rows. + # Once set, no node of it is revived. + metadata_retired_at = models.DateTimeField(null=True, blank=True) + # Reserved. Nothing in this release writes key_hash, so it is NULL on + # every row. NULL rather than "" because the unique index has to let + # any number of workflows carry no value. + key_hash = models.CharField(max_length=64, null=True, blank=True, unique=True) + # What completion recorded: counts, callback ids and when, under the + # generation as key. Written by the compare-and-set that completes the + # workflow, and never rewritten. + completions = models.JSONField(default=dict, blank=True) + # Nodes added, incremented by add() in the statement that takes this + # row's lock, so it counts exactly the nodes that committed. + node_count = models.PositiveIntegerField(default=0) + + class Meta: + indexes = [ + models.Index(fields=["status", "created_at"], name="oxpull_wf_sweep_idx"), + ] + + def __str__(self) -> str: + return f"{self.name} [{self.status}]" + + +class OxpullWorkflowNode(models.Model): + """ + One task's membership of one workflow, under a name dependencies are + declared with, and what the workflow recorded about how it finished. + + task_id is a bare UUID for the reason recorded on OxpullBatchMember: + ox_prune deletes finished OxTask rows, and a foreign key would either + block the prune or cascade the membership away. It is unique across + workflows, because a task is one node: add() refuses unique tasks, so no + enqueue can hand two workflows the same row. + + A name is identified by name_key, the SHA-256 digest of its UTF-8 bytes, + so two names are one name only when they are spelled the same. The + name column's collation plays no part, and MySQL's case, accent and + trailing-space folding cannot refuse a name or resolve the wrong one. + + The foreign key to the workflow is PROTECT: a workflow row cannot be + deleted while any of its nodes remain. + """ + + workflow = models.ForeignKey( + OxpullWorkflow, + on_delete=models.PROTECT, + related_name="nodes", + db_index=False, + ) + name = models.CharField(max_length=128) + name_key = models.CharField(max_length=64) + task_id = models.UUIDField() + created_at = models.DateTimeField() + + # The settled status the workflow recorded for this node's task row, and + # the lease epoch it was recorded at. NULL until something records one, + # which lets a record decide for a row that has since been pruned. + outcome = models.CharField(max_length=10, null=True, blank=True) + outcome_epoch = models.BigIntegerField(null=True, blank=True) + # True once the record says SUCCESSFUL. Its own column so the open-row + # index can lead with it. + succeeded = models.BooleanField(default=False) + + # Reserved. Nothing in this release writes these three, so they are + # NULL on every row. A node the workflow discarded is found by its + # DISCARDED task row and its recorded outcome. + cancel_cause = models.CharField(max_length=16, null=True, blank=True) + cancelled_by_task_id = models.UUIDField(null=True, blank=True) + cancelled_generation = models.PositiveIntegerField(null=True, blank=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["workflow", "name_key"], name="oxpull_wf_node_name_uniq" + ), + models.UniqueConstraint( + fields=["task_id"], name="oxpull_wf_node_task_uniq" + ), + ] + indexes = [ + # The nodes of one workflow not yet recorded as succeeded, in key + # order: a page of them without reading the ones that are done. + models.Index( + fields=["workflow", "succeeded", "id"], name="oxpull_wf_node_open_idx" + ), + ] + + def __str__(self) -> str: + return f"{self.workflow_id}:{self.name} -> {self.task_id}" + + +class OxpullWorkflowEdge(models.Model): + """ + One dependency: downstream_task_id may not run until upstream_task_id + has succeeded. + + satisfied is the workflow's record that it saw the upstream succeed. + SUCCESSFUL is final in django-ox, so a satisfied edge stays satisfied + after the upstream row is pruned. An edge not marked satisfied blocks + unless its upstream row is there and SUCCESSFUL. A missing upstream row + with no record is not taken as success. + + Both sides are bare task UUIDs rather than node references, because + this table is read inside the worker's claim statement and a join + through the node table would be a second index probe on the hottest + query the package has. + + The index on (downstream_task_id, satisfied) is the one the claim uses: + it asks "does anything unsatisfied point at this candidate", with no + workflow in hand. A satisfied edge costs the claim one index entry and + no read of the task table. + """ + + workflow = models.ForeignKey( + OxpullWorkflow, + on_delete=models.PROTECT, + related_name="edges", + db_index=False, + ) + upstream_task_id = models.UUIDField() + downstream_task_id = models.UUIDField() + satisfied = models.BooleanField(default=False) + created_at = models.DateTimeField() + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["workflow", "upstream_task_id", "downstream_task_id"], + name="oxpull_wf_edge_uniq", + ), + ] + indexes = [ + models.Index( + fields=["downstream_task_id", "satisfied"], + name="oxpull_wf_edge_down_idx", + ), + # The reverse question: "what did this task unblock". The unique + # constraint above cannot answer it: its leading column is the + # workflow, and the outcome write has a task id and no workflow + # in hand, exactly as the claim does not. + models.Index(fields=["upstream_task_id"], name="oxpull_wf_edge_up_idx"), + # The edges of one workflow not yet satisfied, in key order. + models.Index( + fields=["workflow", "satisfied", "id"], name="oxpull_wf_edge_open_idx" + ), + ] + + def __str__(self) -> str: + return f"{self.upstream_task_id} -> {self.downstream_task_id}" + + +class OxpullWorkflowScan(models.Model): + """ + Where the reconciler's backstop is in one workflow that is not COMPLETE. + + The backstop pages through a workflow's open edges and nodes in key + order, a page per visit. This row holds its place between visits, so a + pass survives a crash or a timed-out tick. The high-water marks are the + largest edge and node ids when the pass began, and the pass stops there, + so rows added during it wait for the next pass instead of extending it. + + workflow_id is a bare UUID, not a foreign key: this is bookkeeping, and + nothing about it may stop a workflow row from being dealt with. + """ + + class Phase(models.TextChoices): + EDGES = "EDGES" + NODES = "NODES" + RELEASE = "RELEASE" + + workflow_id = models.UUIDField(primary_key=True) + phase = models.CharField(max_length=10, choices=Phase.choices, default=Phase.EDGES) + # The last key visited in this phase. + position = models.BigIntegerField(default=0) + edge_hwm = models.BigIntegerField(null=True, blank=True) + node_hwm = models.BigIntegerField(null=True, blank=True) + # Keys a visit could not write because another transaction held them, + # tried again on a later visit. + skipped = models.JSONField(default=list, blank=True) + # The compare-and-set target that moves the position. + version = models.BigIntegerField(default=0) + + def __str__(self) -> str: + return f"{self.workflow_id} {self.phase}@{self.position}" + + +class OxpullWorkflowRound(models.Model): + """ + The backstop's round-robin position across workflows: one row. + + Workflows are visited in id order, wrapping, and this row names the last + one visited. version is the compare-and-set target, so two reconcilers + cannot both move it. + """ + + id = models.PositiveSmallIntegerField(primary_key=True, default=1) + workflow_id = models.UUIDField(null=True, blank=True) + version = models.BigIntegerField(default=0) + + def __str__(self) -> str: + return f"round at {self.workflow_id} (version {self.version})" diff --git a/src/oxpull/py.typed b/src/oxpull/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/oxpull/rate_limits.py b/src/oxpull/rate_limits.py new file mode 100644 index 0000000..1d3697f --- /dev/null +++ b/src/oxpull/rate_limits.py @@ -0,0 +1,218 @@ +""" +Resolving rate limits from a backend's OPTIONS. + +A limit is a name, a capacity of N admissions per period P, and the tasks +or queues it covers: + + "RATE_LIMITS": { + "stripe": { + "limit": 100, + "per": 60, + "tasks": ["myapp.billing.charge", "myapp.billing.refund"], + }, + "sms": {"limit": 500, "per": 60, "queues": ["sms"]}, + } + +Both tasks listed under "stripe" draw on one budget of 100 per minute. + +Settings are the only declaration site, and that is the whole of the +API. A decorator marking the task itself would put the limit in a task +module, and a worker imports no task modules: django.tasks has no +autodiscovery, and `ox_worker` imports the backend rather than the code +it runs. A limit declared that way would be missing from the registry +the worker builds at startup and enforced only when something else +happened to import that module first, which is worse than having no +decorator at all. OPTIONS is read by every process, so a limit declared +here is visible to the worker and to `manage.py check` alike, which is +what lets the check catch a limit that no worker would apply. + +A limit is named once across the whole installation. Its window row is +keyed on the name alone, so two backends configuring the same name share +one window and count each other's admissions. Uniqueness is different and +is scoped per alias, because its key hashes the alias in; a limit's does +not. Give a limit that should apply to one backend a name only that +backend uses. +""" + +from __future__ import annotations + +import math +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +from django.core.exceptions import ImproperlyConfigured + +CONFIG_KEYS = frozenset({"limit", "per", "tasks", "queues"}) + +# The width of OxpullRateWindow.limit_name, which is the primary key of +# the window table. models.py takes the column's length from here, so a +# name this module accepts is a name the row can hold on every database. +MAX_LIMIT_NAME_LENGTH = 128 + +# Seconds between refresh SELECTs, when nothing a worker did has made its +# view stale. run() re-enters the claim on every task completion, not on +# the poll interval, so without a floor a worker with fast in-flight work +# would refresh at completion rate. +RATE_REFRESH_INTERVAL = 0.25 + + +@dataclass(frozen=True) +class RateLimit: + """One resolved limit: its capacity, and what it covers.""" + + name: str + limit: int + per: int + task_paths: frozenset[str] + queues: frozenset[str] + + def covers(self, task_path: str, queue_name: str) -> bool: + return task_path in self.task_paths or queue_name in self.queues + + +def rate_limits_from_options( + options: dict[str, Any], + backend_alias: str, + backend_queues: Iterable[str] = (), +) -> dict[str, RateLimit]: + """ + RATE_LIMITS from a backend's OPTIONS, as name -> RateLimit ordered by + name. + + Raises ImproperlyConfigured on anything malformed, so a bad deploy + fails at the system check and at worker start rather than silently + never throttling; the same idiom as unique_tasks_from_options. + """ + raw = options.get("RATE_LIMITS", {}) + if not isinstance(raw, dict) or not all(isinstance(n, str) for n in raw): + raise ImproperlyConfigured("RATE_LIMITS must be a dict keyed by limit name.") + accepted = frozenset(backend_queues) + limits: dict[str, RateLimit] = {} + for name in sorted(raw): + config = raw[name] + if not isinstance(config, dict): + raise ImproperlyConfigured( + f"RATE_LIMITS[{name!r}] must be a dict with the keys " + f"{sorted(CONFIG_KEYS)}." + ) + unknown = sorted(set(config) - CONFIG_KEYS) + if unknown: + raise ImproperlyConfigured( + f"RATE_LIMITS[{name!r}] has unknown key(s) {unknown}; " + f"the keys are {sorted(CONFIG_KEYS)}." + ) + _check_name(name) + limit, per = _capacity(name, config.get("limit"), config.get("per")) + paths = frozenset(_string_list(name, config, "tasks")) + queues = frozenset(_string_list(name, config, "queues")) + if accepted and not queues <= accepted: + raise ImproperlyConfigured( + f"RATE_LIMITS[{name!r}] names queue(s) " + f"{sorted(queues - accepted)} that backend " + f"{backend_alias!r} does not accept." + ) + if not paths and not queues: + raise ImproperlyConfigured( + f"Rate limit {name!r} covers no tasks and no queues. List " + "the task paths under 'tasks', or the queue names under " + "'queues'." + ) + limits[name] = RateLimit(name, limit, per, paths, queues) + _refuse_names_a_collation_would_fold(limits) + return limits + + +def _refuse_names_a_collation_would_fold(limits: dict[str, RateLimit]) -> None: + """ + Refuse two limit names that differ only by case. + + A limit's name is the primary key of its window row, and a primary key + decides identity with the column's collation rather than with Python's + ``==``. MySQL's usual ``utf8mb4_0900_ai_ci`` folds case, so ``Reports`` + and ``reports`` are two limits to this parser and one row to the + database: they share a window, count each other's admissions, and + throttle traffic they were never meant to see. Nothing raises, and the + two limits are simply wrong from the first admission onwards. + + Refused rather than warned about, and refused on every database. The + configuration is broken on MySQL and confusing everywhere, and a name is + free to change before anything is deployed. + """ + folded: dict[str, str] = {} + for name in sorted(limits): + key = name.casefold() + first = folded.get(key) + if first is not None: + raise ImproperlyConfigured( + f"RATE_LIMITS has {first!r} and {name!r}, which differ only " + "by case. A limit's name is the primary key of its window " + "row, and on a case-insensitive collation the two share one " + "window and throttle each other. Rename one." + ) + folded[key] = name + + +def refresh_interval_from_options(options: dict[str, Any]) -> float: + """ + RATE_REFRESH_INTERVAL from a backend's OPTIONS, in seconds. + + Validated here, beside the limits, so that a bad value is reported by + `manage.py check` rather than raised by the first worker to start on + a deploy that passed its checks. + """ + raw = options.get("RATE_REFRESH_INTERVAL", RATE_REFRESH_INTERVAL) + try: + # An int too large for a float raises here, and would have raised + # the same OverflowError from math.isfinite; the message below is + # the one promised. + seconds = float(raw) if not isinstance(raw, bool) else math.nan + except (TypeError, ValueError, OverflowError): + seconds = math.nan + if not isinstance(raw, int | float) or not math.isfinite(seconds) or seconds < 0: + raise ImproperlyConfigured( + f"RATE_REFRESH_INTERVAL must be a finite number of seconds, zero " + f"or more; got {raw!r}." + ) + return seconds + + +def _check_name(name: str) -> None: + if not name: + raise ImproperlyConfigured("A rate limit's name cannot be empty.") + if len(name) > MAX_LIMIT_NAME_LENGTH: + raise ImproperlyConfigured( + f"Rate limit name {name!r} is {len(name)} characters, and the " + f"maximum is {MAX_LIMIT_NAME_LENGTH}. The name is the primary " + "key of the window table: PostgreSQL and MySQL refuse a longer " + "one at the first admission, and SQLite stores it and disagrees " + "with them." + ) + + +def _capacity(name: str, limit: object, per: object) -> tuple[int, int]: + if limit is None or per is None: + raise ImproperlyConfigured(f"Rate limit {name!r} needs limit and per together.") + if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0: + raise ImproperlyConfigured( + f"Rate limit {name!r}: limit must be a whole number of " + "admissions, zero or more." + ) + seconds = round(per.total_seconds()) if isinstance(per, timedelta) else per + if not isinstance(seconds, int) or isinstance(seconds, bool) or seconds < 1: + raise ImproperlyConfigured( + f"Rate limit {name!r}: per must be a whole number of seconds, or " + "a timedelta of at least one second. Sub-second periods are not " + "supported." + ) + return limit, seconds + + +def _string_list(name: str, config: dict[str, Any], key: str) -> list[str]: + raw = config.get(key, []) + if not (isinstance(raw, list | tuple) and all(isinstance(v, str) for v in raw)): + raise ImproperlyConfigured( + f"RATE_LIMITS[{name!r}][{key!r}] must be a list of strings." + ) + return list(raw) diff --git a/src/oxpull/routing.py b/src/oxpull/routing.py new file mode 100644 index 0000000..4f9b746 --- /dev/null +++ b/src/oxpull/routing.py @@ -0,0 +1,99 @@ +""" +Where the oxpull tables live. + +The uniqueness lock and the batch tables are written in the same +transaction as a django_ox row (the lock beside its task, the membership +row beside its task, the completed batch beside the callbacks it +enqueues), and the batch completion query joins membership to the task +table. Neither a transaction nor a join crosses a database alias, so +those tables must sit on the same database as the django_ox tables. + +The rate window is neither co-written nor joined, and it belongs here +for the other reason: OxpullWorker issues every window read and write +against this alias, so a router that names a different database for +OxpullRateWindow is not the one the worker follows. + +A router that splits any oxpull table from the django_ox tables is a +configuration error, caught at startup and repeated as a system check, +rather than a runtime fallback that loses atomicity quietly. + +The alias is resolved from db_for_write and every read on these paths is +pinned to it: a router that sends reads to a lagged replica must not be +able to answer "this lock is free", "nothing is pending" or "this window +has room" from a stale snapshot. + +The router is asked once, per model and without hints. django-ox saves +each task row through the same router with the row as the instance hint, +so a router that routes by model answers the same both times and the +task row lands on this alias. A router that answers differently with the +hint, one database per backend_name say, writes the task row on a +database this alias never sees: the lock row and the membership row then +commit apart from their task, and an add() that reissues its transaction +after a deadlock enqueues again on the other database. That router is +outside what this check can see, and it is not supported. +""" + +from __future__ import annotations + +from django.core.exceptions import ImproperlyConfigured +from django.db import models, router + +# Why each table has to answer to the same alias as OxTask. The message +# carries the reason for the model that is actually split, so an operator +# reading it gets the constraint that applies rather than the union. +_CO_WRITTEN = ( + "Its rows are written in the same transaction as django_ox rows and " + "joined to them in one query, and neither a transaction nor a join " + "crosses a database alias." +) +# The edge table is read inside the worker's claim statement, against the +# task table, in one query. A router that sends it elsewhere would make +# the claim reference a table that is not on the connection running it. +_EDGE = ( + "The worker's claim statement reads it in the same query as the task " + "table, and a query does not cross a database alias." +) +_PINNED = ( + "OxpullWorker reads and writes the rate window on the database that " + "holds the task tables, so this route is not the one it follows." +) + + +def oxpull_db_alias() -> str: + """ + The database holding both the django_ox and the oxpull tables. + + Raises ImproperlyConfigured if the active routers do not write all of + them to one alias. + """ + from django_ox.models import OxTask + + from .models import ( + OxpullBatch, + OxpullBatchMember, + OxpullRateWindow, + OxpullUniqueLock, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, + ) + + required: tuple[tuple[type[models.Model], str], ...] = ( + (OxpullUniqueLock, _CO_WRITTEN), + (OxpullBatch, _CO_WRITTEN), + (OxpullBatchMember, _CO_WRITTEN), + (OxpullWorkflow, _CO_WRITTEN), + (OxpullWorkflowNode, _CO_WRITTEN), + (OxpullWorkflowEdge, _EDGE), + (OxpullRateWindow, _PINNED), + ) + task_db = router.db_for_write(OxTask) + for model, reason in required: + model_db = router.db_for_write(model) + if model_db != task_db: + raise ImproperlyConfigured( + f"oxpull needs the django_ox and oxpull tables on one " + f"database. The router writes OxTask to {task_db!r} and " + f"{model.__name__} to {model_db!r}. {reason}" + ) + return str(task_db) diff --git a/src/oxpull/unique.py b/src/oxpull/unique.py new file mode 100644 index 0000000..767cd8b --- /dev/null +++ b/src/oxpull/unique.py @@ -0,0 +1,104 @@ +""" +Marking tasks unique, and deriving the uniqueness key. + +A task is unique-while-pending: while a task with the same key is READY +or RUNNING, enqueueing again returns the pending task's result instead of +inserting a second row. Once the pending task leaves those two statuses +(SUCCESSFUL, FAILED, LOST or DISCARDED, or its row is pruned), the key is +free again. The key follows the row's status, not the code: a LOST row +frees the key while the execution it belonged to may still be running, +and django-ox lets that execution write its outcome onto the row later. +Uniqueness deduplicates pending rows; it does not stop two executions of +one key overlapping. A LOST task frees the key even if its original +execution is still running. + +django.tasks.Task is a frozen slots dataclass, so the unique flag cannot +live on the task object. It lives in a module-level registry keyed by +task.module_path, which also survives Task.using() copies (queue or +priority overrides keep the same module_path). +""" + +from __future__ import annotations + +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +import hashlib +import json +from typing import Any + +from django.core.exceptions import ImproperlyConfigured +from django.tasks.base import Task +from django.utils.json import normalize_json + +_registry: set[str] = set() + + +def unique[**P, R](task: Task[P, R]) -> Task[P, R]: + """ + Mark a task as unique-while-pending. Apply above the @task decorator: + + @unique + @task + def send_invoice(order_id): ... + """ + if not isinstance(task, Task): + raise TypeError( + "@unique must wrap a django.tasks Task; apply it above the @task decorator." + ) + _registry.add(task.module_path) + return task + + +def is_registered(task_path: str) -> bool: + return task_path in _registry + + +def unique_tasks_from_options(options: dict[str, Any]) -> frozenset[str]: + """ + UNIQUE_TASKS from a backend's OPTIONS, as a frozenset of dotted task + paths, equivalent to decorating each listed task with @unique. + + Raises ImproperlyConfigured on an invalid shape so a bad deploy fails + in system checks rather than silently at enqueue time; the same idiom + as django_ox.schedules.schedules_from_options. + """ + raw = options.get("UNIQUE_TASKS", []) + if not (isinstance(raw, list | tuple) and all(isinstance(p, str) for p in raw)): + raise ImproperlyConfigured("UNIQUE_TASKS must be a list of dotted task paths.") + return frozenset(raw) + + +def unique_key( + backend_alias: str, task_path: str, queue_name: str, args: Any, kwargs: Any +) -> str: + """ + Derive the uniqueness key: SHA-256 over the JSON encoding of + (backend_alias, task_path, queue_name, normalised args, normalised + kwargs). Uniqueness is scoped per backend alias: the alias is the + isolation boundary everywhere else in django.tasks, and a key shared + across aliases would return a result whose backend contradicts the + task it was enqueued on. + + Arguments are normalised with the same django.utils.json.normalize_json + the backend uses to store them, so key equality is exactly equality of + what would be written to the task row: tuples equal lists, dict order + is irrelevant (sort_keys), and 1 and 1.0 differ just as they do in the + stored JSON. queue_name is part of the key; the same call on two queues + is two tasks. priority and run_after are not part of the key. + + Dict keys pass through JSON's own coercion before sorting (int keys + become strings, exactly as the row's JSONField stores them), so a + payload mixing int and str keys derives the key of what the row will + actually hold instead of raising over an unsortable dict. + """ + raw = [ + backend_alias, + task_path, + queue_name, + normalize_json(args), + normalize_json(kwargs), + ] + canonical = json.loads(json.dumps(raw)) + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() diff --git a/src/oxpull/worker.py b/src/oxpull/worker.py new file mode 100644 index 0000000..e6f5126 --- /dev/null +++ b/src/oxpull/worker.py @@ -0,0 +1,1119 @@ +""" +The worker that applies rate limits. + + TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": { + "stripe": { + "limit": 100, + "per": 60, + "tasks": ["myapp.billing.charge"], + }, + }, + }, + } + } + +ox_worker runs this class when WORKER_CLASS selects it. The supervisor +behind --processes N starts each child through ox_worker, so select the +worker in settings rather than only in the parent process. Otherwise, +children can use the base django_ox.worker.Worker and bypass rate-limit +admission. + +The budget is consulted before the claim, never after it. django-ox folds +`attempts = attempts + 1` and `lease_epoch = lease_epoch + 1` into the +claim UPDATE itself, on all three claim paths, so a design that claimed a +task and then discovered it had no budget would have spent an attempt on +a task that never ran, and the lease epoch is a fencing token that cannot +be decremented. A throttled task is therefore never claimed at all: it +stays READY, holds no lease, and is indistinguishable from any other +queued task. + +The pre-claim view is necessarily stale, and that is what bounds the +guarantee rather than a shortcut in it: + +For a limit of N admissions per period P, with W ox_worker processes +claiming from the queues that carry the limit's tasks, the number of +task attempts started in any one window is at most N + W - 1 + U. U is +the number of admissions whose count did not land: one for each count +that raised (_count_failed) and one for each count the window rolled +out from under MAX_RATE_ATTEMPTS times (the tail of _count), each +logged as rate_limit_uncounted. Add one for each worker process that +dies between claiming a limited task and recording it. With a single +worker process, no such death and every count landed, the limiter is +exact: at most N per window. A window is one generation of the shared +counter row, opened by a roll and lasting P, not every interval of +length P. Windows are contiguous, so an arbitrary interval of length P +that spans a boundary can carry up to 2(N + W - 1 + U). A count that +cannot be written is logged and the attempt runs; three admissions in a +row that the counter did not record, whether the increment raised or +matched no row, close the limit until a write lands. + +The proof rests on refreshing the view after every admission, before the +next limited claim, which is why _record() marks the view dirty and why a +limited claim costs a SELECT as well as an UPDATE. +""" + +from __future__ import annotations + +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +import inspect +import json +import logging +import random +import time +import uuid +from contextlib import nullcontext, suppress +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.db import ( + DatabaseError, + Error, + IntegrityError, + InterfaceError, + OperationalError, + connections, + transaction, +) +from django.db.models import F, Q, QuerySet +from django.db.models.functions import Now + +from django_ox.models import OxTask +from django_ox.worker import POSTGRES_CLAIM_SQL, Worker, _lease_now + +from .batches import ADD_RETRY_JITTER +from .models import OxpullRateWindow, OxpullWorkflowNode +from .rate_limits import rate_limits_from_options, refresh_interval_from_options +from .routing import oxpull_db_alias +from .workflows import ( + NODE_FLAG, + lost_lock_conflict, + node_flag, + node_flag_for_a_locking_read, + node_flag_sql, + record_node_outcome, + release_downstream, + runnable_filter_q, + runnable_filter_sql, + session_timeouts, + transaction_timeouts, + workflows_enabled_from_options, + workflows_on_database, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from .rate_limits import RateLimit + +logger = logging.getLogger("oxpull") + +# Bound on the increment's retry loop. A pass only fails when the window +# rolled between the refresh and the increment, so reaching the bound +# takes that many rolls inside one claim. Mirrors MAX_UNIQUE_ATTEMPTS. +MAX_RATE_ATTEMPTS = 5 + +# Tries of one node outcome's transaction that loses a lock conflict, and the +# seconds each try waits for a lock. Holding the task row for that long stalls +# this worker's renewal of its other leases, so it stays far below a third of +# the default LOCK_TIMEOUT. +OUTCOME_ATTEMPTS = 5 +OUTCOME_LOCK_TIMEOUT = 5.0 + +# Seconds between looks for the workflow node table, by a worker that records +# only because another backend on its database declares WORKFLOWS, while the +# table is not there yet. +NODE_TABLE_RECHECK = 30.0 + +# The outcomes a node records from the worker. A retry write puts the row back +# to READY and records nothing. +RECORDED_BY_THE_WORKER = (OxTask.Status.SUCCESSFUL, OxTask.Status.FAILED) + +# Unrecorded admissions in a row that close a limit: a count that raised, +# or one that matched no row after every attempt. One is a lost count +# and the task runs: the claim is already committed, and losing a +# count is better than losing the task. A run of them is a counter that +# is not being written at all, and admitting against a counter that never +# advances is not rate limiting, it is a worker at full speed with a +# warning in the log. +MAX_COUNT_FAILURES = 3 + + +@dataclass +class _Seen: + """One limit's window as this worker last read it.""" + + window_id: int + used: int + # time.monotonic() at which the window ends, derived from the database + # clock read in the same statement as the row. Local only in its zero + # point, so a worker whose clock disagrees with the database's still + # measures the right number of seconds. + ends_at: float + + +def _class_hook(worker: Worker, name: str) -> Any: + """ + The hook `name` as the worker's class defines it, bound to the worker. + + Looked up past the instance, where OxpullWorker binds its own answer + under the same name, and bound the way attribute access binds it, so + a hook written as a static method or a class method is called as one. + """ + raw = inspect.getattr_static(type(worker), name) + bind = getattr(raw, "__get__", None) + return raw if bind is None else bind(worker, type(worker)) + + +class OxpullWorker(Worker): + """ + django-ox's Worker, plus rate limits. + + An install with no limits configured pays nothing: the claim filter + adds nothing to what the class's hooks return, so the emitted SQL is + what django-ox emits on its own, and claim_one() delegates without a + statement of its own. + + A subclass narrows what it claims through django-ox's claim filter + hooks, claim_filter_q() and claim_filter_sql(). Those hooks carry only + the subclass's own condition. The worker adds its rate limits to + whatever they return, so a subclass that never calls super() still + claims inside every limit. + """ + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._limits: dict[str, RateLimit] = rate_limits_from_options( + self.backend.options, self.backend.alias, self.backend.queues + ) + self._rate_alias = oxpull_db_alias() + self._refresh_interval = refresh_interval_from_options(self.backend.options) + self._workflows = workflows_enabled_from_options(self.backend.options) + # Whether this worker records node outcomes and releases what they + # held: wherever a backend on its database declares WORKFLOWS, not + # only when its own backend does, because a claim is scoped to queues + # and not to backends. + self._records = self._workflows or workflows_on_database() + # A worker whose own backend declares WORKFLOWS needs the workflow + # tables for its claim in any case. Any other looks for them first. + self._node_table_seen = self._workflows + self._node_table_looked_at: float | None = None + self._seen: dict[str, _Seen] = {} + self._blocked_paths: frozenset[str] = frozenset() + self._blocked_queues: frozenset[str] = frozenset() + # Limits whose counter could not be written MAX_COUNT_FAILURES + # times running, and the run length that got them there. + self._closed: set[str] = set() + self._count_failures: dict[str, int] = {} + # Stored spelling -> configured name, for a limit whose window row + # was written under a spelling the column's collation treats as + # the same name. Learned by _adopt, once; empty everywhere else. + self._folded: dict[str, str] = {} + # The view is stale until the first read, and again after every + # admission this worker makes. That rule is what bounds the + # overshoot at one per worker rather than at N per worker. + self._view_stale = True + self._refresh_after = 0.0 + # Column names come from the model rather than from SQL text here, + # so a django-ox migration that renamed one would be a failure to + # import a field, not a fragment that no longer matches. + self._path_column = OxTask._meta.get_field("task_path").column + self._queue_column = OxTask._meta.get_field("queue_name").column + # django-ox reads the claim filter by calling claim_filter_q() and + # claim_filter_sql() on the worker. Bound here, on the instance, + # those names answer with what the class's hooks return and what + # oxpull adds, the rate limits and the workflow gate. Left to the + # class, they would be a subclass's to override, and one that + # replaced a hook without calling super() would drop every limit + # and the gate from its claims. + self._class_claim_filter_q: Callable[[], Q | None] = _class_hook( + self, "claim_filter_q" + ) + self._class_claim_filter_sql: Callable[[], tuple[str, dict[str, Any]]] = ( + _class_hook(self, "claim_filter_sql") + ) + self.claim_filter_q = self._q_with_oxpull_conditions # type: ignore[method-assign] + self.claim_filter_sql = self._sql_with_oxpull_conditions # type: ignore[method-assign] + + # -- start-up ---------------------------------------------------------- + + def run(self) -> None: + """ + django-ox's poll loop, after checking the workflow schema. + + A worker on a backend that declares WORKFLOWS refuses to start on a + MySQL database whose workflow task-id columns are on another collation + than OxTask.id, because every claim it made would fail. + """ + if self._workflows: + self._check_workflow_schema() + super().run() + + def _check_workflow_schema(self) -> None: + from .checks import collation_mismatches + + try: + mismatches = collation_mismatches(self._rate_alias) + except DatabaseError: + logger.warning( + "Could not check the workflow schema before starting", + exc_info=True, + extra={ + "event": "workflow_schema_check_failed", + "worker_id": self.worker_id, + }, + ) + return + if mismatches: + raise ImproperlyConfigured( + " ".join( + f"{mismatch.message()} {mismatch.hint()}" for mismatch in mismatches + ) + + " The worker did not start. `manage.py check --database " + f"{self._rate_alias}` reports the same as oxpull.E009." + ) + + # -- the claim pass ---------------------------------------------------- + + def claim_one(self) -> OxTask | None: + """ + Refresh the budget, claim, then count the admission. + + The refresh and the increment run outside the claim, because the + SKIP LOCKED path claims inside a transaction that is holding row + locks on the task table and neither statement belongs in there. + + The claim is the line between fail-closed and fail-open. A + database error before it propagates: nothing is spent, and the + poll loop skips the pass. A database error after it, anywhere in + the accounting, is logged and the claimed task is returned: the + attempt is spent and the row is RUNNING, and a task claimed but + never handed over runs nowhere until the reaper finds it. + """ + # Decided before the claim, which on the SKIP LOCKED path opens a + # transaction that should not carry a look at the schema. + self._recording() + if not self._limits: + return super().claim_one() + self._refresh_view() + db_task = super().claim_one() + if db_task is not None: + self._record(db_task) + return db_task + + def _q_with_oxpull_conditions(self) -> Q | None: + """ + claim_filter_q() as django-ox reads it from this worker. The class's + condition and oxpull's own, ANDed. + """ + condition = self._class_claim_filter_q() + oxpull = self._oxpull_condition_q() + if condition is None: + return oxpull + if oxpull is None: + return condition + return condition & oxpull + + def _sql_with_oxpull_conditions(self) -> tuple[str, dict[str, Any]]: + """ + claim_filter_sql() as django-ox reads it from this worker. The + class's fragment, followed by oxpull's own. + + Empty, rate limits and workflow gate and all, when the class has a + condition in claim_filter_q() and no fragment for it. django-ox takes + the single-statement PostgreSQL claim for any non-empty fragment, and + that statement applies the fragment alone, so a fragment of ours there + would drop the class's condition. An empty one sends django-ox to the + path that applies claim_filter_q(), which carries both. A class that + wrote both hooks, or neither, keeps the single statement. + """ + fragment, params = self._class_claim_filter_sql() + if not fragment and self._class_claim_filter_q() is not None: + return "", {} + ours, our_params = self._oxpull_condition_sql() + return fragment + ours, {**params, **our_params} + + def _oxpull_condition_q(self) -> Q | None: + """ + Rate limits and workflow dependencies, conjoined, or None when + neither applies. + + Two independent reasons a READY row may not be claimed, and both + have to hold at once: a throttled task is not claimable however + satisfied its dependencies are, and a workflow node whose parents + have not succeeded is not claimable however much budget is left. + + Each term is present only where its feature is configured, so a + backend with neither returns None and the emitted SQL is what + django-ox emits on its own. The workflow term is switched by + OPTIONS['WORKFLOWS'] rather than by looking for workflow rows, + because a workflow is created at run time: a cached "there are no + workflows" would be a window in which a node runs out of order, + and a query per claim pass to close that window would cost more + than the conjunct it was avoiding. Where it is on, it costs one + probe of oxpull_wf_edge_down_idx, the edge table's index over + (downstream_task_id, satisfied), per candidate row, which finds + nothing for a task that is not in a workflow. + """ + if not (self._workflows or self._blocked_paths or self._blocked_queues): + return None + condition = runnable_filter_q() if self._workflows else Q() + if self._blocked_paths: + condition &= ~Q(task_path__in=sorted(self._blocked_paths)) + if self._blocked_queues: + condition &= ~Q(queue_name__in=sorted(self._blocked_queues)) + return condition + + def _oxpull_condition_sql(self) -> tuple[str, dict[str, Any]]: + """ + The same two conditions for the PostgreSQL claim, which builds + its own SQL. + + This and claim_filter_q must stay equivalent: PostgreSQL is the + only database that reads this one, so a divergence would be a + rule that applies on two databases and not the third, and + nothing would report it. The suite exercises the ordering + guarantee through a real claim, so whichever path the database + under test uses is the one proved. + + Each rate-limit array is one bound parameter however many names + it holds, so a limit covering hundreds of task paths cannot + approach the statement's parameter ceiling. Both columns are NOT + NULL, so `<> ALL` has no three-valued-logic trap. + """ + conjuncts: list[str] = [] + params: dict[str, Any] = {} + if self._workflows: + workflow_clause, workflow_params = runnable_filter_sql() + conjuncts.append(workflow_clause) + params.update(workflow_params) + if self._blocked_paths: + conjuncts.append(f' AND "{self._path_column}" <> ALL(%(ox_blocked_paths)s)') + params["ox_blocked_paths"] = sorted(self._blocked_paths) + if self._blocked_queues: + conjuncts.append( + f' AND "{self._queue_column}" <> ALL(%(ox_blocked_queues)s)' + ) + params["ox_blocked_queues"] = sorted(self._blocked_queues) + return "".join(conjuncts), params + + # -- the node flag the claim carries ---------------------------------- + + def _ready_queryset(self) -> QuerySet[OxTask]: + """ + django-ox's candidates, each flagged with whether it is a workflow node. + + Only where the claim is the SKIP LOCKED path, which returns the + candidate it read. The flag is a plain subquery in the select list, so + the locking read takes no lock on a node row. On MySQL it is NULL for a + row committed after the claim's read view opened, and the outcome looks + the node up (node_flag_for_a_locking_read). + """ + queryset = super()._ready_queryset() + connection = connections[self._db_alias] + if self._recording() and connection.features.has_select_for_update_skip_locked: + queryset = queryset.annotate( + **{NODE_FLAG: node_flag_for_a_locking_read(connection.vendor)} + ) + return queryset + + def _claim_one_postgresql(self, run_after_cutoff: datetime) -> OxTask | None: + """ + django-ox's single-statement PostgreSQL claim with the workflow node + flag added to its RETURNING list. + + This copies the base worker's method and adds the node flag after + RETURNING *. Tests compare the task row, returned object, lease + fence and recorded outcome with the base worker's behaviour so a + change to the original claim must also be reflected here. + """ + if not self._recording(): + return super()._claim_one_postgresql(run_after_cutoff) + queue_clause = 'AND "queue_name" = ANY(%(queues)s)' if self.queues else "" + extra_clause, extra_params = self.claim_filter_sql() + lease_clock = "STATEMENT_TIMESTAMP()" if settings.USE_TZ else "%(lease_now)s" + sql = _with_node_flag( + POSTGRES_CLAIM_SQL.format( + table=OxTask._meta.db_table, + queue_clause=queue_clause, + extra_clause=extra_clause, + lease_clock=lease_clock, + ) + ) + rows = OxTask.objects.db_manager(self._db_alias).raw( + sql, + { + "running": OxTask.Status.RUNNING, + "ready": OxTask.Status.READY, + "worker_id": self.worker_id, + "worker_id_json": json.dumps([self.worker_id]), + "now": run_after_cutoff, + "lease_now": _lease_now() if not settings.USE_TZ else None, + "lease_ttl": timedelta(seconds=self.lock_timeout), + "queues": self.queues, + **extra_params, + }, + ) + return next(iter(rows), None) + + def _reload_claimed(self, pk: uuid.UUID, granted_epoch: int) -> OxTask | None: + """ + django-ox's re-read of a row the compare-and-set claim took, pinned to + the epoch it was granted, with the node flag. + """ + if not self._recording(): + return super()._reload_claimed(pk, granted_epoch) + return ( + OxTask.objects.using(self._db_alias) + .filter(pk=pk, lease_epoch=granted_epoch) + .annotate(**{NODE_FLAG: node_flag()}) + .first() + ) + + # -- recording a node's outcome ---------------------------------------- + + def _write_outcome( + self, + db_task: OxTask, + *, + status: OxTask.Status, + duration_ms: int, + **fields: Any, + ) -> bool: + """ + django-ox's outcome write, and for a workflow node, the node's record + and the marks on the edges out of it, in the same transaction. + + This is the hook and not a task_finished receiver for the reason + recorded all over batches.py: the reaper settles a dead worker's task + through a queryset update that emits no signal. It is also the only + place in django-ox that writes SUCCESSFUL. + + Only SUCCESSFUL and FAILED take this path, and only for a row the claim + flagged as a node or could not flag. A retry write, and a write for a + row flagged as no node, is django-ox's write alone. For a row claimed + elsewhere, which carries no flag, and for a row MySQL's claim read from + outside its read view, whose flag is None, the node lookup inside the + transaction decides. + + The transaction writes the task row first, through django-ox and its + lease fence. Then it locks the node row and records the outcome at the + epoch the write leaves on the row. On SUCCESSFUL it then locks the + edges out of the task in key order and marks them satisfied. The + outcome, the record and the marks commit together or not at all. The + nodes those edges point at are released after the commit, never inside + it. A release inside would lock their task rows while this one is held. + + When the call owns its transaction, meaning autocommit was on and no + atomic block was open when it was called, each try waits at most + OUTCOME_LOCK_TIMEOUT seconds for a lock. Whether it owns the transaction + is decided before the first try, never from what an error leaves. A try + that loses a lock conflict puts the row instance back as it was and runs + the whole transaction again, fence included. It tries up to + OUTCOME_ATTEMPTS times, and not once less than a renewal interval of the + lease is left. Putting the instance back is what keeps a stuck-thread + handover on the epoch it started from. When the tries run out, or + another database error arrives, the write fails closed. It is logged as + workflow_outcome_unrecorded and returns False, so no task_finished is + sent, and the reaper takes the row back as it does after a lost lease. + Django's InterfaceError, which is not a DatabaseError, fails closed the + same way. + + One error is raised instead: the one a lost connection leaves, + _outcome_connection_gone. The instance is put back first. django-ox's + _write_outcome_reconnecting then closes the connection and calls this + method once more on another, after asking whether the row already holds + this write. The task row, the node's record and the marks commit + together, so a row that holds it is a transaction that committed whole + before the connection went. That path runs no release of its own: the + edges are marked, and the nodes they held stay WAITING until the + reconciler's backstop releases them. A second try that fails as well is + django-ox's task_outcome_unrecorded, and the row is the reaper's. The + stuck-thread handover calls this method directly, with no second try: + the error reaches the watchdog, which logs task_stuck_unrecorded, and + the row is the reaper's there too. + + Inside a caller's transaction nothing is bounded or tried again, and + errors from the outcome and the record propagate. The release still + runs after the commit, the caller's, through transaction.on_commit on + the database the record was written to. A caller that rolls back + releases nothing, and a release error after the commit is logged and + cannot reach the caller's transaction. Running a workflow node inside a + caller's transaction is not supported. It holds every lock above until + the caller commits. + """ + flag = getattr(db_task, NODE_FLAG, None) + if not ( + self._recording() + and status in RECORDED_BY_THE_WORKER + and (flag is None or flag) + ): + return super()._write_outcome( + db_task, status=status, duration_ms=duration_ms, **fields + ) + epoch = int(fields.get("lease_epoch", db_task.lease_epoch)) + alias = self._rate_alias + connection = connections[alias] + owned = connection.get_autocommit() and not connection.in_atomic_block + if not owned: + with transaction.atomic(using=alias): + if not super()._write_outcome( + db_task, status=status, duration_ms=duration_ms, **fields + ): + return False + downstream = record_node_outcome(db_task.pk, status, epoch, alias) + if downstream: + held = downstream + transaction.on_commit( + lambda: self._release_after_outcome(db_task, held), + using=alias, + robust=True, + ) + return True + + saved = { + "status": db_task.status, + **{name: getattr(db_task, name) for name in fields}, + } + # The driver connection the tries start on, which get_autocommit() + # opened if none was. _outcome_connection_gone compares against it. + driver = connection.connection + started = time.monotonic() + attempt = 0 + downstream = None + try: + with session_timeouts(alias, lock_wait=OUTCOME_LOCK_TIMEOUT): + while True: + attempt += 1 + try: + with transaction.atomic(using=alias): + transaction_timeouts(alias, lock_wait=OUTCOME_LOCK_TIMEOUT) + if not super()._write_outcome( + db_task, + status=status, + duration_ms=duration_ms, + **fields, + ): + return False + downstream = record_node_outcome( + db_task.pk, status, epoch, alias + ) + break + except BaseException as exc: + _put_back(db_task, saved) + if not ( + lost_lock_conflict(exc, alias) + and attempt < OUTCOME_ATTEMPTS + and self._lease_allows_a_retry(started) + ): + raise + logger.info( + "Task id=%s: recording its workflow outcome lost a lock " + "conflict on try %d of %d. Trying again", + db_task.id, + attempt, + OUTCOME_ATTEMPTS, + extra=self._log_extra( + "workflow_outcome_retry", db_task, try_number=attempt + ), + ) + time.sleep(random.uniform(0, ADD_RETRY_JITTER)) # noqa: S311 + except Error as exc: + _put_back(db_task, saved) + if self._outcome_connection_gone(exc, alias, driver): + raise + logger.error( + "Could not record the %s outcome of workflow node task id=%s after " + "%d tries. The task row is unchanged, and the reaper will take it " + "back", + status, + db_task.id, + attempt, + exc_info=True, + extra=self._log_extra( + "workflow_outcome_unrecorded", + db_task, + dropped_status=str(status), + tries=attempt, + ), + ) + return False + if downstream is not None and status == OxTask.Status.SUCCESSFUL: + self._release_after_outcome(db_task, downstream) + return True + + def _recording(self) -> bool: + """ + Whether this worker flags claimed nodes and records their outcomes now. + + Only where a backend on the database declares WORKFLOWS. A worker whose + own backend does not also waits for the workflow node table, because + before 0004_workflows is applied no row can be a node and the flag has + no table to read. It looks for the table at most every + NODE_TABLE_RECHECK seconds until it is there, and then stops looking. + """ + if not self._records or self._node_table_seen: + return self._records + now = time.monotonic() + looked_at = self._node_table_looked_at + if looked_at is not None and now - looked_at < NODE_TABLE_RECHECK: + return False + self._node_table_looked_at = now + tables = connections[self._rate_alias].introspection.table_names() + self._node_table_seen = OxpullWorkflowNode._meta.db_table in tables + return self._node_table_seen + + def _outcome_connection_gone(self, exc: Error, alias: str, driver: Any) -> bool: + """ + Whether a node outcome's own transaction failed because its connection + was lost, so that the error goes to django-ox's + _write_outcome_reconnecting, which closes the connection and writes + once more on another. + + Only an InterfaceError or an OperationalError, only on the database + django-ox's check and close act on, and only when that check, + _outcome_connection_lost, says the connection is gone. A lock timeout, + a deadlock whose tries ran out and every other error leave a connection + that answers, and the write fails closed as before. + + Django drops a connection whose rollback fails, which is what a + transaction on a lost connection ends in, and then opens another to + turn autocommit back on. That one never saw the transaction, and it + answers, so the check would call the lost connection alive. It is + closed first, when the driver connection is no longer the one the + tries started on, and the check then sees what happened: no connection + open. Nothing is lost by that: it holds no transaction, and the next + statement opens one. A connection that could not be reopened is not + open either. + """ + if not isinstance(exc, (InterfaceError, OperationalError)): + return False + if alias != self._db_alias: + return False + connection = connections[alias] + if connection.connection is not None and connection.connection is not driver: + with suppress(Error): + connection.close() + return self._outcome_connection_lost() + + def _lease_allows_a_retry(self, started: float) -> bool: + """ + Whether another try still leaves the lease more than a renewal interval. + + The lease was renewed at most one interval before the first try, and a + try that holds the task row stalls its renewal. + """ + spent = time.monotonic() - started + return spent < self.lock_timeout - 2 * self.renew_interval + + def _release_after_outcome( + self, db_task: OxTask, downstream: list[uuid.UUID] + ) -> None: + """ + Release the nodes a recorded success was holding, after the commit that + recorded it. + + A failure is logged and swallowed. It costs liveness, not correctness: + the edges are marked, and the nodes stay WAITING for the reconciler. + Raising here would lose the outcome of a task that ran. + + When commit callbacks run inside a transaction that is still open, as + django.test's captureOnCommitCallbacks runs them, the release takes a + savepoint, so its error does not end that transaction. + """ + alias = self._rate_alias + inside = connections[alias].in_atomic_block + try: + with transaction.atomic(using=alias) if inside else nullcontext(): + released = release_downstream(downstream, alias) + except Error: + logger.warning( + "Could not release the dependents of task id=%s. The reconciler " + "will release them", + db_task.id, + exc_info=True, + extra={ + "event": "workflow_release_failed", + "task_id": str(db_task.id), + "worker_id": self.worker_id, + }, + ) + return + if released: + logger.debug( + "Task id=%s released %d workflow node(s)", + db_task.id, + released, + extra={ + "event": "workflow_nodes_released", + "task_id": str(db_task.id), + "released_count": released, + "worker_id": self.worker_id, + }, + ) + + # -- the budget -------------------------------------------------------- + + def _refresh_view(self) -> None: + if not self._view_stale and time.monotonic() < self._refresh_after: + return + self._read_windows() + + def _read_windows(self) -> None: + """ + One SELECT over every configured limit, then a roll attempt for + each window the database's own clock says has expired. + """ + started = time.monotonic() + if self._closed: + self._reopen_what_counts_again() + rows = self._select() + missing = [name for name in self._limits if name not in rows] + if missing: + for name in missing: + self._seed(name) + rows = self._select() + for name in missing: + if name not in rows: + rows[name] = self._adopt(name) + seen: dict[str, _Seen] = {} + blocked: set[str] = set() + for name, config in self._limits.items(): + row = rows[name] + db_now = row.db_now + window_id, used = row.window_id, row.used + ends_at = row.window_started_at + timedelta(seconds=config.per) + if ends_at <= db_now and self._roll(name, window_id, config.per): + # This worker opened the window, so the whole capacity is + # free. The roll takes no token: it runs before any claim, + # which is what makes it free to lose. + window_id, used = window_id + 1, 0 + ends_at = db_now + timedelta(seconds=config.per) + seen[name] = _Seen( + window_id=window_id, + used=used, + ends_at=started + max(0.0, (ends_at - db_now).total_seconds()), + ) + if used >= config.limit or name in self._closed: + blocked.add(name) + self._seen = seen + self._blocked_paths = frozenset( + path for name in blocked for path in self._limits[name].task_paths + ) + self._blocked_queues = frozenset( + queue for name in blocked for queue in self._limits[name].queues + ) + self._view_stale = False + self._refresh_after = started + self._refresh_interval + if blocked and len(blocked) == len(self._limits) and not self._closed: + # Nothing this worker can claim is admissible until the + # earliest of these windows ends, and a window's end is known + # exactly. That is the payoff for a window over a bucket, + # whose exhaustion has no knowable expiry. A limit that is + # closed is the exception: it reopens when the database takes + # a write again, which no window's end predicts, so the + # ordinary cooldown paces the probe instead. + self._refresh_after = max( + self._refresh_after, min(seen[name].ends_at for name in blocked) + ) + + def _select(self) -> dict[str, Any]: + # Now() rides along in the same statement so the window's age is + # measured entirely on the database's clock. Comparing a stored + # window against timezone.now() would be wrong under USE_TZ=False, + # where SQLite writes UTC into a column every other writer fills + # with naive local time. + # + # Keyed by the configured name, which is the stored spelling except + # for a row _adopt has attributed to a limit spelled differently. + rows = ( + OxpullRateWindow.objects.using(self._rate_alias) + .filter(limit_name__in=self._limits) + .annotate(db_now=Now()) + ) + return {self._folded.get(row.limit_name, row.limit_name): row for row in rows} + + def _seed(self, name: str) -> None: + """ + Open the first window for a limit no worker has used yet. + + The loser of the race takes the winner's row, the same pattern as + the uniqueness lock and OxScheduleTick. The savepoint keeps a + caller's transaction usable after the conflict. + """ + try: + with transaction.atomic(using=self._rate_alias): + OxpullRateWindow.objects.using(self._rate_alias).create( + limit_name=name, window_started_at=Now() + ) + except IntegrityError: + pass + + def _adopt(self, name: str) -> Any: + """ + Read the row the database holds for a limit under another spelling. + + _seed found a row in its way and _select did not return one under + this name, so the column's collation treats some other spelling + as this name: a limit deployed as "Stripe" and renamed to "stripe" + on MySQL. Only the database knows what its collation folds, so ask + it with the same predicate every write to the row uses. From here + on _select reads that row as this limit, and _roll and _count + already land on it by the same collation, so the count continues + across the rename rather than the worker dying on it. + """ + row = ( + OxpullRateWindow.objects.using(self._rate_alias) + .filter(limit_name=name) + .annotate(db_now=Now()) + .first() + ) + if row is None: + raise ImproperlyConfigured( + f"Rate limit {name!r}: the window table refused a row for it " + "and holds none that matches it." + ) + # The row another configured limit already owns is not this + # limit's to adopt. The parser refuses two names that differ only + # by case, which every case-insensitive collation folds; what + # else a collation folds (MySQL's utf8mb4_0900_ai_ci makes one + # name of `Café` and `cafe`) only the database can say, and this + # is where it says it. Two limits on one row would count each + # other's admissions and throttle traffic neither was configured + # for, so the worker refuses here, before its first claim. A + # single limit renamed to a spelling the collation folds is the + # case above, and it stays a rename. + other = self._folded.get(row.limit_name) + if other is None and row.limit_name in self._limits: + other = row.limit_name + if other is not None and other != name: + raise ImproperlyConfigured( + f"RATE_LIMITS has {other!r} and {name!r}, and the database " + f"holds one window row ({row.limit_name!r}) for both: the " + "column's collation treats the two names as one. Two limits " + "on one row would share a window and throttle each other. " + "Rename one." + ) + self._folded[row.limit_name] = name + logger.info( + "Rate limit %r: its window row is stored as %r; the column's " + "collation treats the two as one name, so that row is the limit's", + name, + row.limit_name, + extra={ + "event": "rate_limit_row_adopted", + "limit": name, + "stored_as": row.limit_name, + "worker_id": self.worker_id, + }, + ) + return row + + def _roll(self, name: str, window_id: int, period: int) -> bool: + """ + Open the next window, if this worker is the one that gets to. + + window_id is the compare-and-set: the winner's write moves it, so + every loser's predicate stops matching and their rowcount is 0. + The period comparison is evaluated on the database clock on both + sides, so a worker whose clock is wrong cannot roll a live window. + """ + return bool( + OxpullRateWindow.objects.using(self._rate_alias) + .filter( + limit_name=name, + window_id=window_id, + window_started_at__lte=Now() - timedelta(seconds=period), + ) + .update(window_id=window_id + 1, used=0, window_started_at=Now()) + ) + + def _reopen_what_counts_again(self) -> None: + """ + Probe each closed limit, and reopen the ones that answer. + + A closed limit admits nothing, so no claim will run another count + and no count will succeed on its own: the only way to learn that + the counter can be written again is to write one. The probe is + the statement that failed with no increment in it, so a limit + that reopens has proved the exact write it is about to need. + """ + for name in sorted(self._closed): + try: + OxpullRateWindow.objects.using(self._rate_alias).filter( + limit_name=name + ).update(used=F("used")) + except DatabaseError: + continue + self._closed.discard(name) + self._count_failures.pop(name, None) + logger.warning( + "Rate limit %r: its counter takes writes again; admitting again", + name, + extra={ + "event": "rate_limit_reopened", + "limit": name, + "worker_id": self.worker_id, + }, + ) + + def _record(self, db_task: OxTask) -> None: + counted = False + for name, config in self._limits.items(): + if config.covers(db_task.task_path, db_task.queue_name): + self._count(name, db_task) + counted = True + if counted: + # Refresh before the next limited claim. Drop this and a + # worker that saw an empty window could admit the whole + # capacity on its own before looking again, and the bound + # would degrade from N + W - 1 to N x W. + self._view_stale = True + + def _count(self, name: str, db_task: OxTask) -> None: + """ + Count one admission. Unconditional, and never a gate. + + The task is already claimed by the time this runs, so a + compare-and-set that could refuse would have burned the attempt + this whole design exists to protect. Two workers taking the last + token both get it; what stops either taking a second is that both + read the post-increment count at their next refresh. + + A statement that raises is the one case that does gate, at the + next refresh rather than here: see _count_failed. + """ + for _ in range(MAX_RATE_ATTEMPTS): + seen = self._seen.get(name) + if seen is None: + return + try: + counted = ( + OxpullRateWindow.objects.using(self._rate_alias) + .filter(limit_name=name, window_id=seen.window_id) + .update(used=F("used") + 1) + ) + except DatabaseError: + self._count_failed(name, db_task) + return + if counted: + # The admission is on the counter, so any earlier run of + # unrecorded admissions is over. Only a matched row says + # so: a statement that ran and matched nothing has + # recorded no more than one that raised. + self._count_failures.pop(name, None) + return + # The window rolled between the refresh and here, so the + # increment landed on nothing. Re-read and count against the + # window that is open now. The re-read is accounting too: + # the claim is committed, so a database that goes away here + # costs the count, not the task. Left uncaught, the error + # would escape claim_one() with the task claimed and + # unreturned, and the row would sit RUNNING until the reaper + # took it back, one attempt poorer. ImproperlyConfigured is + # the re-read finding the limit's row gone and the seed + # refused (_adopt); before the claim that propagates, here it + # is the same loss of a count and is logged the same way. + try: + self._read_windows() + except (DatabaseError, ImproperlyConfigured): + self._count_failed(name, db_task) + return + logger.warning( + "Rate limit %r: window rolled under %d attempts to count task " + "id=%s; it runs uncounted", + name, + MAX_RATE_ATTEMPTS, + db_task.id, + extra={ + "event": "rate_limit_uncounted", + "limit": name, + "task_id": str(db_task.id), + "worker_id": self.worker_id, + }, + ) + # Rolled out is an admission that was not recorded, the same as + # one whose increment raised, and it counts toward the run. + self._unrecorded(name) + + def _count_failed(self, name: str, db_task: OxTask) -> None: + """ + Record one failed count, and close the limit on a run of them. + + A single failure is fail-open on purpose: the claim is committed + and the attempt is spent, so refusing to run the task would lose + work to save a counter. A run of MAX_COUNT_FAILURES is a + different thing. The counter is not advancing, every worker's + next refresh reads a used that never moves, and the limiter + admits at full speed for as long as the database stays that way. + Closing the limit fails the other way: throttled work waits, + which is what a limit is for. + """ + logger.warning( + "Rate limit %r: could not count task id=%s; it runs uncounted", + name, + db_task.id, + exc_info=True, + extra={ + "event": "rate_limit_uncounted", + "limit": name, + "task_id": str(db_task.id), + "worker_id": self.worker_id, + }, + ) + self._unrecorded(name) + + def _unrecorded(self, name: str) -> None: + """ + One more admission under this limit went unrecorded; close the + limit when that makes MAX_COUNT_FAILURES in a row. The run ends + only when an increment lands (_count), so it is a run of + admissions the counter does not know about, whichever way each + one was lost. + """ + failures = self._count_failures.get(name, 0) + 1 + self._count_failures[name] = failures + if failures < MAX_COUNT_FAILURES or name in self._closed: + return + # Once, on the transition. Every later claim is excluded before + # it reaches the count, so there is nothing per-claim to log. + self._closed.add(name) + logger.warning( + "Rate limit %r: %d counts failed in a row; admitting nothing " + "under it until its counter takes a write again", + name, + failures, + extra={ + "event": "rate_limit_closed", + "limit": name, + "failures": failures, + "worker_id": self.worker_id, + }, + ) + + +def _put_back(db_task: OxTask, saved: dict[str, Any]) -> None: + """Restore the instance fields a failed outcome try mirrored onto it.""" + for name, value in saved.items(): + setattr(db_task, name, value) + + +def _with_node_flag(sql: str) -> str: + """The formatted PostgreSQL claim, with the node flag after RETURNING *.""" + head, returning, tail = sql.rpartition("RETURNING *") + if not returning or tail.strip(): + raise ImproperlyConfigured( + "django-ox's PostgreSQL claim must end in RETURNING * so " + "OxpullWorker can append the workflow node flag. Update " + "OxpullWorker's claim adapter to match the base worker's SQL." + ) + return f"{head}{returning}{node_flag_sql()}{tail}" diff --git a/src/oxpull/workflows.py b/src/oxpull/workflows.py new file mode 100644 index 0000000..fc7483a --- /dev/null +++ b/src/oxpull/workflows.py @@ -0,0 +1,3926 @@ +""" +Workflows: a DAG of tasks, where a node is claimable only once every +node it depends on has succeeded. + + from oxpull import workflows + + flow = workflows.create("nightly", on_complete=summarize) + flow.add("extract", extract) + flow.add("transform-a", transform, "a", depends_on="extract") + flow.add("transform-b", transform, "b", depends_on="extract") + flow.add("load", load, depends_on=["transform-a", "transform-b"]) + flow.seal() + +Workflows create blocked task rows with status WAITING. Release moves a +WAITING task to READY and updates run_after. Cancellation moves a READY or +WAITING task to DISCARDED. These writes use a compare-and-set on the task +row's status and lease epoch; the workflow node stores no lease epoch. +run_after is never NULL, and a DISCARDED node is not revived. + +The workflow generation field is reserved and remains 0. It is not a +workflow lease epoch. + +Every move out of WAITING is a compare-and-set on the row's status and +lease epoch. Two releasers cannot both move a row, and a release decided +before a cancellation matches nothing. The worker's claim still checks the +edges as a second line of defence. It refuses a READY row while any edge +into it is not marked satisfied and its upstream row is absent or not +SUCCESSFUL. A missing upstream row with no record is never taken as +success. + +Completion reads every node in one grouped statement, beside its task row. +A node is pending while its row is there and not settled (SUCCESSFUL, FAILED, +LOST or DISCARDED). WAITING is pending, and so is a status this package does +not know. A settled node counts by its record, the outcome a worker, the +reconciler or the cancellation wrote on the node row, so a success or a +failure still counts after ox_prune deletes its row. A record behind its row +is repaired first, and the workflow completes on a later tick. A row gone with +no record is finished and unattributed. State is never accumulated from +task_finished signals, for the reason written down in batches.py: the +django-ox reaper settles a dead worker's task through a queryset update +that emits no signal at all, so a workflow that counted signals would hang +in exactly the case workflows exist to survive. + +The claim's dependency check lives in the worker, so it applies only +where OPTIONS names one that carries it: + + TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "SCHEDULES": { + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, + }, + }, + } + } + +A workflow is created at runtime, not declared in settings, so no system +check can see that one is in use. add() therefore resolves the task's own +backend and refuses a worker that would not apply the gate, on the line +that creates the work rather than three hours later out of order. + +A unique task cannot be a node, and add() refuses one before it enqueues +anything, asking the backend the task enqueues through, as the enqueue +does. A unique enqueue returns whichever task already holds the key, so +the node could be another workflow's node or a call the application +enqueued itself, and one workflow's failure would cancel it. A shared +expensive job belongs outside the workflow, as a unique task. To make a +task unique that is already used as a node, drain its pending nodes +first: from then on add() refuses it, and a node added before the change +holds no uniqueness lock, so a unique enqueue of the same call is not +deduplicated against it and runs beside it. + +The reconciler does three things no single write can. Its backstop pages +through each open workflow's unsatisfied edges, then its nodes, then its +waiting nodes. It marks the edges whose upstream succeeded, records the +outcomes no worker recorded, and releases what is left waiting. It cancels +the downstream cone of a node that settled without succeeding. And it runs +the compare-and-set that completes a sealed workflow and enqueues its +callbacks. Each is idempotent, and a crash leaves the next tick to redo it. +""" + +from __future__ import annotations + +# The gate raises on Django 5.2, before django.tasks is imported below. +from oxpull import _requires_django_6 # noqa: F401 # isort: skip + +import bisect +import hashlib +import logging +import math +import random +import re +import time +import uuid +from collections.abc import Callable, Iterable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING, Any, cast + +from django.conf import settings +from django.core.exceptions import EmptyResultSet, ImproperlyConfigured +from django.db import ( + DatabaseError, + Error, + IntegrityError, + InterfaceError, + OperationalError, + connections, + transaction, +) +from django.db.models import ( + BooleanField, + Case, + CharField, + Count, + Exists, + F, + Max, + Model, + OuterRef, + Q, + QuerySet, + Subquery, + Value, + When, +) +from django.db.models.functions import Coalesce +from django.db.models.lookups import Exact, In +from django.db.models.sql.datastructures import BaseTable +from django.tasks.base import Task, TaskResult +from django.tasks.signals import task_enqueued +from django.utils import timezone +from django.utils.datastructures import OrderedSet + +from django_ox import _waiting, remaining +from django_ox.actions import RETRYABLE_STATUSES +from django_ox.models import OxTask + +from .batches import ( + ADD_RETRY_JITTER, + DEADLOCK, + NAME_MAX_LENGTH, + ReconcilerSchedule, + _binds, + _build_task_on_a_queue, + _enqueue_callback, + _ox_backend_options, + _reissue, + _reissue_add, + _require_ox_backend, + _resolve_callback, + _sealed_without_a_total, + _sweep, + _timed_out, + reconcile_schedule, + schedule_entry_hint, + task_backend_classes, + unbound_reconciler_hint, + unbound_reconciler_message, +) +from .batches import PENDING_STATUSES as PENDING_STATUSES +from .batches import SETTLED_STATUSES as SETTLED_STATUSES +from .batches import UNATTRIBUTED_STATUSES as UNATTRIBUTED_STATUSES +from .exceptions import UnknownWorkflowNode, WorkflowNodeExists, WorkflowSealed +from .models import ( + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, + OxpullWorkflowRound, + OxpullWorkflowScan, +) +from .routing import oxpull_db_alias + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from django.db.models import Field + + # Built by __getattr__ at the bottom of this module, for the reason + # given there; declared here so the public name is visible to type + # checkers and linters. + reconcile: Task[[], int] + +__all__ = [ + "RECONCILE_TASK_PATH", + "Workflow", + "WorkflowProgress", + "create", + "get", + "name_key", + "node_flag", + "node_flag_for_a_locking_read", + "node_flag_sql", + "reconcile", + "record_node_outcome", + "release_downstream", + "runnable_filter_q", + "runnable_filter_sql", + "workflows_enabled_from_options", + "workflows_on_database", +] + +# The dotted path the reconciler's task row carries, and therefore the +# string a SCHEDULES entry must name. Written once here and read by +# OxpullBackend._check_workflow_reconciler, so the check and the schedule +# cannot drift apart: a rename that missed one of them would leave the +# check warning about a schedule that is present, or silent about one +# that is not. +RECONCILE_TASK_PATH = f"{__name__}.reconcile" + +# The schedule name every hint gives the reconciler. +RECONCILE_SCHEDULE_NAME = "oxpull-workflow-reconcile" + +# The hint for a project whose schedules are rows; batches' +# RECONCILE_SCHEDULABLE_HINT says why it is not the settings block. +RECONCILE_SCHEDULABLE_HINT = ( + "Expose the reconciler to the schedule rows: add to a django-ox " + 'backend\'s OPTIONS "SCHEDULABLE_TASKS": ' + f'{{"{RECONCILE_SCHEDULE_NAME}": "{RECONCILE_TASK_PATH}"}}, then add a ' + "schedule for that key." +) + +# What happens to workflows with nothing running the reconciler. +UNSCHEDULED_CONSEQUENCE = ( + "a sealed workflow would never complete, and the dependents of a failed " + "node would never be cancelled" +) + +# An upstream in one of these has settled without succeeding, so nothing +# downstream of it can ever run. LOST is included deliberately: it says +# the attempt was never observed, and "we do not know that it worked" is +# not "it worked". DISCARDED says an operator closed it. +DEAD_STATUSES = ( + OxTask.Status.FAILED, + OxTask.Status.LOST, + OxTask.Status.DISCARDED, +) + +# The dead statuses django_ox.actions.retry moves back to READY. +RETRYABLE_DEAD_STATUSES = RETRYABLE_STATUSES + +# A node in one of these has not run, so the cancellation may close it. +CANCELLABLE_STATUSES = (OxTask.Status.READY, OxTask.Status.WAITING) + +# Completion and progress() read SETTLED_STATUSES, through _classify: +# SUCCESSFUL and the dead statuses above are settled, and every other status +# is pending, including one a later django-ox adds. A status this package does +# not know therefore holds a workflow open rather than completing it and +# attributing an outcome to it. The suite asserts that SETTLED_STATUSES is +# exactly SUCCESSFUL plus DEAD_STATUSES, and that it covers every OxTask.Status +# with PENDING_STATUSES. + +# Most task ids one statement carries as a literal IN list. A fan-out or a +# doomed cone can be any width, and one list of all of it runs into a +# limit on every engine: PostgreSQL refuses more than 65,535 bound +# parameters under server-side binding, SQLite refuses more than its +# variable limit (32,766 by default, 999 before 3.32), and MySQL past its +# range optimizer's memory budget (about 6,000 ids at the 8.4 default) +# plans a scan of the whole task table. The same size as django-ox's +# UPDATE_CHUNK_SIZE, for the same reason. +ID_CHUNK_SIZE = 1000 + +# Bound parameters a chunked statement carries besides its ids. +PARAMETERS_BESIDE_IDS = 8 + +# Bound parameters one id takes on SQLite, where an edge read spells each id +# as a range and binds both of its ends (_KeysProbe). +SQLITE_PARAMETERS_PER_ID = 2 + + +def _id_chunk_size(db_alias: str) -> int: + """How many ids one statement on this database carries at most.""" + size = ID_CHUNK_SIZE + connection = connections[db_alias] + limit = connection.features.max_query_params + if limit: + per_id = SQLITE_PARAMETERS_PER_ID if connection.vendor == "sqlite" else 1 + size = max(1, min(size, (limit - PARAMETERS_BESIDE_IDS) // per_id)) + return size + + +def _id_chunks(ids: Iterable[Any], db_alias: str) -> Iterator[list[Any]]: + """`ids` sorted, in lists short enough for one statement on this database.""" + size = _id_chunk_size(db_alias) + ordered = sorted(ids) + for start in range(0, len(ordered), size): + yield ordered[start : start + size] + + +# Rows one page of the reconciler's backstop reads: edges to mark satisfied, +# nodes whose record to repair, or nodes to release. +BACKSTOP_PAGE = 1000 + +# Most pages one visit gives one workflow. A visit pages on only while its +# pages come back full, so a workflow with many open rows gets up to this many +# pages a visit, and a workflow with few gets one short page a phase. +VISIT_PAGES = 8 + +# Seconds after which a reconciler tick starts no further backstop visit or +# page. Under a task timeout it is a quarter of what is left, if that is less. +# It is checked only between visits and between pages, so it does not bound a +# tick: see _release_backstop for what is bounded. +BACKSTOP_BUDGET = 5.0 + +# Seconds a backstop write waits for a row lock, and runs one statement. +BACKSTOP_LOCK_TIMEOUT = 1.0 +BACKSTOP_STATEMENT_TIMEOUT = 2.0 + +# Bound on the cancellation fixpoint, per reconciler pass. A pass only +# iterates while it is still changing rows, and each iteration cancels at +# least one node, so this is reached only by a DAG deeper than this many +# levels, which the next tick continues from where this one stopped. +MAX_CANCEL_PASSES = 64 + + +@dataclass(frozen=True) +class WorkflowProgress: + """ + Live counts for one workflow, computed on demand from node task rows. + + total is the sealed snapshot once the workflow is sealed, and the + live node count while it is open. blocked is the part of pending that + cannot run yet: nodes WAITING for an upstream, plus READY nodes the + claim's dependency check refuses. The rest of pending is runnable or + running. The split is a second query and is the number an operator + actually wants when a DAG looks stuck. + + succeeded and failed count how each node settled: its record where the + record is the last word, so a failure whose row ox_prune deleted still + counts as failed. pending, succeeded and failed need not add up to + total. The difference is unattributed, and it means the same three + things it means for a batch: a node whose task row ox_prune deleted + before anything recorded how it finished, a node the reaper marked + LOST, and a node an operator (or this workflow's own cancellation) + discarded. All are finished, and none is attributable to + success or failure, which is why on_success stays withheld while it + is above zero. + """ + + total: int + pending: int + blocked: int + succeeded: int + failed: int + + @property + def unattributed(self) -> int: + return max(self.total - (self.pending + self.succeeded + self.failed), 0) + + @property + def finished(self) -> int: + return self.total - self.pending + + +# -- the gate ---------------------------------------------------------- + + +def name_key(name: str) -> str: + """ + A node name's identity: the SHA-256 hex digest of its UTF-8 bytes. + + Two names are one name only when they are spelled the same. The name + column's collation plays no part, so MySQL's folding of case, accents + and trailing spaces neither refuses a second name nor resolves the wrong + node. + """ + return hashlib.sha256(name.encode("utf-8")).hexdigest() + + +class _KeyProbe(Exact): + """ + `column = key`, with a range on SQLite that holds the same one key. + + From 3.45 to 3.49 SQLite marks an index whose statistics show one key + for every row, which is what one wide fan-in or fan-out alone in the edge + table leaves after ANALYZE. It does not probe a marked index with =, IN + or IS, and the mark stays on the connection until its schema is read + again, whatever statistics come later. A range is still allowed, so + SQLite gets `column BETWEEN key AND key` beside the equality. It holds + exactly the rows the equality holds. A planner that may use the equality + keeps its plan and its estimate, and one that may not seeks the range + instead of reading the whole index. + + Every other database compiles what `exact` compiles. The lookup keeps + that name, which the backends read for the operator and the casts, and + is registered on the edge columns as `probe`. + """ + + def as_sqlite(self, compiler: Any, connection: Any) -> tuple[str, Any]: + equality, equality_params = self.as_sql(compiler, connection) + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + sql = f"({equality} AND {lhs} BETWEEN {rhs} AND {rhs})" + return sql, [*equality_params, *lhs_params, *rhs_params, *rhs_params] + + +class _KeysProbe(In): + """ + `column IN (keys)`, which SQLite gets as one range per key. + + The list form of _KeyProbe, for the same reason. Each key is its own + `column BETWEEN key AND key`, so the ranges hold exactly the keys in the + list and never one that sorts between two of them. There is no IN beside + them: given both, SQLite 3.53 spends longer choosing between a long IN + and a long OR than either takes to run, and then tests each row it finds + against the whole OR. The terms are nested as a balanced tree, because + SQLite refuses an expression more than 1,000 levels deep and a flat OR is + as deep as it is long. Each key binds twice (SQLITE_PARAMETERS_PER_ID). + + The ranges are what a marked index may be read by; whether the planner + reads it at all is a second question, and a read of more columns than the + index carries is where it answers no. So the three reads that take a chunk + of ids name their index as well (_NamedIndex). + + A list that is a subquery, or that holds an expression, compiles as `in` + does. So does every other database. Registered as `probe_in`. + """ + + def as_sqlite(self, compiler: Any, connection: Any) -> tuple[str, Any]: + if not self.rhs_is_direct_value() or any( + hasattr(key, "resolve_expression") for key in self.rhs + ): + return self.as_sql(compiler, connection) + # As `in` reads its list: NULL equals nothing, and a key counts once. + keys = OrderedSet(self.rhs) + keys.discard(None) + if not keys: + raise EmptyResultSet + lhs, lhs_params = self.process_lhs(compiler, connection) + placeholders, values = self.batch_process_rhs(compiler, connection, keys) + ranges = [f"{lhs} BETWEEN {mark} AND {mark}" for mark in placeholders] + params: list[Any] = [] + for value in values: + params += [*lhs_params, value, value] + return _any_of(ranges), params + + +def _any_of(terms: list[str]) -> str: + """`terms` joined by OR, nested no deeper than the logarithm of their number.""" + if len(terms) == 1: + return terms[0] + half = len(terms) // 2 + return f"({_any_of(terms[:half])} OR {_any_of(terms[half:])})" + + +class _AsideProbe(Exact): + """ + `column = value`, put beyond every index's reach on SQLite. + + oxpull_wf_edge_down_idx is (downstream_task_id, satisfied), and Django + spells `satisfied=False` as an equality from 6.1 on where 6.0 negated the + column. A marked index may not be probed with an equality, and one offered + on its second column costs the planner the branch an id that the ranges on + the first give it: it reads the named index whole instead, for every row + the list is long. SQLite's unary plus is a no-op that puts a term beyond + any index, which is where this term belongs. + + Unary plus also drops the column's affinity, so this goes on a column + whose values and whose keys are of one type, and on no other. Every other + database compiles what `exact` compiles. Registered on the edge's + `satisfied` as `aside`. + + The comparison is written out rather than taken from `exact`, which up to + Django 6.0 gives a boolean column the column itself and a plus in front of + a NOT is not an expression SQLite reads. So SQLite is sent one spelling + whatever Django's version. + """ + + def as_sqlite(self, compiler: Any, connection: Any) -> tuple[str, Any]: + lhs, lhs_params = self.process_lhs(compiler, connection) + rhs, rhs_params = self.process_rhs(compiler, connection) + return ( + f"+{lhs} {self.get_rhs_op(connection, rhs)}", + [*lhs_params, *rhs_params], + ) + + +def _register_probes() -> None: + """ + Give the two columns an edge is looked up by with no workflow in hand + their `probe` and `probe_in`. Each leads an index that one wide fan-in, + or one wide fan-out, fills with a single key. The lookups go on these two + fields and on no other UUIDField, and `aside` goes on the column that + follows one of them in its index. + """ + for column in ("downstream_task_id", "upstream_task_id"): + field = cast("Field[Any, Any]", OxpullWorkflowEdge._meta.get_field(column)) + field.register_lookup(_KeyProbe, "probe") + field.register_lookup(_KeysProbe, "probe_in") + settled = cast("Field[Any, Any]", OxpullWorkflowEdge._meta.get_field("satisfied")) + settled.register_lookup(_AsideProbe, "aside") + + +_register_probes() + + +def _blocking_edges(downstream: Any) -> QuerySet[OxpullWorkflowEdge]: + """ + The edges into `downstream` that still hold it back. + + An edge holds its downstream back unless it is marked satisfied or its + upstream row is there and SUCCESSFUL. A missing upstream row with no + mark blocks: a row can be missing because ox_prune deleted a DISCARDED + or FAILED upstream, or because someone deleted it by hand, and neither + says it succeeded. + + The upstream status is a scalar subquery on the task table's primary + key, coalesced to "" when the row is gone. A satisfied edge costs one + index entry on (downstream_task_id, satisfied) and no read of the task + table. The scalar form, rather than a nested NOT EXISTS, is what keeps + PostgreSQL from planning a hash anti-join over the whole task table. + + The downstream id is matched with _KeyProbe, here rather than in + _EdgeProbe, so that progress(), which reads these edges with no hint, + seeks the index on SQLite as the claim does. + """ + upstream_status = Subquery( + OxTask.objects.filter(pk=OuterRef("upstream_task_id")).values("status")[:1] + ) + return ( + OxpullWorkflowEdge.objects.filter( + downstream_task_id__probe=downstream, satisfied=False + ) + .alias(upstream_status=Coalesce(upstream_status, Value(""))) + .exclude(upstream_status=OxTask.Status.SUCCESSFUL) + ) + + +# The index the claim's dependency check reads, one probe per candidate row. +EDGE_DOWNSTREAM_INDEX = "oxpull_wf_edge_down_idx" + +# The index the reads out of a node take, on the other task id column. +EDGE_UPSTREAM_INDEX = "oxpull_wf_edge_up_idx" + + +class _EdgeProbe(Exists): + """ + Exists over _blocking_edges that tells SQLite and MySQL to probe + EDGE_DOWNSTREAM_INDEX for each candidate row. + + Their planners choose from statistics, and statistics taken while one wide + fan-in fills the edge table say a downstream id matches thousands of edges. + SQLite then scans the whole edge table for each candidate row, and MySQL + reads the whole table into a temporary table for each claim. A candidate + has no edge, or a few, so both are wrong. SQLite gets INDEXED BY, which + names the index the check reads. It does not say how: a read of the whole + named index satisfies it. On SQLite 3.45 to 3.49 an index whose + statistics show one key cannot be probed with =, IN or IS, so the range + _KeyProbe adds to the equality is what keeps the read a probe there. + MySQL gets FORCE INDEX, and a NO_SEMIJOIN hint, which keeps the check a + probe per candidate row instead of a join against the whole table. + PostgreSQL has no hints, and 0004 sets the column's n_distinct so its + estimate stays small (see that migration). + + The hint is added to the SQL Django compiles, so the probe keeps whatever + alias Django gives the outer table. Where it goes depends on how Django + renders the edge table's own FROM item, and the alias there is quoted on + some versions and bare on others: see pin_edge_probe. + """ + + def as_sql( + self, + compiler: Any, + connection: Any, + template: str | None = None, + **extra_context: Any, + ) -> tuple[str, Any]: + sql, params = super().as_sql(compiler, connection, template, **extra_context) + return pin_edge_probe(sql, connection), params + + +def pin_edge_probe(sql: str, connection: Any) -> str: + """ + The dependency check's SQL, with the hints _EdgeProbe describes for this + connection. SQL that does not read the edge table the expected way is + returned unchanged. + + Both engines want the hint after the FROM item, alias and all. Django + quotes a table alias there from 6.1 on and did not before, so the + alias is matched in either spelling. The WHERE ahead of it is what + stops an unaliased FROM item handing the keyword over as its alias. + """ + if connection.vendor not in ("sqlite", "mysql"): + return sql + quote = connection.ops.quote_name + quoted = quote(OxpullWorkflowEdge._meta.db_table) + table = re.escape(quoted) + mark = re.escape(quoted[0]) + alias = rf"(?: (?:{mark}\w+{mark}|[A-Za-z]\w*))?" + found = re.search(rf"FROM {table}{alias}(?= WHERE )", sql) + if found is None: + return sql + index = quote(EDGE_DOWNSTREAM_INDEX) + if connection.vendor == "sqlite": + hint = f" INDEXED BY {index}" + else: + hint = f" FORCE INDEX ({index})" + sql = sql[: found.end()] + hint + sql[found.end() :] + if connection.vendor == "mysql": + sql = re.sub(r"^EXISTS\(SELECT ", "EXISTS(SELECT /*+ NO_SEMIJOIN() */ ", sql) + return sql + + +class _NamedIndex(BaseTable): + """ + The edge table in a FROM clause, naming on SQLite the index the read takes. + + A chunk of ids reaches the edge table as one range an id (_KeysProbe). A + planner whose statistics show one key estimates every one of those ranges + at the whole row count, so a branch an id looks dearer than a read of the + table, and it reads the table and tests every row against the whole OR: + the row count times the list length, and it grows with both. INDEXED BY + leaves it no table to read. The claim's check names the same index for the + same reason, in the same words (_EdgeProbe). + + Every other database gets the FROM item Django writes. + """ + + def __init__(self, table_name: str, alias: str | None, index: str) -> None: + super().__init__(table_name, alias) + self.index = index + + @property + def identity(self) -> tuple[Any, ...]: + """What BaseTable compares by, and the index besides: two FROM items + that name different indexes are not the same FROM item.""" + return (*super().identity, self.index) + + def relabeled_clone(self, change_map: dict[str | None, str]) -> _NamedIndex: + alias = change_map.get(self.table_alias, self.table_alias) + return type(self)(self.table_name, alias, self.index) + + def as_sql(self, compiler: Any, connection: Any) -> tuple[str, Any]: + sql, params = super().as_sql(compiler, connection) + if connection.vendor == "sqlite": + sql += f" INDEXED BY {connection.ops.quote_name(self.index)}" + return sql, params + + +def _reading_index[Row]( + queryset: QuerySet[OxpullWorkflowEdge, Row], index: str +) -> QuerySet[OxpullWorkflowEdge, Row]: + """`queryset` with its edge table naming `index` on SQLite (_NamedIndex).""" + pinned = queryset.all() + alias = pinned.query.get_initial_alias() + pinned.query.alias_map[alias] = _NamedIndex( + OxpullWorkflowEdge._meta.db_table, alias, index + ) + return pinned + + +def runnable_filter_q() -> Q: + """ + The claim's extra condition: nothing still holds this candidate back. + + A node that waits for an upstream is WAITING, which the claim never + reads, so on the normal path this condition only ever meets READY nodes + whose edges are all satisfied. It is kept in the claim for the READY + rows nothing in this package released: a django-ox retry, an + application's own UPDATE, or a faulty release. Fail-closed: see + _blocking_edges. It probes the edge index once per candidate row, whatever + the planner's statistics say (_EdgeProbe). + """ + return Q(~_EdgeProbe(_blocking_edges(OuterRef("pk")))) + + +def runnable_filter_sql() -> tuple[str, dict[str, Any]]: + """ + runnable_filter_q() as a WHERE fragment, for the single-statement + PostgreSQL claim, which builds its own SQL. + + The fragment is appended to the candidate select's conditions as a + bare AND conjunct against the task table, per Worker.claim_filter_sql's + contract. Table and column names come from the models' _meta, the same + way OxpullWorker takes its rate-limit columns, so a rename in a future + django-ox is an import error rather than a fragment that silently + matches nothing. + + The unqualified task-table reference resolves to the candidate + subselect's own FROM item, which is the innermost scope carrying that + name. Both correlated sub-selects use aliases, so nothing else is + ambiguous. + """ + task_table = OxTask._meta.db_table + edge_table = OxpullWorkflowEdge._meta.db_table + task_pk = OxTask._meta.pk.column + status = OxTask._meta.get_field("status").column + downstream = OxpullWorkflowEdge._meta.get_field("downstream_task_id").column + upstream = OxpullWorkflowEdge._meta.get_field("upstream_task_id").column + satisfied = OxpullWorkflowEdge._meta.get_field("satisfied").column + # S608: every interpolated name comes from a model's own _meta, and + # every value in the statement is a bound parameter. The same + # construction as OxpullWorker.claim_filter_sql and as django-ox's + # POSTGRES_CLAIM_SQL, both of which format table and column names + # into a statement for the same reason: a literal here would be a + # table name that a future release could rename without a test + # noticing. + fragment = ( + f" AND NOT EXISTS (" # noqa: S608 + f'SELECT 1 FROM "{edge_table}" ox_wf_edge' + f' WHERE ox_wf_edge."{downstream}" = "{task_table}"."{task_pk}"' + f' AND ox_wf_edge."{satisfied}" = %(ox_wf_unsatisfied)s' + f' AND COALESCE((SELECT ox_wf_up."{status}" FROM "{task_table}" ox_wf_up' + f' WHERE ox_wf_up."{task_pk}" = ox_wf_edge."{upstream}"), \'\')' + f" <> %(ox_wf_success)s)" + ) + return fragment, { + "ox_wf_success": OxTask.Status.SUCCESSFUL, + "ox_wf_unsatisfied": False, + } + + +def _cancellable_edges( + workflow_pk: uuid.UUID, db_alias: str +) -> QuerySet[OxpullWorkflowEdge]: + """ + The edges of this workflow that doom a node the cancellation may close. + + An edge not marked satisfied, into a READY or WAITING row, from an upstream + that settled without succeeding: its row is there in a dead status, or its + row is gone and its node's record holds a dead status. A row that is gone + with no record dooms nothing, because a pruned row says nothing about how + it finished. + + Not the negation of the gate. A node whose upstream is merely still + running is blocked but not doomed, and cancelling it would be a verdict + on work still in progress. An edge marked satisfied never dooms anything: + its upstream succeeded. + + Each row it reads is read by key with a scalar subquery: the upstream's + status, its node's record and the downstream's status. An EXISTS beside an + OR is what PostgreSQL turns into a hashed subplan over the whole task + table. + """ + return ( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(workflow_id=workflow_pk, satisfied=False) + .annotate( + upstream_status=Coalesce(_status_of("upstream_task_id"), Value("")), + ) + .alias( + upstream_record=Coalesce( + Subquery( + OxpullWorkflowNode.objects.filter( + task_id=OuterRef("upstream_task_id") + ).values("outcome")[:1] + ), + Value(""), + ), + downstream_status=_status_of("downstream_task_id"), + ) + .filter( + Q(upstream_status__in=DEAD_STATUSES) + | Q(upstream_status="", upstream_record__in=DEAD_STATUSES) + ) + .filter(downstream_status__in=CANCELLABLE_STATUSES) + ) + + +def _status_of(column: str) -> Subquery: + """The status of the task row a column names, as a scalar subquery by key.""" + return Subquery(OxTask.objects.filter(pk=OuterRef(column)).values("status")[:1]) + + +# -- counting ---------------------------------------------------------- + + +def _node_task_ids(workflow_pk: uuid.UUID, db_alias: str) -> Any: + return ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id=workflow_pk) + .values("task_id") + ) + + +def _node_task_counts(workflow_pk: uuid.UUID, db_alias: str) -> dict[str, int]: + """ + Node task rows grouped by status: the join the whole design rests on. + + Deliberately the same query batches runs over its membership rows, + written out again rather than shared, for the reason recorded in + oxpull_prune: the two features must be free to diverge without one + silently changing the other. Both sides are pinned to the one alias + oxpull_db_alias() resolves, because a join does not cross aliases. + + A node whose task row is gone joins to nothing and appears in no + bucket: pruned reads as finished, which is what keeps a long-lived + workflow completing after ox_prune has been through. + """ + rows = ( + OxTask.objects.using(db_alias) + .filter(id__in=_node_task_ids(workflow_pk, db_alias)) + .values("status") + .annotate(count=Count("id")) + .order_by() + ) + return {row["status"]: row["count"] for row in rows} + + +# -- the handle -------------------------------------------------------- + + +class Workflow: + """ + A handle on one OxpullWorkflow row. Build one with create() or get(). + + The handle holds no derived state: status and progress are read from + the database on each call, because the reconciler that completes a + workflow runs in another process. + """ + + def __init__(self, row: OxpullWorkflow) -> None: + self._row = row + + def __repr__(self) -> str: + return f"" + + @property + def id(self) -> uuid.UUID: + return self._row.pk + + @property + def name(self) -> str: + return self._row.name + + @property + def status(self) -> str: + """The status as stored now, not as it was when this handle was built.""" + db_alias = oxpull_db_alias() + return str( + OxpullWorkflow.objects.using(db_alias) + .values_list("status", flat=True) + .get(pk=self._row.pk) + ) + + def add( + self, + node_name: str, + task: Task[..., Any], + *args: Any, + depends_on: str | Iterable[str] | None = None, + **kwargs: Any, + ) -> TaskResult[Any, Any]: + """ + Enqueue a task as a named node, with its dependencies, in one + transaction. + + Deliberately not ParamSpec-typed, unlike Batch.add: a keyword-only + parameter cannot follow ``*args: P.args``, so keeping the task's + own argument types would mean spelling dependencies as a second + positional, ``flow.add("load", None, load_task, 42)``, with a + `None` on every node that has none. The declaration reads more + than it type-checks here, so the argument types go. ``depends_on`` + is reserved as a result: a task with a keyword argument of that + name must be wrapped or renamed. + + depends_on names nodes already added to this workflow. That + restriction is what makes a cycle unconstructible: every edge + points backwards in insertion order, so there is no cycle + detector here because there is nothing for one to find. + + One transaction, taking its locks in the order every workflow + operation takes them: the workflow row first, then this node's + task row, then the upstream nodes, then its own node and edges. A + node with no dependencies is enqueued READY. A node with + dependencies is inserted WAITING, which no worker claims, and each + of its edges is inserted satisfied when the upstream row reads + SUCCESSFUL. When every edge is satisfied, the node is released in + the same transaction. Otherwise it is checked again once the + transaction commits: an upstream can succeed after this read and + before this node's edges are visible to that upstream's worker. A + run_after the caller gave stays on the row, and the release keeps + it. + + The task row, the node row and every edge commit or roll back + together, so no worker sees a node without the edges that hold it. + + When add() owns its transaction, a lost deadlock (InnoDB's 1213, + PostgreSQL's 40P01) or PostgreSQL serialization failure (40001, which + two overlapping add()s raise at REPEATABLE READ) reissues the whole of + it, enqueue included, as Batch.add does. add() owns its transaction + when the connection is in autocommit and no atomic block is open. + Inside a caller's transaction the error propagates and the caller + retries its own transaction. + + The caller's side of that contract. A transaction that adds to or + seals several workflows takes them in ascending id order, because two + transactions that take the same workflow rows in opposite orders + deadlock. It takes no locking read or UPDATE of an existing task row. + And it does not claim or run workflow nodes. A worker's claim or + execution inside a caller's transaction is not supported for them. + + Raises WorkflowSealed if the workflow is no longer OPEN, + UnknownWorkflowNode for a dependency this workflow does not have, + and WorkflowNodeExists for a duplicate node name or task. A unique + task raises ImproperlyConfigured before anything is enqueued; the + module docstring says why, and what to do instead. + """ + db_alias = oxpull_db_alias() + _require_ox_backend(task, role="nodes", label="Workflow") + _require_workflow_backend(task) + _refuse_unique_node(task) + if len(node_name) > NAME_MAX_LENGTH: + raise ValueError(f"Node name exceeds {NAME_MAX_LENGTH} characters.") + wanted = _dependency_names(depends_on) + return _reissue_add( + lambda: self._add_once(db_alias, node_name, task, wanted, args, kwargs), + db_alias, + label="Workflow", + owner_pk=self._row.pk, + task_path=task.module_path, + log=logger, + ) + + def _add_once( + self, + db_alias: str, + node_name: str, + task: Task[..., Any], + wanted: list[str], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> TaskResult[Any, Any]: + """One attempt of add(): the transaction that add() may reissue.""" + with transaction.atomic(using=db_alias): + status = self._take_for_add(db_alias) + if status != OxpullWorkflow.Status.OPEN: + raise self._not_open_error(status) + if wanted: + # Born WAITING, never inserted READY and then moved: a row + # moved out of READY leaves index entries in the range every + # claim scans. django-ox's helper sends no task_enqueued, so + # it is sent here, where task.enqueue() would have sent it. + result = _waiting.enqueue(task, args, kwargs, using=db_alias) + task_enqueued.send_robust(type(task.get_backend()), task_result=result) + else: + result = task.enqueue(*args, **kwargs) + upstream_ids = self._resolve_dependencies(wanted, db_alias) + # A plain read. SUCCESSFUL is final, so an upstream this read + # sees SUCCESSFUL stays so, and its edge can be marked now. One + # it misses is found by the check after commit. + succeeded = _succeeded_among(upstream_ids, db_alias) + now = timezone.now() + try: + # Its own savepoint, so a duplicate does not poison a + # caller's surrounding transaction; the confinement the + # unique enqueue path uses. + with transaction.atomic(using=db_alias): + OxpullWorkflowNode.objects.using(db_alias).create( + workflow_id=self._row.pk, + name=node_name, + name_key=name_key(node_name), + task_id=result.id, + created_at=now, + ) + OxpullWorkflowEdge.objects.using(db_alias).bulk_create( + [ + OxpullWorkflowEdge( + workflow_id=self._row.pk, + upstream_task_id=upstream_id, + downstream_task_id=result.id, + satisfied=upstream_id in succeeded, + created_at=now, + ) + for upstream_id in upstream_ids + ] + ) + except IntegrityError as exc: + raise WorkflowNodeExists( + f"Workflow {self._row.pk} already has a node named " + f"{node_name!r} or holding task {result.id}." + ) from exc + if upstream_ids: + node_id = uuid.UUID(str(result.id)) + if succeeded.issuperset(upstream_ids): + _release_added_node(node_id, db_alias) + else: + transaction.on_commit( + lambda: _recheck_added_node(node_id, db_alias), + using=db_alias, + robust=True, + ) + return result + + def _take_for_add(self, db_alias: str) -> str | None: + """ + Take the workflow row's lock, count the node, and return the status. + + The UPDATE matches only an OPEN row and increments node_count, so + either seal() waits for this add() and counts its node, or this add() + finds the workflow no longer OPEN and its whole transaction rolls + back, the increment with it. The UPDATE always changes the row it + matches, so its rowcount is right on a MySQL connection without + CLIENT_FOUND_ROWS. A zero rowcount is answered by a status read under + the row lock, which is current even under REPEATABLE READ. + """ + rows = OxpullWorkflow.objects.using(db_alias).filter(pk=self._row.pk) + if rows.filter(status=OxpullWorkflow.Status.OPEN).update( + node_count=F("node_count") + 1 + ): + return OxpullWorkflow.Status.OPEN + return rows.select_for_update().values_list("status", flat=True).first() + + def _resolve_dependencies( + self, wanted: list[str], db_alias: str + ) -> list[uuid.UUID]: + """ + Node names to task ids, read under the workflow row lock add() + has already taken. + + Read under the lock and not before it: a concurrent add() of the + named node commits before this transaction gets the lock, or + after this one releases it, so the name either resolves or does + not, and never resolves to a row that then disappears. + + Names are matched by name_key, so a name resolves only to the node + spelled exactly that way, whatever the column's collation. + + A locking read, which is a current read. Under MySQL's REPEATABLE + READ a plain SELECT answers from the read view fixed by the + transaction's first consistent read, a caller's for one, and that + view can predate a node committed before the lock was taken. The + rows it locks are this workflow's own node rows, which nothing else + writes while add() holds the workflow row. PostgreSQL has no current + read at REPEATABLE READ, and needs none here: the other add() + rewrote the workflow row, so the lock UPDATE above has already + raised a serialization failure. + """ + if not wanted: + return [] + keys = {name_key(name): name for name in wanted} + known: dict[str, uuid.UUID] = {} + for chunk in _id_chunks(keys, db_alias): + nodes = ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id=self._row.pk, name_key__in=chunk) + .order_by("pk") + ) + if connections[db_alias].features.has_select_for_update: + nodes = nodes.select_for_update() + known.update(nodes.values_list("name_key", "task_id")) + missing = [name for key, name in keys.items() if key not in known] + if missing: + raise UnknownWorkflowNode( + f"Workflow {self._row.pk} has no node(s) {sorted(missing)}; " + "depends_on may only name nodes already added, which is " + "also what makes a cycle unconstructible." + ) + return [known[name_key(name)] for name in wanted] + + def seal(self) -> None: + """ + Freeze membership and snapshot total_count. Completion is + possible only after this; a workflow that is never sealed never + completes, though its runnable nodes do run. + + Inside a caller's transaction the contract add() describes applies, + including the ascending id order across workflows. + + Raises WorkflowSealed if it was already sealed or completed. + There is no un-seal. Raises ImproperlyConfigured if no django-ox + backend schedules the reconciler, as Batch.seal() does, since the + workflow would then stay SEALED forever; the status is checked + first, so a workflow that is already sealed says so whatever the + settings. + """ + db_alias = oxpull_db_alias() + + def retrying(conflict: str, attempt: int, bound: int) -> None: + logger.info( + "Workflow %s: seal() lost a serialization failure on attempt %d " + "of %d; reissuing its transaction", + self._row.pk, + attempt, + bound, + extra={ + "event": "workflow_seal_serialization_retry", + "workflow_id": str(self._row.pk), + "attempt": attempt, + }, + ) + + # At PostgreSQL REPEATABLE READ the status write below, having + # waited on an add() in flight, raises a serialization failure when + # that add() commits: the row changed after this transaction's + # snapshot. Owning the transaction, seal() runs it again on a fresh + # snapshot. Inside a caller's transaction the error propagates. + _reissue( + lambda: self._seal_once(db_alias), + db_alias, + deadlocks=False, + retrying=retrying, + ) + + def _seal_once(self, db_alias: str) -> None: + """ + One attempt of seal(): the transaction that seal() may reissue. + + total_count is copied from node_count by the status write itself, and + no node row is read. add() increments node_count in the statement that + takes the workflow row's lock, and its whole transaction commits or + rolls back with it, so node_count counts exactly the nodes that + committed. The status write waits on any add() still holding the row, + and an UPDATE reads the row's latest committed version, so the copy is + current even inside a caller's transaction whose read view is older. + At PostgreSQL REPEATABLE READ such a row raises a serialization failure + instead. No node row is locked, so seal() never stalls a node's outcome + or an add() to another workflow. + """ + with transaction.atomic(using=db_alias): + sealed = ( + OxpullWorkflow.objects.using(db_alias) + .filter(pk=self._row.pk, status=OxpullWorkflow.Status.OPEN) + .update( + status=OxpullWorkflow.Status.SEALED, + sealed_at=timezone.now(), + total_count=F("node_count"), + ) + ) + if not sealed: + raise self._not_open_error(self._stored_status(db_alias)) + # After the status, so a workflow already SEALED or COMPLETE + # reports that rather than the settings. Inside the block, so a + # refusal here rolls the status write back and the workflow + # stays OPEN. + _require_reconcile_schedule() + + def node_ids(self) -> dict[str, uuid.UUID]: + """This workflow's node names, mapped to their task ids.""" + db_alias = oxpull_db_alias() + return dict( + OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id=self._row.pk) + .values_list("name", "task_id") + ) + + def progress(self) -> WorkflowProgress: + """ + Counts for this workflow, queried now from the node task rows and the + nodes' records, classified as completion classifies them (_classify). + """ + db_alias = oxpull_db_alias() + row = OxpullWorkflow.objects.using(db_alias).get(pk=self._row.pk) + if row.total_count is not None: + total = row.total_count + classes = _classify(row.pk, db_alias) + else: + # An open workflow has no snapshot, so the live node count is + # the only honest denominator. Read it first and let the + # classified nodes raise it: an add() committing between the two + # then makes this reading lag by a node, which is harmless. + # The other order inflates the denominator instead. + nodes = ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id=row.pk) + .count() + ) + classes = _classify(row.pk, db_alias) + total = max(nodes, classes.nodes) + blocked = ( + OxTask.objects.using(db_alias) + .filter(id__in=_node_task_ids(row.pk, db_alias)) + .filter( + Q(status=OxTask.Status.WAITING) + | Q( + Q(status=OxTask.Status.READY), + Exists(_blocking_edges(OuterRef("pk"))), + ) + ) + .count() + ) + return WorkflowProgress( + total=total, + pending=classes.pending, + blocked=blocked, + succeeded=classes.succeeded, + failed=classes.failed, + ) + + def _stored_status(self, db_alias: str) -> str | None: + return ( + OxpullWorkflow.objects.using(db_alias) + .values_list("status", flat=True) + .filter(pk=self._row.pk) + .first() + ) + + def _not_open_error(self, status: str | None) -> WorkflowSealed: + if status is None: + raise OxpullWorkflow.DoesNotExist( + f"Workflow {self._row.pk} no longer exists." + ) + return WorkflowSealed( + f"Workflow {self._row.pk} is {status}; membership is frozen." + ) + + +def _dependency_names(depends_on: str | Iterable[str] | None) -> list[str]: + if depends_on is None: + return [] + names = [depends_on] if isinstance(depends_on, str) else list(depends_on) + if not all(isinstance(name, str) for name in names): + raise ImproperlyConfigured( + "depends_on must be a node name or an iterable of node names." + ) + # Order preserved, duplicates dropped: the unique constraint on the + # edge would refuse the second copy and abort the whole add(). + return list(dict.fromkeys(names)) + + +def workflows_enabled_from_options(options: dict[str, Any]) -> bool: + """ + WORKFLOWS from a backend's OPTIONS, as a bool. + + The gate is declared rather than inferred, for one measured reason: + it is a conjunct in the claim statement, and a worker cannot decide + at run time whether to include it without either a query per claim + pass or a cached answer that is a window in which a node runs out of + order. Declaring it keeps a backend with no workflows emitting + exactly the SQL django-ox emits on its own, the property rate + limits already hold to, and it gives + the configuration a name a system check can see, which is what turns + "your nodes will run out of order" from a run-time surprise into a + start-up error. + + Raises ImproperlyConfigured on a non-bool so a bad deploy fails in + system checks rather than silently at claim time; the same idiom as + django_ox.schedules.schedules_from_options. + """ + raw = options.get("WORKFLOWS", False) + if not isinstance(raw, bool): + raise ImproperlyConfigured( + f"WORKFLOWS must be True or False, not {type(raw).__name__}." + ) + return raw + + +# -- the release ------------------------------------------------------- + + +def _succeeded_among(task_ids: list[uuid.UUID], db_alias: str) -> set[uuid.UUID]: + """The ids among task_ids whose row is there and SUCCESSFUL, by a plain read.""" + found: set[uuid.UUID] = set() + for chunk in _id_chunks(task_ids, db_alias): + found.update( + uuid.UUID(str(pk)) + for pk in OxTask.objects.using(db_alias) + .filter(pk__in=chunk, status=OxTask.Status.SUCCESSFUL) + .values_list("pk", flat=True) + ) + return found + + +def _release_added_node(task_id: uuid.UUID, db_alias: str) -> None: + """ + Release a node add() just inserted with every edge satisfied, inside + add()'s transaction. + + The epoch comes from the row itself rather than being assumed, so the + move stays pinned the way every other release is. + """ + epoch = ( + OxTask.objects.using(db_alias) + .filter(pk=task_id, status=OxTask.Status.WAITING) + .values_list("lease_epoch", flat=True) + .first() + ) + if epoch is not None: + _waiting.release(task_id, lease_epoch=epoch, using=db_alias) + + +def _recheck_added_node(task_id: uuid.UUID, db_alias: str) -> None: + """ + Release a node add() left WAITING, if its upstreams have all succeeded + by the time add() committed. + + add() read its upstreams inside its own transaction. An upstream that + succeeded after that read ran its release before this node's edges were + visible to it, so nothing else would move this node until the + reconciler's backstop. A failure here must not surface from the caller's + commit, so it is logged and left to the backstop. + """ + try: + _satisfy_edges_into([task_id], db_alias) + release_downstream([task_id], db_alias) + except DatabaseError: + logger.warning( + "Could not re-check workflow node task id=%s after add(). The " + "reconciler will release it", + task_id, + exc_info=True, + extra={"event": "workflow_release_failed", "task_id": str(task_id)}, + ) + + +def _mark_satisfied(edge_ids: Iterable[int], db_alias: str) -> int: + """ + Mark edges satisfied by primary key. Returns how many changed. + + Only a caller that saw each edge's upstream row SUCCESSFUL may pass it. + The UPDATE carries no subquery on the task table: on MySQL at REPEATABLE + READ that would take shared locks on every task row it reads. + """ + marked = 0 + for chunk in _id_chunks(edge_ids, db_alias): + marked += ( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(pk__in=chunk, satisfied=False) + .update(satisfied=True) + ) + return marked + + +def _succeeded_or_recorded(task_ids: Iterable[Any], db_alias: str) -> set[uuid.UUID]: + """ + The ids among task_ids known to have succeeded, by plain reads: the row is + there and SUCCESSFUL, or the row is gone and its node's record says + SUCCESSFUL. + + This is the proof a writer needs before it marks an edge satisfied. A row + that is gone with no such record proves nothing. The reads can only be + stale in the safe direction: SUCCESSFUL is final, a pruned row does not + come back, and only a revival clears a SUCCESSFUL record, which needs the + row. + """ + ids = sorted({uuid.UUID(str(task_id)) for task_id in task_ids}) + proven: set[uuid.UUID] = set() + gone: list[uuid.UUID] = [] + for chunk in _id_chunks(ids, db_alias): + found = { + uuid.UUID(str(pk)): status + for pk, status in OxTask.objects.using(db_alias) + .filter(pk__in=chunk) + .values_list("pk", "status") + } + proven.update( + pk for pk, status in found.items() if status == OxTask.Status.SUCCESSFUL + ) + gone.extend(pk for pk in chunk if pk not in found) + for chunk in _id_chunks(gone, db_alias): + proven.update( + uuid.UUID(str(task_id)) + for task_id in OxpullWorkflowNode.objects.using(db_alias) + .filter(task_id__in=chunk, outcome=OxTask.Status.SUCCESSFUL) + .values_list("task_id", flat=True) + ) + return proven + + +def _satisfiable_edges_into( + task_ids: Iterable[Any], db_alias: str +) -> dict[int, uuid.UUID]: + """ + The edges into task_ids not yet marked whose upstream is known to have + succeeded, mapped to the node each points at. + """ + ids = sorted({uuid.UUID(str(task_id)) for task_id in task_ids}) + open_edges: list[tuple[int, Any, Any]] = [] + for chunk in _id_chunks(ids, db_alias): + open_edges.extend( + _reading_index( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(downstream_task_id__probe_in=chunk, satisfied__aside=False) + .values_list("pk", "upstream_task_id", "downstream_task_id"), + EDGE_DOWNSTREAM_INDEX, + ) + ) + proven = _succeeded_or_recorded({up for _, up, _ in open_edges}, db_alias) + return { + pk: uuid.UUID(str(down)) + for pk, up, down in open_edges + if uuid.UUID(str(up)) in proven + } + + +def _satisfy_edges_into(task_ids: Iterable[Any], db_alias: str) -> int: + """ + Mark satisfied the edges into task_ids whose upstream is known to have + succeeded (_succeeded_or_recorded). Returns how many changed. + + The edges are locked in key order before the marks, waiting for a held + lock (_write_locked). When the call owns its transaction and loses a lock + conflict, the nodes behind the edges it could not mark go to their + workflows' skip queues. + """ + wanted = _satisfiable_edges_into(task_ids, db_alias) + if not wanted: + return 0 + marked: list[int] = [] + _, skipped = _write_locked( + OxpullWorkflowEdge, + wanted, + lambda pks: marked.append(_mark_satisfied(pks, db_alias)), + db_alias, + skip_locked=False, + ) + queue_skipped({wanted[pk] for pk in skipped}, db_alias) + return sum(marked) + + +# -- what a node's record says --------------------------------------------- + +# The statuses a node's record can hold: a task row's settled statuses. +RECORDED_STATUSES = frozenset( + (OxTask.Status.SUCCESSFUL, *DEAD_STATUSES), +) + + +def outcome_accepts_q(status: str, epoch: int) -> Q: + """ + The node records that a write of `status` at lease epoch `epoch` replaces. + + A record is (outcome, outcome_epoch). A write lands when the record has no + epoch, when its epoch is lower, or when its epoch is the same and either it + holds no outcome or it holds LOST and the write is not LOST. So a later + incarnation always wins, an earlier one never does, and at one epoch only + an observation of LOST gives way, to the outcome the same execution wrote. + A failure recorded at an epoch stays failed when a discard at that epoch + arrives, and a settled record at an epoch is never rewritten at it. + """ + if status not in RECORDED_STATUSES: + raise ValueError(f"{status!r} is not a settled status a node can record.") + same_epoch = Q(outcome__isnull=True) + if status != OxTask.Status.LOST: + same_epoch |= Q(outcome=OxTask.Status.LOST) + return ( + Q(outcome_epoch__isnull=True) + | Q(outcome_epoch__lt=epoch) + | (Q(outcome_epoch=epoch) & same_epoch) + ) + + +def record_takes( + outcome: str | None, outcome_epoch: int | None, status: str, epoch: int +) -> bool: + """outcome_accepts_q() for one record read into memory.""" + if outcome_epoch is None or outcome_epoch < epoch: + return True + if outcome_epoch > epoch: + return False + return outcome is None or ( + outcome == OxTask.Status.LOST and status != OxTask.Status.LOST + ) + + +def _record_many(records: dict[int, tuple[str, int]], db_alias: str) -> int: + """ + record_outcome() for many nodes, keyed by node pk. One guarded UPDATE per + chunk of nodes that share a status and an epoch. Returns how many took it. + """ + groups: dict[tuple[str, int], list[int]] = {} + for node_pk, (status, epoch) in records.items(): + groups.setdefault((status, int(epoch)), []).append(node_pk) + taken = 0 + for (status, epoch), node_pks in sorted(groups.items()): + for chunk in _id_chunks(node_pks, db_alias): + taken += ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(pk__in=chunk) + .filter(outcome_accepts_q(status, epoch)) + .update( + outcome=status, + outcome_epoch=epoch, + succeeded=status == OxTask.Status.SUCCESSFUL, + ) + ) + return taken + + +def record_outcome(node_pk: int, status: str, epoch: int, db_alias: str) -> bool: + """ + Write (status, epoch) onto one node's record if the record accepts it. True + when it did. + + `succeeded` is set by the same guarded UPDATE, so it changes only when the + record does. + """ + return bool( + OxpullWorkflowNode.objects.using(db_alias) + .filter(pk=node_pk) + .filter(outcome_accepts_q(status, epoch)) + .update( + outcome=status, + outcome_epoch=epoch, + succeeded=status == OxTask.Status.SUCCESSFUL, + ) + ) + + +def record_node_outcome( + task_id: Any, status: str, epoch: int, db_alias: str +) -> list[uuid.UUID] | None: + """ + Record a node's outcome and, on success, mark the edges out of it. Returns + the ids of the nodes those edges point at, or None when the task is not a + node. + + Runs inside the transaction that wrote the task row's outcome, straight + after that write landed, so the row, the record and the marks commit + together or not at all. It locks in the order every workflow operation + locks: the task row is already this transaction's, then the node row, + then the edges out of it in key order. At READ COMMITTED the node lock is + what serialises this against an add() that names this node as an upstream: + whichever commits first, the other sees its rows. At PostgreSQL REPEATABLE + READ the edge read answers from the snapshot this transaction's first write + opened. An edge committed after that snapshot is invisible here, so this + marks nothing for it and its downstream node waits for a backstop page. + + Nothing here waits in any way of its own. A lock wait, a deadlock or a + serialization failure raises, and the caller decides whether the whole + transaction runs again. + """ + locks = connections[db_alias].features.has_select_for_update + nodes = OxpullWorkflowNode.objects.using(db_alias).filter(task_id=task_id) + if locks: + nodes = nodes.select_for_update() + found = list(nodes.values_list("pk", flat=True)) + if not found: + return None + record_outcome(found[0], status, epoch, db_alias) + if status != OxTask.Status.SUCCESSFUL: + return [] + edges = ( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(upstream_task_id__probe=task_id, satisfied=False) + .order_by("pk") + ) + if locks: + edges = edges.select_for_update() + pairs = list(edges.values_list("pk", "downstream_task_id")) + _mark_satisfied([pk for pk, _ in pairs], db_alias) + return sorted({uuid.UUID(str(downstream)) for _, downstream in pairs}) + + +# The attribute a claimed task row carries when the claim computed whether it +# is a workflow node. +NODE_FLAG = "ox_wf_node" + + +def node_flag() -> Exists: + """Whether the task row is a workflow node, as an annotation for a claim.""" + return Exists(OxpullWorkflowNode.objects.filter(task_id=OuterRef("pk"))) + + +def node_flag_for_a_locking_read(vendor: str) -> Case | Exists: + """ + node_flag() for the SKIP LOCKED claim on `vendor`. On MySQL it is NULL + wherever the claim cannot tell. + + MySQL answers the subqueries of a locking read from a read view. The view + opens at the statement's first plain read, which can be the gate on a row + the scan passes before the one it claims. The locking read still returns a + row committed after the view opened, but that row's node is outside the + view. So a node that add() commits while the scan runs would read as no + node, and its outcome would be written without its record or its marks. + + Whether the claimed row itself is in the view settles it. add() commits a + task row and its node together, and nothing deletes the node of a READY + row. So where the row is in the view, its node is too, and the flag is + exact. Where the row is not, the flag is NULL, and the outcome's locking + read of the node decides. The row is looked for first, so the flag is never + read from an older view than the look. + + PostgreSQL reads the whole statement from one snapshot, and its scan + returns no row committed after that snapshot. There the flag is node_flag(). + """ + if vendor != "mysql": + return node_flag() + return Case( + When(Exists(OxTask.objects.filter(pk=OuterRef("pk"))), then=node_flag()), + default=Value(None), + output_field=BooleanField(null=True), + ) + + +def node_flag_sql() -> str: + """ + node_flag() as an extra column of the PostgreSQL claim's RETURNING list. + + The unqualified task table resolves to the row the UPDATE wrote. Names come + from the models' _meta, as in runnable_filter_sql(). + """ + task_table = OxTask._meta.db_table + node_table = OxpullWorkflowNode._meta.db_table + task_pk = OxTask._meta.pk.column + task_id = OxpullWorkflowNode._meta.get_field("task_id").column + # S608: every interpolated name comes from a model's own _meta, as in + # runnable_filter_sql(). + return ( + f', EXISTS (SELECT 1 FROM "{node_table}" ox_wf_n' # noqa: S608 + f' WHERE ox_wf_n."{task_id}" = "{task_table}"."{task_pk}")' + f' AS "{NODE_FLAG}"' + ) + + +def release_downstream(task_ids: Iterable[Any], db_alias: str) -> int: + """ + Move to READY every WAITING row among task_ids whose incoming edges are + all marked satisfied. Returns how many moved. + + A node leaves WAITING only when every edge into it is satisfied, which + no later write can undo short of a revival that also clears the marks. + So the plain reads that decide can only be stale in the safe direction. + They can miss a mark and leave a node for a later release. They cannot + release a node early. + + Each chunk is one transaction. It reads the edges into the chunk not yet + marked, then the WAITING rows among the rest with their epochs, then + moves those rows through django-ox's release_many, pinned to the epochs + it read. release_many locks the chunk's rows in key order before its + UPDATE, inside the same transaction. A row cancelled, revived or + released since the read matches nothing. Split in that order the two + reads are as safe as one: a node gains no incoming edge after add() + commits it, and nothing clears a mark on an edge whose downstream still + waits. One statement with NOT EXISTS plans on MySQL as a hash join that + scans the chunk's edges. + + When the call owns its transaction, a chunk waits at most + RELEASE_LOCK_TIMEOUT seconds for a lock, and a statement runs at most + RELEASE_STATEMENT_TIMEOUT seconds. MySQL applies the statement bound to + the reads only. A chunk that loses a lock conflict is decided again once, + from fresh reads. A lock conflict is a deadlock, a serialization failure, + a lock wait that timed out or a statement that ran past its bound, and + each rolls the chunk's transaction back whole. A chunk that loses twice + is logged, and its ids go to their workflows' skip queues, which the + reconciler retries first. The chunks after it still run. + + Inside a caller's transaction nothing is bounded and nothing is decided + again. Every error propagates, because the caller's work is lost with + it. + """ + ids = {uuid.UUID(str(task_id)) for task_id in task_ids} + if not ids: + return 0 + connection = connections[db_alias] + if not (connection.get_autocommit() and not connection.in_atomic_block): + return sum( + _release_chunk(chunk, db_alias) for chunk in _id_chunks(ids, db_alias) + ) + released = 0 + deferred: list[uuid.UUID] = [] + with session_timeouts( + db_alias, lock_wait=RELEASE_LOCK_TIMEOUT, statement=RELEASE_STATEMENT_TIMEOUT + ): + for chunk in _id_chunks(ids, db_alias): + for attempt in range(1, RELEASE_CHUNK_ATTEMPTS + 1): + try: + with transaction.atomic(using=db_alias): + transaction_timeouts( + db_alias, + lock_wait=RELEASE_LOCK_TIMEOUT, + statement=RELEASE_STATEMENT_TIMEOUT, + ) + moved = _release_chunk(chunk, db_alias) + except OperationalError as exc: + if not lost_lock_conflict(exc, db_alias): + raise + if attempt < RELEASE_CHUNK_ATTEMPTS: + time.sleep(random.uniform(0, ADD_RETRY_JITTER)) # noqa: S311 + continue + logger.warning( + "Could not release up to %d waiting workflow node task " + "row(s) after %d lock conflicts. The reconciler will retry " + "them", + len(chunk), + attempt, + exc_info=True, + extra={ + "event": "workflow_release_deferred", + "task_count": len(chunk), + }, + ) + deferred.extend(chunk) + else: + released += moved + break + if deferred: + queue_skipped(deferred, db_alias) + return released + + +# Tries of one release chunk that loses a lock conflict: the first, and one +# more from fresh reads. A chunk that loses both goes to the skip queue. +RELEASE_CHUNK_ATTEMPTS = 2 + +# Seconds one release chunk waits for a row lock, and runs one statement. +RELEASE_LOCK_TIMEOUT = 1.0 +RELEASE_STATEMENT_TIMEOUT = 2.0 + +# PostgreSQL's query_canceled, which a statement that runs past +# statement_timeout raises. +PG_QUERY_CANCELED = "57014" + +# MySQL's ER_QUERY_TIMEOUT, past max_execution_time, and MariaDB's +# ER_STATEMENT_TIMEOUT, past max_statement_time. +ER_QUERY_TIMEOUT = 3024 +ER_STATEMENT_TIMEOUT = 1969 + + +def _release_chunk(chunk: list[uuid.UUID], db_alias: str) -> int: + """One chunk of release_downstream, from its two reads to its release.""" + held = { + uuid.UUID(str(task_id)) + for task_id in _reading_index( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(downstream_task_id__probe_in=chunk, satisfied__aside=False) + .values_list("downstream_task_id", flat=True), + EDGE_DOWNSTREAM_INDEX, + ) + } + free = [task_id for task_id in chunk if task_id not in held] + if not free: + return 0 + pairs = list( + OxTask.objects.using(db_alias) + .filter(pk__in=free, status=OxTask.Status.WAITING) + .values_list("pk", "lease_epoch") + ) + if not pairs: + return 0 + changed, _ = _waiting.release_many(pairs, using=db_alias) + return changed + + +def lost_lock_conflict(exc: BaseException, db_alias: str) -> bool: + """ + Whether a transaction lost to another transaction's locks: a deadlock, a + serialization failure, a lock wait that timed out, or a statement that ran + past its time bound. + + Each of these rolls back the whole transaction, or leaves it to be rolled + back, and running the transaction again from the start is what every + engine's manual says to do. + """ + from .backend import ( + _is_lock_wait_timeout, + is_deadlock, + is_serialization_failure, + ) + + if not isinstance(exc, OperationalError): + return False + vendor = connections[db_alias].vendor + if ( + vendor == "postgresql" + and getattr(exc.__cause__, "sqlstate", None) == PG_QUERY_CANCELED + ): + return True + if ( + vendor == "mysql" + and exc.args + and exc.args[0] + in ( + ER_QUERY_TIMEOUT, + ER_STATEMENT_TIMEOUT, + ) + ): + return True + return ( + is_deadlock(exc, db_alias) + or is_serialization_failure(exc, db_alias) + or _is_lock_wait_timeout(exc, db_alias) + ) + + +@contextmanager +def session_timeouts( + db_alias: str, *, lock_wait: float, statement: float | None = None +) -> Iterator[None]: + """ + On MySQL, bound this session's lock waits, and its statements when + `statement` is given, for the block. Seconds. Other databases pass + through. + + MySQL has no bound scoped to one transaction, so the session's own values + are read first and put back when the block ends, however it ends. A + session whose values cannot be put back is closed, so that no later use + of the connection runs under bounds its owner never set, and that is + logged as workflow_session_bounds_not_restored. + + Putting them back never takes the place of an error leaving the block: + that error is raised, and a failure to put the values back beside it is + logged. Nothing is put back on a session that is gone, because its bounds + went with it: one Django has dropped, having opened another connection or + not, and one that no longer answers after the block's error was a lost + connection's, which is closed for its owner to reopen. + """ + connection = connections[db_alias] + if connection.vendor != "mysql": + yield + return + names = ["innodb_lock_wait_timeout"] + values: list[Any] = [max(1, math.ceil(lock_wait))] + if statement is not None: + if getattr(connection, "mysql_is_mariadb", False): + names.append("max_statement_time") + values.append(statement) + else: + names.append("max_execution_time") + values.append(max(1, math.ceil(statement * 1000))) + read = "SELECT " + ", ".join(f"@@SESSION.{name}" for name in names) + assign = "SET SESSION " + ", ".join(f"{name} = %s" for name in names) + with connection.cursor() as cursor: + cursor.execute(read) + saved = list(cursor.fetchone()) + cursor.execute(assign, values) + session = connection.connection + try: + yield + except BaseException as exc: + if connection.connection is not session: + raise + if _lost_with(connection, exc): + with suppress(Error): + connection.close() + raise + try: + _put_back_session_bounds(connection, assign, saved) + except Exception: + _session_bounds_not_restored(connection) + raise + if connection.connection is not session: + return + try: + _put_back_session_bounds(connection, assign, saved) + except BaseException as exc: + _session_bounds_not_restored(connection) + if not isinstance(exc, DatabaseError): + raise + + +def _lost_with(connection: Any, exc: BaseException) -> bool: + """Whether `exc` is a lost connection's error and `connection` no longer answers.""" + if not isinstance(exc, (InterfaceError, OperationalError)): + return False + try: + return not connection.is_usable() + except Exception: + return True + + +def _put_back_session_bounds(connection: Any, assign: str, saved: list[Any]) -> None: + with connection.cursor() as cursor: + cursor.execute(assign, saved) + + +def _session_bounds_not_restored(connection: Any) -> None: + """Close a session whose bounds could not be put back, and say so.""" + with suppress(Error): + connection.close() + logger.warning( + "Could not put back the session's lock wait and statement " + "bounds. The connection was closed", + exc_info=True, + extra={"event": "workflow_session_bounds_not_restored"}, + ) + + +def transaction_timeouts( + db_alias: str, *, lock_wait: float, statement: float | None = None +) -> None: + """ + On PostgreSQL, bound lock waits, and statements when `statement` is given, + for the rest of the current transaction. Seconds. Other databases pass + through. + """ + connection = connections[db_alias] + if connection.vendor != "postgresql": + return + bounds = [("lock_timeout", lock_wait)] + if statement is not None: + bounds.append(("statement_timeout", statement)) + params: list[str] = [] + for name, seconds in bounds: + params.extend((name, f"{max(1, round(seconds * 1000))}ms")) + sql = "SELECT " + ", ".join("set_config(%s, %s, true)" for _ in bounds) + with connection.cursor() as cursor: + cursor.execute(sql, params) + + +# -- the skip queue ------------------------------------------------------------ + +# Ids one workflow's skip queue holds. Ids past it are dropped with a log line. +# Nothing is lost by that: the nodes stay WAITING, and the reconciler's pass +# over waiting nodes still finds them. +SKIP_QUEUE_LIMIT = 100 + +# Tries of the compare-and-set that adds ids to one workflow's queue. +SKIP_QUEUE_ATTEMPTS = 5 + + +def queue_skipped(task_ids: Iterable[Any], db_alias: str) -> None: + """ + Put node task ids a release could not move into their workflows' skip + queues, OxpullWorkflowScan.skipped. Each id is kept once, oldest first, + up to SKIP_QUEUE_LIMIT per workflow. + + An id that is not a node is dropped. A database error is logged and not + raised. The release that called this has already given up on those rows, + and the reconciler's pass over waiting nodes still finds them. + """ + ids = sorted({uuid.UUID(str(task_id)) for task_id in task_ids}) + try: + by_workflow: dict[uuid.UUID, list[str]] = {} + for chunk in _id_chunks(ids, db_alias): + for workflow_id, task_id in ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(task_id__in=chunk) + .values_list("workflow_id", "task_id") + ): + by_workflow.setdefault(workflow_id, []).append(str(task_id)) + for workflow_id in sorted(by_workflow): + _add_to_skip_queue(workflow_id, sorted(by_workflow[workflow_id]), db_alias) + except DatabaseError: + logger.warning( + "Could not queue %d waiting workflow node(s) for the reconciler to " + "retry. Its pass over waiting nodes still finds them", + len(ids), + exc_info=True, + extra={"event": "workflow_release_skip_unqueued", "task_count": len(ids)}, + ) + + +def _add_to_skip_queue(workflow_id: uuid.UUID, ids: list[str], db_alias: str) -> None: + """Add ids to one workflow's skip queue by compare-and-set on its version.""" + scans = OxpullWorkflowScan.objects.using(db_alias) + for _ in range(SKIP_QUEUE_ATTEMPTS): + row = ( + scans.filter(workflow_id=workflow_id) + .values_list("skipped", "version") + .first() + ) + queued: list[str] = list(row[0] or []) if row is not None else [] + merged = list(dict.fromkeys([*queued, *ids])) + kept, dropped = merged[:SKIP_QUEUE_LIMIT], merged[SKIP_QUEUE_LIMIT:] + if row is None: + try: + with transaction.atomic(using=db_alias): + scans.create(workflow_id=workflow_id, skipped=kept) + except IntegrityError: + continue + elif not scans.filter(workflow_id=workflow_id, version=row[1]).update( + skipped=kept, version=row[1] + 1 + ): + continue + if dropped: + logger.warning( + "Workflow %s: its skip queue is full, and %d waiting node(s) were " + "not queued. The reconciler's pass over waiting nodes still finds " + "them", + workflow_id, + len(dropped), + extra={ + "event": "workflow_release_skip_overflow", + "workflow_id": str(workflow_id), + "dropped_count": len(dropped), + }, + ) + return + logger.warning( + "Workflow %s: could not queue %d waiting node(s) for the reconciler to " + "retry. Its pass over waiting nodes still finds them", + workflow_id, + len(ids), + extra={ + "event": "workflow_release_skip_unqueued", + "workflow_id": str(workflow_id), + "task_count": len(ids), + }, + ) + + +def _take_skipped(db_alias: str) -> list[str]: + """ + Take up to one chunk of ids out of the workflows' skip queues, by + compare-and-set, starting after the workflow the round last visited. A + queue that moved under the compare-and-set keeps its ids for a later tick. + """ + scans = OxpullWorkflowScan.objects.using(db_alias) + queues = list( + scans.order_by("workflow_id").values_list("workflow_id", "skipped", "version") + ) + after = ( + OxpullWorkflowRound.objects.using(db_alias) + .filter(pk=ROUND_ROW) + .values_list("workflow_id", flat=True) + .first() + ) + if after is not None: + start = bisect.bisect_right([uuid.UUID(str(q[0])) for q in queues], after) + queues = queues[start:] + queues[:start] + taken: list[str] = [] + for workflow_id, queued, version in queues: + room = ID_CHUNK_SIZE - len(taken) + if room <= 0: + break + if not queued: + continue + take, rest = list(queued[:room]), list(queued[room:]) + if scans.filter(workflow_id=workflow_id, version=version).update( + skipped=rest, version=version + 1 + ): + taken.extend(take) + return taken + + +def _revisit(task_ids: Iterable[Any], db_alias: str) -> int: + """ + Everything the backstop does for a node, for nodes whose rows it could not + write when it paged past them. Returns how many were released. + + Marks the edges into them whose upstream is known to have succeeded, + repairs their records, and releases the ones left waiting with every edge + satisfied. These writes wait for a held lock, up to BACKSTOP_LOCK_TIMEOUT, + rather than skipping it. A write that still loses puts the ids back in the + queue, so a row is tried again on every tick until its lock is released. + """ + ids = sorted({uuid.UUID(str(task_id)) for task_id in task_ids}) + wanted = _satisfiable_edges_into(ids, db_alias) + if wanted: + _, skipped = _write_locked( + OxpullWorkflowEdge, + wanted, + lambda pks: _mark_satisfied(pks, db_alias), + db_alias, + skip_locked=False, + ) + queue_skipped({wanted[pk] for pk in skipped}, db_alias) + nodes: list[tuple[int, Any, str | None, int | None]] = [] + for chunk in _id_chunks(ids, db_alias): + nodes.extend( + OxpullWorkflowNode.objects.using(db_alias) + .filter(task_id__in=chunk) + .values_list("pk", "task_id", "outcome", "outcome_epoch") + ) + _repair_node_records(nodes, db_alias, skip_locked=False) + return release_downstream(ids, db_alias) + + +def _write_locked( + model: type[Model], + pks: Iterable[int], + write: Callable[[list[int]], object], + db_alias: str, + *, + skip_locked: bool = True, +) -> tuple[list[int], list[int]]: + """ + Lock rows of `model` by primary key in key order and run `write` over the + ones this transaction locked, in one transaction. Returns (written, + skipped). + + With skip_locked, a row another transaction holds is skipped at once. + Without it, the lock is waited for. When the call owns its transaction, a + lock wait is bounded by BACKSTOP_LOCK_TIMEOUT and a statement by + BACKSTOP_STATEMENT_TIMEOUT, and a lost lock conflict or a timeout skips + every row of the call. Inside a caller's transaction nothing is bounded and + every error propagates. + """ + wanted = sorted(set(pks)) + connection = connections[db_alias] + owned = connection.get_autocommit() and not connection.in_atomic_block + bounds: AbstractContextManager[None] = ( + session_timeouts( + db_alias, + lock_wait=BACKSTOP_LOCK_TIMEOUT, + statement=BACKSTOP_STATEMENT_TIMEOUT, + ) + if owned + else nullcontext() + ) + locked = wanted + try: + with bounds, transaction.atomic(using=db_alias): + if owned: + transaction_timeouts( + db_alias, + lock_wait=BACKSTOP_LOCK_TIMEOUT, + statement=BACKSTOP_STATEMENT_TIMEOUT, + ) + locked = _lock_in_key_order( + model, wanted, db_alias, skip_locked=skip_locked + ) + if locked: + write(locked) + except OperationalError as exc: + if not (owned and lost_lock_conflict(exc, db_alias)): + raise + logger.info( + "The workflow backstop could not lock %d %s row(s) and will try them again", + len(wanted), + model._meta.model_name, + exc_info=True, + extra={ + "event": "workflow_backstop_skipped", + "row_count": len(wanted), + "table": model._meta.db_table, + }, + ) + return [], wanted + written = set(locked) + return sorted(written), [pk for pk in wanted if pk not in written] + + +def _lock_in_key_order( + model: type[Model], pks: Iterable[int], db_alias: str, *, skip_locked: bool = False +) -> list[int]: + """ + Lock rows of `model` by primary key, in key order, inside the current + transaction. Returns the keys this transaction locked, in key order. + + Almost every multi-row lock on a workflow's node or edge rows is taken this + way before the rows are written. Workflow._resolve_dependencies is the + exception: it finds its upstreams by (workflow_id, name_key) and only sorts + by primary key, so MySQL locks those rows in unique-index order. It can + deadlock there with _repair_node_records, which takes the same rows by key + and waits. Both retry their whole call, so no work is lost. + + An UPDATE locks rows in the order its plan reads them, which on PostgreSQL + can be table order, so two writers of the same rows could otherwise lock + them in opposite orders. PostgreSQL sorts before it locks. MySQL locks as it + reads the primary key range, which is key order. With skip_locked, a row + another transaction holds is left out. A database without row locks returns + every key and locks nothing. + """ + wanted = sorted(set(pks)) + connection = connections[db_alias] + if not connection.features.has_select_for_update: + return wanted + skip = skip_locked and connection.features.has_select_for_update_skip_locked + locked: list[int] = [] + for chunk in _id_chunks(wanted, db_alias): + locked.extend( + model._default_manager.using(db_alias) + .filter(pk__in=chunk) + .order_by("pk") + .select_for_update(skip_locked=skip) + .values_list("pk", flat=True) + ) + return locked + + +def _repair_node_records( + nodes: Iterable[tuple[int, Any, str | None, int | None]], + db_alias: str, + *, + skip_locked: bool = True, +) -> None: + """ + Record on each node what its task row says now, where the transition table + takes it. `nodes` are (node pk, task id, outcome, outcome_epoch). + + The rows are read by primary key with a plain read, and each record is + written by the guarded UPDATE record_outcome() uses, so a stale reading + lands nowhere a later one already did. A node row another transaction + holds is skipped and its task id queued. + """ + nodes = list(nodes) + rows: dict[uuid.UUID, tuple[str, int]] = {} + for chunk in _id_chunks([task_id for _, task_id, _, _ in nodes], db_alias): + rows.update( + (uuid.UUID(str(pk)), (status, epoch)) + for pk, status, epoch in OxTask.objects.using(db_alias) + .filter(pk__in=chunk) + .values_list("pk", "status", "lease_epoch") + ) + wanted: dict[int, tuple[str, int, uuid.UUID]] = {} + for node_pk, task_id, outcome, outcome_epoch in nodes: + key = uuid.UUID(str(task_id)) + row = rows.get(key) + if row is None or row[0] not in RECORDED_STATUSES: + continue + if record_takes(outcome, outcome_epoch, row[0], row[1]): + wanted[node_pk] = (row[0], row[1], key) + if not wanted: + return + _, skipped = _write_locked( + OxpullWorkflowNode, + wanted, + lambda pks: _record_many( + {pk: (wanted[pk][0], wanted[pk][1]) for pk in pks}, db_alias + ), + db_alias, + skip_locked=skip_locked, + ) + queue_skipped({wanted[pk][2] for pk in skipped}, db_alias) + + +# The one row that holds the backstop's round-robin position. +ROUND_ROW = 1 + + +# Where a pass stands: (phase, position, edge_hwm, node_hwm). +Place = tuple[str, int, int | None, int | None] + +# The place of a workflow with no pass under way, and of a scan row that holds +# only a skip queue. +AT_REST: Place = (OxpullWorkflowScan.Phase.EDGES, 0, None, None) + + +@dataclass +class _Scan: + """One workflow's place in its backstop pass, as a visit carries it.""" + + phase: str + position: int + edge_hwm: int | None + node_hwm: int | None + # The place this visit last read or stored, which its next store must + # still find. AT_REST when no row was found. + stored: Place + + @property + def place(self) -> Place: + return (self.phase, self.position, self.edge_hwm, self.node_hwm) + + +def _release_backstop(db_alias: str) -> int: + """ + The reconciler's backstop: satisfy, record and release what no worker did. + Returns how many nodes were released. + + First the skip queues, up to one chunk of ids a tick, retried by _revisit. + The budget below does not count this. However long it takes, the round + below still runs at least one page. + + Then a round over the workflows that are OPEN or SEALED, in id order, + starting after the one the last tick stopped at, and wrapping. Each + workflow gets a visit (_visit). The deadline is BACKSTOP_BUDGET after the + round starts, or a quarter of what the task timeout leaves if that is + sooner. The round starts no visit after the deadline, and a visit starts no + page after it, but the first visit and its first page always run. A tick + that stops part-way stores the last workflow it visited, by compare-and-set + on OxpullWorkflowRound, so the next tick goes on from there and every + workflow gets a visit each round. + + What the budget does not bound. A page that starts before the deadline runs + to its end. Its reads carry no time bound. Each of its writes waits at most + BACKSTOP_LOCK_TIMEOUT for a lock and runs each statement for at most + BACKSTOP_STATEMENT_TIMEOUT, and each release chunk it runs may try twice + with RELEASE_LOCK_TIMEOUT and RELEASE_STATEMENT_TIMEOUT. MySQL applies the + statement bound to SELECTs only. So a tick can run past its deadline by a + page's work, and the skip queue's retries come on top. + + Task rows are read by primary key only, with plain reads. The writes lock + only oxpull's own edge and node rows, and release_downstream moves task + rows through django-ox's pinned helper after each write commits. + + How long a row can wait, if the locks on it are eventually released. A pass + over workflow w takes at most ceil(E / BACKSTOP_PAGE) + 2 * ceil(N / + BACKSTOP_PAGE) + 3 pages, for its E open edges and N open nodes. Each round + gives w one visit of at least one page. A row open when a pass begins is + read in that pass, and a row added after it began is read in the next one, + so any open row is read within two passes of w, which is at most twice + that many rounds. A row that was locked is queued and tried again on every + tick, and every later pass reads it again, so no row is ever given up on. + """ + released = _revisit_skipped(db_alias) + return released + _backstop_round(db_alias, time.monotonic() + _backstop_budget()) + + +def _revisit_skipped(db_alias: str) -> int: + """Retry up to one chunk of queued ids (_take_skipped, _revisit).""" + taken = _take_skipped(db_alias) + return _revisit(taken, db_alias) if taken else 0 + + +def _backstop_budget() -> float: + """Seconds this tick may spend paging: BACKSTOP_BUDGET, or less under a timeout.""" + left = remaining() + if left is None: + return BACKSTOP_BUDGET + return max(0.0, min(BACKSTOP_BUDGET, left / 4)) + + +def _backstop_round(db_alias: str, deadline: float) -> int: + """One tick's share of the round-robin over open workflows (_release_backstop).""" + live = [ + uuid.UUID(str(pk)) + for pk in OxpullWorkflow.objects.using(db_alias) + .filter(status__in=(OxpullWorkflow.Status.OPEN, OxpullWorkflow.Status.SEALED)) + .order_by("pk") + .values_list("pk", flat=True) + ] + if not live: + return 0 + rounds = OxpullWorkflowRound.objects.using(db_alias) + stored = rounds.filter(pk=ROUND_ROW).values_list("workflow_id", "version").first() + after, version = stored if stored is not None else (None, None) + start = bisect.bisect_right(live, after) if after is not None else 0 + visiting = live[start:] + live[:start] + marks = ( + _highest_pk(OxpullWorkflowEdge, db_alias), + _highest_pk(OxpullWorkflowNode, db_alias), + ) + released = 0 + visited = 0 + for workflow_id in visiting: + if visited and time.monotonic() >= deadline: + break + released += _visit(workflow_id, marks, deadline, db_alias) + visited += 1 + if visited < len(visiting): + _move_round(visiting[visited - 1], version, db_alias) + return released + + +def _highest_pk(model: type[Model], db_alias: str) -> int: + top = model._default_manager.using(db_alias).aggregate(top=Max("pk"))["top"] + return int(top or 0) + + +def _move_round(workflow_id: uuid.UUID, version: int | None, db_alias: str) -> None: + """Store where the round stopped, by compare-and-set. A loser leaves it.""" + rounds = OxpullWorkflowRound.objects.using(db_alias) + if version is None: + with suppress(IntegrityError), transaction.atomic(using=db_alias): + rounds.create(pk=ROUND_ROW, workflow_id=workflow_id, version=1) + return + rounds.filter(pk=ROUND_ROW, version=version).update( + workflow_id=workflow_id, version=version + 1 + ) + + +def _visit( + workflow_id: uuid.UUID, marks: tuple[int, int], deadline: float, db_alias: str +) -> int: + """ + One backstop visit to one workflow. Returns how many nodes it released. + + A pass over a workflow has three phases, each paged in key order up to the + high-water marks the pass recorded when it began: the edges not marked + satisfied (_edge_page), then the nodes not recorded as succeeded + (_node_page), then the same nodes again for release (_release_page). A + visit runs pages from where the workflow's pass stands. It pages on while + its pages come back full, up to VISIT_PAGES, and a short page ends its + phase. It stops when the pass ends, or once the tick's deadline has passed + after its first page. + + The place is stored on OxpullWorkflowScan by compare-and-set on the place + this visit read, after every full page and when the visit stops part-way, + and a visit that loses the compare-and-set stops. A pass that ends puts a + stored place back at rest. A pass whose pages all come back short, and + which begins and ends inside one visit, stores nothing. A page always moves + the position past every row it read, whether it could write them or not: a + row it could not lock goes to the skip queue, which the same row holds + beside the place. + """ + stored = ( + OxpullWorkflowScan.objects.using(db_alias) + .filter(workflow_id=workflow_id) + .values_list("phase", "position", "edge_hwm", "node_hwm") + .first() + ) + place: Place = AT_REST if stored is None else cast("Place", tuple(stored)) + scan = _Scan(*place, stored=place) + if scan.edge_hwm is None or scan.node_hwm is None: + scan.phase, scan.position = OxpullWorkflowScan.Phase.EDGES, 0 + scan.edge_hwm, scan.node_hwm = marks + released = 0 + pages = 0 + while pages < VISIT_PAGES and not (pages and time.monotonic() >= deadline): + pages += 1 + full, last, moved = _PAGES[scan.phase](workflow_id, scan, db_alias) + released += moved + if full: + scan.position = last + if not _store_scan(workflow_id, scan, db_alias): + return released + continue + following = _FOLLOWING_PHASE.get(scan.phase) + if following is None: + if scan.stored != AT_REST: + scan.phase, scan.position, scan.edge_hwm, scan.node_hwm = AT_REST + _store_scan(workflow_id, scan, db_alias) + return released + scan.phase, scan.position = following, 0 + _store_scan(workflow_id, scan, db_alias) + return released + + +def _store_scan(workflow_id: uuid.UUID, scan: _Scan, db_alias: str) -> bool: + """ + Store a visit's place by compare-and-set on the place it last saw. False + when another visit moved it. + + The skip queue shares the row and moves only its own column and the + version, so a visit's own queued ids do not make it lose its place. + """ + scans = OxpullWorkflowScan.objects.using(db_alias) + if scan.place == scan.stored: + return True + phase, position, edge_hwm, node_hwm = scan.stored + fields = { + "phase": scan.phase, + "position": scan.position, + "edge_hwm": scan.edge_hwm, + "node_hwm": scan.node_hwm, + } + moved = scans.filter( + workflow_id=workflow_id, + phase=phase, + position=position, + edge_hwm=edge_hwm, + node_hwm=node_hwm, + ).update(version=F("version") + 1, **fields) + if not moved: + if scan.stored != AT_REST: + return False + try: + with transaction.atomic(using=db_alias): + scans.create(workflow_id=workflow_id, **fields) + except IntegrityError: + return False + scan.stored = scan.place + return True + + +def _edge_page( + workflow_id: uuid.UUID, scan: _Scan, db_alias: str +) -> tuple[bool, int, int]: + """ + Mark satisfied the edges on this page whose upstream is known to have + succeeded, then release what they held. Returns (full, last key, released). + + The page reads the workflow's unmarked edges on oxpull_wf_edge_open_idx. + The marks commit before the release runs. + """ + rows = list( + OxpullWorkflowEdge.objects.using(db_alias) + .filter( + workflow_id=workflow_id, + satisfied=False, + pk__gt=scan.position, + pk__lte=scan.edge_hwm, + ) + .order_by("pk") + .values_list("pk", "upstream_task_id", "downstream_task_id")[:BACKSTOP_PAGE] + ) + if not rows: + return False, scan.position, 0 + proven = _succeeded_or_recorded({up for _, up, _ in rows}, db_alias) + wanted = { + pk: uuid.UUID(str(down)) + for pk, up, down in rows + if uuid.UUID(str(up)) in proven + } + released = 0 + if wanted: + marked, skipped = _write_locked( + OxpullWorkflowEdge, + wanted, + lambda pks: _mark_satisfied(pks, db_alias), + db_alias, + ) + queue_skipped({wanted[pk] for pk in skipped}, db_alias) + downstream = {wanted[pk] for pk in marked} + if downstream: + released = release_downstream(downstream, db_alias) + return len(rows) == BACKSTOP_PAGE, rows[-1][0], released + + +def _open_node_page( + workflow_id: uuid.UUID, scan: _Scan, db_alias: str +) -> list[tuple[int, Any, str | None, int | None]]: + """The page of nodes not recorded as succeeded, on oxpull_wf_node_open_idx.""" + return list( + OxpullWorkflowNode.objects.using(db_alias) + .filter( + workflow_id=workflow_id, + succeeded=False, + pk__gt=scan.position, + pk__lte=scan.node_hwm, + ) + .order_by("pk") + .values_list("pk", "task_id", "outcome", "outcome_epoch")[:BACKSTOP_PAGE] + ) + + +def _node_page( + workflow_id: uuid.UUID, scan: _Scan, db_alias: str +) -> tuple[bool, int, int]: + """Repair the records on this page of open nodes. Returns (full, last key, 0).""" + rows = _open_node_page(workflow_id, scan, db_alias) + if not rows: + return False, scan.position, 0 + _repair_node_records(rows, db_alias) + return len(rows) == BACKSTOP_PAGE, rows[-1][0], 0 + + +def _release_page( + workflow_id: uuid.UUID, scan: _Scan, db_alias: str +) -> tuple[bool, int, int]: + """ + Release the waiting nodes on this page of open nodes whose edges are all + satisfied. Returns (full, last key, released). + """ + rows = _open_node_page(workflow_id, scan, db_alias) + if not rows: + return False, scan.position, 0 + waiting: list[uuid.UUID] = [] + for chunk in _id_chunks([task_id for _, task_id, _, _ in rows], db_alias): + waiting.extend( + uuid.UUID(str(pk)) + for pk in OxTask.objects.using(db_alias) + .filter(pk__in=chunk, status=OxTask.Status.WAITING) + .values_list("pk", flat=True) + ) + released = release_downstream(waiting, db_alias) if waiting else 0 + return len(rows) == BACKSTOP_PAGE, rows[-1][0], released + + +_PAGES: dict[str, Callable[[uuid.UUID, _Scan, str], tuple[bool, int, int]]] = { + OxpullWorkflowScan.Phase.EDGES: _edge_page, + OxpullWorkflowScan.Phase.NODES: _node_page, + OxpullWorkflowScan.Phase.RELEASE: _release_page, +} + +_FOLLOWING_PHASE: dict[str, str] = { + OxpullWorkflowScan.Phase.EDGES: OxpullWorkflowScan.Phase.NODES, + OxpullWorkflowScan.Phase.NODES: OxpullWorkflowScan.Phase.RELEASE, +} + + +def _require_workflow_backend(task: Task[..., Any]) -> None: + """ + Refuse a node whose backend would not run it under the gate. + + Two ways that happens, and both are silent without this. A backend + that does not declare WORKFLOWS emits the ungated claim, so every + node is claimed as soon as it is enqueued, in no particular order. A + backend that declares it but names a WORKER_CLASS that is not an + OxpullWorker does the same. The second is also oxpull.E007, a + start-up error; this is the line that catches the first, and it + catches both for anyone who reaches add() anyway. + + Ordering is the whole product here, so it fails on the add() call + rather than three hours later, out of order, with a healthy-looking + queue. + """ + from django.tasks import task_backends + + from django_ox.worker import worker_class + + from .worker import OxpullWorker + + alias = task.backend + if not workflows_enabled_from_options(task_backends[alias].options): + raise ImproperlyConfigured( + f"Backend {alias!r} does not declare OPTIONS['WORKFLOWS'] = True, " + "so its claim carries no dependency gate: every node of this " + "workflow would be claimed as soon as it was enqueued, in no " + "particular order." + ) + runs = worker_class(alias) + if not issubclass(runs, OxpullWorker): + raise ImproperlyConfigured( + f"Workflow nodes on backend {alias!r} would be run by " + f"{runs.__module__}.{runs.__qualname__}, which does not apply " + "dependency gating: every node would be claimed as soon as it " + "was enqueued, in no particular order. Set " + "OPTIONS['WORKER_CLASS'] to 'oxpull.worker.OxpullWorker'." + ) + + +def _refuse_unique_node(task: Task[..., Any]) -> None: + """ + Refuse a unique task as a node, before anything is enqueued. + + A unique enqueue returns whichever task already holds the key, so the + node's row could be another workflow's node or a call the application + enqueued on its own, and a later enqueue of the same call would return + it too. Every holder's edges would gate that one row and every + holder's reconciler could discard it: one workflow's failure cancels + another's node, the application's call is discarded without running, + a row already running becomes a node ahead of its upstream, and two + workflows that add the same calls in opposite orders wait on each + other for good. Refusing only when the key is already held would not + do, because the application's enqueue can come after the node. + + Decided by the backend the task enqueues through, with the check its + enqueue() makes, so a task listed in UNIQUE_TASKS on another backend + is an ordinary node here. A refused add() calls no enqueue(), sends no + task_enqueued and writes no row. + """ + from .backend import OxpullBackend + + backend = task.get_backend() + if isinstance(backend, OxpullBackend) and backend._is_unique(task): + raise ImproperlyConfigured( + f"Workflow node {task.module_path!r} is a unique task on backend " + f"{task.backend!r}, and a unique task cannot be a workflow node. " + "Its enqueue can return a task that another workflow or the " + "application already holds, and this workflow's failures would " + "cancel that task. A shared expensive job belongs outside the " + "workflow as a unique task; add a task that is not unique as the " + "node." + ) + + +def declaring_aliases() -> list[str]: + """The OxpullBackend aliases whose OPTIONS declare WORKFLOWS = True, sorted.""" + from .backend import OxpullBackend + + found = [] + for alias, backend_class in task_backend_classes().items(): + if not issubclass(backend_class, OxpullBackend): + continue + options = settings.TASKS[alias].get("OPTIONS") + if isinstance(options, dict) and options.get("WORKFLOWS") is True: + found.append(alias) + return sorted(found) + + +def workflows_on_database() -> bool: + """ + Whether any backend declares WORKFLOWS on the database the task rows live + on. + + django-ox writes every backend's task rows to one database, the one + oxpull_db_alias() resolves, so one declaring backend puts workflow nodes + on it. Any OxpullWorker there may claim a node, because claims are scoped + to queues and not to backends, so every one of them records a node's + outcome and releases what it held. + """ + return bool(declaring_aliases()) + + +def reconcile_schedule_hint(schedule: ReconcilerSchedule | None = None) -> str: + """ + What to do about a missing workflow reconciler schedule. + + A project that reads schedule rows is told to expose the reconciler to + them, when a backend reading them can run it; batches' + reconcile_schedule_hint says why. Otherwise it gets one SCHEDULES + entry and the backend to add it to: a backend declaring WORKFLOWS + first, then any django-ox backend, and only one the reconciler binds + to. The warning is printed once and names one backend, so following it + cannot define one schedule name on two backends (django_ox.E003) or put + the reconciler on a queue its backend refuses (django_ox.E002). + """ + if schedule is None: + schedule = reconcile_schedule(RECONCILE_TASK_PATH) + if schedule.row_readers and not schedule.exposed: + try: + public = _build_task_on_a_queue(_reconcile) + except Exception: + public = None + if public is not None and any( + _binds(public, alias) for alias in schedule.row_readers + ): + return RECONCILE_SCHEDULABLE_HINT + declaring = declaring_aliases() + others = sorted(alias for alias in _ox_backend_options() if alias not in declaring) + return schedule_entry_hint( + RECONCILE_TASK_PATH, RECONCILE_SCHEDULE_NAME, [*declaring, *others] + ) + + +def _require_reconcile_schedule() -> None: + """ + Refuse to create or seal a workflow that nothing would ever complete. + + The same refusal batches make, for the same reasons: completion and + the cancellation of a failed node's dependents happen only in the + reconciler, only a django-ox schedule runs it, and oxpull.W003 can be + silenced or never read. + """ + schedule = reconcile_schedule(RECONCILE_TASK_PATH) + if schedule.aliases: + return + if schedule.unbound: + raise ImproperlyConfigured( + f"Workflows need a schedule for {RECONCILE_TASK_PATH} that a worker " + "can run. " + + unbound_reconciler_message( + RECONCILE_TASK_PATH, schedule, UNSCHEDULED_CONSEQUENCE + ) + + " " + + unbound_reconciler_hint( + RECONCILE_TASK_PATH, RECONCILE_SCHEDULE_NAME, schedule + ) + + " The system check reports the same condition as oxpull.E008." + ) + raise ImproperlyConfigured( + f"Workflows need a schedule for {RECONCILE_TASK_PATH}, and no django-ox " + f"backend in settings.TASKS declares one, so {UNSCHEDULED_CONSEQUENCE}. " + f"{reconcile_schedule_hint(schedule)} The system check reports the same " + "condition as oxpull.W003." + ) + + +def create( + name: str, + *, + on_complete: Task[..., Any] | str | None = None, + on_complete_args: list[Any] | tuple[Any, ...] | None = None, + on_complete_kwargs: dict[str, Any] | None = None, + on_success: Task[..., Any] | str | None = None, + on_success_args: list[Any] | tuple[Any, ...] | None = None, + on_success_kwargs: dict[str, Any] | None = None, +) -> Workflow: + """ + Create an OPEN workflow. + + on_complete fires once every node has finished, whatever the outcome, + cancellations included. on_success fires in addition, and only when + every node's outcome is known and none of them FAILED, so a + workflow with a cancelled branch completes without it. Each is a Task + or its dotted path, with optional arguments; both are enqueued as + ordinary tasks and are not supervised beyond storing their ids. + + Raises ImproperlyConfigured if no django-ox backend schedules the + reconciler, as batches.create() does: a workflow nothing completes + should fail on the line that creates it. + """ + _require_reconcile_schedule() + if len(name) > NAME_MAX_LENGTH: + # SQLite would store it, MySQL and PostgreSQL would not; reject + # it identically everywhere. + raise ValueError(f"Workflow name exceeds {NAME_MAX_LENGTH} characters.") + complete_path, complete_args, complete_kwargs = _resolve_callback( + on_complete, on_complete_args, on_complete_kwargs, label="Workflow" + ) + success_path, success_args, success_kwargs = _resolve_callback( + on_success, on_success_args, on_success_kwargs, label="Workflow" + ) + db_alias = oxpull_db_alias() + row = OxpullWorkflow.objects.using(db_alias).create( + name=name, + status=OxpullWorkflow.Status.OPEN, + on_complete_path=complete_path, + on_complete_args=complete_args, + on_complete_kwargs=complete_kwargs, + on_success_path=success_path, + on_success_args=success_args, + on_success_kwargs=success_kwargs, + created_at=timezone.now(), + ) + return Workflow(row) + + +def get(workflow_id: uuid.UUID | str) -> Workflow: + """ + The workflow with this id. Raises OxpullWorkflow.DoesNotExist if + there is none. + """ + db_alias = oxpull_db_alias() + return Workflow(OxpullWorkflow.objects.using(db_alias).get(pk=workflow_id)) + + +# -- the reconciler ---------------------------------------------------- + + +def _reconcile() -> int: + """ + Satisfy, record and release what no worker did, cancel what can no longer + run, complete every SEALED workflow whose nodes have all finished, and + return how many were completed. + + Run it through django-ox's schedule dispatcher once a minute. Workflow + completion and callback enqueue share a transaction, so a crash leaves + either an incomplete workflow or a completed workflow with its callbacks + enqueued. A later pass can revisit an incomplete workflow. The release + backstop (_release_backstop) resumes from its recorded position. + + Cancellation is not on the critical path of anything that succeeds. + The worker that writes an upstream's SUCCESSFUL releases what it held. + The backstop here is for what did not happen. A backstop that fails is + logged, and the tick still cancels and completes workflows, because + neither of those waits on a release. + + A tick under a task timeout visits sealed workflows in a new order, + stops between workflows when the next would probably not finish in + time, and lets a TaskTimeout out when one lands anyway, so django-ox + records that attempt as timed out (see batches._sweep). + """ + db_alias = oxpull_db_alias() + try: + released = _release_backstop(db_alias) + except Exception as exc: + if _timed_out(exc): + raise + released = 0 + logger.exception( + "The workflow release backstop failed. This tick still cancels and " + "completes workflows", + extra={"event": "workflow_release_backstop_failed"}, + ) + if released: + logger.info( + "Released %d waiting workflow node(s) no worker released", + released, + extra={"event": "workflow_nodes_released", "released_count": released}, + ) + sealed = ( + OxpullWorkflow.objects.using(db_alias) + .filter(status=OxpullWorkflow.Status.SEALED) + .order_by() + .values_list("pk", flat=True) + ) + completed = 0 + for workflow_pk in _sweep(list(sealed), "workflow", "workflows", logger): + try: + _cancel_unreachable(workflow_pk, db_alias) + if _complete_if_finished(workflow_pk, db_alias): + completed += 1 + except Exception as exc: + # The tick ran out of time; this workflow did not fail. + if _timed_out(exc): + raise + logger.exception( + "Workflow %s could not be reconciled", + workflow_pk, + extra={ + "event": "workflow_reconcile_failed", + "workflow_id": str(workflow_pk), + }, + ) + return completed + + +def _cancel_unreachable(workflow_pk: uuid.UUID, db_alias: str) -> int: + """ + Discard every READY or WAITING node of this workflow that depends, + directly or transitively, on a node that settled without succeeding. + + Without this those rows wait forever: the workflow never completes and + ox_prune never reaches them, because it prunes only settled rows. + + Only READY and WAITING nodes are cancelled. A node that already ran + holds its own outcome, and a RUNNING node belongs to the worker holding + its lease. + + It first asks, with a plain read and no transaction, whether anything is + cancellable at all (_cancellable_edges). A workflow whose cones are + already closed costs one read a tick, and no lock and no write. The read + can only be stale in one direction that matters, a node dying just after + it, and the next tick cancels that cone. + + Then one transaction per level of the cone (_cancel_one_level), until a + level moves nothing or MAX_CANCEL_PASSES levels have run. A level that + loses a deadlock or a serialization failure runs its transaction again + from its first read, as completion does. At PostgreSQL REPEATABLE READ a + retry that commits after the level's snapshot and before its lock raises + the serialization failure, and the level run again decides on the row as + it is now. + """ + if not _cancellable_edges(workflow_pk, db_alias).exists(): + return 0 + + def retrying(conflict: str, attempt: int, bound: int) -> None: + _log_reissue(workflow_pk, "the cancellation", conflict, attempt, bound) + + cancelled = 0 + for _ in range(MAX_CANCEL_PASSES): + moved = _reissue( + lambda: _cancel_one_level(workflow_pk, db_alias), + db_alias, + deadlocks=True, + retrying=retrying, + ) + cancelled += moved + if not moved: + break + if cancelled: + logger.info( + "Workflow %s cancelled %d unreachable node(s)", + workflow_pk, + cancelled, + extra={ + "event": "workflow_nodes_cancelled", + "workflow_id": str(workflow_pk), + "cancelled_count": cancelled, + }, + ) + return cancelled + + +def _cancel_one_level(workflow_pk: uuid.UUID, db_alias: str) -> int: + """ + Cancel the nodes doomed by the upstreams that are dead now. Returns how + many moved. + + The concurrency argument, which is the whole reason this is not one + statement: + + An operator may call django_ox.actions.retry on a FAILED or LOST upstream + at any moment, and a retry that then succeeds makes the downstream node + legitimately runnable. So the decision rests only on upstream rows this + transaction holds. In order: + + 1. The workflow row, by a no-op UPDATE under SEALED. It makes this + transaction a writer at once, which keeps SQLite off its read-to-write + lock upgrade, and two reconcilers cancelling one workflow serialise. + 2. A plain read of the upstreams that doom a cancellable node + (_cancellable_edges), noting which rows it found. + 3. A locking read of the rows it found, by primary key in key order, and + nothing else. Only the ones still in a dead status under that lock are + doomed. retry() changes a row with an UPDATE that needs the same lock, + so a retry either committed before the lock and the row is not dead, or + waits until this transaction commits. An upstream that died after step + 2 is not in the set, and its cone waits for the next tick. + 4. The rows step 2 did not find, whose node records a dead status. A + pruned row does not come back, and nothing writes the record of a + missing row, so they stay doomed. + 5. A plain read of the READY and WAITING rows behind an unmarked edge from + those upstreams, and only those, with their lease epochs. + 6. django-ox's cancel_many over those pairs. It locks the rows in key + order and moves only a row still READY or WAITING at the epoch read. A + row claimed since is RUNNING, and a row reaped back to READY has a new + epoch, so neither is discarded. + 7. The DISCARDED record on the nodes that moved, at their epochs. + + The lock order is the one every workflow operation takes: the workflow + row, then task rows, then node rows. The locking read in step 3 names + task rows by primary key alone, so on MySQL it locks those rows and no + node of the workflow still running. SQLite has no row locks and needs + none: writers are serialised. + """ + locks = connections[db_alias].features.has_select_for_update + with transaction.atomic(using=db_alias): + if not ( + OxpullWorkflow.objects.using(db_alias) + .filter(pk=workflow_pk, status=OxpullWorkflow.Status.SEALED) + .update(status=OxpullWorkflow.Status.SEALED) + ): + return 0 + # upstream_status is annotated by _cancellable_edges, which the type + # of its return value cannot carry. + fields = ("upstream_task_id", "upstream_status") + candidates: dict[uuid.UUID, bool] = { + uuid.UUID(str(upstream)): status != "" + for upstream, status in _cancellable_edges( + workflow_pk, db_alias + ).values_list(*fields) + } + if not candidates: + return 0 + doomed: set[uuid.UUID] = set() + found = sorted(pk for pk, present in candidates.items() if present) + for chunk in _id_chunks(found, db_alias): + rows = OxTask.objects.using(db_alias).filter(pk__in=chunk).order_by("pk") + if locks: + rows = rows.select_for_update() + doomed.update( + uuid.UUID(str(pk)) + for pk, status in rows.values_list("pk", "status") + if status in DEAD_STATUSES + ) + gone = sorted(pk for pk, present in candidates.items() if not present) + for chunk in _id_chunks(gone, db_alias): + doomed.update( + uuid.UUID(str(task_id)) + for task_id in OxpullWorkflowNode.objects.using(db_alias) + .filter(task_id__in=chunk, outcome__in=DEAD_STATUSES) + .values_list("task_id", flat=True) + ) + if not doomed: + return 0 + cone: set[uuid.UUID] = set() + for chunk in _id_chunks(doomed, db_alias): + cone.update( + uuid.UUID(str(downstream)) + for downstream in OxpullWorkflowEdge.objects.using(db_alias) + .filter( + workflow_id=workflow_pk, + satisfied=False, + upstream_task_id__in=chunk, + ) + .values_list("downstream_task_id", flat=True) + ) + pairs: list[tuple[uuid.UUID, int]] = [] + for chunk in _id_chunks(cone, db_alias): + pairs.extend( + (uuid.UUID(str(pk)), int(epoch)) + for pk, epoch in OxTask.objects.using(db_alias) + .filter(pk__in=chunk, status__in=CANCELLABLE_STATUSES) + .values_list("pk", "lease_epoch") + ) + if not pairs: + return 0 + moved = _waiting.cancel_many(pairs, using=db_alias) + if moved: + _record_cancelled([pk for pk, _ in pairs], db_alias) + return moved + + +def _record_cancelled(task_ids: list[uuid.UUID], db_alias: str) -> None: + """ + Record DISCARDED on the nodes among task_ids whose rows are now DISCARDED, + locking their node rows in key order first. + """ + records: dict[int, tuple[str, int]] = {} + for chunk in _id_chunks(task_ids, db_alias): + epochs = dict( + OxTask.objects.using(db_alias) + .filter(pk__in=chunk, status=OxTask.Status.DISCARDED) + .values_list("pk", "lease_epoch") + ) + if not epochs: + continue + for node_pk, task_id in ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(task_id__in=list(epochs)) + .values_list("pk", "task_id") + ): + epoch = epochs.get(task_id) + if epoch is None: + epoch = epochs[uuid.UUID(str(task_id))] + records[node_pk] = (OxTask.Status.DISCARDED, int(epoch)) + _lock_in_key_order(OxpullWorkflowNode, records, db_alias) + _record_many(records, db_alias) + + +class _NodePending(Exception): + """ + Raised inside the completing transaction to roll it back. + + Not an error anyone sees: it is how a node that became pending again + between the cheap filter and the compare-and-set undoes the CAS. + Django rolls an atomic block back on any exception, which is the + shortest way out of a write already issued. batches._MemberPending, + for the same reason. + """ + + +# How completion reads one node: its task row is there and not settled +# (pending), settled with a record the transition table would change (repair), +# settled with a record it would not change or gone with a record (recorded), +# or gone with no record (unknown). +PENDING_NODE = "pending" +REPAIR_NODE = "repair" +RECORDED_NODE = "recorded" +UNKNOWN_NODE = "unknown" + + +def _row_taken_q() -> Q: + """outcome_accepts_q() against the node's own task row (_classified_nodes).""" + same_epoch = Q(outcome__isnull=True) | ( + Q(outcome=OxTask.Status.LOST) & ~Q(row_status=OxTask.Status.LOST) + ) + return ( + Q(outcome_epoch__isnull=True) + | Q(outcome_epoch__lt=F("row_epoch")) + | (Q(outcome_epoch=F("row_epoch")) & same_epoch) + ) + + +def _classified_nodes(workflow_pk: uuid.UUID, db_alias: str) -> Any: + """ + This workflow's nodes, each annotated with its task row's status and epoch, + how completion reads it (kind), and the settled status it counts as + (settled_as): the record's for a recorded node, the row's for one the + record does not yet say. + """ + return _with_classes( + OxpullWorkflowNode.objects.using(db_alias).filter(workflow_id=workflow_pk) + ) + + +def _with_rows(nodes: QuerySet[OxpullWorkflowNode]) -> Any: + """`nodes` annotated with their task row's status and epoch, NULL if gone.""" + row = OxTask.objects.filter(pk=OuterRef("task_id")) + return nodes.annotate( + row_status=Subquery(row.values("status")[:1]), + row_epoch=Subquery(row.values("lease_epoch")[:1]), + ) + + +def _with_classes(nodes: QuerySet[OxpullWorkflowNode]) -> Any: + """`nodes` annotated as _classified_nodes describes.""" + return ( + _with_rows(nodes) + .annotate( + kind=Case( + When( + Q(row_status__isnull=True, outcome__isnull=True), + then=Value(UNKNOWN_NODE), + ), + When(row_status__isnull=True, then=Value(RECORDED_NODE)), + When(~Q(row_status__in=SETTLED_STATUSES), then=Value(PENDING_NODE)), + When(_row_taken_q(), then=Value(REPAIR_NODE)), + default=Value(RECORDED_NODE), + output_field=CharField(), + ), + ) + .annotate( + settled_as=Case( + When(kind=REPAIR_NODE, then=F("row_status")), + When(row_status__isnull=True, then=F("outcome")), + When(outcome__isnull=True, then=F("row_status")), + default=F("outcome"), + output_field=CharField(null=True), + ), + ) + ) + + +@dataclass(frozen=True) +class _Classes: + """One workflow's nodes, counted the way completion reads them.""" + + nodes: int + pending: int + repair: int + succeeded: int + failed: int + # Nodes whose task row is there and DISCARDED. + discarded_rows: int + # Nodes whose task row is there and FAILED or LOST. + dead_rows: int + + +def _classify(workflow_pk: uuid.UUID, db_alias: str) -> _Classes: + """ + Count this workflow's nodes by how completion reads them, in one statement. + + succeeded and failed count how each node settled. That is its record where + the record is the last word, so a failure that was then discarded, or whose + row was pruned, still counts as failed, and a success whose row was pruned + still counts as succeeded. Where the record is behind the row, it is the + row. A node that is gone with no record counts in neither. + """ + nodes = pending = repair = succeeded = failed = discarded = dead = 0 + for kind, settled_as, row_status, count in ( + _classified_nodes(workflow_pk, db_alias) + .values("kind", "settled_as", "row_status") + .annotate(count=Count("pk")) + .order_by() + .values_list("kind", "settled_as", "row_status", "count") + ): + nodes += count + if row_status == OxTask.Status.DISCARDED: + discarded += count + elif row_status in RETRYABLE_DEAD_STATUSES: + dead += count + if kind == PENDING_NODE: + pending += count + continue + if kind == REPAIR_NODE: + repair += count + if settled_as == OxTask.Status.SUCCESSFUL: + succeeded += count + elif settled_as == OxTask.Status.FAILED: + failed += count + return _Classes( + nodes=nodes, + pending=pending, + repair=repair, + succeeded=succeeded, + failed=failed, + discarded_rows=discarded, + dead_rows=dead, + ) + + +def _log_reissue( + workflow_pk: uuid.UUID | None, what: str, conflict: str, attempt: int, bound: int +) -> None: + """Log a reissue, for one workflow or, with no workflow_pk, for several.""" + kind = "a deadlock" if conflict == DEADLOCK else "a serialization failure" + owner = "Workflows" if workflow_pk is None else f"Workflow {workflow_pk}" + extra: dict[str, Any] = { + "event": "workflow_reconcile_conflict_retry", + "conflict": conflict, + "attempt": attempt, + } + if workflow_pk is not None: + extra["workflow_id"] = str(workflow_pk) + logger.info( + "%s: %s lost %s on attempt %d of %d. Running its transaction again", + owner, + what, + kind, + attempt, + bound, + extra=extra, + ) + + +def _repair_records(workflow_pk: uuid.UUID, db_alias: str) -> None: + """ + Record, in a transaction of its own, the outcome of every node of this + workflow whose record the transition table would change, and mark the edges + out of each that succeeded. The nodes those edges held are released after + the commit. + + The node rows, then the edge rows, are locked in key order before they are + written (_lock_in_key_order). A repair that loses a deadlock or a + serialization failure reads the nodes again and runs its transaction + again. At PostgreSQL REPEATABLE READ a writer that records the same node + while the repair waits for its lock raises the serialization failure. + """ + + def retrying(conflict: str, attempt: int, bound: int) -> None: + _log_reissue(workflow_pk, "the record repair", conflict, attempt, bound) + + downstream = _reissue( + lambda: _repair_records_once(workflow_pk, db_alias), + db_alias, + deadlocks=True, + retrying=retrying, + ) + if downstream: + release_downstream(downstream, db_alias) + + +def _repair_records_once(workflow_pk: uuid.UUID, db_alias: str) -> set[uuid.UUID]: + """One attempt of _repair_records. Returns the nodes the new marks point at.""" + rows = list( + _classified_nodes(workflow_pk, db_alias) + .filter(kind=REPAIR_NODE) + .values_list("pk", "task_id", "row_status", "row_epoch") + ) + if not rows: + return set() + succeeded = sorted( + uuid.UUID(str(task_id)) + for _, task_id, status, _ in rows + if status == OxTask.Status.SUCCESSFUL + ) + downstream: set[uuid.UUID] = set() + with transaction.atomic(using=db_alias): + records = {pk: (status, int(epoch)) for pk, _, status, epoch in rows} + _lock_in_key_order(OxpullWorkflowNode, records, db_alias) + _record_many(records, db_alias) + edges: dict[int, uuid.UUID] = {} + for chunk in _id_chunks(succeeded, db_alias): + edges.update( + (pk, uuid.UUID(str(down))) + for pk, down in _reading_index( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(upstream_task_id__probe_in=chunk, satisfied__aside=False) + .values_list("pk", "downstream_task_id"), + EDGE_UPSTREAM_INDEX, + ) + ) + if edges: + _lock_in_key_order(OxpullWorkflowEdge, edges, db_alias) + _mark_satisfied(edges, db_alias) + downstream.update(edges.values()) + logger.info( + "Workflow %s recorded %d node outcome(s) no worker recorded", + workflow_pk, + len(rows), + extra={ + "event": "workflow_records_repaired", + "workflow_id": str(workflow_pk), + "node_count": len(rows), + }, + ) + return downstream + + +def _complete_if_finished(workflow_pk: uuid.UUID, db_alias: str) -> bool: + """ + Complete one sealed workflow if every node has finished and every record + is written. True if this call is the one that completed it. + + The nodes are classified first, outside any transaction (_classify). A + node whose record is behind its task row is repaired in a transaction of + its own, and the workflow waits for the next tick, so the records commit + before the completion that reads them. A pending node, one whose row is + READY, WAITING, RUNNING or a status this package does not know, holds the + workflow. A node whose row is gone with no record does not hold it: it is + finished, and unattributed, so on_success is withheld. + + The classification is a filter, not the authority. django_ox.actions.retry + moves a FAILED or LOST row back to READY and takes no workflow lock, so + "nothing is pending" is not monotone. The compare-and-set therefore locks + the FAILED and LOST rows and classifies again under its locks + (_complete_once). + + Everything after the CAS rides in the same transaction, so a crash + between completing the workflow and enqueueing its callbacks rolls + both back and the next tick redoes them. + + At PostgreSQL REPEATABLE READ a reconciler whose CAS waited on another + one completing the same workflow gets a serialization failure when that + one commits, not a rowcount of 0. The transaction is reissued, and the + CAS then matches no SEALED row. + """ + classes = _classify(workflow_pk, db_alias) + if classes.repair: + _repair_records(workflow_pk, db_alias) + return False + if classes.pending: + return False + + def retrying(conflict: str, attempt: int, bound: int) -> None: + logger.info( + "Workflow %s: completion lost a serialization failure on attempt %d " + "of %d; reissuing its transaction", + workflow_pk, + attempt, + bound, + extra={ + "event": "workflow_complete_serialization_retry", + "workflow_id": str(workflow_pk), + "attempt": attempt, + }, + ) + + outcome = _reissue( + lambda: _complete_once(workflow_pk, db_alias), + db_alias, + deadlocks=False, + retrying=retrying, + ) + if outcome is None: + return False + failed, cancelled, unattributed = outcome + logger.info( + "Workflow %s complete (%d failed, %d cancelled, %d unattributed)", + workflow_pk, + failed, + cancelled, + unattributed, + extra={ + "event": "workflow_completed", + "workflow_id": str(workflow_pk), + "failed_count": failed, + "cancelled_count": cancelled, + "unattributed_count": unattributed, + }, + ) + return True + + +def _complete_once( + workflow_pk: uuid.UUID, db_alias: str +) -> tuple[int, int, int] | None: + """ + The completing transaction: (failed, cancelled, unattributed) when this + call completed the workflow, None when it did not. + + The lock order is the workflow row, then task rows. The CAS takes the + workflow row. Then the FAILED and LOST node rows a plain read finds are + locked by primary key in key order, and each must still be in the status + and at the epoch that read saw. A retry that committed first fails that + check, and one that comes later waits for this transaction and lands on a + COMPLETE workflow, the case that waits for django-ox's retry seam. Then the + nodes are classified again. A pending node, a record to repair, or a FAILED + or LOST row that is not one of the rows locked rolls the completion back. + The last is a node that was retried and claimed before the read and failed + again after it: under READ COMMITTED the classification sees it settled, + and nothing holds it against another retry. + + What the completion counted, its callback ids and when are written into + completions under this generation, once, and never rewritten. + """ + locks = connections[db_alias].features.has_select_for_update + try: + with transaction.atomic(using=db_alias): + # The callbacks are enqueued once because of this rowcount: + # racing reconcilers all attempt the same UPDATE, one matches + # a SEALED row, and the losers' transactions change nothing. + # Each enqueued task then runs like any other, which is at + # least once. Never check-then-act. + completed_at = timezone.now() + won = ( + OxpullWorkflow.objects.using(db_alias) + .filter(pk=workflow_pk, status=OxpullWorkflow.Status.SEALED) + .update( + status=OxpullWorkflow.Status.COMPLETE, completed_at=completed_at + ) + ) + if not won: + return None + dead = { + uuid.UUID(str(pk)): (status, int(epoch)) + for pk, status, epoch in OxTask.objects.using(db_alias) + .filter( + id__in=_node_task_ids(workflow_pk, db_alias), + status__in=RETRYABLE_DEAD_STATUSES, + ) + .values_list("pk", "status", "lease_epoch") + } + if locks: + for chunk in _id_chunks(dead, db_alias): + for pk, status, epoch in ( + OxTask.objects.using(db_alias) + .filter(pk__in=chunk) + .order_by("pk") + .select_for_update() + .values_list("pk", "status", "lease_epoch") + ): + if dead[uuid.UUID(str(pk))] != (status, int(epoch)): + # Retried since the read. The next tick decides. + raise _NodePending + classes = _classify(workflow_pk, db_alias) + # A FAILED or LOST row the classification counts that is not one + # of the rows locked above settled after that read, so a retry of + # it would not wait for this commit. The next tick decides. + if classes.pending or classes.repair or classes.dead_rows != len(dead): + raise _NodePending + row = OxpullWorkflow.objects.using(db_alias).get(pk=workflow_pk) + failed = classes.failed + cancelled = classes.discarded_rows + fields: dict[str, Any] = { + "failed_count": failed, + "cancelled_count": cancelled, + } + total = row.total_count + if total is None: + total = _sealed_without_a_total( + OxpullWorkflowNode.objects.using(db_alias).filter( + workflow_id=workflow_pk + ) + ) + fields["total_count"] = total + unattributed = max(total - (classes.succeeded + failed), 0) + if row.on_complete_path: + fields["callback_task_id"] = _enqueue_callback( + row.on_complete_path, + row.on_complete_args, + row.on_complete_kwargs, + label="Workflow", + ) + # An outcome nobody recorded is not a success, and neither is + # a cancelled branch: on_success is the operator's "the run + # was clean" signal. A cancelled node is DISCARDED, which is + # unattributed, so the shortfall below already withholds it + # and there is no second rule here. + if failed == 0 and unattributed == 0 and row.on_success_path: + fields["success_task_id"] = _enqueue_callback( + row.on_success_path, + row.on_success_args, + row.on_success_kwargs, + label="Workflow", + ) + completions = dict(row.completions or {}) + generation = str(row.generation) + if generation not in completions: + completions[generation] = { + "total": total, + "succeeded": classes.succeeded, + "failed": failed, + "cancelled": cancelled, + "unattributed": unattributed, + "callback_task_id": _id_text(fields.get("callback_task_id")), + "success_task_id": _id_text(fields.get("success_task_id")), + "completed_at": completed_at.isoformat(), + } + fields["completions"] = completions + OxpullWorkflow.objects.using(db_alias).filter(pk=workflow_pk).update( + **fields + ) + # A complete workflow has nothing left for the backstop to visit. + OxpullWorkflowScan.objects.using(db_alias).filter( + workflow_id=workflow_pk + ).delete() + except _NodePending: + return None + return failed, cancelled, unattributed + + +def _id_text(value: Any) -> str | None: + return None if value is None else str(value) + + +# -- retention: what oxpull_prune deletes --------------------------------------- + +# The task row statuses no retry moves. A node whose row is in one of these, or +# whose row is gone, and whose outcome is recorded, can have its node and the +# edges into it deleted. +RETIRABLE_STATUSES = (OxTask.Status.SUCCESSFUL, OxTask.Status.DISCARDED) + + +@dataclass(frozen=True) +class RetiredRows: + """What one oxpull_prune pass over one workflow did.""" + + nodes: int = 0 + edges: int = 0 + # True when a node was still pending, so nothing was deleted. + pending: bool = False + + +@dataclass(frozen=True) +class RetainedRows: + """ + Node and edge rows of workflows oxpull_prune has begun to retire and still + keeps. + + They are the nodes a retry can still move, their rows FAILED or LOST, the + nodes whose outcome nothing recorded before their row was deleted, and the + edges into both. A number that only grows means rows that ox_prune + --include-failed has not removed, or workflows with a node still pending. + """ + + workflows: int + nodes: int + edges: int + + +def retained_rows() -> RetainedRows: + """ + Count the node and edge rows oxpull_prune keeps, for monitoring. Three + plain counts on the database oxpull_db_alias() names. + """ + db_alias = oxpull_db_alias() + retired = OxpullWorkflow.objects.using(db_alias).filter( + metadata_retired_at__isnull=False + ) + ids = retired.values("pk") + return RetainedRows( + workflows=retired.count(), + nodes=OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id__in=ids) + .count(), + edges=OxpullWorkflowEdge.objects.using(db_alias) + .filter(workflow_id__in=ids) + .count(), + ) + + +def retirable_workflows(cutoff: Any, db_alias: str) -> list[uuid.UUID]: + """ + The COMPLETE workflows that completed before `cutoff` and that + retire_workflow can still act on, in id order. + + Every such workflow not yet stamped is one. A stamped one counts only + while a node of it is pending, has a record to repair, or can be deleted + (_prune_can_act_q). So a workflow whose node rows are all gone is left out, + and so is one that keeps only recorded FAILED or LOST nodes and nodes gone + with no record. Workflow rows are never deleted, and without this a + scheduled run would visit every workflow that ever completed. A retry of a + kept node, or ox_prune deleting its row, makes the workflow count again. + + Plain reads decide. One reads the workflows not yet stamped. Another reads + the stamped workflows that still have a node row. Then one read per chunk + of those reads their nodes, with a task row probed by key for each node. So + the reads grow with the workflows past the cutoff and with the nodes those + workflows keep. + + The two workflow reads stay apart. Joined by OR, the node read is a + subquery PostgreSQL can only hash. Once the node table is too large for + work_mem to hold that hash, PostgreSQL reads the whole node table again for + each stamped workflow. + """ + past = OxpullWorkflow.objects.using(db_alias).filter( + status=OxpullWorkflow.Status.COMPLETE, completed_at__lt=cutoff + ) + unstamped = { + uuid.UUID(str(pk)) + for pk in past.filter(metadata_retired_at__isnull=True).values_list( + "pk", flat=True + ) + } + stamped = sorted( + uuid.UUID(str(pk)) + for pk in past.filter(metadata_retired_at__isnull=False) + .filter( + Exists( + OxpullWorkflowNode.objects.using(db_alias).filter( + workflow_id=OuterRef("pk") + ) + ) + ) + .values_list("pk", flat=True) + ) + acting: set[uuid.UUID] = set() + for chunk in _id_chunks(stamped, db_alias): + acting.update( + uuid.UUID(str(pk)) + for pk in _with_rows( + OxpullWorkflowNode.objects.using(db_alias).filter(workflow_id__in=chunk) + ) + .filter(_prune_can_act_q()) + .order_by() + .values_list("workflow_id", flat=True) + .distinct() + ) + # A workflow another run stamps between the two reads can be in both. + return sorted(unstamped | acting) + + +def _prune_can_act_q() -> Q: + """ + A node of a stamped workflow that retire_workflow can act on (_with_rows). + Its row is gone and its outcome recorded, or its row is in any status but + FAILED and LOST, which covers pending and deletable rows. Or its row is + FAILED or LOST and its record is one completion would repair. + + Left out: a node gone with no record, and a FAILED or LOST node recorded as + its row says. Completion classifies nodes the same way (_with_classes). + """ + kept = RETRYABLE_DEAD_STATUSES + return ( + Q(row_status__isnull=True, outcome__isnull=False) + | (Q(row_status__isnull=False) & ~Q(row_status__in=kept)) + | (Q(row_status__in=kept) & _row_taken_q()) + ) + + +def retire_workflow( + workflow_pk: uuid.UUID, db_alias: str, *, chunk_size: int = ID_CHUNK_SIZE +) -> RetiredRows: + """ + Delete the node and edge rows of one COMPLETE workflow that nothing can move + again. oxpull_prune calls it for each workflow retirable_workflows returns. + + 1. The stamp. metadata_retired_at is set under the workflow row's lock, once, + and it stays set. Nothing that revives a node may touch a stamped + workflow. + 2. The nodes are classified as completion classifies them. If any is + pending, such as a node an operator retried after completion, nothing is + deleted and the workflow is tried again on the next run. + 3. Records behind their task rows are repaired (_repair_records), which + also marks the edges out of each node the repair finds succeeded. + 4. The deletes, chunk by chunk (_retire_chunk). A node goes, with the edges + into it, only when its outcome is recorded and its task row is + SUCCESSFUL, DISCARDED or gone, which no retry can move. A node whose row + is FAILED or LOST stays with the edges into it while the row exists, so + a retry of it is still gated. It goes once ox_prune --include-failed + deletes the row. A node with no record is never deleted. The edges out + of a successful node that goes, into a node that stays, are marked + satisfied in the same transaction, because the node's record is the + proof of that success once its row is pruned. The nodes those marks + point at are released after the commit. + + The workflow row stays, and so do batch members, which this never reads. + """ + if not _stamp_retired(workflow_pk, db_alias): + return RetiredRows() + if _classify(workflow_pk, db_alias).pending: + return RetiredRows(pending=True) + _repair_records(workflow_pk, db_alias) + + def retrying(conflict: str, attempt: int, bound: int) -> None: + _log_reissue(workflow_pk, "a prune delete", conflict, attempt, bound) + + size = max(1, min(chunk_size, _id_chunk_size(db_alias))) + nodes = edges = 0 + after = 0 + while True: + chunk = list( + OxpullWorkflowNode.objects.using(db_alias) + .filter(workflow_id=workflow_pk, outcome__isnull=False, pk__gt=after) + .order_by("pk") + .values_list("pk", "task_id", "outcome")[:size] + ) + if not chunk: + break + after = chunk[-1][0] + deleted_nodes, deleted_edges, marked = _reissue( + partial(_retire_chunk, workflow_pk, chunk, db_alias, size=size), + db_alias, + deadlocks=True, + retrying=retrying, + ) + nodes += deleted_nodes + edges += deleted_edges + if marked: + release_downstream(marked, db_alias) + return RetiredRows(nodes=nodes, edges=edges) + + +def retirable_rows(workflow_pk: uuid.UUID, db_alias: str) -> RetiredRows: + """ + What retire_workflow would delete from this workflow now, by plain reads, + stamping nothing. A node whose row is SUCCESSFUL or DISCARDED counts even + before its record is written, because retire_workflow repairs records + first. + """ + if _classify(workflow_pk, db_alias).pending: + return RetiredRows(pending=True) + going = _classified_nodes(workflow_pk, db_alias).filter( + Q(row_status__in=RETIRABLE_STATUSES) + | Q(row_status__isnull=True, outcome__isnull=False) + ) + return RetiredRows( + nodes=going.count(), + edges=OxpullWorkflowEdge.objects.using(db_alias) + .filter( + workflow_id=workflow_pk, + downstream_task_id__in=going.values("task_id"), + ) + .count(), + ) + + +def _stamp_retired(workflow_pk: uuid.UUID, db_alias: str) -> bool: + """ + Set metadata_retired_at on a COMPLETE workflow if it is not set, under the + workflow row's lock. True when the workflow is COMPLETE and stamped. + + A stamp that loses a deadlock or a serialization failure runs its + transaction again when it owns it. At PostgreSQL REPEATABLE READ a stamp + that waited on another oxpull_prune stamping the same workflow raises the + serialization failure once that one commits, and the stamp run again finds + the workflow stamped. + """ + + def retrying(conflict: str, attempt: int, bound: int) -> None: + _log_reissue(workflow_pk, "the prune stamp", conflict, attempt, bound) + + return _reissue( + partial(_stamp_retired_once, workflow_pk, db_alias), + db_alias, + deadlocks=True, + retrying=retrying, + ) + + +def _stamp_retired_once(workflow_pk: uuid.UUID, db_alias: str) -> bool: + """One attempt of _stamp_retired.""" + with transaction.atomic(using=db_alias): + rows = OxpullWorkflow.objects.using(db_alias).filter( + pk=workflow_pk, status=OxpullWorkflow.Status.COMPLETE + ) + if rows.filter(metadata_retired_at__isnull=True).update( + metadata_retired_at=timezone.now() + ): + return True + locked = rows.filter(metadata_retired_at__isnull=False) + if connections[db_alias].features.has_select_for_update: + locked = locked.select_for_update() + return locked.exists() + + +def _retire_chunk( + workflow_pk: uuid.UUID, + chunk: list[tuple[int, Any, str | None]], + db_alias: str, + *, + size: int | None = None, +) -> tuple[int, int, set[uuid.UUID]]: + """ + One transaction of retire_workflow's deletes, over recorded nodes. Returns + (nodes deleted, edges deleted, the nodes of the edges it marked). Each + edge DELETE names at most `size` rows, and at most what one statement on + this database carries. + + The locks follow every workflow operation's order: the workflow row, then + the chunk's task rows by primary key in key order, then node rows, then + edge rows, each in key order. Under the task row lock each node's row is + read again, and only a row that is SUCCESSFUL, DISCARDED or gone lets its + node go. So a row a revival moved between the classification and this + transaction keeps its node. The node DELETE repeats outcome IS NOT NULL. + + The edges out of a node that goes and succeeded, not yet marked and not + going with it, are marked satisfied here. The node's record is the proof of + its success once its row is pruned, and it is deleted in this transaction. + """ + locks = connections[db_alias].features.has_select_for_update + with transaction.atomic(using=db_alias): + workflow = OxpullWorkflow.objects.using(db_alias).filter( + pk=workflow_pk, metadata_retired_at__isnull=False + ) + if locks: + if not workflow.select_for_update().exists(): + return 0, 0, set() + elif not workflow.update(metadata_retired_at=F("metadata_retired_at")): + return 0, 0, set() + by_task = { + uuid.UUID(str(task_id)): (pk, outcome) for pk, task_id, outcome in chunk + } + present = sorted( + uuid.UUID(str(pk)) + for pk in OxTask.objects.using(db_alias) + .filter(pk__in=sorted(by_task)) + .values_list("pk", flat=True) + ) + # Only the rows the plain read found are locked: a pruned row does not + # come back, and a locking read of a missing key would lock a gap on + # MySQL. + rows = OxTask.objects.using(db_alias).filter(pk__in=present).order_by("pk") + if locks: + rows = rows.select_for_update() + statuses = { + uuid.UUID(str(pk)): status + for pk, status in rows.values_list("pk", "status") + } + going: dict[int, uuid.UUID] = {} + succeeded: set[uuid.UUID] = set() + for task_id, (node_pk, outcome) in by_task.items(): + status = statuses.get(task_id) + if status is None or status in RETIRABLE_STATUSES: + going[node_pk] = task_id + if status == OxTask.Status.SUCCESSFUL or ( + status is None and outcome == OxTask.Status.SUCCESSFUL + ): + succeeded.add(task_id) + if not going: + return 0, 0, set() + _lock_in_key_order(OxpullWorkflowNode, going, db_alias) + # Under the node lock, keep only the nodes this transaction will really + # delete. The node DELETE below takes recorded nodes, so an unrecorded + # one stays; computing the edges from anything wider would delete the + # edges into a node that stays, and leave it with nothing gating it. + # That is the harm the fail-closed gate exists to prevent, so both + # DELETEs read from the same set. + recorded = set( + OxpullWorkflowNode.objects.using(db_alias) + .filter(pk__in=sorted(going), outcome__isnull=False) + .values_list("pk", flat=True) + ) + going = {pk: task_id for pk, task_id in going.items() if pk in recorded} + if not going: + return 0, 0, set() + succeeded &= set(going.values()) + into: dict[int, Any] = dict( + OxpullWorkflowEdge.objects.using(db_alias) + .filter( + workflow_id=workflow_pk, downstream_task_id__in=sorted(going.values()) + ) + .values_list("pk", "downstream_task_id") + ) + out: dict[int, Any] = {} + for ids in _id_chunks(succeeded, db_alias): + out.update( + (pk, downstream) + for pk, downstream in OxpullWorkflowEdge.objects.using(db_alias) + .filter( + workflow_id=workflow_pk, satisfied=False, upstream_task_id__in=ids + ) + .values_list("pk", "downstream_task_id") + if pk not in into + ) + _lock_in_key_order(OxpullWorkflowEdge, {*into, *out}, db_alias) + deleted_nodes = ( + OxpullWorkflowNode.objects.using(db_alias) + .filter(pk__in=sorted(going), outcome__isnull=False) + .delete()[0] + ) + # going is already the recorded set, so the filter above removes + # nothing more. It stays as the statement's own guard. + deleted_edges = 0 + step = min(ID_CHUNK_SIZE if size is None else size, _id_chunk_size(db_alias)) + ordered = sorted(into) + for start in range(0, len(ordered), step): + ids = ordered[start : start + step] + deleted_edges += ( + OxpullWorkflowEdge.objects.using(db_alias) + .filter(pk__in=ids) + .delete()[0] + ) + if out: + _mark_satisfied(out, db_alias) + return deleted_nodes, deleted_edges, {uuid.UUID(str(d)) for d in out.values()} + + +def clean_scan_rows(db_alias: str, *, chunk_size: int = ID_CHUNK_SIZE) -> int: + """ + Delete the backstop's scan rows of workflows that are COMPLETE or gone. + Returns how many. The backstop visits only OPEN and SEALED workflows, so + these rows hold nothing it reads again. Each DELETE names at most + `chunk_size` rows, and at most what one statement on this database carries. + + Plain reads decide, and each DELETE names scan rows by key: the scan rows + first, then, per chunk, their workflows' statuses, so a workflow a scan row + names is already committed when its status is read. A chunk opens no + transaction for its read and its DELETE. A chunk that loses a deadlock or a + serialization failure reads the statuses again and runs again when the call + owns its transaction. At PostgreSQL REPEATABLE READ a DELETE that waited on + the skip queue's compare-and-set of the same row raises the serialization + failure once that commits. + + No transaction holds the read open for the DELETE. It would guard nothing, + because only a status read under the workflow row's lock could, and on + SQLite it breaks the DELETE. A connection that has read inside a + transaction cannot take the write lock while another connection is + writing, and SQLite refuses it at once with "database is locked". + """ + + def retrying(conflict: str, attempt: int, bound: int) -> None: + _log_reissue(None, "the scan row cleanup", conflict, attempt, bound) + + scans = OxpullWorkflowScan.objects.using(db_alias) + ordered = sorted( + uuid.UUID(str(pk)) for pk in scans.values_list("workflow_id", flat=True) + ) + step = max(1, min(chunk_size, _id_chunk_size(db_alias))) + deleted = 0 + for start in range(0, len(ordered), step): + ids = ordered[start : start + step] + deleted += _reissue( + partial(_clean_scan_chunk, ids, db_alias), + db_alias, + deadlocks=True, + retrying=retrying, + ) + return deleted + + +def _clean_scan_chunk(ids: list[uuid.UUID], db_alias: str) -> int: + """One attempt of clean_scan_rows over one chunk. It opens no transaction.""" + live = { + uuid.UUID(str(pk)) + for pk in OxpullWorkflow.objects.using(db_alias) + .filter( + pk__in=ids, + status__in=(OxpullWorkflow.Status.OPEN, OxpullWorkflow.Status.SEALED), + ) + .values_list("pk", flat=True) + } + done = [pk for pk in ids if pk not in live] + if not done: + return 0 + return int( + OxpullWorkflowScan.objects.using(db_alias) + .filter(workflow_id__in=done) + .delete()[0] + ) + + +# The task row stores f"{func.__module__}.{func.__qualname__}", and both +# the schedule and the worker resolve the reconciler by that path, so the +# function carries its public name however it is spelled above. +_reconcile.__name__ = _reconcile.__qualname__ = "reconcile" + + +def __getattr__(name: str) -> Any: + """Resolve the module's lazily built public names.""" + if name == "reconcile": + return cast("Task[[], int]", _build_task_on_a_queue(_reconcile)) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/settings_typecheck.py b/tests/settings_typecheck.py new file mode 100644 index 0000000..6030720 --- /dev/null +++ b/tests/settings_typecheck.py @@ -0,0 +1,6 @@ +from .settings import * # noqa: F403 + +# Settings for strict type checking and migration checks with both apps. +# Run these checks on Django 6.0 or later because oxpull is installed here. +# The default test settings omit oxpull so tests/ can also run on Django 5.2. +INSTALLED_APPS = [*INSTALLED_APPS, "oxpull"] # noqa: F405 diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py new file mode 100644 index 0000000..159fef8 --- /dev/null +++ b/tests/test_ci_changes.py @@ -0,0 +1,211 @@ +""" +The CI selection logic for the oxpull feature suite (tools/ci/changes.py) and +its shard list (tools/ci/shard.py). + +Nothing here runs a database. A wrong answer from `classify` is a pull request +that skips tests it needed, which no later job can notice, so each rule has a +case and every "false" is asserted from a diff that should produce one. +""" + +import ast +import json +from pathlib import Path + +import pytest + +from tools.ci import changes, shard + + +def flags(event, paths, ref=""): + return changes.classify(event, ref, paths) + + +@pytest.mark.parametrize( + ("paths", "feature", "hazard"), + [ + (["src/django_ox/worker.py"], True, True), + (["src/django_ox/models.py", "tests/test_x.py"], True, True), + (["src/django_ox/migrations/0009_x.py"], True, True), + (["src/oxpull/workflows.py"], True, True), + (["src/django_ox/new_module.py"], True, True), # unknown source runs more + (["src/django_ox/admin.py"], True, True), + (["src/django_ox/management/commands/ox_health.py"], True, True), + (["src/django_ox/actions.py"], True, True), + (["src/django_ox/views.py"], True, True), + (["src/django_ox/urls.py"], True, True), + (["src/django_ox/metrics.py"], True, True), + (["src/django_ox/stats.py"], True, True), + (["src/django_ox/durations.py"], True, True), + (["src/django_ox/testing.py"], True, False), + ( + ["src/django_ox/management/commands/ox_import_beat_schedules.py"], + True, + False, + ), + (["src/django_ox/management/_beat_cron.py"], True, False), + ( + ["src/django_ox/testing.py", "src/django_ox/actions.py"], + True, + True, + ), + (["docs/index.md", "mkdocs.yml", "CHANGELOG.md"], False, False), + (["tests/test_batches.py"], False, False), # the base suite covers it + (["README.md"], True, True), # a hazard test reads the README + ([".github/workflows/ci.yml"], True, True), + (["tools/ci/shards.json"], True, True), + (["pyproject.toml"], True, True), + (["tests_oxpull/test_batches.py"], True, False), + (["tests_oxpull/test_workflows_hazard_cancel_cas.py"], True, True), + (["tests_oxpull/conftest.py"], True, True), + (["tests_oxpull/fixtures/archive.json"], True, True), + (["tools/some_new_script.sh"], True, False), # unknown path: feature runs + (["docs/a.md", "src/django_ox/worker.py"], True, True), # one bad path decides + ], +) +def test_a_pull_request_runs_what_its_paths_can_reach(paths, feature, hazard): + result = flags("pull_request", paths) + assert (result["feature"], result["hazard"], result["full"]) == ( + feature, + hazard, + False, + ) + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +COMMANDS = { + path.stem: path.relative_to(ROOT).as_posix() + for path in SRC.glob("*/management/commands/*.py") + if not path.name.startswith("_") +} + + +def source_path(module): + base = SRC.joinpath(*module.split(".")) + for path in (base.with_suffix(".py"), base / "__init__.py"): + if path.is_file(): + return path.relative_to(ROOT).as_posix() + return None + + +def imported_modules(tree, package): + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + yield alias.name + elif isinstance(node, ast.ImportFrom): + module = node.module + if node.level: + parts = package.split(".") + base = ".".join(parts[: len(parts) - node.level + 1]) + module = f"{base}.{module}" if module else base + yield module + for alias in node.names: + yield f"{module}.{alias.name}" + + +def string_argument(call): + if call.args and isinstance(call.args[0], ast.Constant): + value = call.args[0].value + return value if isinstance(value, str) else None + return None + + +def references(path): + tree = ast.parse(path.read_text(), filename=str(path)) + found = [ + (name, source_path(name)) for name in imported_modules(tree, "tests_oxpull") + ] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = string_argument(node) + if name is None: + continue + func = node.func + called = func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + if called == "call_command" and name in COMMANDS: + found.append((f"call_command({name!r})", COMMANDS[name])) + elif called == "include" and (target := source_path(name)): + found.append((f"include({name!r})", target)) + package = ( + name if target.endswith("/__init__.py") else name.rpartition(".")[0] + ) + urls = ast.parse((ROOT / target).read_text(), filename=target) + found += [ + (f"include({name!r}) imports {module}", source_path(module)) + for module in imported_modules(urls, package) + ] + return [(reference, source) for reference, source in found if source] + + +def test_no_hazard_test_reaches_a_module_that_skips_the_hazard_jobs(): + hazard_files = sorted((ROOT / "tests_oxpull").glob("*hazard*.py")) + assert hazard_files, "no tests_oxpull/*hazard*.py file found" + reached = [ + (path.relative_to(ROOT).as_posix(), reference, source) + for path in hazard_files + for reference, source in references(path) + ] + assert reached, "the scan resolved no reference to a file under src/" + excluded = [ + f"{hazard_file}: {reference} reaches {source}, excluded by SRC_NOT_HAZARD" + for hazard_file, reference, source in reached + if changes.SRC_NOT_HAZARD.match(source) + ] + assert not excluded, "\n".join(excluded) + + +def test_a_diff_that_cannot_be_computed_runs_everything_but_the_full_matrix(): + assert flags("pull_request", None) == { + "feature": True, + "hazard": True, + "full": False, + } + + +@pytest.mark.parametrize( + ("event", "ref", "full"), + [ + ("schedule", "refs/heads/main", True), + ("workflow_dispatch", "refs/heads/main", True), + ("push", "refs/heads/main", False), + ("push", "refs/tags/rc1", True), + ("push", "refs/tags/v1.9.0", True), + ("something_new", "", False), + ], +) +def test_other_events_run_the_hazard_suite_and_a_candidate_runs_the_matrix( + event, ref, full +): + result = flags(event, None, ref) + assert result["feature"] and result["hazard"] + assert result["full"] is full + + +def test_the_full_matrix_adds_the_grid_and_the_pooled_server(): + plain = changes.render(flags("pull_request", ["src/django_ox/worker.py"])) + full = changes.render(flags("schedule", None)) + assert len(json.loads(plain["sqlite_cells"])) == 2 + assert len(json.loads(plain["postgres_cells"])) == 1 + assert len(json.loads(full["sqlite_cells"])) == 6 + labels = [cell["label"] for cell in json.loads(full["postgres_cells"])] + assert labels.count("postgres") == 6 + assert labels.count("postgres pooled") == 1 + + +def test_every_feature_test_file_is_in_exactly_one_shard(): + assert shard.problems(shard.load()) == [] + + +def test_a_file_missing_from_every_shard_is_named(): + shards = shard.load() + victim = shards["shard2"].pop() + assert any(victim in line and "no shard" in line for line in shard.problems(shards)) + + +def test_a_file_in_two_shards_is_named(): + shards = shard.load() + shards["shard1"].append(shards["shard2"][0]) + victim = shards["shard2"][0] + assert f"{victim} is in shard1 and shard2" in shard.problems(shards) diff --git a/tests/test_claim_fence.py b/tests/test_claim_fence.py index 39679b8..8c12101 100644 --- a/tests/test_claim_fence.py +++ b/tests/test_claim_fence.py @@ -129,8 +129,8 @@ def run(): class CountsAfterClaiming(Worker): """ - Counts every claim it makes on a row of its own after the base claim has - returned, as Oxpull Pro counts a rate-limited admission: an UPDATE whose + Counts every claim it makes on its own row after the base claim has + returned, as OxpullWorker counts a rate-limited admission: an UPDATE whose row lock a caller's transaction holds until it commits. """ @@ -230,11 +230,13 @@ def a(): ) def test_an_override_waiting_on_a_callers_transaction_holds_up_no_claim(settings): """ - X3-1 without Oxpull Pro. One Worker shared by two threads, whose - claim_one() override updates a row after the base claim. B's transaction - holds that row's lock after B's first run_once(); A's run_once() claims - and then waits for it; B's second run_once() must not wait for A, or - neither ever finishes: the database cannot see a lock held in Python. + A post-claim row update must not hold up another claim in the same worker. + + Two threads share one Worker whose claim_one() override updates a row + after the base claim. B's transaction holds that row's lock after B's + first run_once(). A claims a task and then waits for the row lock. + B's second run_once() must not wait for A: the database cannot detect + a dependency on a Python lock, so that cycle would leave both waiting. """ _budget(settings, 3) counter = tasks.record.enqueue("counter") @@ -266,7 +268,7 @@ def a_is_waiting(): def test_a_claim_waiting_on_sqlites_write_lock_holds_up_no_claim(settings, monkeypatch): """ - X3-2. On SQLite, B's transaction holds the write lock after its first + On SQLite, B's transaction holds the write lock after its first run_once(); A's claim waits for it in the busy handler; B's second run_once() must go ahead, so that B can commit and A's claim can land, rather than wait for A until A's busy timeout gives up. PostgreSQL and diff --git a/tests/test_claim_recovery.py b/tests/test_claim_recovery.py index 6c2a1b3..1420c4e 100644 --- a/tests/test_claim_recovery.py +++ b/tests/test_claim_recovery.py @@ -448,8 +448,8 @@ def _write_outcome(self, db_task, **kwargs): if not self._mine(db_task): return super()._write_outcome(db_task, **kwargs) if self.how == "declined": - # An override that fails closed: nothing written, and False, as - # Oxpull Pro answers when a workflow node's record fails. + # Fail closed with no writes, as OxpullWorker does when recording + # a workflow node fails. return False if self.how == "raised": raise DatabaseError("refused, and the connection still answers") diff --git a/tests/test_llms_full.py b/tests/test_llms_full.py index ac77b63..cbe94a4 100644 --- a/tests/test_llms_full.py +++ b/tests/test_llms_full.py @@ -26,3 +26,12 @@ def test_every_nav_page_is_included(): def test_changelog_snippet_is_resolved(): assert "--8<--" not in OUTPUT.read_text() + + +def test_a_section_index_page_uses_its_directory_url(): + assert page_url("index.md") == "https://oxpull.com/django-ox/" + assert page_url("features/index.md") == "https://oxpull.com/django-ox/features/" + assert ( + page_url("features/batches.md") + == "https://oxpull.com/django-ox/features/batches/" + ) diff --git a/tests/test_monitoring_docs.py b/tests/test_monitoring_docs.py index 88868eb..ac5335f 100644 --- a/tests/test_monitoring_docs.py +++ b/tests/test_monitoring_docs.py @@ -20,11 +20,46 @@ _ROW = re.compile(r"^\| `([a-z_]+)` \|", re.MULTILINE) +#: Names built with an f-string, by their template, and every name each one +#: produces. A template missing here fails the scan rather than going unread. +FSTRING_NAMES = { + "{}_add_{}_retry": { + "batch_add_deadlock_retry", + "batch_add_serialization_retry", + "workflow_add_deadlock_retry", + "workflow_add_serialization_retry", + }, + "{}_reconcile_stopped": {"batch_reconcile_stopped", "workflow_reconcile_stopped"}, + "{}_id": {"batch_id", "workflow_id"}, +} + + +def fstring_names(node: ast.JoinedStr, where: str) -> set[str]: + template = "".join( + part.value if isinstance(part, ast.Constant) else "{}" for part in node.values + ) + if template not in FSTRING_NAMES: + raise AssertionError( + f"{where}: f-string name the scanner cannot expand: {template!r}" + ) + return FSTRING_NAMES[template] + + def emitted_events() -> set[str]: events: set[str] = set() for path in SRC.rglob("*.py"): text = path.read_text() events |= set(_INLINE.findall(text)) | set(_HELPER.findall(text)) + for node in ast.walk(ast.parse(text, filename=str(path))): + if not isinstance(node, ast.Dict): + continue + for key, value in zip(node.keys, node.values, strict=True): + if ( + isinstance(key, ast.Constant) + and key.value == "event" + and isinstance(value, ast.JoinedStr) + ): + events |= fstring_names(value, f"{path}:{value.lineno}") return events @@ -63,11 +98,34 @@ def _literal_str_keys(node: ast.AST) -> set[str] | None: for key in node.keys: if isinstance(key, ast.Constant) and isinstance(key.value, str): keys.add(key.value) + elif isinstance(key, ast.JoinedStr): + keys |= fstring_names(key, f"line {key.lineno}") else: return None return keys +def _variable_dict_keys(func: ast.AST) -> dict[str, set[str]]: + """Keys of each dict a function builds in a local variable.""" + found: dict[str, set[str]] = {} + for node in ast.walk(func): + if isinstance(node, ast.Assign | ast.AnnAssign): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + for target in targets: + if isinstance(target, ast.Name): + keys = _literal_str_keys(node.value) if node.value else None + if keys is not None: + found.setdefault(target.id, set()).update(keys) + elif ( + isinstance(target, ast.Subscript) + and isinstance(target.value, ast.Name) + and isinstance(target.slice, ast.Constant) + and isinstance(target.slice.value, str) + ): + found.setdefault(target.value.id, set()).add(target.slice.value) + return found + + def log_extra_fixed_keys(helper: ast.FunctionDef) -> set[str]: """The keys a _log_extra() definition always attaches, from its return.""" returns = [node for node in ast.walk(helper) if isinstance(node, ast.Return)] @@ -116,14 +174,25 @@ def emitted_extra_keys() -> set[str]: fixed = log_extra_fixed_keys(helpers[0]) keys: set[str] = set() for path, tree in trees.items(): + enclosing: dict[ast.AST, dict[str, set[str]]] = {} + for func in ast.walk(tree): + if isinstance(func, ast.FunctionDef | ast.AsyncFunctionDef): + local = _variable_dict_keys(func) + for inner in ast.walk(func): + enclosing[inner] = local for node in ast.walk(tree): if not isinstance(node, ast.Call): continue extra_kw = next((kw for kw in node.keywords if kw.arg == "extra"), None) if extra_kw is not None: parsed = _literal_str_keys(extra_kw.value) + local = enclosing.get(node, {}) if parsed is not None: keys |= parsed + elif ( + isinstance(extra_kw.value, ast.Name) and extra_kw.value.id in local + ): + keys |= local[extra_kw.value.id] else: value = extra_kw.value helper = None @@ -217,6 +286,22 @@ def test_fixed_keys_refuse_an_entry_the_scanner_cannot_read(entry): log_extra_fixed_keys(helper) +def test_an_fstring_name_without_a_template_entry_is_refused(): + node = ast.parse('f"{noun}_unlisted_{event}"', mode="eval").body + assert isinstance(node, ast.JoinedStr) + with pytest.raises(AssertionError, match="cannot expand"): + fstring_names(node, "here") + + +def test_a_dict_built_in_a_variable_is_read_with_its_later_keys(): + func = ast.parse( + "def log():\n" + " extra: dict = {'event': 'e', 'first': 1}\n" + " extra['second'] = 2\n" + ).body[0] + assert _variable_dict_keys(func) == {"extra": {"event", "first", "second"}} + + def test_every_emitted_extra_key_is_documented(): # This guard requires emitted keys to be documented. It does not # require every documented key to have an emitter the scanner can see. diff --git a/tests/test_oxpull_gate.py b/tests/test_oxpull_gate.py new file mode 100644 index 0000000..403048f --- /dev/null +++ b/tests/test_oxpull_gate.py @@ -0,0 +1,142 @@ +""" +oxpull needs Django 6.0 or later; django-ox does not. + +These tests live in tests/, not tests_oxpull/, because they must run on the +Django 5.2 legs, where the oxpull suite is not collected at all. Each case runs +a fresh interpreter, since the gate acts at import time. On Django 6.0 and +later a case pretends to be 5.2 by setting django.VERSION before oxpull is +imported; on a real 5.2 the same case runs against the real thing. +""" + +import subprocess +import sys +import textwrap + +import django +import pytest + +REAL_5_2 = django.VERSION[:2] < (6, 0) +GATED_MODULES = [ + "oxpull.backend", + "oxpull.worker", + "oxpull.unique", + "oxpull.batches", + "oxpull.workflows", + "oxpull.exceptions", + "oxpull.management.commands.oxpull_prune", +] +ENGINE = "django.db.backends.sqlite3" +PRELUDE = """ +ENGINE = {engine!r} +import sys +import django +from django.conf import settings + +def pretend_5_2(): + if {fake}: + django.VERSION = (5, 2, 18, "final", 0) + +settings.configure( + INSTALLED_APPS={apps!r}, + DATABASES={{"default": {{"ENGINE": ENGINE, "NAME": ":memory:"}}}}, + USE_TZ=True, +) +""" + + +def run(body, *, fake, apps=("django_ox", "oxpull")): + code = PRELUDE.format(engine=ENGINE, fake=fake, apps=list(apps)) + textwrap.dedent( + body + ) + return subprocess.run( # noqa: S603 + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + +# On Django 5.2 only the real thing means anything; elsewhere only the pretence. +MODES = [pytest.param(True, id="simulated-5.2")] +if REAL_5_2: + MODES = [pytest.param(False, id="real-5.2")] + + +@pytest.mark.parametrize("fake", MODES) +def test_ready_refuses_with_a_message_that_names_the_fix(fake): + result = run( + """ + from django.apps import apps + django.setup() + pretend_5_2() + apps.get_app_config("oxpull").ready() + """, + fake=fake, + ) + assert result.returncode != 0 + assert "ImproperlyConfigured" in result.stderr + assert "oxpull needs Django 6.0 or later" in result.stderr + assert "remove 'oxpull' from INSTALLED_APPS" in result.stderr + assert "ModuleNotFoundError" not in result.stderr + + +@pytest.mark.parametrize("fake", MODES) +@pytest.mark.parametrize("module", GATED_MODULES) +def test_each_module_that_needs_django_tasks_refuses_on_import(module, fake): + result = run( + f""" + import importlib + pretend_5_2() + try: + importlib.import_module({module!r}) + except Exception as exc: + print(type(exc).__name__) + print(exc) + else: + print("IMPORTED") + """, + fake=fake, + ) + assert result.returncode == 0, result.stderr + name, _, message = result.stdout.partition("\n") + assert name == "ImproperlyConfigured", result.stdout + assert "Django 6.0" in message + + +@pytest.mark.parametrize("fake", MODES) +def test_the_database_check_does_not_import_workflows(fake): + result = run( + """ + django.setup() + pretend_5_2() + from oxpull import checks + found = checks.check_workflow_schema(databases={"default"}) + print(found, "oxpull.workflows" in sys.modules) + """, + fake=fake, + apps=("django_ox",), + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "[] False" + + +@pytest.mark.parametrize("fake", MODES) +def test_the_modules_that_need_no_tasks_framework_still_import(fake): + result = run( + """ + pretend_5_2() + import oxpull, oxpull.apps, oxpull.rate_limits, oxpull.routing + print(oxpull.__version__ == __import__("django_ox").__version__) + """, + fake=fake, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "True" + + +@pytest.mark.skipif(REAL_5_2, reason="needs a Django that supports oxpull") +def test_a_supported_django_sets_up_with_oxpull_installed(): + result = run("django.setup()\nprint('ok')", fake=False) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" diff --git a/tests/test_release_check.py b/tests/test_release_check.py index a10871e..abfd332 100644 --- a/tests/test_release_check.py +++ b/tests/test_release_check.py @@ -22,6 +22,7 @@ VERSION = "0.2.0" MIGRATIONS = {"0001_initial.py", "0002_oxscheduletick.py", "0003_lease_epoch.py"} +OXPULL_MIGRATIONS = {"0001_initial.py", "0002_oxpullbatch.py"} LICENCE = "BSD 3-Clause License\n\nCopyright (c) 2026, Oxpull\n" URLS = { @@ -39,7 +40,16 @@ def _payload(version, migrations): "django_ox/worker.py": "class Worker:\n pass\n", "django_ox/models.py": "class OxTask:\n pass\n", "django_ox/migrations/__init__.py": "", + "oxpull/__init__.py": "from django_ox import __version__\n", + "oxpull/backend.py": "class OxpullBackend:\n pass\n", + "oxpull/models.py": "class OxpullBatch:\n pass\n", + "oxpull/py.typed": "", + "oxpull/migrations/__init__.py": "", } + for name in sorted(OXPULL_MIGRATIONS): + payload[f"oxpull/migrations/{name}"] = ( + "class Migration:\n dependencies = []\n" + ) for name in sorted(migrations): payload[f"django_ox/migrations/{name}"] = ( "class Migration:\n dependencies = []\n" @@ -95,6 +105,7 @@ def _repo( urls=None, readme_pin=None, docs_pin=None, + oxpull_init="from django_ox import __version__\n", ): repo = tmp_path / "repo" (repo / "src" / "django_ox").mkdir(parents=True, exist_ok=True) @@ -109,6 +120,9 @@ def _repo( (repo / "src" / "django_ox" / "__init__.py").write_text( f'__version__ = "{dunder or version}"\n', encoding="utf-8" ) + if oxpull_init is not None: + (repo / "src" / "oxpull").mkdir(parents=True, exist_ok=True) + (repo / "src" / "oxpull" / "__init__.py").write_text(oxpull_init, "utf-8") if changelog: refs = ( f"\n[{changelog}]: https://github.com/oxpull/django-ox/releases/tag/" @@ -152,6 +166,26 @@ def test_mismatched_dunder_version_fails(tmp_path): assert "__version__ is 0.1.2 but pyproject version is 0.2.0" in _messages(problems) +def test_oxpull_version_is_derived_from_django_ox(tmp_path): + problems, notes = check_source(_repo(tmp_path)) + assert problems == [] + assert "oxpull.__version__ is django_ox.__version__" in notes + + +@pytest.mark.parametrize( + "init", + [ + '__version__ = "0.2.0"\n', + '__version__ = "0.1.0"\n', + "", + "from somewhere import __version__\n", + ], +) +def test_a_hardcoded_or_missing_oxpull_version_fails(tmp_path, init): + problems, _ = check_source(_repo(tmp_path, oxpull_init=init)) + assert "does not take __version__ from django_ox" in _messages(problems) + + def test_stale_changelog_entry_fails(tmp_path): problems, _ = check_source(_repo(tmp_path, changelog="0.1.2")) assert "newest CHANGELOG entry is [0.1.2]" in _messages(problems) @@ -247,14 +281,41 @@ def test_an_empty_unreleased_heading_does_not_block(tmp_path): def test_clean_build_passes(tmp_path): - problems, notes = check_dist(_dist(tmp_path), VERSION, MIGRATIONS) + problems, notes = check_dist( + _dist(tmp_path), VERSION, MIGRATIONS, OXPULL_MIGRATIONS + ) assert problems == [] assert any("django_ox-0.2.0.tar.gz" in note for note in notes) assert any("django_ox-0.2.0-py3-none-any.whl" in note for note in notes) +def test_a_dropped_oxpull_migration_fails_the_build(tmp_path): + dist = _dist(tmp_path) + for path in dist.glob("*.whl"): + with zipfile.ZipFile(path) as source: + kept = { + n: source.read(n) + for n in source.namelist() + if n != "oxpull/migrations/0002_oxpullbatch.py" + } + with zipfile.ZipFile(path, "w") as target: + for name, body in kept.items(): + target.writestr(name, body) + problems, _ = check_dist(dist, VERSION, MIGRATIONS, OXPULL_MIGRATIONS) + assert "missing oxpull migration 0002_oxpullbatch.py" in _messages(problems) + + +@pytest.mark.parametrize( + "member", ["__init__.py", "backend.py", "models.py", "py.typed"] +) +def test_an_archive_without_the_oxpull_package_fails(tmp_path, member): + dist = _dist(tmp_path, omit=(f"oxpull/{member}",)) + problems, _ = check_dist(dist, VERSION, MIGRATIONS, OXPULL_MIGRATIONS) + assert f"does not contain oxpull/{member}" in _messages(problems) + + def test_missing_dist_directory_fails(tmp_path): - problems, _ = check_dist(tmp_path / "dist", VERSION, MIGRATIONS) + problems, _ = check_dist(tmp_path / "dist", VERSION, MIGRATIONS, OXPULL_MIGRATIONS) assert "does not exist" in _messages(problems) @@ -267,7 +328,7 @@ def test_a_dropped_migration_fails(tmp_path, dropped): that is not there. """ dist = _dist(tmp_path, migrations=MIGRATIONS - {dropped}) - problems, _ = check_dist(dist, VERSION, MIGRATIONS) + problems, _ = check_dist(dist, VERSION, MIGRATIONS, OXPULL_MIGRATIONS) text = _messages(problems) assert f"is missing migration {dropped}" in text # Caught in both archives; either one can be the thing that gets installed. @@ -277,28 +338,32 @@ def test_a_dropped_migration_fails(tmp_path, dropped): def test_two_versions_staged_together_fails(tmp_path): _dist(tmp_path, version="0.1.2") - problems, _ = check_dist(_dist(tmp_path), VERSION, MIGRATIONS) + problems, _ = check_dist(_dist(tmp_path), VERSION, MIGRATIONS, OXPULL_MIGRATIONS) text = _messages(problems) assert "holds a django_ox wheel at 0.1.2 as well as 0.2.0" in text assert "holds a django_ox sdist at 0.1.2 as well as 0.2.0" in text def test_no_artifact_at_the_declared_version_fails(tmp_path): - problems, _ = check_dist(_dist(tmp_path, version="0.1.2"), VERSION, MIGRATIONS) + problems, _ = check_dist( + _dist(tmp_path, version="0.1.2"), VERSION, MIGRATIONS, OXPULL_MIGRATIONS + ) text = _messages(problems) assert "has no django_ox wheel at 0.2.0" in text assert "has no django_ox sdist at 0.2.0" in text def test_missing_licence_fails(tmp_path): - problems, _ = check_dist(_dist(tmp_path, licence=None), VERSION, MIGRATIONS) + problems, _ = check_dist( + _dist(tmp_path, licence=None), VERSION, MIGRATIONS, OXPULL_MIGRATIONS + ) assert "does not carry LICENSE" in _messages(problems) @pytest.mark.parametrize("module", ["worker.py", "models.py", "__init__.py"]) def test_missing_module_fails(tmp_path, module): dist = _dist(tmp_path, omit=(f"django_ox/{module}",)) - problems, _ = check_dist(dist, VERSION, MIGRATIONS) + problems, _ = check_dist(dist, VERSION, MIGRATIONS, OXPULL_MIGRATIONS) assert f"does not contain django_ox/{module}" in _messages(problems) diff --git a/tests_oxpull/__init__.py b/tests_oxpull/__init__.py new file mode 100644 index 0000000..de62a69 --- /dev/null +++ b/tests_oxpull/__init__.py @@ -0,0 +1,25 @@ +""" +The oxpull feature suite: unique tasks, batches, workflows and rate limits. + +oxpull needs Django 6.0, so this suite cannot run on the Django 5.2 legs and +has its own settings, apart from the base suite's. Run it as + + pytest tests_oxpull --ds=tests_oxpull.settings + +pyproject.toml points pytest at tests.settings, which has no oxpull app, and +testpaths is tests alone, so the Django 5.2 legs, which run `pytest tests`, +never collect this directory. Pointing pytest at the repository root loads this +package's conftest and refuses with the message below. +""" + +import os + +import pytest + +# A child process the suite starts names a settings module of its own, so only +# the base suite's settings are refused. +if os.environ.get("DJANGO_SETTINGS_MODULE", "").startswith("tests."): + raise pytest.UsageError( + "tests_oxpull runs on its own settings, not the base suite's. Run it as: " + "pytest tests_oxpull --ds=tests_oxpull.settings" + ) diff --git a/tests_oxpull/_historical.py b/tests_oxpull/_historical.py new file mode 100644 index 0000000..2b2dc64 --- /dev/null +++ b/tests_oxpull/_historical.py @@ -0,0 +1,54 @@ +""" +The released oxpull 1.2.0 source, read from a vendored archive. + +Two hazard test files start a real worker process on the 1.2.0 package to +prove that a fleet still running it stays correct beside the current code. +The archive holds src/oxpull exactly as released, so those tests need no git +history, no tag and no network. See fixtures/PROVENANCE.md. +""" + +import hashlib +import tarfile +from pathlib import Path + +ARCHIVE = Path(__file__).resolve().parent / "fixtures" / "oxpull-1.2.0-src.tar.gz" +ARCHIVE_SHA256 = "395c0deecf4310e1fce2dab537d88637084a98ed35bb8fa7b2c2742941c365a2" +TREE_SHA256 = "57a24e1f3cf301ecab07fb3478d075c3668a3099da5c02536641f7d884dba518" +FILE_COUNT = 18 + + +def _tree_digest(root: Path) -> tuple[str, int]: + """SHA-256 of 'sha256 path' lines for every file under root/src, sorted by path.""" + paths = sorted( + (p for p in (root / "src").rglob("*") if p.is_file()), + key=lambda p: p.relative_to(root).as_posix(), + ) + lines = [ + f"{hashlib.sha256(p.read_bytes()).hexdigest()} " + f"{p.relative_to(root).as_posix()}\n" + for p in paths + ] + return hashlib.sha256("".join(lines).encode()).hexdigest(), len(lines) + + +def extract_released_oxpull(destination: Path) -> Path: + """ + Unpack oxpull 1.2.0 into destination and return its `src` directory. + + Raises AssertionError, never skips, when the archive or the unpacked + tree does not match the recorded digests: a damaged fixture must fail + the tests that depend on it rather than quietly drop them. + """ + data = ARCHIVE.read_bytes() + actual = hashlib.sha256(data).hexdigest() + assert actual == ARCHIVE_SHA256, ( + f"{ARCHIVE.name} has SHA-256 {actual}, expected {ARCHIVE_SHA256}" + ) + with tarfile.open(ARCHIVE, mode="r:gz") as tar: + tar.extractall(destination, filter="data") + tree, count = _tree_digest(destination) + assert (tree, count) == (TREE_SHA256, FILE_COUNT), ( + f"the unpacked tree has digest {tree} over {count} files, " + f"expected {TREE_SHA256} over {FILE_COUNT}" + ) + return destination / "src" diff --git a/tests_oxpull/conftest.py b/tests_oxpull/conftest.py new file mode 100644 index 0000000..5a85f06 --- /dev/null +++ b/tests_oxpull/conftest.py @@ -0,0 +1,235 @@ +import contextlib +import sqlite3 + +import pytest +from django.core.checks.registry import registry as check_registry +from django.db import DEFAULT_DB_ALIAS, connections +from django.test import TestCase + +from django_ox.worker import Worker +from oxpull.routing import oxpull_db_alias + +from . import isolation, tasks + +# The marker that keeps a worker's commit callbacks waiting for a commit that +# never comes, in a test that runs inside one transaction. +COMMIT_CALLBACKS_WAIT = "commit_callbacks_wait" + +# Literal statements, not a formatted table name: these are the only two +# statistics tables current SQLite writes. +CLEAR_SQLITE_STATISTICS = { + "sqlite_stat1": "DELETE FROM sqlite_stat1", + "sqlite_stat4": "DELETE FROM sqlite_stat4", +} + +# What asks a server for its release, by Django's name for the vendor. MariaDB +# answers the MySQL one with its own name in the string. +SERVER_VERSION = { + "postgresql": "SHOW server_version", + "mysql": "SELECT VERSION()", +} + + +def engine_versions() -> str: + """ + The database engines this run plans with, on one line. + + The SQLite library comes with the operating system or the Python build, + and a PostgreSQL or MySQL server comes from an image tag that moves. + None of them is a package the project pins, and the plans the suite + asserts differ between their releases, so every run says which ones it + ran on. + + SQLite is loaded into every interpreter, so it is always named. The + server is asked only where the default database is on one. It is asked + through the driver, on a connection of its own that is closed again: + pytest-django blocks Django's connection outside a test, and the header + is printed before any test runs. The connection names no database, which + is how Django connects to create the test database: the database in the + settings need not exist before the run or after it. A server that does + not answer is said so here, and the tests that need it say the rest. + """ + versions = f"sqlite: {sqlite3.sqlite_version}" + connection = connections[DEFAULT_DB_ALIAS] + query = SERVER_VERSION.get(connection.vendor) + if query is None: + return versions + server_wide = type(connection)({**connection.settings_dict, "NAME": None}) + try: + raw = server_wide.get_new_connection(server_wide.get_connection_params()) + try: + with contextlib.closing(raw.cursor()) as cursor: + cursor.execute(query) + (server,) = cursor.fetchone() + finally: + raw.close() + except connection.Database.Error as exc: + server = f"no answer ({type(exc).__name__})" + return f"{versions}, {connection.vendor}: {server}" + + +def pytest_report_header(config): + return engine_versions() + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + # -q prints no header, and CI runs with -q. + if config.get_verbosity() < 0: + terminalreporter.write_line(engine_versions()) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "hazard: a test in a file named for a hazard; slow, and run on the " + "paths and schedules CI selects for it", + ) + config.addinivalue_line( + "markers", + f"{COMMIT_CALLBACKS_WAIT}: a test inside one transaction keeps a worker's " + "commit callbacks waiting, as a caller's open transaction does", + ) + + +def pytest_collection_modifyitems(config, items): + """ + Mark every test in a file with "hazard" in its name as a hazard test. + + CI selects on the marker (-m hazard, -m "not hazard") instead of on file + names, so the split does not depend on how a command line spells a glob. + The files are named for the hazard they pin, which is why the name is the + rule: a new hazard file is marked without anyone remembering to. + """ + for item in items: + if "hazard" in item.path.name: + item.add_marker(pytest.mark.hazard) + + +@pytest.fixture(autouse=True) +def commit_callbacks_after_a_worker_pass(request, monkeypatch): + """ + Run a worker's commit callbacks when its pass returns, in a test that runs + inside one transaction. + + A test marked django_db without transaction=True runs inside a transaction + that is rolled back at its end, so nothing in it ever commits. A worker's + pass there is inside a caller's transaction, where the release of a node's + dependents waits for the commit (transaction.on_commit). An ox_worker runs + in autocommit and commits that pass itself, so these tests run the + callbacks the pass registered when it returns, standing in for that commit. + + A test that is about the wait itself carries the commit_callbacks_wait + marker, and nothing here touches it. A transactional test commits for real + and is left alone too. + """ + marker = request.node.get_closest_marker("django_db") + if marker is None or request.node.get_closest_marker(COMMIT_CALLBACKS_WAIT): + return + if marker.kwargs.get("transaction") or "transactional_db" in request.fixturenames: + return + real = Worker.execute + + def execute(self, db_task, *, inline=False): + with TestCase.captureOnCommitCallbacks(using=oxpull_db_alias(), execute=True): + return real(self, db_task, inline=inline) + + monkeypatch.setattr(Worker, "execute", execute) + + +@pytest.fixture(autouse=True) +def task_state(): + tasks.STATE.clear() + yield + tasks.STATE.clear() + + +@pytest.fixture(autouse=True) +def sqlite_planner_statistics(request, django_db_blocker): + """ + Drop the SQLite planner statistics a test leaves behind. + + Some tests run ANALYZE. On SQLite the statistics go into sqlite_stat1 + (and sqlite_stat4, on a build with STAT4), which the flush between tests + does not touch, so every later test in the process plans with them. + Statistics taken while a wide fan-in filled the edge table make the + workflow dependency check scan that whole table once per candidate row: + a test that takes 7 s on its own took four minutes after one that ran + ANALYZE. + + After each test that used the database, empty those tables and run + ANALYZE sqlite_schema, which makes the connection reload the now empty + statistics. A test that runs ANALYZE in its body still plans with what + it gathered, because this runs after the test. PostgreSQL and MySQL are + left alone: the server keeps their statistics current, and the tests + that ANALYZE there do it on purpose before they measure. + """ + yield + connection = connections[DEFAULT_DB_ALIAS] + if connection.vendor != "sqlite": + return + uses_db = request.node.get_closest_marker("django_db") is not None or not { + "db", + "transactional_db", + }.isdisjoint(request.fixturenames) + if not uses_db: + return + with django_db_blocker.unblock(), connection.cursor() as cursor: + cursor.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + " AND name IN ('sqlite_stat1', 'sqlite_stat4')" + ) + present = [name for (name,) in cursor.fetchall()] + for table in present: + cursor.execute(CLEAR_SQLITE_STATISTICS[table]) + if present: + cursor.execute("ANALYZE sqlite_schema") + + +@pytest.fixture(autouse=True) +def pooled_sessions_are_not_passed_on(): + """ + Leave no connection in Django's pool at an isolation level a test set. + + A test that sets a level which outlives the transaction says so through + isolation.outlives_the_transaction(): the session's own level on the + server, or the driver's isolation_level that a changed OPTIONS puts on + the connection. Unpooled, the close() that ends such a test ends both. + Pooled, close() hands the connection back to Django's pool as it is, + and a checkout in a later test runs at that level. So after such a test + every pool is renewed. + """ + isolation.CHANGED.clear() + yield + if not isolation.CHANGED: + return + isolation.CHANGED.clear() + isolation.renew_pools() + + +@pytest.fixture(autouse=True) +def check_registry_is_left_as_it_was_found(): + """ + Leave django.core.checks holding what it held before the test. + + A test that adds an app to INSTALLED_APPS runs that app's + AppConfig.ready(), which registers the app's checks for the rest of the + process. Nothing takes them off again when INSTALLED_APPS goes back, so + a later `manage.py check` runs a check belonging to an app that is no + longer installed, and it raises. The admin tests install + django.contrib.auth and django.contrib.admin on purpose, and the + rate-limit name tests run the real command on purpose. Both are right, + so the registry is what has to be put back. + """ + registered = set(check_registry.registered_checks) + deployment = set(check_registry.deployment_checks) + yield + check_registry.registered_checks.clear() + check_registry.registered_checks.update(registered) + check_registry.deployment_checks.clear() + check_registry.deployment_checks.update(deployment) + + +@pytest.fixture +def worker(): + """A worker with no backoff delay, suitable for inline run_once() tests.""" + return Worker(backoff_initial=0, poll_interval=0.05) diff --git a/tests_oxpull/fixtures/PROVENANCE.md b/tests_oxpull/fixtures/PROVENANCE.md new file mode 100644 index 0000000..327f367 --- /dev/null +++ b/tests_oxpull/fixtures/PROVENANCE.md @@ -0,0 +1,23 @@ +# oxpull-1.2.0-src.tar.gz + +This archive contains `src/oxpull` from the historical `oxpull` 1.2.0 release. +Two hazard-test files start worker processes using this source to exercise +mixed-version scenarios. `_historical.py` verifies and unpacks the archive; +the test loader uses the unpacked source without installing it. + +- Source: earlier `oxpull` 1.2.0 source snapshot, dated 2026-09-12. +- Contents: 18 files, `src/oxpull/**`, `__version__ = "1.2.0"`. No LICENSE, README or tests. +- Archive SHA-256: `395c0deecf4310e1fce2dab537d88637084a98ed35bb8fa7b2c2742941c365a2` +- Tree digest: `57a24e1f3cf301ecab07fb3478d075c3668a3099da5c02536641f7d884dba518` + (SHA-256 of the lines sorted by path ` \n` for the 18 files) + +Build from a tar stream of the historical `oxpull` 1.2.0 `src/oxpull` tree: + + python build_oxpull_1_2_0_src.py oxpull-1.2.0-src.tar.gz < INPUT.tar + +The builder sorts entries, zeroes mtimes, uids and gids, fixes modes and writes a +gzip header with no name and no timestamp. Rebuilding gives the same bytes on one +zlib; if a different zlib compresses differently, the tree digest is the stable +check and the archive digest in `_historical.py` is updated with it. + +Do not edit the archive. A change to what 1.2.0 shipped is a different fixture. diff --git a/tests_oxpull/fixtures/build_oxpull_1_2_0_src.py b/tests_oxpull/fixtures/build_oxpull_1_2_0_src.py new file mode 100644 index 0000000..16ea5d6 --- /dev/null +++ b/tests_oxpull/fixtures/build_oxpull_1_2_0_src.py @@ -0,0 +1,49 @@ +""" +Rebuild oxpull-1.2.0-src.tar.gz from a tar stream of the historical +`oxpull` 1.2.0 `src/oxpull` tree on standard input. + + python build_oxpull_1_2_0_src.py OUT.tar.gz < INPUT.tar + +With the same zlib, output is byte-for-byte reproducible: entries sorted +by name, zero mtimes, uids and gids, empty owner names, fixed modes, and +a gzip header with no name and no timestamp. Across zlib versions, the +tree digest is the stable check. +""" + +import gzip +import io +import sys +import tarfile +from pathlib import Path + + +def build(source: bytes) -> bytes: + with tarfile.open(fileobj=io.BytesIO(source)) as tar: + members = sorted( + (m for m in tar.getmembers() if m.isreg() or m.isdir()), + key=lambda m: m.name, + ) + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w", format=tarfile.USTAR_FORMAT) as out: + for member in members: + info = tarfile.TarInfo(member.name.rstrip("/")) + info.mtime = 0 + info.uid = info.gid = 0 + info.uname = info.gname = "" + if member.isdir(): + info.type = tarfile.DIRTYPE + info.mode = 0o755 + out.addfile(info) + else: + info.size = member.size + info.mode = 0o644 + out.addfile(info, tar.extractfile(member)) + packed = io.BytesIO() + with gzip.GzipFile(filename="", mode="wb", fileobj=packed, mtime=0) as gz: + gz.write(raw.getvalue()) + return packed.getvalue() + + +if __name__ == "__main__": + with Path(sys.argv[1]).open("wb") as handle: + handle.write(build(sys.stdin.buffer.read())) diff --git a/tests_oxpull/fixtures/oxpull-1.2.0-src.tar.gz b/tests_oxpull/fixtures/oxpull-1.2.0-src.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1e72a9abe493cacdb9184cb258a236c284169419 GIT binary patch literal 42942 zcmV(*K;FL}iwFP!00002|LnbcciTpm2bjP4DKPZuHJPE%k`pIA)M!pCveNBozgm(f zJL~mfAQF<$CIL1;T2?&X&%XCrb*lhUvg4$CccSTuB@!qUs&3u){oSUX{{D+!_-7Y> zzJ2(R{)M0T-;UqEdHB`WUnCE|_!WNE%_^(mkzf4p{ApnSt9Q%weE$1?p8en3-F^7& zZ~Ol%@Bd^{l*MW?NtZXDfB(Pv`s>a6|IJ>s{}1-QdH6-L``iBi;y;tg+q`ays+>%c zgQUNgK1g@_z20P!&FA#dtG>CU|GM|ZZ$JM(J^$Hq*?ji1f0y(B;G3BL?|t>{gWt~o zukmwHSBqr!x2(LZQY3(+SS+i0l^iaYM^$-IT=sgs={#$iiYo568U(erEYa zww|vh*?Lt?E{c3ULvAAD&d$?CHOuEs`dx8(2v>ZM7xd}IGF#+$DbpZ-2JGg%YVwJG z$4^dP)K&jC_Vde~|MP76CNF27p9O3^|8Um7*^AHrgWU(eo&R6ur{C}Q&aQGiT!|7u za(Z2?rdLIInGkze!S@$+o+qm!ualxlsxnWSe6@lrnk1hUt5LGZW_hnDlN=83O|q=3 zX%0MNb(O7>y!`t*U*~XJod7E-7R`Q=xl5W=0cX1^fxmpm&+ox??vX!K^&5ETbzQ7r zgtAKJ)$~nLSJz2a&Jy5Q^Flr^7B~<5w5(P*d7KW8Gs{-ldDg&`R~0h2)woA&=IrqF zFQ>p?-qTD%Hhe<|I?^kAmCMGE&-JWp9gWUH6(*T})T=n*=rspDr5)TbwP@ z8IlTbdL+;3BW{a+;6n;s#zjr;TlzUUebUZ{8$Laam?_e9p95#fx0F9c3Ctl70R}B&;x#pN%hXGwhWLg;NZIZ=q{Pc!80V!ia+ZGPfdyv zuFhs@56Qkq#66i@tXJzg2W3}8UseJEWh+p48@Xm$&FA@)K1s9lsb2UPo{VcUN}d8C z!v2kt(;Su#)`ss}-QcHAq?R|MWSTWA8MQ;|;_tziQxLNHrJlIx0-z4L0O)D@ru}uA zznkVun!TI(0*>D@uje-`#o_&D!F{uHy>viy`$*2vaa~t66ZzwEm0!ZKyfL3WhcRj8 zvbp&bUZRxc;}2GKRyG-ZGU^SZ2aB$t&p&<#I}AtU`5IRHB)+I^1&k8GJIL z|DNOx9OaJ7JC>t-o6UR< z(0Z+BetcoaBwv;V(AF`Z&67-NEQJM!Mv%|?05im{zzsx)Q8mz10&9rll{lAmQRW~ObOtcans`0G{}_Zg z%n+ya_}S_4$=UtJZxi#PWS(blQRXDgRff!6hLKH0<~GN*LxO^l-=RdP>KRM%)inzH zhQ-4U&yn8$0vSH7g4;WI^rPb^$7jb0u6J^s6)Sy6g=7KWn;E3?1~3GWL^Xe#Grdf2 zrZC?54X79MIX(&xDNzaUFpQFys{$4%U|{!n=i&IlZmzRAT!wO{Ueou<1?=SBgT1ei zHdd9A2XDU_C+)?X1Dl8U;r`(+t*%fXy3THJJtLh4UzpT2QJt9N6A;H)0w;~O8HOQh z<;~LNp%upEDQ58MOlb1ebpdP}zXRo?S^(cz!TN#brEhFig}+sW96F>$y34K+9t9wR zIjC(lN=Tqd4*eyT3P}+Q3JZ@U!Q>Lms}ROOY5a-{nw-9VS?{6C#^GI4OwY`@gJWZPriG3`rrF_%t?RE z->p!WnDL-f5J*fva%z`#vB+xvVq`~{oIX2zar)=yXShU*iqHLGlw4mGK$cnZ!{L)1 z6mQ5EW;?5DhxA%_t;;&YPqU)I&k8 zqj6)Roi+4{JFL^ek3lK<8|)}fts!F7nd5AD4T%jCb>bPP>C=TUnMw9GE9R(>p-^yh zb#ITi;_ASxt7?Q(0ZOcICcy2cAQ5J;Jm2o_eLdXgO~!LyH1w9uGM|E8J;n2jN;ixG z#J56b0A~f)0@bPPGDE3SXV+=Z&;;>dKEjTyh@$kcI-&sGFh{Ox7L=!g)B<`m+>rAN zT-Ozl4T?>i5^LCSBT-YW9I_pe1vM@+uQ}m(~&el(zv}=7+0X zcFdT&mmqtQEm+~8L}5tmJ|DFdO(OQb(Hb0VT_V=&GoTu{5MOvQp}!^*U0@vsmJ${c zsLZS-k^cdx7;r{c^?>di>2;GFUl}jyj}o-yhVdR04a!$MwFBnBptKHY>1p8qBiViT z{rC7}LxvsG+OWfS-cU=$!;bNn%nCan((&rVu=JI2fX)-qYtdrppA|D=;0%hPBNDdI zoB_R@uN%?86E=+4Epo`_d*QykzRF8bZR$5-ohN@37YKi^>OLy0sO6)T%~l?|C&bPH zB6AX1bX0P`^k1imm^7r@*$KgeD>-YZAi!YR968|)D5>)q>S`#27Hv&+{f=@s}AdinT3)K^VDw0txLGl3Ala{kjH%>&snTaT%VH%Wx%DtQQatWK< zple38-xN1-$F42u{4C=@jVM<=cMQm$S(mG7jqadS%)AmsB0RF0f|(9`1xHpbMmmSG zS2DW^nh}E`X_%<;mim}v0eqpFMe8~x4@W|j2lUOP3sMeMOSD-m_ph0ExK{%XgqrU8 zdhRsIS(W>>A%lncP_8%O0Q8!$8)T`_l}A+#>i&w}6c%vh^v(v|{)-$PUPZGot}GC; zsOc2t&I(oG3t>a*M+2z@+UwZAENmrk%M7>!6FZr1)@i}oFU^guVD_Ss?Xm+?DaGty zGb!*qIyV=6044#h6i9a^e%ymt1BQEULkOW;g2fo9CJvXwIuuLNMc4x>%(|_Y4`nOx?f6Wqmyo<`Xg>#=;zWz6 zan7B==bdid7qh}!8o_eOXEZ*k)S_8mWINMVQ%P+z*QqA zO^Q2dN|k?WB0?rUI2=o~XlAU%0WIq5haJFJ~!t;rAVd~d; zojN9~;V}0SgC?I}j1n)tN4Q7X0{)nRS^0`SIV^8plUPUY@1+GWm%JVfQ#Tfkq62ur zcn=j`nD|QO4d&b&hz5-Jb9k#aEa87o8<4H41<=RpWlJVC zMNuOykT1e9I6p&pxpWe!Cm`7)ztF(W^zX4#lG ziND!RMO&N7A49YCpxW_GviW%Hn4bSCO;fYoUW?LCxVqR~7dzAs)85~CZI#r@S<9@NC;*@E#Y`8D-n1$6y%()pW{8sV{ zyQkN7QdL50ARHrY0^bKYJ&H0b?^R{+0`syW&j|3I%PSpaFeJ8GRx$x%mLwJm)`rS7 zcomSN@I7+!^PHSf8DC8Y`b~-7N?b3=&czsAjF|1af#w-9KR1Jf;!$`%2;MiqDjJ~? zk*{!Sipv75ES61~=PsZ%;ky_e287Br`bMv-`VED8RIGW`2>lspa_9!}(wQcK?6&co z8I}RYYI5=J@N$c}Pv*F%tcKkV-cCpFOSTZVWwf)80=*0P0n-LLd{Atx_!Wf3C(VNN zf<#ihV8zTD6hO|zOL6_mJfAhB`~`{-e&7@#roi0Ohu3V#a&R8UMm?`HC)dfpL*yjh zp~7zmN?Tp=Y^81)BurG-3pSOFXOou_lte2@Q4*k*E>gd_T}w{_?(3BFQE(iJaynlF zp`@Ls6d2xJm>r2h6NVZDU*pe3A3smt$HL)X>JQ#l6ICcY$Nkn=eNky}p=kr7Bg&Co zI+2$Zl91&B9mB~=(B3ddATvPdGc*Leo(f7gQg+99(XqO&)@bi}9UEVg&e{=eG_Ht0 z^Bv=nQw#M3JYy8xLAN>Cw8{yHgvFo@M`+8JP$cm(vwR^DU=@K-3jISwF304;pep+c z*wlQ2F&U#)J<8|Bc}|RCR;@vh?TGRPhiu%6$Aw4aUQ1pNl_1t-Gjfc~8LGesRkb5s z#f&8UFn19H;3QyO)cK+!`v~2jL>LP^v^Iq6$C!b3$U#KOXjrwC*WmelINb%v_YO?R zk`+Et!sC-DpYVnwd)}m{s9Fp|hw5`LQ0Rm3A6biycg1gwNcmCll>colR+Dpyr^yfK z)1j@PWv;GEX6vx6%*TjVz$41f3!rXUxMz~VrE-L2qQq& zzj#irO*lH_oWK-N9xxg|twBf{1?Xb;rntC0pb@u}@>fMkH#0OvQ`-inh!!H(rP zI}Wrdw7z8+4!WP;xbd9aBYm0T*xQ$v6ERQ*`Uox#sJQ?U{cT&9b$yXd(e!x%5-8y( zd$+FS9|-RnZ5c?7-#AhUlPqT_ZHei!k-B9VS^;OmZDx6nt5dMC`zX$veA|&j?#8~% zk>saRsK;b6Io7*n6r+W4JibHWRaJM(FlA+I<|eRfm};zG|5RZJK(oSi`j6?i<0HW zSGKMSv@urZ=+|Da&F{e?Lt0VIT|b!c(A_V`t4SG;x*8)@ArC@TvbSCCh`ec?x=j+N z*pg=8C$m9SU@@hsWa||=VP-D(2E8Y&`ID1v13SLwlgZchTO42<=M2%ALWYKi3S-T^ zO}rwqRz?z-QZ13<)WFg-9y!`DkZAc`0WoruQ9;7G;Y6Zm)rL_x1r;L~UThODFO<3a zM{FE|4O`>@$OzRAN}4M&Y^PUMh0!T;)xc|R7-ouLa?Uf<3W!#&unV0pa(LNI3zJ=l zMFg+b%{ACj)f#^Dq2^=a2@vE#lU?4TNb-RRXmk|WT(_N@?Rwjgy&_$*1ddE(-NCxz z4s@p;-Fk{>#cpLeT{FQHv!YywH^cNC6B0LQ7&44lo?hI$Z3lUyFbCM}5pRQgKUuV3 z>c-WzWsf&TP+Ty~f7_h^)$*NiB-MTV5h_r8riv#raH*4}VZh9D}N)uBa~REhzUceHB#b9}b@+lKUxMe2O@8hAk(68n?O2 z<&%`WL06{TuscE24P$flMruVkrmpbY^Qsd+i1*9+&*)eo|99~*-_{8rrmH=h-nDs>Y9@0Q4v)kkabHL zn>o$W&u$K+m3;{ zdCflQR)%eO4e&a1Hq6H0N!vdA{JW;>-tkU4h*T(2Ium+cD3wrHAi5uvnOeK}e&=#K zPlpKw+o}xA+Y9S5A}-Rsh2nbYT-^PDb07n-8r#OmmvN_rJ_KQvqI_@MI!W4y1Z!8e zhHR<5({K-pAUh$lZGhxoJ7}U0^0jC@Sf!w~!`9*s{y_tIEJ$sVF?j@on5TD(4Q{K@f)J;-~l! zdF^85vjcNeG!+jfelExFeJM|nhws?)d48E$R%%1afr`F4y{xNdAkcj`vEMG%3yi`? z@k37Bk8bWvP~aDS-x0L>pwYx8c->FXUqW8f%JY}{0CzzYo9h-$|9a5Lhn^H~urQL8 zn{m=jDbc~WMVByClqn5$5=|%5D4Okrq$&33#c|Gw?K7e*38r;^aG`T)?#MlC7DVVJ zyL%c01w|;^A+-Cz-cxbJ2HxgG2nly~+k7W1Y&F-+hfL*Fw2HW0Lu}mabw1CtCRc+- z%ZVvTg@>VdYv6XGr7f%l@uR%YkDP~pZ7O;;d8&x8vKniOgQP7!=0 z=ci{ZW6Mgt+5?wdPI3LbPVwF$JN+J08cP`M03uybE+6H=;tMu!iY1E{8dA~%-c1l~ zBX?EcnC9dLfdf((msc28Lq{ViiwjOm!zFM|E^GHzxEq}U@RItqsJQ6c+|K3)&CgM&C%Np}?;g4$&KpWn;V5a2ASG14 zy+qRr-fvN0xv?X$z_G-I%tT9asOx-()=AyNQ&jy)@j$VIkQ^xP%xUJ!x3ghdLA>2h z$EL;Vs!+Bh@(-K|qTU=mPP8lx<3yUGfQkD)G4q^T;H@Vd>{L}^F2}4g(c_AEAEzL6%9n$w!X}o9A6Xy$0NvjG0IXFs5sSx zuY-C>G*E_;H?I-$ptGE~(f2Dg0X+1tY2+F09%t4T9D~nieOj4C#gn|Q9H&I(H0{N? zHQiZWgJI|c`3{R3R?AKk9Xdt-H(bixttfNl(n% zWY;ctR+r2GwYRn#oZ|g4y;LNk8p)6q>{j5x=&oKdGjYR|e^XRrHJ1P7d{m46ywJ9Ob%{_&_Icv6KzV#Uo9qZc&z+})D`DO86vhrA zZv|1Uk=kMkbx8z#MFMCbQG(Q^s0VU9BKFMr$6C?p$I%0+ua575Op5go&1RP}oo-nf z!9~UC+d-albS-bO$E`wbjz8n#wb^;^las^Ggki5aPsjnAhnAJcVBMJGa`qsTq$DrmNdMy`&LBAmp!=*jg* z58UsH?;Ark%t80YuQNFGTx0^KbQY^ztE$%)Fg86QGp8FX!PvHH#msu$FF80od;awC z(d6X#uP={J&rWg78?RH47BG}!Mi5|r7|n-_Jn`b#7!{WU&&s|PtRf?~c(r61jN|H&$bfEP>-dorjX-DHoiT~WWj{s@3Ie_N zC_t+&pAE^5o^*sVG#iTl91vNE5oyJ!u)zbutr!NyT`+{FGEHirb>WXwukFYw4~`j* zK!n9|lYUnL54!Foo2Xl3ut6*6Qd2F7!2i1ad>zWF@w2dxr7`Z1PCuO-i(Wf3R}D6Y z5eH02LOP+7PoyHfZ>*j*>Yx}0B9M>B71dlKgUS3;3}}Wu%9Jjs)QTidB^Nvn*ZF{4 zSk_?LH=nTpH{+Yd^3Na?wCHna;qI_MS6Du>?_X8Dr zJ0gM3b1Af8cg6S5Poi;NzIb#9qLdPG(Qgtgd8LM_cZes4E28LVn4{-UpFTc2JAO3s zDH@W=?6}Ukg1S=aBvOBia@a+obFldq&T0-Nl5hn+#cnevyqOzV%y0{Th>!aVQ5LR? zat6YP_A>>cOm5i(I8^1`erN`vc^qZq-gA{u>~E4l2716?32<;KjMgBRXd#F-(d)=1 zB*}>$tU!s{_g@NUG!@CQrOo;l=2$&J}4qU^Cv3A$sFL ze-ShWm7rk)M-4308OAZ4^s26-aWwc?uW@;_YFpianDIDWtzjNFv}sMJN`ip7gMMuJ z^haHK?SK!;Fwns^$d%9Gg*r3tRjvSZg=d^sB#t#7e`iAJ#bpoPJQ!Own3L!c_{g0c zRwHD|H+TeSUCCuc?mJ+~diScT2r!id)G0jV=!`KvW8E|kd)`fqjoX}+qa7>GtD7mm zck2P7>HEXSPmsF+x1DRetOjka?m8rft+Vsprp&M$Vs=P_vST8as>UAK+(p_@qpIjJzjHbM$i0IUaFlZQ`QocH_)^ znSEYPrLI8}AvKloI&e$IM_+TR#SC|BLS$bn$(rU*dUZ>VVRTAfM}Zhra@8!85T5Ds zk{=+-aO{C3dap`yPE(vnfd^b5M^bw5bINAq#HL|L z=3PhihpCw5-mIFg(JO$2AQTLy9byxQeC^Y8=CgX8593VWk5rl4Ny7KsnwY{O;|NFB zE>STWqt0oRb&Rt?ISvr8NOPrlobZdZ*IjcU;Vt$x!@Kc@!}~BmnX78^MBXuwx@=&Z z_oJ$FhT7B^hL7z=XVVEWbTy3x?SdpLhFlGhRNL$Q*#y`mbNVCF&62z$Q(Yo^yX{g;mK10p2Tnm80q)-iB6O_m)Aw z6Ndjh2v<1;tOY? z>CpBfjC^J!oAq6Bo>6E|aE^NNwg81HN;>k0C4RS9)5un_vM~~e6S6W+`4u{^d5HL-l!e1ilYM@u@sJqb2>XhhhlD4okgbGgUX{*^q^o71rN~|D z%G8Udnv=Ad%;BDS|1~FtQFi-vfzjPC5@k|g-jK+lW*%&iP{;;Gm!vVEL)5>cbXe&v z6J&&pG_}B9K~4d%C`s9L7`U|F-Z2zZ>16Kdn3KhV$q5#Ey`NpW(wX|%1>)Ta96k!;IPVCFn;;cLY$SFhYL3!%mZ4skK6 zfD;G?RdNFU?3V`*s{(z2_Rdi&*M)1v8BC8EM+YJMTmyoTK*`@t%z!G3oh4p0lmPkZ z!EKYLzRJ9$o#3DTEoQFSBz~S=<}3bS&_^A{=zT|E64Yob(AVhTrA{jB-pph(_eJQe z&#@fgCV9T(8r0N2Q1<;m3_Ca(a13=KDSzvqEJ{@8P4SfZ+WxRL2y^p*3ko35#DC<~ zj^0M}B_cr>hz3lnWs}k;1f3IH2yAmhU6ydV_jKbRc)j6=q|b4Xzh~p*^qbv3Bs+2z zx`9A0`rn^Cf2w2Aa==Rvjx>(daXlL%a)a9pxcDIXU9%}K-7$E6?b~>J2Y-j<7$vXV zCgYyLM3=ycCvUU)+F?3Y(ogy!GnA9oP7 z4yK(SBORZ@0Vo%OknlQuq~zWGORf-IC{0wsVx;%=bj6`Sk`grFel!q4UQ2c`9H6#fww)%b`pU`gCmHs>9hVxzFR?$J?U; z1qL-)6qnL9?rsadck6-=qXkd7bCG~V&|*wl2;+m{U2fwn^t)Xvk8q7r@Aw4Y=9$>eF~&Ox0}q*jrZcvcV()9k)738c2!Q<309)g>0!E^Evbtk_N+XG ziYptV+#1ja$25_T+X)NM;4o018(`JMvM^&Z(du4y#^+*d_*TUADKpaGDN|b)Vt7N32?b;!iSi-=juNuE&KCOgb9a7k`{ z@Sr=R{ncxuOyUs$qo^puw`t3+wz{QgvvL4}=~8({$4kHh7hq;L9T2Gnc5Q3c=XB*D zxNj7m(ntn{ODC=+>h3)11_$A{{+9!zr{1lL2y7!FM^*>X8~C`3X0bBoBpu1 zXhyU%?yWtU!|~_$h(x{+$FLhZC7!sDdc7r>*~Xb%d#Rqt1W}JI}1+pl?x!= zw?gawYxEt6A3^wNYPaeB-Hy~ZA91Aoxrv2-$5Dw&CFLBNRwpO0vm~HjN3F8%t|#yc zOS4VNJmu$r2>+3K|NO|F8!ejCesO_q#v8N{{Y-%TC=rVW1_P;5x ztVy{=u9ubg&oC|0mo2;e6i!~k!g|a?%jT=A8&mU$R=c1(fn|i9A8(}Fz!HJz$Q{bg z!ZbNWE1NX}dTtY-LD-(Gd>l=V4Bn}>SwVisw^**-E#QU8ZQdVXW7bkp`xuvvcRnQ{ z(6VwaY>PW{hEq>T9T2P=8&Fi|e}kFuc29uHP}8SYdL5w8%2O3{WZQ_XrWygWbd8JQ z;t@9_>>lhEf8^acr{`WqXuo>TONT5AxajLw_vs7(r(jK>f6GwsP3I}@BgQIS%5Ajh z;4oSVZSD5q`dUVdDzBakKBVk8fJtuRP$3I^DY*)>iabI@Ed+x2N1WHr488eVbP+e9 zi*>1qDvfxVOzU=?=3@jsbKeu6LSJtI=$!XZV@j}c&}pcD7qH21GRi{{>n#{HZ?7e}})PW@0!ddBj~*|>3d&gFb+m;f|_?k4^ZfcMOK0(mux@_cE!c` zEXuoBWH&t<0WbjyI*2?RcWCWe3Tzc7t_1NJVRFf!e|q%i<3}%_9G~__$@>q(-e%SX zBaFyY+7uL^%0QCpbuKMX<*08;&F~fmdlo-VXmaO67Wmw+?pk95;S8nQHDAfEjSJNu za>&5FAzvUb$sx#=k{#Y5y+tv15$U0QC_k-(c5r+;8s*7q-9=Pvj4hZ%E8Ud$qNV#I z5+GrV0NLNh+dI!g;|>G|5A_Ue87Dj*TKijIIuGUrM<-y*K=I+YRm7dfY@qA-Z(V~w z$|6*j?FeVUNNB^A$UqQadt~M6j*{<(?tF?nNgt5#`_#bgm_d zJ!BAkzA@DrgPe^bk<>XKJ2veW`QqG_PZl1WLLgAZfkr_b(PzZRCiKSNbMDB2eyIH> z`z2OC&u?~$@@-Xs=vKiWIVzy(>w<*g?})5#KA6%$RMuS zZ09`BY66GPr$s}}l_h}&GuADn?ztp{e zIK1>#bH773V#XraM4&YXNpN)@M?+K}}^+moO+X z!f1gyD&s3cZue__aX>gAFQO_fLX`Wt!*~Kg2rR-jmPXarH zMx<5gIj(TiXg-~m(W&gJo?g` zOI=@^e48Cwi5<5c8?R?A70r?H=3JdE8e$)h2;Tc;BLBDH;t%bQqmRs=AA0=X-dRXf zXQIS1WGP3%5m zR!cz9{6JU7F1(xa@Ch{gM$;#Ij1m6Vm&Y%UPm_wIhohpADFMKjlzuycL&aiBQazEr z83jDQxWKebG2+FOnf$nEH-D7;7ZwjaL3W)1JNv1a-;DHRcRlpJnX1I>1SqA4=S_h* z1jL#1)WpM}?rf<&63%>|%OijEovnZDImjjA9W2T@5Fw16*gvDNxoNqR!G6GEHSSsQ ze_jv@N^B1)u!T|q;-yrXr2GipaW9isFlaDDZHkB0=~;tWC|t8XGwq=g+9R!HGPzh| zX4Pb(S|SNc&TjAZvb{Xfze?mxr#cn`-rbt-4Kwp ze0lcYFODa$c1M3fTz@W$#M&*Hr#NZRHKBKCCRB7mou}++#5d4uyOt)nZ+7l34SG9| z+N}|7RoEziwG9W1%IuC%aS1MApMXJ)ZGW3G47e;2=^P%|B)EJH0?H32jn>;&qZg%N*rO4>wkGg>ZP zHL`#3C~^yWvdxSmr?gxo{t106>!RxQkmLz9NN6AZWl!j{!DPajCtzEEU4qawBmuy{ z8B)U%Owq|rwQYcF$yyurz+OoKeeYhP9Zp9E!WcAdxMWF5gAOC;x^jB_h&^WrnuVG; zjDpxl49A|SX+bC+Z?RYXg4Hi}154kw0F#xIyR)7^e@t(tb8%D^+{}{F!yNWIz4uB` z&Pi_s(L!%UcQ!DN!Mvu`A#E4-w;|>$LEh8 zpB^2aJi?no>T>w2ArbK&Reoi;+ipn+j8uIHg8(new0=oLzG6-y(d7lcbINulRazNe z34wPJ5%dtGlZtO!(^}Luf#3Ee)=DS)cxG>EXh`kGJltO^S zcs0(JnzK~pgxqNj5M0Si#I^`Oq9}X!#KktD&m49y8O@n^X<%Yvd+-M&AFhU`(qk8JWAq3FXJ40* zqhLSCG~xuLM%WXV%#p$~a;*nD1xeaFIX-&+>L#>C&pC3Ov zbEl5#QcdQLse-xhKy1bmi@6^f=bP>D`+WKPJ6PRa*7b)E>3u$?XBx4U3THH4M*^;? zrFzm1*9VpAoROCd6UBAb>IO{KD9DSI&*!XjV21)N{Hu_A!SLSNlJ=e4X$SduHUqVX zxGg9umDts9$$FwhrbRZ()kBDOD3U$*8)vVXAmzeH=?D7;#4}H0NqZ!` ze?G6S85~dy!L+^4h#nJd^)Zds@?lkh1xHIarZtGjLTEs!$cE!rMt2Gfo9zX7UGnu8s5UKrE_S~M+LUPJ-xZLdWLNUaq~Jw)tr)X<&$fXNKw zIC{)q_FA8OLNbNCKj^r+R9es!9SZ;3T~7uLNKp%;|A`^z_1^6ALQi(|537WzPHS69e6gA!)rxS=s$7lddbkvHWfWQgzVcs%Kq;F|y(&tpq4lclC*@`qBK2RFv1bK zU9g&Ae-Qz@RSVxFPwy&6x528oTy|LN*Wn2c^QO)%JX_v~J3&obx5S+YO()c}8yT6$ zPQbQ2Bi8{vfbLN=kC8%oD2xzU4>t`csRHgG{oIk5sR-Pr;IdU?Q6U ze2xo7gnBK&vA;wBpxB2gdCWLDO=JPFix9;nM)+jz_Ru)jBrOoEXW6O2in1f7pYp@d zS7Shx8)+TmrJGlwvYxTKNf2nWcsjgWBnr+Idr7&bgT~zj|C_!K}{-maatOGDubZOaz&PFGr) z=QHiF?rKeZ-s~AGvLr!;Fy(N%_^Dv9sa~ksGy&X@gT5zS2iVOC7qQ@m*Xz{BwMWxJ z)JjU}P;hWn(VZAauH617c-JTYQJvsGD(LB?oCv1ViO;<;40*A4GxykOggkD=U(TiiWVJT=gzz*pU7 zGYxN5$QWu&aOY!7Dx-MItP)Eg(|Cy8d5>O+ZEi~^6(|bN`K!Vy4x^Be>~iC{U{M>n zNr!{@gGr-hhD0`m+G9=};+rItS9QS+v{-{7p(yNpUsVLL$jJ5JB1i4G5vKk<_Aw$x zEGH@Nd;;24QCCL6jn#DA&ia}Ei4<-*2P9@qBT)d04n4hRb48p=ut&zW!)I;S9IF~c zEk>%??Gr!SQE1=AtZh6#0&J%@G0{rUnfKdqM9w94SCQbsDwdxxP$To$HU5_ksWt~feN ztrpnaL(wb@=I;T!;tZ8UluC}!E7+HE!_$fr?#BN&mAPF!;Vsviq220A+DwY_6b#!IC`qR}{ z%uCXSqv-9Xh_3@r?3=2CJHT?f=mHttoN{HWMGRPB-?p}P1fRU$D6>9r_JJh=Gdp#{ zIX&OIjWJ8ygJ`CGCnEpo(v$~Wsv@l**{>C(=!FOS3HCM=Q-PKdQ`tC872jmNVmO;w zzQAQ9$!3bntgn%(+q}t2OXV6`SK5x;mD=zXCUq1h!s3F5=VTmB3e^73sn))j$~hW> z^Li538`Srp)oCiP$BwDkwxg-s!*9HOg?Rx~*W}+x%Hc%&Ws|^WC5>Ia5Gx1zL>4wt zI^f7%ERCkHDtVFBXq!+rm+7Wdu#=*d*5#7A_yu&pj*KpcV9>5q0`+{F)nNJ_QY-;| zsJ!G^V;V+sH_npt*Su*rb)DqTbH5{;0#>rkB+H|B_u zN!1d8LOs;s=-u-iF-~108&7$zoU#j zg9CR)OG@&v@jb#*pB@+zON&t4K!2;K$z|mm$GagmkdS75+SZ67v2GgmXJ=f&{7;4G*-Ls|q$W+2lfncqS@qYxZj zDb1(p2~`=d(Dl`gNekpmzo4BL_tq^F#14~Cx?u~XHyDAx{cJfN*HBvm!Qua-EZ zGatrm(qk!yT9i~RcxSbigL@7bj=2pCR5#T5t=2k_sO_WoZmXi#NMd}pQq)ct)f^xW zXcu0n@1oICMTcBF4U4#D|Lg$wNrIhIH~{~7`uy1raOnc~1jQ@A(~UnWVTDwfAdG?8 zp*NBV@yE2k-R?q7NbbAR+O?PcgIO^ZIEQG*o^ZMb%bphh~1c9cd%V7L~&3`|6YIV z0Mttc@8Mw|hH2EZOd629o3e9p-UMsqUQb8f6nT{^TbOoteV^^`vKgu_2H@#a3D|Z> zx=pxqQI7}b+VH*=^PY7z&ky>pQoZl)LHEIgD0t#%*WDLgMNWn2OYA2OTus^yELoc7Fx7&=53NOpw`DGW z+xzlW?eP9*e^Z%`f7kdmnRWVC6e+dkB?G>}3~xdY(R55FKKrAH;}XZFRYW)0tJgNE zSw@lxZ%pls2E^}W&N$@z4?3i9sKFV$loa~@7djB>_hE}${US6baw?;K>}?9O^6Dot z2isgN;%M5ny5pH6yBDKyIaTdq!-aD~*tIbrkw002&V|L^>C2aoA9Xdb_4|MYkD(g$ zG;(_|3KFv>E=O}=MYbc5U!n148qEu83A`Bjo5{b8KAAjhkN5oKX)8Sewi25~ukv@R zv4sA%_m>Ureujqxots*t22W!!8dPTrY6#mnQ76;F zH-+td=z&yp?QAfNC7n|_HSb-hKdBSQwqtG8nQ!Vq@9xl6^n6^3*@2!LbBP2f`qIp_ zAA}dVExnxr6~Nu~xy1}Iqz~j}Iv%RcZ~Zrd{D4{3aMBM5uz>senv1A~+vNPFD;D}5 zo1p5zug3}fNrUxYlNCc$QPR78P!=PG9>B* zp|nqunc{{pe_1>%|JR452U~t)>0TSJt#i5?=X$uuhtgeJ7sMPy!h@$u{~0HgQcHPFq(C&he1(1J ze1Zy(#LhKL`xCe~!haj3kF38rsNG|GR!!W@#Dh5U z=J6OLS1R)5(lHT0cxfn1nJ;E7qFAH$_DCn(1sUC!W~e$})-qHW^tYua{eS0Ge(wg- z?gjRv<(m)x(ck3zDU8?0DGS}c9~o@ZNciOK^9^UYX&m0;4I@c$nbLtX&P!}8LI#*i zWEA8A48${0;fDkUm0D++vS-Ze-H3~=FR+Thtj+e`%JmIrpODI|TzHgR%(In*u!5`c zbS&Q-*wr7#oe^=#vf1p_mtVdZB`2@r#xV>SO|Cckew2L4ZnGCD>y*Cl^@~&#OV>}v zmMV<1~3k$$TjN(gt}}Z?FEh~2DC{`VK@erb>*EN z5llK~u}WXMNM*Uos5PkzCuBCtqJ3nulYJ-JV@ZW=B-Z9 z25Br}4zk=JHs~=MpM#Xz-jp*a{G%SuWd)?BFfzT5BkL~z0i!0e??6`HC?NMZRok?= z%~sW-n8GJ>4C`Qy!&qf*Az&_&l41tQ=GO#505pKGkw3VjO3tem;AydZJ1%sbNFE9^I0nd0hHPj#tAZgbXV2VE4XG5Qgxr+ zfX#BhQ4nWFs-Uzi>8`%_VDGCG=1r@^?qv)c*oBeRq!;4QGBuNx;FG+b-Y8^QRbi~? zp7&w$mGnNqCOK8gS51+R4;gTFiK&08qVO0^_BLNq69vXobk0o>h?EU1m=my*W8I4@ zDw1;=FG#!}=4C`v1PyRRUucI&h0URJ-P-512G+GiZj?s2VPQZ=&Hi{v;HFy_=fmg@ zeQgkoh@b;T$=|U3IX6~9ZC`6_(A;C7wZ^uw=opES)n=7sfHGVhc4LEeq|8$CMp00> z5MZaY-Dj;>`!gm(}01adP_A-d^OzJR~3~pi^$C z(qESZ^@c8~v4GK4OW{@IHKp57bbF&hnb#&=G4K%nlB(nL(!)1Z7ktLYov}wb|J+D{ zb|FySxrEWElPn4bq=O$gMi=s@dro7e@`zDFK;KeOf~g<^I_N;$MG1@sQCvMWcwf?8 zPb%qxg7ODhjJ8ObRVJU=kD@)@R3F4!iK!X5SY87Hed3FD{-ZSt&6sbbWCqMdsb>Wh zg^Lov2;^}~nbup03QxE(To~=2-vm#>l1p+2h{5bfajcon(a5E6A*+4?TpTQSbnmjM zAPrcej_a^XCzB`7kNz@wbbR;-f5Dqc#Y)-fOl~gUpIMcico?d_n!^rUR9I-G7|!gN z>{xlN7r(Beft;q&bQFEF#wn|t#vxe5#yw!(D3=VjDh$ZV%4!Er^vf5I4$qDanV^k; z&a+uD%?NrND5NwBc{GkOpdLRvJw7=bInS&{)pOlpE%4{!mNXja@yd`qV@WBkN);x3 zJu)no8HB5bK$BnC9zjBhfNZ5 zIFdIU0!39#z@nVa*E0#M&KbhRYRo(kga4l%KRG@+lQWDWWLAx351;99Ae3dB z#&AL)-e!US`%~Q_)+rwOJ4WK`Ik002!W^z#$3Wb0Vem2c4ZC{b$s3M46%Cl31JFa> zBWUxK9rJ(Xb>&e4fqG(!&n5wakhF~c*aPnWKuReYQ^KWl-uNq1gy!Mb2m z)-h7Ra>Q*yV}fb;h~i+bZl~k!qbHA#pPfy=>jCR^m!s zr}F}s@8n{hUG`t2+`#f}pN}7ob6Pp;a@}aPjbWq8yT-m+v||)m3R%CAC|aDfqx^(e zj^P;hoBfWF9MIXUm~p6p?&FFoVygu@C$7`v*qPNG!;r_tl@d?abxlBiYD_!FfU=HJ z=Tvb2c_~hgUmPEvVO33nK(Z#)8I#6|H3=GW$3Vi692aWugDn5S22=J^_yiqs;1HOd z0(3Z$)4t)tZlVIk-#XO&@FdmxE*jU)o|yi!BO!JwZR)B$vkYc<=Cb zKL$#0M!j*K9ZV~nt(S9BC9JAI0W7Wkjt*#5G}JJ~(;84AI)tc}%}UN~_7$dLln&uY z)k>AM+>^Y?n>I;(lFI+lF=Fktw9$HXG2`1E1nMNft63LJi`nK&B?oK*RHqp6DKLh? z@GcG2G>0z$$lvH^VzO!hym!C#MW50o-@8}%V7Z{}trC|EMSZe^$kaaJ>pCKw#gxDn z?Zz&xeF-`cGY$1TRwc2iZu@A42{>UA4#MvEU=*@A7m5m_ch(U7zRvTXf+Iu;6so`( z%_>d84o1tWA*gci{iH%b(a(xzKdgT%JmNJ46T&jA<|85x*QV)>3|9G(3^PB|PEIA( z+wS1CsgB;+&w}(P(`hLJ$~3jZVHj2maLgRFJyl7?489bATmDE9;(jFR8SH(g=Di6A z$r74M@ZkN)V-J_3c|_}KOKZCWGk1!!w4)~M;HPlOeiSb8;$5O7H37S~@`3n$kt`N<G zK-i=MD~Yt-)}G|$4*FdZO&b&X5@vKD|HdR^nq>zAbb6b=bi<7a$ZdNq*qQfgxH``2 zB1i*4$8woeb2M}GTpRI7F=w%0DBv8$gWB84m2#0O9~geZb$mz@oo1?sh$$>g3vDbn z2)cA!(-w6``eI?@qG2CqbC+3QQ!;FoQ*tUAJKGTWL;O%2toG_cWC`M=$vS;oIVBat;fpjT=YdbdmOV2Paf>v$LJnnL5TNO95!Z$&5~=sHh4U(GjPqP7=?N?GT>R_ z{HXIC%0M*llLt9EN6%kAJCm)-u*+t19~{yTTVO&%gHaV0ul;Ns6U*h?i*iN!Hn4(Dl}uPwru8E ztIPRx09b@(A!I}`AaL zQpgu+=VJw1@ZRUFVS_=%@f&VlTEZ^#%=cYy+n;i~O^VE9mKxOVB}*}P3Zx;L9Q8C! zlOO(kd~)1tIq-<%ct7<*fJeMV_SYx4kJ9Z&&mx}b_}n`cN4>5H24!~Fi#cZF z6W`#ZXPQ zk!P4vd$FFQ`-5rJLY6roB?s{>wUr}ue2=6d_yUa7Y~^vC15TI7smd8y;hm9)X^T=});{<(4oLB1V_Qw%`BsZ1;E@lhfy%IG7IP28Nd zr#B~4BHYf*M_&l#flI0z-#(@0?#nCZ8^8gaThJ^Oy`gKpfK%hrcJ9lr>%yHR39Tr9 zhs~lng&qS`OVB(y-$}wBY4SZ|!MNf#Nt%{z(sDb8c)GcJ$Stj<^jk?~16JrrdsD*$ z`M$dk)W;U+zsGa(!2ur_sQRwHi~Fpw#ufVS;}m5qm!l3I`g(kY{Dt4fLkt*JK{0GR zuEKC+DqT@Md5b6k<}3!l3yx8A0&;8uPTG!HYCwyoSzQw@S&Ula5o^N6DVXr=5RN?Z z&Bz06n&tyL(y`AeSOJ>JnWmAtd{YMLJxu7&5vWH+g#l_xRbH19awjnC9q6BQKHO{t zQy`<*(H4xIQQ&NM>tNE}h)_7NTvk-+gY6iA3`=r3oX}3-DXzTk>dp_LEKyjO@D#&y zsmWf7fkl8z$igBW#|)#!6nj?gd~hWRvr9d`=Lb zm?OmCM09n8Ul`d1 z`j7S4S;$vq7=vb*yuUw6?x!g62eJ*r59&{HxDL`J0~cd#(O#>RMt$?5=BsFm4@KW#LA^7XHY`a*D<4XWeE zMlc=79pQvbRe{?_2f!B2*kyOroB+^Vu`Zj3SYBLU7b=xfu+Pggb2=tf9kag1au}Ez zYc4rz|BXXXA)}r99B^7(IdQ!?EWR-qI=K%mg^1rG24r(*N?L)u7#Te27OJ0zBhZN1 zEMLwmEG)u&w*Cg=O?V znu?cHmV-q4p}VB5BTjvykd@j(C*TvfpVGKa@uUEgnF4jT=}qz^|Y zOwwJT}LTw_xnJGD5;yaa{xew#7+ya~|}l1Q{T3X6@pONmL;ph~hta0^yPDnN*NdgE!;@L^0TM!FURdWS# z95~M2AVFJqA;nU*@7Gpm-e)gxcKaWikr^Gl#`mY$udnBm>xU43_!j*nu$vJ9{ zW*iLK%wS`SDRrbF%DN82z$}bBxS?}Ml4*ch-WXDSA5HcH8b)El1Cjp*2rSzX<`Ez4(b28;~X45dcZOK=eVJ?dCWesr)!U02a?!HRcXC65jN z^jF?e-VlJ*(ZXTaN!1GszG#@sfLpRO?hu4$-V4n-7Rrd3%U<6UM36*z&3Z3=(fRFE zWM}>nv2J}U+hN&t2#s}iAfvn!!{(!GS#t#}IK!?z z5>7RO0K?{HMgz(?J3K{WMr|@+wk0Vc+;v(#I^E9W@PJT8!6`Sa)PkaDWNpAW|bpmVo=YHE^(z z6OLfMV4u^#EyFPKlObQ8*^r$vlY{78#RzbM?&lr0k`Y?G9|Sb|2r1Ap&e$O1bN8nqT!l&hKMI3V?PFz8+y>O{8d zHa-FuBb)}4DZ^b1(C?4MEx@0!u>>UqN{YJH*brlk zoK>XJO2*8oPS5wjivWE7m(gxgkSTAFFlV@Nq?30#@S2I$YIzS{d zny~_1&_x)m*nV)dFVS+s#(MJ$T#2=5Q!f&9>mj}c8$0MFAIe*%D$^W|gA$t0?eJc4 zd0CTCU_04W#VZL=M`vWZI3QF_(3M0mPIMLc+cq>Fd6Es4`u5L^a*`#@HFmw`*5@!o zcJ4}cyg65xJ8d-yUem|U@k7z&_3DbEMrqQ2)_U*0Yaw`!u6ejg;PSOAE*a=R^9za% zwEHYE+MER7>zm1fFGdwc>}mF4)UMl(;y_X=^nXjiA1_@U)!m`yU^@^}&b@S7Ky}nz zgyBL-E0dfki6w}yn$dlcRalha_RcsrEDlONc?qv2aLnG9Ea;wQc}ISb4o;>1e3xM$ zz$FgbkkLs_ICtOUYwJvG36$?jMKy;~|0oTzThyb_K7_|yicNf?xFE`y~a7&ijvQrTG4tTUk-!QDJ@rN<>>ak3+a86dK~^BC<1MrsVLL(P69TN&&3*fZP1FN7?W$O%UMi$` zz#A-byrXCqiBxl_189e~uM-9fDNNX_ARac>Mv+O3``?DXOG;9jwjS;Zzl$AH|C*<{-r^2%j~4r)vTQ z{p>Zj{wUtve2FV=yHO-J4yX01-QW&RIk*cN#2qSjuVcfb#bWN94BWxCT2@qNZf+-@ zl4<7YT-Yl9tSYWbI2N&VjKh3`jA#^pFS$Fo&A;0>yg%%ZX#Bj)iMlvFRZPveUMSsA z_#E0g9jU}rynkylaSIU6sXIO>!(B+JLm-AqY`aUwW+pxyXU}iwtjg5^b#{44?cyoa z;ww|Lco^<^f>9at(1^zt=i+gJG2bibdYH5rf<~7V#F%Y?pt-B-ph>I~5s#w+K$nkS zYNtxDI(PKdsx6?rfsP)fJPKE#yMPRJ)MP8H19;8VcESod zW9foSfl^Wtf(DJz+MuOJR})80+lP~AsVm1GTQu5+5aE#~y3?Ba-Jxj|flAfOXNPBJ zCy&2-c^0>U=;kdSUDwlYv%>EiHVf(6f>E-ZubYmh!ZEso?++W@VkygkYLlLS|)XuG=ry11=;f`HhE>pJ}mSr}V`Pzg?04VnQH8&fj7rXqx*o3$N>Zn?@g@e!Br0 z1r>Qx(s?4i$)(s1Pd3_1O(%LmZpccbqn1=_DKFQTF{?x>yHlHLGgc4bn1XULVcd>0vNoXl(E}*TIW4`xn-;fp)Q$NAxT-m&XNVps< zZ|)TRu1cSG6_b8&V0Z^nB$$XyS(zwB(?aK0blYrdYjdS#5E+I%sy51Hq2AG4s^BY~li}gH>DasyKd{9W~`l)QM4Vi5vJH7DJsV)qlMjr+=*AW8UOmZIwAmTW; zYP(;4>0UBu(HH7K8_Zeb`7X+f%D=vKc8k+(@VlAY>)(aDGBmF5w#!>?>UD9hQEwZr zdh8u{%6Q~z-{>NbF6-oI(KU82v3D_OSP$&%Z$7-&|?bLMU6k$-N4dTVJ3jrV707wtxE?ck-T-a zVhncjn6=X=tG6bN67%EO$zN&;D5nv!nn-i8A{sd+d)n}$Q4F&zOFZVf0?lz@oNY=@ z%E>ZUEIa35Bwry{{IHv!Jn%H5YqTJc`vwRBj4eV@58}0qK`frED62Fv8-cY#conm6 zv|&OImD{(tY9Xjnh)Q~VdjDtZq-!!LH+0dxQ}0_pvs-%KhDqGP5o$~U!$z>~w3uO> z{$AL$hz>)8B}Q0;mQE&;A3OPw>L-6+XLG3)>a)E=-o<-?U&^CVgMvwUaBN#eH8|&@ zoY}Gh7|9jDX+l39#O3;YUQ9WA=&HIV9g&?r&615PGO5#^j{&->CQiulpZa%zZ~E>V z7k0rOTQd_&#Pu1trU0+7r**sEDEEkq-tdf^T#{-aXXwh5CZezh)c^#JaF~E>w86Sd zyHjKj9y=AANtA84*TjE zlm;PTM)H9art!4(!;-Vl@HS~wv;h8^UDEriI;NHGwlN}AyE9Nb4JfVtq~rjZqUfVs zU0^~AT*yi5fXr}*nc)tmlkl-v793*8+WQ_9kn;97W>`Z%a-gVhE4~@p>G_OG8)y(& zK5o^(!5%$>_ZcepM_(`Xqp2dncivb>DdeY3%^lx*bo~9{%O_`)u|N6l@aQkc&mK(< zpFBP^yFV6Vo(->6^Me`QxjSQ1@~IevrEnVTvXJd_)$efDg_I5 zw%F2mI=Tn_6I*tPI#IW482BM{bDt?n#EqvN^wNM_8JPRf<-+gF%Q+mr+I<~JB3e&B z#)1EJ7%a1}-HDXnO(p}Ec~4vy-qrEi6C+Ca%);W&v!4oBZC*|zDVo$oviLOuhTo!q z4Xoc3s~xH$t68l)iR8Ts8`J^>IK?Tw$$RN)3G4dBukh2<)88|9{=G}gXwv1)=X}O4 z{CxBE*Yq#^#Q*Mn{gwNUKi_-s&ED5vB)h-H0<1BO4Icl+|IVKlFZYRAj;*7Bc3Dg= z&53&eiGg&IfwCN2H;fzSMqtV=KA#EtA_Mwy>7RsJ*=&&;1e2z6P{nvg-{d!G za++P_Y%Z+on|*N#W?U$Z-ALe&gV43zYB+DkH2D=IjO{Prp`;}vW?(U-hu&b)6qq_I zwW*se`9?n!@ph>0x-Tk5g^!c{e^~%12l7kAa zCc#8j&sHypOrfK9n;MC<}2p4NXj&(Q8qU zsd_Fdz$r;o>2+O9b4|c9X+XFR{~OM~;s1-Q1U(ncgWrGNXCPDm_Tj@#{J;JC-h;hw zzxpD1_#6NK2kpPM_x?PJz%BB>WBE(vj5-geH+_<-|p@G#{Yjk`)_r#BtIsb zdxzysuV?MTDVQ~>r;e0xr{&`B;P)e@mQfUP{@oXYli!H)9qL9Nfi6V2#Nu)Exu*xF950P7`5RFuu?x} zac1?+4vID*!hKek|fWbpCyl%O_8IVUOxbK^Pe8lF&CO z{HeQ!$pCZIDpH0H*$iKr zA{)$-dKQ%WA}<@z>j&3@-XV(2Da}O5W@26@2=GFnrTUe1P0x*a4aE z@#S-^>}5P&{u|~#k?`q2A2aGcRKwo389$QY-^+70&(Xsix2F(1V&t}M?8Nx1c>7+D z5>+(t=32Pl4mabN)nDDubI02Xtt$f{%Or!ntaEJ^SvoIyH&_9@T+WNUiI&;# ziEGwws|1ax#lB3Lf+yMxd2Cy@1H7oXC{!82no!v!Ow=7yr=toUL`2c7>$e3Z{!%7= zW%-MgsPVE;zLH1vLjS##UD(z54s`&na?M=@ByVIfD=Mm}(j4vks#2K#$H>>_#W~VK z4qVSWv@!IGAdpiqrkAMtbAWsf^M4;SjTM@!q_g7L@4q>b1Z*E=Ad*4}JSAKR9DSKE-K-HUcFU^<%v>vYiRVyE%B(5{2Cb z3w=;U`rY-cyb$TpRaSqGgANwiy9sD7m#eFTZ@wl}_(cYm28=*bdgMAR<_Am>EE#&9T`-vEN zP7y-+R6kRC6tnHZ!*lY^tE@+y5T5WqUyn9ki}~Pav9WyFd|gX4ZM-rPS=|@jRgi6x z9d{5E-OH;D!0slWH@@Wbsw$>=*S3c9N4Qg@z2`um{o&}As~LF#ujlN;+wV~P47^7X zy0+c*dUPw*$ecG`XP18W=k2aZVDy{(LVWm-*@geHT1)q~$92nhB>?+ew?Ltyg4)j= z-#4z4_AmXPr_Y~lej<8shPRH$#aVY96N>>pZUJox?muk(Y{~CuS-!Ab_-C(PSmye< zh`L?3|MN8axE}lGjrQ>xxy*u?+?LxuW)21_=w}TVaYzZLM77Z@sX^m-mpA=!@@m^* zc#M~AmL#el_|}wmJztMR!H}#QaoQpt9qik?emDE=-NN=iv<#w8)^<31{1?`FzAG*d z*Q@FlnVi8T6Ir`kbd=Am%}|+PUY}Qu)GGBKKkENX(YX0!e%3?Sy&>lvifpte-&b{B zT$X>yZ#M4;uvd2MA8a%T(xb!Eqr*qXn-{ju=k#K{umYE{ML$_pdRkIrqJHvr+?^lz z0)tz50{fbd+zq#~hk{|?`jT89I!~cu( z|9^P>*H>Tt=KufI_n!~r=dJy}eg7ZqKKwS8|9js)_)Y%*DnCI$XM=PXiO!u9m<&kFfxpHqlu%dN=iByA4l+q6)Jo~#dG}c#w={=2alayq^HAWj)|Bl&DgZh5UGzJyCJDyK3P@;qnsn!L3!!3 zc+Xb|F2t?s(d2~xS6Fr|fdJtRI-g8az}Ms}O@ZaiP+4HyHqPRt9`c}byhV6QDjhPQ zU?XiLj9)O~CTxg8P5I7Q3}dUgOCJm*aw#dk0y;JWy)#W7Gjh{N^J&9cAJ@Z*lW%eFq!xfiLM#Y#gSH^% zsvyEkpd?w&R;4Bhy|~N=pC^~vnmn!hk&V*KeIki-%wgr8FAOy2y*T3- zZP?c`F>C|TGNs*We~uaK-kcjCT;w(3_);l`rW*HkfWC#S=B698ocK6G^}u?AW|~QI zFdq)=YQy=ig49OfrV;kw4uJ|9(l;O%f@c2b!<`2Yze%q0cd)6K*g1#*L$u8@qkIZZ z1Y^nhw3U$z!uWTJekVX)H;DZEkr_S1U{4O(&OPUs9wpvgJ=B+Zl$TY#fEhP}1*;fR z+<-)Tdr(Ao-zCdu^NMzLir`&o&s32RDkN^E!D>Pj5gLt(E_~+^N5qBFu&xfB9#@qj z7$8I(Ktw>H@Km6&)W5nk(!MRHsEs0Zt2xxlDH>YrO*BgUD3MWOiW?9Hax$T+rVsEB*wQ{MC5x!RrsN`;o-swuoDV-65Hov*fa_fS~cb6PB85y5)M7 zP~JM>WOlSJcKtkU`{L7=M@Pq}r{BMPGD;L>pUMdk>`X!GT`yx{9$ewXaN68Q8CFC3 z1Hf+@eKyG3+;}HIu2FzEgI;4>cW}oYq#y&HjWaK^WphRSf%iMw<(lreS=jxNOD?ij zi?XWpIR1+Mt}9Dzb)?G+33yyz5WY>c?y50JAL(f>*y-A~{9Ky zC2fK{#_2fdX~Ix%i1JP~R_9GlXf_TG+u@A57I5dlpK-EyQt2`CIjpt90`?Kgy-!Qi z$Lq1503rc1Xg{)&yj(97#coz=nvR^ zC(;|N&M=b3b+X}1Jx%V4NOEeh3hOAvDk-;l5)2|C1z9`RsepmfYxwTnjumTD(xaf4 zk$@6YS2XFbO*atNB|5>_w(qY!at*NQ7KENej)CHS+zCS*LJ$jU<<5BmPZy>VGcQiv*l`q`PD%eIJHTTKYjMZjXH zQKek<$?6xiY?iJe+<9D@&)idbbZnDb`@zAu0ycqt=ga0gulGfZpy5rQHD&5pCRJ98 zqNKJIrf(lf1V%JtIoDEAO%IcM_E@m#=HVXv!|wI9!C zoY=&ba7x`xuD6YWnJV3{W%GGH!&}+RoKJeGRv+|_eW!cL4`P=iv5eF@@9v>0Bu*}Z zq^}$hNfAhOLB{b-zUl|90^u@}yz4s0L=EQhnJxzEYrCovldJ$1RxA)&8jVpi}i+veyXdbT%keL^;&5lH!Aasg1kp`A57G=r(C3n(gCLQ zO95~^zPt6>r#UD0_*t=X60eDveQ$=#xeJq1?~2m)e83S^t_aio6Zp^+XEvy*1cGhM zU5jJbBFw`ys-cwXr&e=)MLtxtSw~Sal-;zN`LrX$DWW&ILL3f1%)QYkeWJXjjs&`K zzLFOrojj#iAC{-|3y2USqV z#HWqV@!-f%S0=$r=joN_=T*}%U&53TIDd@W2?z!vk4|y!mH77Lq?)PsrJTcsfb+Xn zv(noYgfI#W5ILhDhydgv3t{93rC9TUm^mfoxZT7c@IXsvpB_E&N`^blPER_MOUv#1 znZh=aVqvL?bj~2S|Kyw8Cm*;C_*rwW9pg|tqEVI{Gcg5B22j!Ug;VDqIYG3kHkaP) zt{2<$B|5rT6m**u99v+G(QjOz7od;aNJxQ3Ivm@SsSss|W_S^LadzpOQS%?JI!MIA zEv*>+ije`Ii`zWdE8GsQ$H91C6*KL(tS`-Xk@a{~@PIJ+&j{L4Ej?_%UNJTlH`<7> z(#?*?{t39!^>#gDAYr}TcoH&QV3}i)z3jH^gbTSsjAA|rS7YBH(NK>cE57tFrI&gI zf&Ey1+71DFncTVHCoz^aS>blO2EkIsI)o$EY6f!9&6-` zKix{l+hn;ZlhvIFHePq}8=Rh;uu(KyP&tT}#W|bXghs@Y)&XTyK{t}PaSDaAiCus~ zA*haIGQ%rZ3(nN(x+JYjGJ7HW>KaYpR({`KIo{!E7ISjfVebght0vt0pMQx2tlt;mz!=_gTGOZ&!LQaJM(x2kRl>?0bVoUi)6AS$8Wx^*diI zo&THie^!@c0bQK`({G#q)0h8s_scIn@qhf*>!03~zg_;<-7mhl+s6O-;}@UuzkZW{ zqDzq|2`aF+Z5ks{)69w65)wXq^z_LKrva+AN)HNkt|D9tPT_nKnK-x?-2YXzQ)@>_ z9#oUL8!iNjX1Et76u^7W?;k#-lJsJ4l7|wUJ)vz}mC0uzzuJ9#w+#)u=icr<|Em2l zx**j%d*_$MYBfHtpy`fhXQBm(S(|exLHs6%wBUYZN2l|_|kPFdUZ~K6uwWM zeWgEspJgEAWb|Jg=5S(2Slfo0&)3C;C^>M3Z0Dk)xmk7t#pIq>%k4%Y%rDed@aT0= zT1ZGn)5qO>rL95XuV1GyFD&CIyZ`Kw*ducx--Mq>2*yLQiDW}<g~MD zT(_K=+;}4^FtjNk;AJqLuxW*A*>c;~<*8c{XIw`dm0;y!(|{0aUOIhkR?Z_P z>83s}sd<*d0ZjjxytlU0Nh;dp1%od)wmR!|Q*tb41&*hil&~xUcWxk)PV{VKK~cxP zsBL=$H->Fz=;Ta8Yh`-wCK+)jZSt?DE# zHko}P{?>5@^BEg_Iz8DsC05u?bw3tM_dXT@czhRU#-H(daiNq7Tv{B@>_?BBu_C6F z<_|j?2wu0+CkXqG{1YWo`Row5|8|=BrpRuLM&BhV>!ae$vIkF}eEaC%rr$mMj~C7W zVDQ`7xdRi^DxpVq83Zx?erTMx7u8(olutTd#BuCPOR_*>cUFC(l()LEV0DgjpKb3x zcwM%N!FERqT^E1XET#L~<@D5X?FqXTAOhH$iuQ|0fys z3pkJ(w_gFOYtB7Imu*IrZswqgQev1`eJ&vlx5r&e!pF+Y*RcrU!BAjc-5C-89!89y zsqdjBN2@=cF`0U)6S$IGX#x9Qzmcr62nM^zPZl+Om?~z(*L5PvSJiU=WC2!L`6{z$ ztqlnYK;Hi^`p-Cn=JEXDx6dEG`1k3fCx;K8|Ly)`fo2{5{>v=O{^GaufE!AEHFP!0 z_IND&!h4nBA0`*2dITqs0H!eUYJ(*8O3>K*xIDjQ@Gu4R<^!u=9GRXkG%Q$BeK9(rM(;iS}R#JOLM#xmQ z|5R(R2U%2*-R5o|X^bYy@nyf0&us!sTw(me`!F137*J@`FJgc=CAFkyhM zN#79JV3FbFmsNeK&js?n=@noAFIIo5J5~#)VVqksJQDlpl=ayKP)cyIT{<=LTrXs( zV3v4Aoh8wf$dtIyj+JCsT-Te8XubD(V2~MNL*PDLrIaUFl-2`VGjhhpN)cCi>l4+spt>_hR9p)|w&;%4;haUUW!0eP85Nt^~1;y8xZPC*MlQn*Ia ztu+isG})qIH@P3bd(YL9e%mP^H*Putsw(aW?&Fv8c>QFE;q_sIpW=){B7(~}=CC-IG)vJR58PvVTMVdfaC&&C zsLBk}4@IpR#&jbhfaAR@&P{i|i5MO?>Xo7B=bmu;X1`Y*YM88-KS79B1Oo1_5ByiZ z{8};$LfTTdj#q6~?asdM&G0>SZBw6nf1KPp!szT{w`^wT!s{vK!lYC3v!=nja6FZ? zUhQUe+uQVI+HX|*y<4Yuu`?obQlu5XQ*d->gsnkRYz zo_prBk9w(fCPzmH2S-Qa{I8cJuh}-+V&OjBZchGme*DGs&i}k~XIfO#S#@-zVRji4 z6qchSG7>dMM?S~k=xA-9(Pw3o)X*Uz)kok-(jr9xODDA0S~i=IO>8C&;>MS>;>8MJ z$n$CgKF(l=wp02f3p937dX_9nkYXZ#S%z>T4rZt;VN+oQNY8iHsd$6db?j7vrziBp z4eX*=&n-uQ8zyyTx$nhsgzQqjaIH^utvfMVv!Ft3)%7RL%f>0thTx-; zW-e%M7Qv0+YkEp}iKOK6v^277ubiSTRCH0J)v)a^yTA$d=)|bny;~8|lNzKT%kQ9# z$#%j?46=i>d7y8??dN`w2wFa(ITtB4)z7+UpenpW>mp}m)=cP{M<71pJ(AH*{RbcT zp0>OU0Og4^yDxM#OQp#Qha7_4ZB!l?IaWQbMSd`UKEg>FyKSmorgEoL=Zd^^oqwh4 zXI37q@2MH{sj0Wa%7c7$c7CzxG0HmT`fl?Kb|)Mhdf@)1IU#fPwH%k8x`ta(gZ1KZ zh1JEVG#sugY+1AFx(?I$5M!J9ieK(YHOfS^z%Hwq^;qa>;#*hZO`FqcAkE&;&T7wz zv$%Rc*Sy2^WlX|!U#u#Td9^a$F-HvZuaYkhvck%xgwqS7Y%>JOR(a|KjQjlsXx+F# z;#(Y7jijA!w20MRj!gjxv!Hi3Z3&qM?x*sSaYRE!kJSM?2wQ5GSt)*tAw2SY{j%Vk z)2EmZ4lh@wQds^?7Fz#1>3t-e!mE?kcD!8X^gr22Z`vl4Fwpk!t+oAie&^a!LCt|8qZJ#YatWP`2@UuW>us z@Xo*SyVkTbC5CCn@?s@UHI%lcnyWS4IXH8_^^^mr;r?*Ah={(7w)y?K~UE1AG zEr6N?P!E7_&}vK$oO+d7dJD(mnvhL|Oh$xpL3CC%$XRN;2lh<&JWn=`?ULBWRnMb( zJW2jHqQ{eeVE=5kt3~-zctYi9z4BYh$CYG*fuB{vk4@|Hzhb5PEh%Q+@) zJN9;Yv($WX9mJ{benXK|TmJCJ9`|mCC;E3d^Ngs4Bi~w|SUrXi&_mxj=RxePg*t7p zk`f}+4Cpt5AtZ1#SeS1<*rOuYs(QP#=7mwuy!r3^%mq<-_ZOQx1lM=YobN;3IWt8V zraT!{f|vTUasFa^ysw$M%#NVNAd}E)t=X=qX6zESx{F#ex9y+N14XN#<3bk|Gl*sR z5&YwmNW+NSOTeOe`6}7)gZ5n~jkwDWM|4P&B6Z@viJZ7^%!xbv=$o$X{CWA#b+hHb zM4x2wFgeZc8EvRUWd9%Ke>vMy(p-LV0I=)$-~afB4*%QT-+%Qf|LeEP|N7Badvhlk zxwVuaI1{N4l*^PD)W-4oT=xGup+v zl1wpK`pk-Wn0?Mt;fba@{HU2*O(V{6x|+=A2- z7NmKt>0pe2Azg->O>;iZq=s=ELBur0o3hE^qVhGeR>lM|6S;TdbgXhBu*b!cI~}>WxE1dkdHD2FLburdgD)3W$;H( z^OjNwG8kIjlnM{X>>w#K0)>dHtC-`>9yJD{#AG3*QtaNg#gLtnx&MH$M*OD*)0z+SH-|sp>~`5m0aMRq+6#e z)XQ-;eeiVpF+JJLh@D-+wa{JsxabK3qYXdmrwLIvw6hFDsc z?gn2=kc-|Z{r{J%@?~6S{)(ZmQ9d4zU-=yc@X${4d31WBBKbiAGI0GfI>4)+jK-Vm zejkq7fCNr_9q6i#V^w6p)foC%9!5Me@dG#oot-Sf8Q9$-zD~k-{s;ejV7>V{0X*H> z^yPJTe`*A2)kR#XUZqzQ^xu)Z%5&-*H0U z_l4Tb<-S=D^j$Z1TSWFhZM@$c|NG7#zxu;h9sKVv|M1EF|E=u*zgmG0l7qMnc!vU( z(26A@AHeI9)f)zm%&+zVA-AtKz(f({Qb6tq(b)YCP|8_=QXZKKh`e$rf!y30?obGD zh52y?k|1$a#BG+VMZ(?d4U^K^=BJ|8w_6aOEI|%y&CekMkuxH02A~ShZ$*tJ z0Obxx9*ehgHoVe3f@T`uE+-baUvJCVmgya4 z(^Dyu5^jPhAD3c=3iR?Q!UK*FPv)sy88RSJUw>WrD3m8 zySkSj9wLsFY1KSFrE2LUXLSr(8>i$dN-A`PH+S8}jjPSZhq{Lg!$xctwHv-q^7okQ z3M_Qu%w>%;W&r81gbHbgktr0>6G0ONajezYUpQ-P;Va5~vN4RrF`rRTY6Y7U+(d;A zZ2-|ZMyAd{aFkY}dJhyUe4VY_rJKn$^1TGLDV(^d{o#k#07<2in^F7or^eWw zf5Bp%_;*|y=&Je?=zQ@bZWB4p>L+L!d_>{0|4~lSziet#J;|-yL2``C&A?P=F6qnP z{qET)fBp)$=LBzhi4UMjhd)z=TBN24x1wJJItw55SsuS2{|g@XMSNQ8D*yh%2EyMP zV-v6f4e{GdtK}QP{T!g2Kp?dyqE03q>qgROnL|%&2UmQuL#in4X?N%hx>y%0@3z$< zO=<0()}M(tVAl{k5f;BHWAPER_kN<$U<$ZWRinR?B8ov;p38nKXFHIJ{r-I?? zO>IaK+L1_9gxZ_=Ol@}&I<5IlA%z(t&UHGGrYDX9 zEfWqmtFG5gOG@0F00~D@z*c3uo`kg|+)T^F!P)c{j^qLH7yhqy&4}R*2aB$8t)cWc z{MUySlEl-!_X~s~S5YW(^nU!!u1ISH zCWDD=U!ya?A6dVH;2S_t_h@<(PpeSBQF;JiN|#Hgrhw6bHvP>=7V?PHfJ_VF19B}W z@j|P-Hmaueb*`L?P3$zcJlv|OHQDi4(i0>}Cgis=^S~`~EYU3K^|j>@&$aFc#mIO; z+4$-~tXyF#C=sWuUrz7btP7DJp+8zUpEA*in~d4s2G6Z20Dgv2t+{jajayfq!Ewh0 zzsxPH8R0vJHX}oH^2Pm-NMwfbMhhVR-;R+ z>?3jJgC*gHqkePh_ZQA$&~2|CVw4aE6AOY6haTwtOD8L2Jn&n~n)eCrfHIj%2>)@A z6luXxM9YHG#ALG%ASQNxt760m0*SV|te4JGS-3vo^vF~-1N`TFd%kKQfQQLM>XZE1 zH!#Zm4)#QhtAgY#2ZOzlM8zgn{-nY}Rp_ykEl!pW^aW!`(Q>f2-JI5 z&EU64K8gH43e5jo`+vUt>W_E2{-3YzeDeSNR{o#E5L9mmb}R$lU>~949K>im#=X;l z(!_f%8vw1+Q$eER&mhtH=E9^ZfQB5GK7ZXBEG zp;+CCD3m7K^`i^j3~A^Fs6!uuK=i8rr|-KTzu!sg{oX)^V(c8j!_inNKnno75>Kw? zX;hngF(7BAJ~dW30A<$r$Kc#*wrQOfV2lQ;y$v*9Wu2eYU@;N-_ynRx*VSWHenX ztE-A2uoS}kO|h!+F+61fO}m*D3&!*^A4@pbOa?`10+i%ZbVPCW@c#F>7lz_VHKW?Z z$qXo-%Z>`CahK4g87Z$)2cK7-EcGBNsIe5V1>( zWys1R!5_y$-<+^R0}b}I!HugA?ig%mYivhC=~}tc-211ohcWqlmmd&t1q6;G6DBlu zNRM*HW`**vMERnU_PHHRtl(|coQ0$!&j6LFUa#Dghse+`_c^(MLP36(Y|mm{N%%c* z0$OMu>KJSm8V?>CGfz{CnlqX5rBmh{N zk)bobUSEc~sN0R9Rs(m*5~FpK*mhNrQg}j_mk}bZa%7#?PSt!L@aU zpyV52Mp)r28UQFN^=RX34TPZ@bv|%y7g^A-#vT&3iu#8Bm zWL_n0&@VcZZXgCH(ekal171#GSYR2`DH-Nb>8h-jw68gOMKmUsucTKIa~fT7w-aoo zXN--oXI3>Rdrgs&p2;YK`(G^Z>U3Lg8_R7e*2k4I%F|yP&eF3?GY5qprU)JbNg?8% zC(7;b2gyDi%1~{~xaA4OM?SMflI6JYD%eZ}$_;!^g>SUA<&b2|v??qV#5uB!&ZPwe zlZpem&{Y1}Mz7Zyz_&|CWK8dDfhd

XXdGg@L{^JFe1}-)FBUBcxn0tuN@Yd4bV_V3d)mEBC#yU?RXS63Wc zgDils##(t*ZnvDID>#W#ka{}fks&fle}Qq)ohCl)$M|fZHq0L2=KfVAj|CL{sYa+< zB8H2)UWv?EQDccYXi5i zvJk)lKjC$%N;RHsI*s;_uO zsFU@c{)j^W2^^k%v1qny%P|F@VW$*vtZdoyW?Hz@f-3VC2M*=zfdA8CzVX(ldo>>z zkwUcV2m(3)Kv|7e#BAn>CSWBNl@XB@dj` z8habb_m+dk%L7FW&k5c)b+W>A$9 z$l5_S%eeh;>w=ZOUy1V3*XMSljfjqDY9LD@@R;x<2-eDZKmKKcTQ=PB%YJ;!9Y4L5 zKUNn}utmv@x|ruJBcLlPV@L}aj+UV1?;x;C(>Q@VKV@RrIXxNU)(~(@&U><4T90|- zR|H8suTnan&g!lF1>+^6Gb+cEb#V&eZW!bWa6uKZY0ry@j;T^>9ax3U^t{{@qOMQT^kC1& zD))wCKHO@Ck;`k_2?5T~kY4rs`_W|yU4Q>0{zb^X{ucC1HkzLPpRHTVGy#PngT1iv z-t3VX@^JnJUzy+A;p zd(4Sy^)0a&5;5N-$(oE>$F)L1AF%pZs(uPGr{@kZPM$`CPd|OWO z59xeXh=V#)^P4y!Du#XmEoG8NWtH%tr$BQEQj42RjKaKmT`5DS>=*Diq_40T!zct( zs<3lexq}s@UC{AcnaA8b>QvcK^AdH)yj#}gaG%i|=+nHJYD2hhZQ01V0toRLe2>gD zK*VZUv*$qsJK-yW?xYfFSw&2^@lMvHuteWV%SW_;LUp!CYuq6kMTACQI;99hIk>V3 z*A07cClZ~#2CgxEdf3L;*3a7qoND#-@8Ohx-#OrxAJ?Na5k#e{5e<9i>WMMWD$pQl zzuXLbpN_asJ4@ONzx^Kv`Tzbtq;`tteBc3rJFN__cugOQYWqDw{_cK}PC#b?d|Nw~ zp!xeEECQ>+pkSmarLX$s?x86;<=dGvzN)B*GlFlT=OqZGhY~X=*6ZTZyzWpTBx%XD zC1?Mvz95#RcycVf%SadS2>cmVeGZX5kw0gyowbBF#bpuTVAb1Qteng$Xk>@eVsi9$ z6ZK$LR)}ur;Je|SM}Pg3r-xu@Pdak+^`G+h>`%XLwPUpH<^`CjS zL_dk6vY7-d8%afa6(J(Kes%JEQ~mFT8<=kzgWhF6IJpg)Q{{z=)r$9fIma~kY*4># z?dN8AdzduL4!+!LKe_iRM#}CUj1N8+Y4^M3Y3X=k{bt92^xU`+?Er?{F+7Eh-Z{P) z^EWz3`g2}OQX`z^!`Wwi<|+;Mc|KmudMAT-MACmzN)YLl(HA><7#|pM%xO>s7p?^Xy~U>>Q_%e^LZ{dLiPD+F5rg2RqR#E2#B9Vf zB*(gZyOP?_{xfWmhz`d8LXkSw?t7ri?Qy6MdE2u3$@ID9OxQV_sPADfm zJXvx=VB!7#C;S}M7fZC7fc9%cX(!{>dLKxEf zQ~7fnW1L1_*YxR#p!M?*8Z`8}0|iubDr34{ZYiv}uMe!h$m8XXc4tq4(W72Pjdj=U z7K`fCS)ifz;1>6eR$|=E0!wvIOOi}wCC{jneqHi@med@{rJ=GO5Tez*>uD;A2z$a| z!8lN?X*tv5&d<8*6Uc-=&+m@%JDs0|#nH+9kkycl?Dso%@nCnK4nprb4@qwiwuo8n z=-`2gywBZaydB&b-x=it>-@fUe%QfT99?nE3Q>m4(w{wkpQZ@4g;<{ijm{{lwc01# ze6%>W1VqF_J3ZK`;;_35EM3~w2Y%b%iECWw ziLdS3V$odZYzrHp->n|sC)P(qA>FRjzlZt30sfA=JM`N4AfBlt_Z{F;KdC zj$=P)tBtZ)OZKppN_d}eqp>$7gv4jJ@u4&fc9nKZ%(Ybis6pNqm-WesdY%nt1zD6% zy>r9gjIy?j5x;c#_N>@8u>aJYkT=2s1Rdlu&_k*dsnv|SbhH;fNrUnZDn=BC4}dQs z(poK9&JDB>%Zd?(#CS?pPAifEB9qLmHjv$g6s6i3aH3c_iCr2faD7}_$;P->BTDz2 zh?*TLNAl=Mm_`Rd9F9#g^&FgGOC$xwR5cW@lBV`o3_<>+z7WDV6f+B<v(;sL`SUKSRdVqa5?2+keZ@!BINGL~L zx5yyVzlRe{luQ}?pj58kCuc%!-BG>^9=&6)tyI=1j{vu39FPm7t01%z>2+CdT8?ay zYHLf&ToWgvEfGnZ!^-gJa?@4zQIDI$No$|9t3X z9M2?jSq^$_r?4pw5(PdO3fz62ABRtrrXWr2o3u2lripeJt+T%E02I1W6Pr)`}xSXHI1YGtd}hFHu^$+!@NDVgIY%76)Aq%XK$| zHneVH&qd;-()icUrXK1;sc+LfcxtG^UprMVznd(pRl20n&sfm|D|AANroeQl+J@#C$j;)Nkq+%y%IIW_XBGjMRGqM#slpe$L-dmi4E^;?Cu8l(qz=s|vs&X$~3{>I2d z3PlkM54U{*>5;ZKroZWJ--5ubM8c>4e9)r6`2`5d&~lfc%!2cNJN7{nQb0siTYzkT zmk!P~N9Ys*6rJ?Y+Yo7W=Ur`tWQq=s_a&RXr?t-hPs#sw6PNY0bptN5c>laiNAJna zLaK?R_K&OON!@}~iUjy)>q(2ikvV2CyxbIgw!W8YI4xF?os-Dvi`68(`+d3!ByX!i zI)z%$yEZmzQg8QLCpZpzDzu&ST}o;6UR37h8lnzb&il%;t(v;~7A?q@8JQMRCw*0y zNG}A*G_B@)6Rw#5U}u4J$GnrIp(u_f&i0n>JZhz*kF>aQLaLe$H+rw#h6*WjzXLrt z_}p}*{H;t3<~q`;gUh>WJOzT=|NCy+?>FUUl3H-o9M5Y&(ruD3JMT-_ffB9vDyJa! zHPi>-4xzB9V8i-ChdblMn$#Lo_N8jbI$27{9(eg}IAjI`fzh)GmzLypHM>*hWtQMz z(F6o)uYlXL3dDZZk*g~!^DF3&*>#rQ$At_!=HYbeUPt{i-|q=xa0RZ62#mKYsahNK zbb#>_#3}CVv`u3bH;y;&2(#WG_WTSUOIG00%^H~}Y}n326JzB^=iz?xOBTrZ;sb1^ zWvy;(!MH&NVo+ZbtB2&s+~Rn)+0JHVIghFfCfXfA=yx{k28H5l<{u_b5uX@dEaQ}L zW(rr(et9%`;RWw)>RSoEPK=)Q%f9j=k5vRVDq>8fJ zZ!>UvT$BmIFiCSZ;kcMCZjbK}jk>DMp7NmYy z=87mzO!-!D|73FA#j*hgvY2n&6M_@++KhXt?+gtA&IL3SfXms?FQdw*`Xuzn9@4u~ zB_Pon*XFTa)y-xxxx|<46&oc>)iBTFc6ewB0CgJ+z+q zIK@RGGOWxK2$J1b>y#Lz%hPhuDWD!sI{7J4{#Ut#d!u{xAXko{6kR`LPlDmjdpn4i z{;T+BDb&|EW+T(<0Auxk?r5g>))zhZsu;`7b%FMMt{Hk)26ZHRlbHr zZ+S&HxzxmqwN{t+G_>t6xNnd@q4A5t7to8xK>Ut94K?*7DW266vxMjh1^CP*IV<{_ z@I7(*>pAb7Vi^ox*7lFDo$hPZryVoSSxrEGc7i3)z1f?F5yZ?wYgLvu?yVJyPH7@&=RYs$vkVxHq zPL0rdqE10jdtRLCfap~r)H+0pW9IUzc6-n5{eDWLre}U_F{I?kUuhc4kj|Z&I~R6a zS+)md!I=SFV#9kKSU4_Fs01738U|=@zKD8mA@Maq$+DfsYf_82{9|st$JfjE-pukI z(Ku+M&Gby-@K+F-SD8d@b0|IH3{`(N59nkYv-GhSTP{|2$9zT}3uLr-D^L5-2^ zE)Itw0s%K*8|})6farE|OX_xpFQac)`G!kSRiu`NZ(H(bD*Dgy5<`=8y#K$D3 z2vZUc?+8_r0Cr(c=p{^HZ;;y3pjO-y@7mSLAfxWvNR&Q^Kz_KS1ZE2KmG1FL1WbuZ zyeQn;!>LJ=;VmhpLb6T2q3|JsFhEm8^6sMr+-?arn@ZS=sM!CH3%6?pO7ztSCig8% zYQ;fV^6Cn?JNbp2pzwx!L8@elPt>xKeSt;a`Bcy?)i>Qamek%Tok$$rDEwNsbl%25 zHHkuRfKn|RBr-P=OhaRbdW`}XaXM=j2TreOr%5^k?f{(4W=DRI*#zDmZkY;Z8ISTi zFaobcNbyA<_Cf4P^KbgPZ$fWWUC~l&Zh@w34MpCQ3Xl7{hX=aptr&g{MWi&1pZfOi%5{hlrlf=5{sfRcPBS_C{NeS}GW~ zy~nHUdbg8o`fKHut=i8JO`MXCkW2P%_e6g;K3*dI)2Dx*{(buQ>EEY+pZ None: + """ + Record that the running test set `level` on a session, or in OPTIONS, + where a later checkout from Django's pool would still find it. + + Unpooled, the close() that ends such a test ends the session and the + driver connection with it. Pooled, close() hands the connection back + as it is: SET SESSION CHARACTERISTICS stays on the server session, and + the isolation_level Django put on the driver connection for a changed + OPTIONS stays on the connection, which Django does not reset when + OPTIONS name no level. The conftest renews the pools after the test. + """ + CHANGED.append(level) + + +def renew_pools() -> None: + """ + Close Django's connection pools, so every later checkout is a session + opened from the settings in force then, by a pool Django builds anew. + + Closing, not draining: a drained pool replaces its connections in the + background, and one it was already opening lands in the pool after the + drain with the session it started with. A closed pool closes those as + they arrive, and each checked-out connection when it comes back. + """ + for alias in connections: + connection = connections[alias] + if getattr(connection, "pool", None) is not None: + connection.close_pool() + + +@contextmanager +def callers_transaction(how: str) -> Iterator[None]: + """ + A transaction the caller opens and ends, around the block. "atomic" is an + atomic block. "autocommit-off" turns autocommit off on this thread's + connection, commits when the block ends, and turns it back on. + """ + if how == "atomic": + with transaction.atomic(): + yield + return + assert how == "autocommit-off", how + transaction.set_autocommit(False) + try: + yield + transaction.commit() + finally: + transaction.rollback() + transaction.set_autocommit(True) + + +def pg_sqlstate(exc: BaseException) -> str | None: + """The SQLSTATE behind a Django OperationalError, from psycopg 3 or psycopg2.""" + cause = exc.__cause__ + return getattr(cause, "sqlstate", None) or getattr(cause, "pgcode", None) + + +def result_fields(result): + """What a caller can read off a TaskResult, as one comparable tuple.""" + return ( + result.id, + result.status, + result.enqueued_at, + result.started_at, + result.last_attempted_at, + result.finished_at, + result.args, + result.kwargs, + result.backend, + result.worker_ids, + [(error.exception_class_path, error.traceback) for error in result.errors], + result.task.module_path, + result.task.queue_name, + result.task.priority, + result.task.run_after, + ) + + +def set_session_isolation(level: str) -> None: + """ + This thread's connection runs the transactions it opens next at + `level`. The caller closes the connection afterwards, so the next use + reopens it at the configured level. + """ + connection = connections["default"] + if connection.vendor == "postgresql": + statement = ( + f"SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL {level}" + ) + elif connection.vendor == "mysql": + statement = f"SET SESSION TRANSACTION ISOLATION LEVEL {level}" + else: + raise AssertionError(f"{connection.vendor} has no session isolation level") + outlives_the_transaction(level) + with connection.cursor() as cursor: + cursor.execute(statement) + + +def set_transaction_isolation(level: str) -> None: + """ + The open transaction runs at `level`. PostgreSQL accepts this only as the + transaction's first statement, and it lasts for that transaction alone. + """ + with connections["default"].cursor() as cursor: + cursor.execute(f"SET TRANSACTION ISOLATION LEVEL {level}") + + +def isolation_for_new_connections(monkeypatch: Any, level: str) -> None: + """ + Every connection opened from here on runs at `level`, as it does for a + project that sets OPTIONS["isolation_level"]. A connection already open + keeps its level. + """ + connection = connections["default"] + if connection.vendor == "postgresql": + from django.db.backends.postgresql.psycopg_any import IsolationLevel + + value: Any = IsolationLevel[level.replace(" ", "_")] + else: + value = level.lower() + outlives_the_transaction(level) + monkeypatch.setitem(connection.settings_dict["OPTIONS"], "isolation_level", value) + + +def session_isolation() -> str: + """The level this thread's connection runs at, spelled as SQL spells it.""" + connection = connections["default"] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SHOW transaction_isolation") + else: + cursor.execute("SELECT @@transaction_isolation") + (level,) = cursor.fetchone() + return str(level).upper().replace("-", " ") + + +@contextmanager +def opened_connections() -> Iterator[list[Any]]: + """ + Every connection created through the handler while the block runs, as + the wrappers themselves. The handler is shared by every thread: enter + this where the threads that run meanwhile already have their own + connections, or those are recorded too. + """ + opened: list[Any] = [] + create_connection = connections.create_connection + + def create(alias: str) -> Any: + wrapper = create_connection(alias) + opened.append(wrapper) + return wrapper + + connections.create_connection = create + try: + yield opened + finally: + connections.create_connection = create_connection + + +@contextmanager +def how_new_connections_run() -> Iterator[list[dict[str, Any]]]: + """ + For every PostgreSQL connection that connects while the block runs, the + level a statement in autocommit runs at, whether autocommit is on, and + the OPTIONS it was built from. Read on the driver's own cursor, so + Django records nothing. + """ + seen: list[dict[str, Any]] = [] + + def record(sender: Any, connection: Any, **kwargs: Any) -> None: + if connection.vendor != "postgresql": + return + with connection.connection.cursor() as cursor: + cursor.execute("SHOW transaction_isolation") + (level,) = cursor.fetchone() + seen.append( + { + "level": str(level).upper(), + "autocommit": connection.get_autocommit(), + "options": dict(connection.settings_dict.get("OPTIONS", {})), + } + ) + + connection_created.connect(record) + try: + yield seen + finally: + connection_created.disconnect(record) + + +# The level, lock_timeout and statement_timeout in force on a PostgreSQL +# session, as current_setting() spells them. +SESSION_SETTINGS = ( + "SELECT current_setting('transaction_isolation'), " + "current_setting('lock_timeout'), current_setting('statement_timeout')" +) + + +def settings_of(wrapper: Any) -> tuple[str, ...]: + """SESSION_SETTINGS on the wrapper's driver connection, which Django won't log.""" + with wrapper.connection.cursor() as cursor: + cursor.execute(SESSION_SETTINGS) + return tuple(str(value) for value in cursor.fetchone()) + + +def new_session_settings() -> tuple[str, ...]: + """SESSION_SETTINGS on a new connection built from the current OPTIONS.""" + probe = connections.create_connection("default") + # Outside Django's pool, as the enqueue's own read is: a pooled checkout + # is a session the pool opened earlier, from the OPTIONS it was made with. + options = probe.settings_dict.get("OPTIONS", {}) + probe.settings_dict = { + **probe.settings_dict, + "OPTIONS": {name: value for name, value in options.items() if name != "pool"}, + } + try: + probe.ensure_connection() + return settings_of(probe) + finally: + probe.close() + + +@contextmanager +def fresh_read_transactions() -> Iterator[list[dict[str, Any]]]: + """ + For every PostgreSQL connection created through the handler while the + block runs: the first word of each statement it ran, SESSION_SETTINGS as + they stood when it first ran a SELECT, and as they stood after a COMMIT. + """ + seen: list[dict[str, Any]] = [] + create_connection = connections.create_connection + + def create(alias: str) -> Any: + wrapper = create_connection(alias) + if wrapper.vendor != "postgresql": + return wrapper + record: dict[str, Any] = {"statements": [], "at_read": None, "after": None} + seen.append(record) + + # Django calls an execute wrapper with these five, positionally. + def recording( + execute: Any, sql: str, params: Any, many: Any, context: Any + ) -> Any: + word = sql.split()[0].upper() + if word == "SELECT" and record["at_read"] is None: + record["at_read"] = settings_of(wrapper) + result = execute(sql, params, many, context) + record["statements"].append(word) + if word == "COMMIT": + record["after"] = settings_of(wrapper) + return result + + wrapper.execute_wrappers.append(recording) + return wrapper + + connections.create_connection = create + try: + yield seen + finally: + connections.create_connection = create_connection + + +def record_holder_checks(monkeypatch: Any) -> list[str | None]: + """ + What each check of a holder row answered, in order: "granted", or the + SQLSTATE it was refused with. A test that expects a refusal asserts it + was met, so a run that never provoked one cannot pass. + """ + answers: list[str | None] = [] + lock = getattr(OxpullBackend, "_lock_holder_nowait", None) + if lock is None: + # A version with no holder check records nothing. + return answers + + def recording(task_pk: Any, db_alias: str) -> None: + try: + lock(task_pk, db_alias) + except OperationalError as exc: + answers.append(pg_sqlstate(exc)) + raise + answers.append("granted") + + monkeypatch.setattr(OxpullBackend, "_lock_holder_nowait", staticmethod(recording)) + return answers diff --git a/tests_oxpull/policy_tasks.py b/tests_oxpull/policy_tasks.py new file mode 100644 index 0000000..19d86e7 --- /dev/null +++ b/tests_oxpull/policy_tasks.py @@ -0,0 +1,146 @@ +""" +Tasks that declare django-ox's per-task policy, for tests_oxpull/test_policy.py. + +A module of their own because Django 6.0's ``task()`` takes no keyword +arguments beyond its own: ``@task(max_attempts=...)`` raises TypeError there +at import, and tests_oxpull/tasks.py is imported by nearly every test. On 6.0 each +task is built with ``dataclasses.replace`` on a bare ``@task`` instead. That +binds the same PolicyTask to the module-level name the worker imports, so the +worker's handling is exercised on both supported versions, while the +decorator spelling is exercised where it exists. +""" + +import dataclasses +import time +from datetime import timedelta + +import django +from django.tasks import task + +from django_ox import remaining +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.unique import unique + +from .tasks import STATE, _record + +#: Whether @task forwards keyword arguments to the backend's task class. +#: Django 6.1 does; 6.0, the other version this package supports, does not. +DECORATOR_TAKES_POLICY = django.VERSION >= (6, 1) + + +def policy_task(**declared): + """``@task(**declared)`` where the decorator takes policy, else the same task.""" + + def wrap(func): + if DECORATOR_TAKES_POLICY: + return task(**declared)(func) + routing = { + name: declared.pop(name) + for name in ("backend", "queue_name") + if name in declared + } + return dataclasses.replace(task(**routing)(func), **declared) + + return wrap + + +def note(**fields): + STATE.setdefault("notes", []).append(fields) + + +def _busy(seconds): + """Pure Python, so an injected TaskTimeout lands inside it.""" + end = time.monotonic() + seconds + while time.monotonic() < end: + pass + + +# -- backoff callbacks --------------------------------------------------- + + +def retry_now(exc, task_result): + return 0 + + +def stop_retrying(exc, task_result): + # The stored budget is the row's; task_result.task is the declaration. + note( + callback="stop", + attempts=task_result.attempts, + max_attempts=OxTask.objects.get(pk=task_result.id).max_attempts, + ) + + +def wait_an_hour(exc, task_result): + note(callback="hour", attempts=task_result.attempts) + return timedelta(hours=1) + + +# -- the default backend --------------------------------------------------- + + +@unique +@policy_task(max_attempts=5) +def unique_with_budget(value): + return value + + +@policy_task(max_attempts=5, backoff=retry_now, timeout=30) +def declares_everything(value): + return value + + +@policy_task(max_attempts=4, backoff=retry_now) +def fails_with_budget_four(): + raise ValueError("four") + + +@policy_task(max_attempts=6) +def callback_with_budget(label): + STATE[label] = STATE.get(label, 0) + 1 + return label + + +@policy_task(timeout=8) +def reconciles_under_its_own_timeout(): + """ + One tick of each reconciler inside an attempt that has a timeout of its + own, and what the workflow backstop budgeted from it. + """ + STATE["backstop_budget"] = workflows._backstop_budget() + STATE["reconciled"] = [batches._reconcile(), workflows._reconcile()] + STATE["remaining_after"] = remaining() + + +# -- workflow nodes, on "flows" -------------------------------------------- + + +@policy_task(backend="flows", max_attempts=5) +def node_with_budget(label): + return _record(label) + + +@policy_task(backend="flows", max_attempts=5, backoff=retry_now) +def failing_node_with_budget_five(label): + _record(label) + raise ValueError(f"{label} boom") + + +@policy_task(backend="flows", max_attempts=5, backoff=stop_retrying) +def failing_node_that_stops(label): + _record(label) + raise ValueError(f"{label} boom") + + +@policy_task(backend="flows", max_attempts=3, backoff=wait_an_hour) +def failing_node_that_waits_an_hour(label): + _record(label) + raise ValueError(f"{label} boom") + + +@policy_task(backend="flows", max_attempts=2, timeout=1) +def node_past_its_timeout(label, seconds): + _record(label) + _busy(seconds) + return label diff --git a/tests_oxpull/racing.py b/tests_oxpull/racing.py new file mode 100644 index 0000000..7592b44 --- /dev/null +++ b/tests_oxpull/racing.py @@ -0,0 +1,84 @@ +""" +Running one callable on many threads at once. + +Shared by the unique and batch race tests: both prove the same class of +property (a rowcount, not a check-then-act, picks the single winner), so +they share the harness rather than two copies of it. +""" + +import threading +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +THREADS = 8 + + +@contextmanager +def pool_for(threads: int) -> Iterator[None]: + """ + On a pooled database, room in this process's pool for `threads` + connections at once beside the test's own. + + Each thread here stands for a process or a request with a connection of + its own. These threads share one process's pool instead, so it is sized + for them, as a worker's pool is sized for its threads (django-ox + docs/production.md). A smaller pool makes a thread wait for a + connection rather than for the lock or row the test is about. + """ + from django.db import connections + + pool = getattr(connections["default"], "pool", None) + if pool is None or pool.max_size >= threads + 1: + yield + return + sizes = (pool.min_size, pool.max_size) + pool.resize(pool.min_size, threads + 1) + try: + yield + finally: + pool.resize(*sizes) + + +def race( + target: Callable[[], Any], threads: int = THREADS +) -> tuple[list[Any], list[BaseException]]: + """The same callable on `threads` threads at once.""" + return race_all([target] * threads) + + +def race_all( + targets: list[Callable[[], Any]], +) -> tuple[list[Any], list[BaseException]]: + """ + Call each target on its own thread, all released from one barrier, + and return (results, errors). Used with django_db(transaction=True) + so each thread's connection sees committed state, mirroring real + worker processes. + """ + from django.db import connections + + threads = len(targets) + barrier = threading.Barrier(threads) + results: list[Any] = [] + errors: list[BaseException] = [] + + def run(target: Callable[[], Any]) -> None: + try: + barrier.wait(timeout=10) + results.append(target()) + except BaseException as exc: + errors.append(exc) + finally: + # Threads get their own connections; close them like the + # worker's pool threads do. + connections.close_all() + + workers = [threading.Thread(target=run, args=(target,)) for target in targets] + with pool_for(threads): + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=30) + assert not worker.is_alive() + return results, errors diff --git a/tests_oxpull/routers.py b/tests_oxpull/routers.py new file mode 100644 index 0000000..89bf22d --- /dev/null +++ b/tests_oxpull/routers.py @@ -0,0 +1,28 @@ +"""Routers used only by tests, activated via override_settings.""" + + +class SplitTaskRouter: + """Sends django_ox writes to another database; oxpull must refuse.""" + + def db_for_write(self, model, **hints): + if model._meta.app_label == "django_ox": + return "tasksdb" + return None + + +class ReplicaReadRouter: + """Sends reads for both apps to the (empty) replica database.""" + + def db_for_read(self, model, **hints): + if model._meta.app_label in {"django_ox", "oxpull"}: + return "replica" + return None + + +class SplitRateWindowRouter: + """Sends the rate window to another database; oxpull must refuse.""" + + def db_for_write(self, model, **hints): + if model._meta.label == "oxpull.OxpullRateWindow": + return "tasksdb" + return None diff --git a/tests_oxpull/settings.py b/tests_oxpull/settings.py new file mode 100644 index 0000000..0360643 --- /dev/null +++ b/tests_oxpull/settings.py @@ -0,0 +1,142 @@ +import os +from pathlib import Path +from typing import Any + +BASE_DIR = Path(__file__).resolve().parent + +SECRET_KEY = "test-only" +USE_TZ = True +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +INSTALLED_APPS = [ + "django_ox", + "oxpull", +] + +# File-based SQLite, mirroring django-ox's test settings: threads in the +# concurrency tests need their own connections, and Django's shared-cache +# in-memory test database raises "database table is locked" across +# threads regardless of the busy timeout. +DATABASES: dict[str, dict[str, Any]] = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + "OPTIONS": {"timeout": 20}, + "TEST": {"NAME": BASE_DIR / "test.sqlite3"}, + }, + # A deliberately separate, never-written database. Tests activate a + # read router pointing at it to prove the dedup path pins its reads + # to the write database (a lagged replica must not fake staleness). + "replica": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "replica.sqlite3", + "OPTIONS": {"timeout": 20}, + "TEST": {"NAME": BASE_DIR / "test_replica.sqlite3"}, + }, +} + +# Two runs of one leg against the same server must not share a test +# database: the second drops the first one's out from under it, or fails +# to create one that is already in use. Given OX_TEST_DB_NAME, the default +# alias tests against exactly that name and the replica against that name +# with "_replica" appended, so concurrent runs stay apart and the replica +# stays the separate, never-written database the routing tests need. The +# default alias takes the name as NAME as well, which is how django-ox +# hands a test database to a process that does not run the test runner. +# With the variable unset, which is how CI runs, every name is the fixed +# one it has always been. +if "OX_TEST_DB_NAME" in os.environ: + DATABASES["default"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["default"]["TEST"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["replica"]["TEST"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + "_replica" + +TASKS = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", "emails"], + "OPTIONS": { + "MAX_ATTEMPTS": 3, + "UNIQUE_TASKS": ["tests_oxpull.tasks.configured_unique"], + # Batches and workflows refuse to be created or sealed unless + # some django-ox backend schedules their reconciler, as a + # project must. The test workers call run_once(), which never + # dispatches a schedule, so these tick nothing here. + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, + }, + }, + }, + # A non-django-ox backend, so the batch guard against members that + # never get an OxTask row has something real to refuse. + "immediate": {"BACKEND": "django.tasks.backends.immediate.ImmediateBackend"}, + # Workflows are declared per backend, and the default backend stays + # free of them on purpose: the rate-limit tests pin that a backend + # configuring neither feature emits exactly the base worker's claim SQL, + # and a gate on "default" would quietly break that promise. + "flows": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", "emails"], + "OPTIONS": { + "MAX_ATTEMPTS": 3, + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "WORKFLOWS": True, + }, + }, + # Workflows with the blocked-node deferral turned on. A separate + # alias on purpose: "flows" stays the default-shaped backend, so the + # matched gated/ungated pair that proves the claim gate is + # load-bearing keeps measuring the gate and not the deferral. + "flows_deferred": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default", "emails"], + "OPTIONS": { + "MAX_ATTEMPTS": 3, + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "WORKFLOWS": True, + "DEFER_BLOCKED_NODES": True, + }, + }, + # Declares workflows but names no gating worker: oxpull.E007. + "flows_ungated": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": {"WORKFLOWS": True}, + }, + # Only a worker for the backend that configures a limit applies it, + # though its window is shared by name. This one carries a limit so + # that "default" can stay genuinely limit-free, which is what the + # zero-cost test measures against. + "limited": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": { + "hourly": { + "limit": 1, + "per": 3600, + "tasks": [ + "tests_oxpull.tasks.limited_echo", + "tests_oxpull.tasks.limited_add", + ], + }, + }, + }, + }, +} + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": {"console": {"class": "logging.StreamHandler"}}, + "loggers": { + "django_ox": {"handlers": ["console"], "level": "WARNING"}, + }, +} diff --git a/tests_oxpull/settings_mysql.py b/tests_oxpull/settings_mysql.py new file mode 100644 index 0000000..38dde48 --- /dev/null +++ b/tests_oxpull/settings_mysql.py @@ -0,0 +1,60 @@ +import os + +from .settings import * # noqa: F403 + +# Mirrors django-ox's tests/settings_mysql.py so the same disposable +# server serves both suites: +# docker run -d --name ox-mysql -e MYSQL_ROOT_PASSWORD=ox \ +# -e MYSQL_DATABASE=oxtest -p 33069:3306 mysql:8 +# +# Django's MySQL backend wants mysqlclient; PyMySQL works once registered +# under the same name. +try: # pragma: no cover - environment shim, not behaviour + import MySQLdb # noqa: F401 +except ModuleNotFoundError: # pragma: no cover + try: + import pymysql + + pymysql.install_as_MySQLdb() + except ModuleNotFoundError: + pass + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oxtest", + "USER": "root", + "PASSWORD": "ox", + "HOST": "127.0.0.1", + "PORT": "33069", + "OPTIONS": { + "init_command": "SET sql_mode='STRICT_TRANS_TABLES'", + "charset": "utf8mb4", + }, + "TEST": {"CHARSET": "utf8mb4", "COLLATION": "utf8mb4_unicode_ci"}, + }, + "replica": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oxtest", + "USER": "root", + "PASSWORD": "ox", + "HOST": "127.0.0.1", + "PORT": "33069", + "OPTIONS": { + "init_command": "SET sql_mode='STRICT_TRANS_TABLES'", + "charset": "utf8mb4", + }, + "TEST": { + "NAME": "test_ox_replica", + "CHARSET": "utf8mb4", + "COLLATION": "utf8mb4_unicode_ci", + }, + }, +} + +# Scoped to OX_TEST_DB_NAME exactly as tests_oxpull/settings.py is, and for the +# reason given there. +if "OX_TEST_DB_NAME" in os.environ: + DATABASES["default"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["default"].setdefault("TEST", {})["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["replica"]["TEST"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + "_replica" diff --git a/tests_oxpull/settings_postgres.py b/tests_oxpull/settings_postgres.py new file mode 100644 index 0000000..d3696c6 --- /dev/null +++ b/tests_oxpull/settings_postgres.py @@ -0,0 +1,33 @@ +import os + +from .settings import * # noqa: F403 + +# Mirrors django-ox's tests/settings_postgres.py so the same disposable +# server serves both suites: +# docker run -d --name ox-pg -e POSTGRES_PASSWORD=ox -p 54329:5432 postgres:16 +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "postgres", + "USER": "postgres", + "PASSWORD": "ox", + "HOST": "127.0.0.1", + "PORT": "54329", + }, + "replica": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "postgres", + "USER": "postgres", + "PASSWORD": "ox", + "HOST": "127.0.0.1", + "PORT": "54329", + "TEST": {"NAME": "test_ox_replica"}, + }, +} + +# Scoped to OX_TEST_DB_NAME exactly as tests_oxpull/settings.py is, and for the +# reason given there. +if "OX_TEST_DB_NAME" in os.environ: + DATABASES["default"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["default"].setdefault("TEST", {})["NAME"] = os.environ["OX_TEST_DB_NAME"] + DATABASES["replica"]["TEST"]["NAME"] = os.environ["OX_TEST_DB_NAME"] + "_replica" diff --git a/tests_oxpull/settings_postgres_pooled.py b/tests_oxpull/settings_postgres_pooled.py new file mode 100644 index 0000000..93c660d --- /dev/null +++ b/tests_oxpull/settings_postgres_pooled.py @@ -0,0 +1,7 @@ +from .settings_postgres import * # noqa: F403 + +# PostgreSQL through Django's connection pool, at psycopg_pool's defaults, on +# both aliases. Worker threads, the reconciler and the enqueue path in one +# process then share a pool, as they do in a project that turns it on. +for _alias in ("default", "replica"): + DATABASES[_alias]["OPTIONS"] = {"pool": True} # noqa: F405 diff --git a/tests_oxpull/subprocesses.py b/tests_oxpull/subprocesses.py new file mode 100644 index 0000000..2abd1de --- /dev/null +++ b/tests_oxpull/subprocesses.py @@ -0,0 +1,17 @@ +"""Environment helpers for tests that start another Python process.""" + +from pathlib import Path + +import django_ox + + +def django_ox_path() -> str: + """ + The directory this process imports django_ox from. + + A child process given a PYTHONPATH of its own otherwise falls back to + whichever django-ox the interpreter has installed, which need not be the + one this process runs against. Put this on the child's PYTHONPATH when it + runs the current oxpull source. + """ + return str(Path(django_ox.__file__).resolve().parent.parent) diff --git a/tests_oxpull/tasks.py b/tests_oxpull/tasks.py new file mode 100644 index 0000000..5af821c --- /dev/null +++ b/tests_oxpull/tasks.py @@ -0,0 +1,172 @@ +"""Module-level task functions; django.tasks requires module-level definitions.""" + +import time + +from django.tasks import task + +from oxpull.unique import unique + +# Mutable per-test state, reset by the `task_state` fixture. +STATE: dict[str, object] = {} + + +@task +def plain_echo(value): + return value + + +@unique +@task +def unique_echo(value): + return value + + +@unique +@task +def unique_add(a, b): + return a + b + + +@unique +@task +def unique_fail_always(): + raise ValueError("boom") + + +@unique +@task +def unique_flaky(succeed_on): + calls = STATE.get("flaky_calls", 0) + 1 + STATE["flaky_calls"] = calls + if calls < succeed_on: + raise RuntimeError(f"failing on call {calls}") + return calls + + +# Not decorated with @unique; made unique via OPTIONS["UNIQUE_TASKS"] in +# tests_oxpull/settings.py. +@task +def configured_unique(value): + return value + + +@task +def plain_fail(): + raise ValueError("boom") + + +@task +def plain_flaky(succeed_on): + calls = STATE.get("plain_flaky_calls", 0) + 1 + STATE["plain_flaky_calls"] = calls + if calls < succeed_on: + raise RuntimeError(f"failing on call {calls}") + return calls + + +@task +def record_callback(label): + """Batch callback target; records that it ran when a worker runs it.""" + STATE[label] = STATE.get(label, 0) + 1 + return label + + +# Runs at enqueue time on the immediate backend, so it never writes an +# OxTask row: batch membership must refuse it. +@task(backend="immediate") +def immediate_echo(value): + return value + + +# The same, but it records that it ran, so a test can prove a batch +# callback on this backend never executes its side effect. +@task(backend="immediate") +def immediate_record(label): + STATE[label] = STATE.get(label, 0) + 1 + return label + + +# On a second queue, so a limit attached to one queue has another queue to +# leave alone. +@task(queue_name="emails") +def emails_echo(value): + return value + + +# On the "limited" backend, whose OPTIONS carry a rate limit covering both +# of these. The default backend has none, which is what the zero-cost test +# measures against. +@task(backend="limited") +def limited_echo(value): + return value + + +@task(backend="limited") +def limited_add(a, b): + return a + b + + +# -- workflow nodes ---------------------------------------------------- +# +# On the "flows" backend, whose OPTIONS declare WORKFLOWS and name the +# worker that gates a claim on its dependencies. + + +def _record(label): + """Append `label` to the recorded execution order.""" + order = STATE.setdefault("order", []) + order.append(label) # type: ignore[attr-defined] + return label + + +@task(backend="flows") +def step(label): + return _record(label) + + +@task(backend="flows") +def slow_step(label, seconds=0.2): + time.sleep(seconds) + return _record(label) + + +@task(backend="flows") +def failing_step(label): + _record(label) + raise ValueError(f"{label} boom") + + +@task(backend="flows") +def wedged_step(label, seconds=30): + """Holds its lease long enough for the reaper to take it away.""" + _record(label) + time.sleep(seconds) + return label + + +# On the ungated backend, so add() has something real to refuse. +@task(backend="flows_ungated") +def ungated_step(label): + return _record(label) + + +@task +def default_step(label): + return _record(label) + + +# Workflow nodes on the backend that also defers blocked nodes +# (OPTIONS["DEFER_BLOCKED_NODES"]). A separate alias rather than a +# setting override, because the gate and the deferral are chosen per +# backend at worker construction and the tests need both live at once. + + +@task(backend="flows_deferred") +def deferred_step(label): + return _record(label) + + +@task(backend="flows_deferred") +def failing_deferred_step(label): + _record(label) + raise ValueError(f"{label} boom") diff --git a/tests_oxpull/test_batches.py b/tests_oxpull/test_batches.py new file mode 100644 index 0000000..b9d3f4e --- /dev/null +++ b/tests_oxpull/test_batches.py @@ -0,0 +1,2353 @@ +""" +Batch creation, membership, sealing, progress and reconciliation. + +The load-bearing tests are the ones that prove state is derived, not +counted: a member the reaper settles emits no signal at all and the batch +must still complete, and a member whose task row was pruned or whose +attempt went unobserved must read as finished without being called a +success or a failure. +""" + +import copy +import dataclasses +import logging +import pickle +import subprocess +import sys +import threading +import time +import warnings +from contextlib import contextmanager +from datetime import timedelta + +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.management import call_command +from django.db import IntegrityError, OperationalError, connections, transaction +from django.db.models import F +from django.tasks import TaskResultStatus +from django.tasks.signals import task_enqueued, task_finished +from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from django_ox.results import public_status +from django_ox.schedules import schedules_from_options +from oxpull import batches +from oxpull import unique as unique_module +from oxpull.backend import OxpullBackend +from oxpull.exceptions import BatchMemberExists, BatchSealed +from oxpull.models import OxpullBatch, OxpullBatchMember +from oxpull.routing import oxpull_db_alias + +from . import isolation, tasks +from .racing import THREADS, race, race_all + +pytestmark = pytest.mark.django_db + +RECONCILE_SCHEDULE = { + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + } + } +} + + +def reconcile(): + """Run one reconciler pass inline, as a scheduled worker would.""" + return batches.reconcile.call() + + +def stored(batch): + return OxpullBatch.objects.get(pk=batch.id) + + +# -- shape and configuration ------------------------------------------- + + +def test_create_returns_an_open_batch(): + batch = batches.create("nightly-invoices") + assert batch.name == "nightly-invoices" + assert batch.status == OxpullBatch.Status.OPEN + assert stored(batch).total_count is None + assert stored(batch).failed_count is None + + +def test_get_returns_the_same_batch(): + batch = batches.create("fetch") + assert batches.get(batch.id).id == batch.id + assert batches.get(str(batch.id)).name == "fetch" + + +def test_get_raises_for_an_unknown_batch(): + with pytest.raises(OxpullBatch.DoesNotExist): + batches.get("6c1b3f52-0000-4000-8000-000000000000") + + +def test_over_long_name_is_rejected_identically_on_every_database(): + with pytest.raises(ValueError, match="128 characters"): + batches.create("x" * 129) + + +def test_callback_that_does_not_import_fails_at_create(): + with pytest.raises(ImproperlyConfigured): + batches.create("bad", on_complete="tests_oxpull.tasks.nope") + + +def test_callback_that_is_not_a_task_fails_at_create(): + with pytest.raises(ImproperlyConfigured): + batches.create("bad", on_success="tests_oxpull.tasks.STATE") + + +def test_callback_arguments_are_stored_normalised(): + batch = batches.create( + "args", + on_complete=tasks.record_callback, + on_complete_args=("done",), + ) + row = stored(batch) + assert row.on_complete_path == "tests_oxpull.tasks.record_callback" + assert row.on_complete_args == ["done"] + assert row.on_complete_kwargs == {} + + +def test_reconcile_is_schedulable_from_backend_options(): + schedules = schedules_from_options(RECONCILE_SCHEDULE, "default") + assert [schedule.name for schedule in schedules] == ["oxpull-batch-reconcile"] + assert schedules[0].task.module_path == "oxpull.batches.reconcile" + + +# A project whose QUEUES omits "default": the reconciler Task must still +# be constructible, because constructing it is what importing +# oxpull.batches used to do. +QUEUES_WITHOUT_DEFAULT = """ +import django +from django.conf import settings + +settings.configure( + INSTALLED_APPS=["django_ox", "oxpull"], + DATABASES={"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}}, + TASKS={ + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["emails", "reports"], + } + }, + USE_TZ=True, + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", +) +django.setup() + +from django_ox.schedules import schedules_from_options + +import oxpull.batches # must not raise + +def queue_for(config): + options = {"SCHEDULES": {"oxpull-batch-reconcile": config}} + return schedules_from_options(options, "default")[0].task.queue_name + +print(oxpull.batches.reconcile.queue_name) +print(queue_for({"task": "oxpull.batches.reconcile", "cron": "* * * * *"})) +print( + queue_for( + { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + "queue_name": "reports", + } + ) +) +""" + + +def test_reconcile_is_importable_when_queues_omit_default(): + """ + Repro in a fresh interpreter, because the failure was at import + time. A module-level @task is validated against its backend the + moment it is built, so `import oxpull.batches` raised InvalidTask + ("Queue 'default' is not valid for backend") in any project whose + QUEUES omits "default" -- naming neither oxpull nor batches, and + before the schedule's own queue_name could apply. + """ + result = subprocess.run( # noqa: S603 + [sys.executable, "-c", QUEUES_WITHOUT_DEFAULT], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + # The first configured queue by default, and the schedule may route it. + assert result.stdout.split() == ["emails", "emails", "reports"] + + +def test_members_must_be_enqueued_on_a_django_ox_backend(): + batch = batches.create("immediate") + with pytest.raises(ImproperlyConfigured, match="django-ox backend"): + batch.add(tasks.immediate_echo, "x") + assert OxpullBatchMember.objects.count() == 0 + + +def test_callbacks_must_be_enqueued_on_a_django_ox_backend(): + # The guard add() applies to members applies to callbacks too, and + # for a sharper reason: a backend that runs the function at enqueue + # time would run it inside the completing transaction and hand back + # an id that is not a UUID. + with pytest.raises(ImproperlyConfigured, match="django-ox backend"): + batches.create("immediate-callback", on_complete=tasks.immediate_echo) + with pytest.raises(ImproperlyConfigured, match="django-ox backend"): + batches.create( + "immediate-callback", on_success="tests_oxpull.tasks.immediate_echo" + ) + assert OxpullBatch.objects.count() == 0 + + +def test_a_stored_callback_on_a_non_ox_backend_never_runs(worker): + """ + Repro for the reconciler loop. A callback path that only became + non-django-ox after create() (a later deploy moved the task onto an + immediate backend) must be refused before it is enqueued: an + immediate backend executes the function inline and returns a + non-UUID id, so recording it raises, the completion rolls back, and + the side effect has already happened. Without the guard in + _enqueue_callback each tick repeats it and the batch never leaves + SEALED. + """ + batch = batches.create( + "deployed-away", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + batch.seal() + OxpullBatch.objects.filter(pk=batch.id).update( + on_complete_path="tests_oxpull.tasks.immediate_record" + ) + + for _ in range(3): + assert reconcile() == 0 + + assert stored(batch).status == OxpullBatch.Status.SEALED + assert stored(batch).callback_task_id is None + assert "complete" not in tasks.STATE # the side effect never fired + + +def test_callback_using_overrides_are_rejected_at_create(): + # Only the dotted path is stored, so an override passed here would + # be dropped in silence and the callback would land on a queue the + # operator is not draining. + with pytest.raises(ImproperlyConfigured, match="queue_name"): + batches.create( + "override", on_complete=tasks.record_callback.using(queue_name="emails") + ) + assert OxpullBatch.objects.count() == 0 + + +def test_a_unique_callback_is_refused_at_create(): + """ + A unique task coalesces while one with the same arguments is + pending, so two batches sharing it as on_complete would point at one + callback task and one of them would never see its callback run. + Refused where the batch is configured, naming the task. + """ + with pytest.raises(ImproperlyConfigured, match="unique") as caught: + batches.create("one", on_complete=tasks.unique_echo, on_complete_args=["same"]) + assert "'tests_oxpull.tasks.unique_echo'" in str(caught.value) + assert OxpullBatch.objects.count() == 0 + + +def test_a_unique_on_success_callback_is_refused_at_create(): + with pytest.raises(ImproperlyConfigured, match="unique") as caught: + batches.create("one", on_success=tasks.unique_echo, on_success_args=["same"]) + assert "'tests_oxpull.tasks.unique_echo'" in str(caught.value) + + +def test_a_callback_made_unique_through_options_is_refused_at_create(): + with pytest.raises(ImproperlyConfigured, match="unique") as caught: + batches.create("one", on_complete=tasks.configured_unique) + assert "'tests_oxpull.tasks.configured_unique'" in str(caught.value) + + +def test_a_callback_made_unique_after_create_is_refused_at_completion( + monkeypatch, caplog +): + """ + The row outlives the deploy that wrote it. A callback that was an + ordinary task at create() and is unique by the time the batch + completes must not be enqueued: the batch stays SEALED and the + reconciler says which task and why, every tick, until the deploy is + fixed. + """ + batch = batches.create( + "later", on_complete=tasks.record_callback, on_complete_args=["c"] + ) + batch.seal() + monkeypatch.setattr( + unique_module, + "_registry", + unique_module._registry | {"tests_oxpull.tasks.record_callback"}, + ) + with caplog.at_level(logging.ERROR, logger="oxpull.batches"): + assert reconcile() == 0 + row = stored(batch) + assert row.status == OxpullBatch.Status.SEALED + assert row.callback_task_id is None + assert not OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback" + ).exists() + assert "'tests_oxpull.tasks.record_callback'" in caplog.text + assert "unique" in caplog.text + + +def test_callback_arguments_must_be_a_list_and_a_dict(): + # list() on a bare string would silently explode it into characters. + with pytest.raises(ImproperlyConfigured, match="args must be a list"): + batches.create( + "strargs", + on_complete=tasks.record_callback, + on_complete_args="admin@example.com", + ) + with pytest.raises(ImproperlyConfigured, match="kwargs must be a dict"): + batches.create( + "strkwargs", + on_success=tasks.record_callback, + on_success_kwargs=[("label", "x")], + ) + assert OxpullBatch.objects.count() == 0 + + +# -- happy path -------------------------------------------------------- + + +def test_seal_then_complete_reports_progress_at_each_stage(worker): + batch = batches.create( + "nightly", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + batch.add(tasks.plain_echo, 1) + batch.add(tasks.plain_echo, 2) + + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 2, 0) + + batch.seal() + assert batch.status == OxpullBatch.Status.SEALED + assert stored(batch).total_count == 2 + + assert worker.run_once() is True + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 1, 1) + + # A batch never completes past a member that has not run. + assert reconcile() == 0 + assert batch.status == OxpullBatch.Status.SEALED + + assert worker.run_once() is True + assert reconcile() == 1 + + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.failed_count == 0 + assert row.completed_at is not None + assert OxTask.objects.filter(id=row.callback_task_id).exists() + assert OxTask.objects.filter(id=row.success_task_id).exists() + + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 0, 2) + assert progress.unattributed == 0 + + +def test_callbacks_are_enqueued_with_their_stored_arguments(worker): + batch = batches.create( + "callback-args", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + batch.seal() + assert reconcile() == 1 + + callback = OxTask.objects.get(id=stored(batch).callback_task_id) + assert callback.task_path == "tests_oxpull.tasks.record_callback" + assert callback.args == ["complete"] + + # An ordinary task: a worker runs it like any other. + assert worker.run_once() is True + assert tasks.STATE["complete"] == 1 + + +def test_zero_member_batch_completes_vacuously_successful(): + batch = batches.create( + "empty", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + batch.seal() + assert stored(batch).total_count == 0 + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.failed_count == 0 + assert row.callback_task_id is not None + assert row.success_task_id is not None + assert batch.progress().total == 0 + + +def test_batch_without_callbacks_still_completes(worker): + batch = batches.create("no-callbacks") + batch.add(tasks.plain_echo, "x") + batch.seal() + assert worker.run_once() is True + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is None + assert row.success_task_id is None + assert OxTask.objects.count() == 1 + + +def test_reconciler_is_idempotent(worker): + batch = batches.create( + "idempotent", on_complete=tasks.record_callback, on_complete_args=["complete"] + ) + batch.add(tasks.plain_echo, "x") + batch.seal() + assert worker.run_once() is True + + assert reconcile() == 1 + callback_id = stored(batch).callback_task_id + assert reconcile() == 0 + assert stored(batch).callback_task_id == callback_id + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 1 + ) + + +def test_reconcile_runs_as_an_ordinary_task(worker): + batch = batches.create("scheduled") + batch.add(tasks.plain_echo, "x") + batch.seal() + assert worker.run_once() is True # the member + + result = batches.reconcile.enqueue() + assert worker.run_once() is True # the reconciler itself + result.refresh() + assert result.status == TaskResultStatus.SUCCESSFUL + assert result.return_value == 1 + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +# -- sealing ----------------------------------------------------------- + + +def test_add_after_seal_raises_and_enqueues_nothing(): + batch = batches.create("sealed") + batch.seal() + + with pytest.raises(BatchSealed): + batch.add(tasks.plain_echo, "late") + + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + assert stored(batch).total_count == 0 + + +def test_add_after_completion_raises(): + batch = batches.create("done") + batch.seal() + assert reconcile() == 1 + + with pytest.raises(BatchSealed): + batch.add(tasks.plain_echo, "late") + + +def test_seal_twice_raises(): + batch = batches.create("twice") + batch.seal() + sealed_at = stored(batch).sealed_at + + with pytest.raises(BatchSealed): + batch.seal() + assert stored(batch).sealed_at == sealed_at + + +def test_unsealed_batch_never_completes(worker): + batch = batches.create("never-sealed") + batch.add(tasks.plain_echo, "x") + assert worker.run_once() is True + + assert reconcile() == 0 + assert stored(batch).status == OxpullBatch.Status.OPEN + + +# -- transactions ------------------------------------------------------ + + +def test_rollback_discards_task_and_membership_together(): + batch = batches.create("rollback") + with pytest.raises(ValidationError), transaction.atomic(): + batch.add(tasks.plain_echo, "x") + raise ValidationError("abort") + + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + batch.seal() + assert stored(batch).total_count == 0 + + +@pytest.mark.django_db(transaction=True) +def test_a_failed_membership_insert_leaves_no_orphan_task(monkeypatch): + """ + add()'s own transaction, exercised in real autocommit. + + Under the ordinary django_db fixture every test already runs inside + a transaction, so a rollback test there passes whether or not add() + opens one of its own: the fixture supplies the rollback. This one + runs with transaction=True and forces the membership INSERT to fail, + so it fails if add()'s atomic block is ever dropped or narrowed -- + the enqueued task would commit on its own and become a task no batch + waits on and no operator can see. + """ + + def refuse_to_save(*args, **kwargs): + raise IntegrityError("membership insert refused") + + monkeypatch.setattr(OxpullBatchMember, "save", refuse_to_save) + + batch = batches.create("orphan") + with pytest.raises(BatchMemberExists): + batch.add(tasks.plain_echo, "x") + monkeypatch.undo() + + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + + +def deadlock_once(monkeypatch): + """ + Make the first membership INSERT the victim of an InnoDB deadlock. + + A real 1213 needs two transactions to close a lock cycle inside + add(), and the only cycle the code once had (batch row before unique + lock) is the one its statement order removed; none has been reached + by racing. So the error is raised from the membership INSERT with + MySQL's own code and text. The one difference from the real thing is + that InnoDB has already rolled the whole transaction back by the + time 1213 reaches the client; here add()'s atomic block does the + rollback on the way out, and the end state is the same. + """ + real_save = OxpullBatchMember.save + calls = [] + + def save_or_deadlock(self, *args, **kwargs): + calls.append(1) + if len(calls) == 1: + raise OperationalError( + 1213, + "Deadlock found when trying to get lock; try restarting transaction", + ) + return real_save(self, *args, **kwargs) + + monkeypatch.setattr(OxpullBatchMember, "save", save_or_deadlock) + return calls + + +@pytest.mark.django_db(transaction=True) +def test_add_reissues_its_own_transaction_after_a_deadlock(monkeypatch): + """ + add() owns its transaction when the caller has not opened one, so a + deadlock that rolls that transaction back left nothing behind and + add() can reissue it: enqueue again, claim the batch row again, + insert the membership again. The enqueue of a unique member runs + inside add()'s transaction and so cannot reissue on its own. + + On MySQL the retry lands and the batch has one member for one task. + On the other databases the code is not a deadlock code and the error + propagates as before, with the rolled-back transaction leaving no + orphan task and no membership. + """ + calls = deadlock_once(monkeypatch) + batch = batches.create("victim") + + if connections["default"].vendor == "mysql": + result = batch.add(tasks.unique_echo, "k") + assert len(calls) == 2 + assert OxTask.objects.count() == 1 + member = OxpullBatchMember.objects.get() + assert str(member.task_id) == result.id + assert member.batch_id == batch.id + else: + with pytest.raises(OperationalError): + batch.add(tasks.unique_echo, "k") + assert len(calls) == 1 + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + + +@pytest.mark.django_db(transaction=True) +def test_add_inside_a_callers_transaction_leaves_the_deadlock_to_the_caller( + monkeypatch, +): + """ + Inside a caller's atomic block a deadlock has rolled back the + caller's work too, and only the caller can decide to redo that. add() + propagates the error after one attempt on every database and the + caller retries its whole transaction, as documented. + """ + calls = deadlock_once(monkeypatch) + batch = batches.create("ambient") + + with pytest.raises(OperationalError), transaction.atomic(): + batch.add(tasks.unique_echo, "k") + assert len(calls) == 1 + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + + +class SimulatedDriverError(Exception): + """The psycopg error a PostgreSQL OperationalError carries as its cause.""" + + +def conflict_once(monkeypatch, code): + """ + Make the first membership INSERT lose a conflict with this database's own + error code: an InnoDB code on MySQL, a SQLSTATE on PostgreSQL, set where + Django's error wrapper leaves it. The server would already have rolled the + transaction back, and add()'s atomic block does that on the way out. + """ + real_save = OxpullBatchMember.save + calls = [] + + def save_or_lose(self, *args, **kwargs): + calls.append(1) + if len(calls) == 1: + if connections["default"].vendor == "mysql": + raise OperationalError(code, "simulated") + cause = SimulatedDriverError("simulated") + cause.sqlstate = code + error = OperationalError("simulated") + error.__cause__ = cause + raise error + return real_save(self, *args, **kwargs) + + monkeypatch.setattr(OxpullBatchMember, "save", save_or_lose) + return calls + + +@pytest.mark.django_db(transaction=True) +def test_add_reissues_its_own_transaction_after_a_postgresql_deadlock(monkeypatch): + """ + PostgreSQL's deadlock detector (40P01) rolls back the whole transaction of + the victim, as InnoDB's 1213 does. add() that owns its transaction runs it + again, and the batch has one member for one task. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("40P01 is PostgreSQL's deadlock code") + calls = conflict_once(monkeypatch, "40P01") + batch = batches.create("pg-victim") + + result = batch.add(tasks.unique_echo, "k") + + assert len(calls) == 2 + assert OxTask.objects.count() == 1 + member = OxpullBatchMember.objects.get() + assert str(member.task_id) == result.id + + +@pytest.mark.django_db(transaction=True) +def test_add_in_a_transaction_opened_by_turning_autocommit_off_is_not_run_again( + monkeypatch, +): + """ + A caller that turned autocommit off opened the transaction, even with no + atomic block open. A conflict rolled back the caller's work too, so add() + runs once and the error reaches the caller, as inside an atomic block. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite raises neither conflict") + code = {"postgresql": "40001", "mysql": 1213}[vendor] + calls = conflict_once(monkeypatch, code) + batch = batches.create("autocommit-off") + connection = connections["default"] + + connection.set_autocommit(False) + try: + with pytest.raises(OperationalError): + batch.add(tasks.unique_echo, "k") + finally: + connection.rollback() + connection.set_autocommit(True) + + assert len(calls) == 1 + assert OxTask.objects.count() == 0 + assert OxpullBatchMember.objects.count() == 0 + + +def test_one_poisoned_batch_does_not_abort_the_sweep(monkeypatch, worker): + """ + "One poisoned batch cannot starve the sweep" needs two batches to + mean anything: with one, skipping it and abandoning the pass look + identical. The first batch's callback explodes; the second must + still complete in the same pass. + """ + poisoned = batches.create("poisoned", on_complete=tasks.plain_fail) + poisoned.seal() + healthy = batches.create( + "healthy", on_complete=tasks.record_callback, on_complete_args=["complete"] + ) + healthy.seal() + + real_enqueue = batches._enqueue_callback + + def explode_for_the_first(path, args, kwargs): + if path == "tests_oxpull.tasks.plain_fail": + raise RuntimeError("callback enqueue failed") + return real_enqueue(path, args, kwargs) + + monkeypatch.setattr(batches, "_enqueue_callback", explode_for_the_first) + assert reconcile() == 1 + monkeypatch.undo() + + assert stored(poisoned).status == OxpullBatch.Status.SEALED + assert stored(healthy).status == OxpullBatch.Status.COMPLETE + assert stored(healthy).callback_task_id is not None + + +def test_callback_enqueue_is_atomic_with_the_compare_and_set(monkeypatch, worker): + batch = batches.create( + "atomic", on_complete=tasks.record_callback, on_complete_args=["complete"] + ) + batch.add(tasks.plain_echo, "x") + batch.seal() + assert worker.run_once() is True + + def explode(*args, **kwargs): + raise RuntimeError("callback enqueue failed") + + monkeypatch.setattr(batches, "_enqueue_callback", explode) + # The pass logs and skips the batch; one poisoned batch must not + # abort the sweep. + assert reconcile() == 0 + + row = stored(batch) + assert row.status == OxpullBatch.Status.SEALED + assert row.completed_at is None + assert row.failed_count is None + assert OxTask.objects.count() == 1 # the member only; no callback row + + monkeypatch.undo() + assert reconcile() == 1 + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +# -- failure, retry and death ------------------------------------------ + + +def test_batch_waits_for_a_retrying_member(worker): + batch = batches.create("retrying") + result = batch.add(tasks.plain_flaky, 2) + batch.seal() + + assert worker.run_once() is True # attempt 1 fails, back to READY + row = OxTask.objects.get(id=result.id) + assert row.status == OxTask.Status.READY + assert row.attempts == 1 + + assert reconcile() == 0 + assert batch.progress().pending == 1 + + assert worker.run_once() is True # attempt 2 succeeds + assert reconcile() == 1 + assert stored(batch).failed_count == 0 + + +def test_running_member_holds_the_batch_open(): + batch = batches.create("running") + result = batch.add(tasks.plain_echo, "x") + batch.seal() + OxTask.objects.filter(id=result.id).update( + status=OxTask.Status.RUNNING, locked_by="worker", locked_at=timezone.now() + ) + + assert reconcile() == 0 + assert batch.progress().pending == 1 + + +def test_failed_member_completes_the_batch_without_on_success(worker): + batch = batches.create( + "failing", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + result = batch.add(tasks.plain_fail) + batch.seal() + for _ in range(3): # MAX_ATTEMPTS in tests_oxpull/settings.py + assert worker.run_once() is True + assert OxTask.objects.get(id=result.id).status == OxTask.Status.FAILED + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.failed_count == 1 + assert row.callback_task_id is not None + assert row.success_task_id is None + + progress = batch.progress() + assert (progress.total, progress.failed, progress.succeeded) == (1, 1, 0) + + +def test_member_reaped_to_lost_still_completes_the_batch(worker): + """ + The marquee case. A dead worker's task is reclaimed by the reaper, + which settles it through a queryset update and emits no signal at all + (django_ox worker.reap). A batch that counted task_finished would hang + here forever; this one asks the rows. + + The reaper writes LOST rather than FAILED, having no evidence either + way about work it never saw finish. django.tasks carries no such value, + so the public status still reads FAILED and callers see what they + always saw. What changes for a batch is attribution: an unobserved + member is no longer counted a failure, so failed_count is 0 and the + member falls to unattributed instead. on_success stays withheld under + either reading, which is the property an operator actually relies on. + """ + batch = batches.create( + "reaped", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + result = batch.add(tasks.plain_echo, "x") + batch.seal() + OxTask.objects.filter(id=result.id).update( + status=OxTask.Status.RUNNING, + attempts=3, + max_attempts=3, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + signals = [] + + def receiver(sender, task_result, **kwargs): + signals.append(task_result.id) + + task_finished.connect(receiver) + try: + assert worker.reap() == 1 + finally: + task_finished.disconnect(receiver) + + assert signals == [] # the reaper is silent; that is the whole point + lost = OxTask.objects.get(id=result.id) + assert lost.status == OxTask.Status.LOST + # Unchanged for anyone reading through django.tasks. + assert public_status(lost.status) == TaskResultStatus.FAILED + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is not None # on_complete fires + assert row.success_task_id is None # on_success does not + assert row.failed_count == 0 + assert batch.progress().unattributed == 1 + + +def test_member_reaped_with_attempts_remaining_keeps_the_batch_open(worker): + batch = batches.create("reclaimed") + result = batch.add(tasks.plain_echo, "x") + batch.seal() + OxTask.objects.filter(id=result.id).update( + status=OxTask.Status.RUNNING, + attempts=1, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + assert worker.reap() == 1 + assert OxTask.objects.get(id=result.id).status == OxTask.Status.READY + + assert reconcile() == 0 + assert batch.progress().pending == 1 + + +def test_pruned_member_counts_as_finished(worker): + batch = batches.create("pruned") + first = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + batch.seal() + + # ox_prune deletes finished task rows outright; the membership row + # then joins to nothing and must read as finished, not as lost. + OxTask.objects.filter(id=first.id).delete() + assert worker.run_once() is True # the surviving member + + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 0, 1) + assert progress.unattributed == 1 + # The deprecated alias still reads the same number off real rows. + with pytest.warns(DeprecationWarning, match="BatchProgress.unattributed"): + assert progress.pruned == 1 + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + # Attribution degrades to "finished": a pruned member cannot be + # counted as a failure. + assert row.failed_count == 0 + + +def test_pruned_member_withholds_on_success(worker): + """ + Repro: `ox_prune --include-failed` deletes a FAILED member out from + under a live batch, and the surviving rows then read as zero + failures. on_success is the operator's "the run was clean" signal, + so it must not fire on a run whose outcome is no longer knowable. + """ + batch = batches.create( + "pruned-failure", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + failing = batch.add(tasks.plain_fail) + batch.add(tasks.plain_echo, "b") + batch.seal() + while worker.run_once(): + pass + assert OxTask.objects.get(id=failing.id).status == OxTask.Status.FAILED + + # Age the failed row past the cutoff and run the real prune command. + OxTask.objects.filter(id=failing.id).update( + finished_at=timezone.now() - timedelta(hours=2) + ) + call_command("ox_prune", "--older-than", "1h", "--include-failed") + assert not OxTask.objects.filter(id=failing.id).exists() + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is not None # on_complete still fires + assert row.success_task_id is None # on_success does not + assert batch.progress().unattributed == 1 + + +PRUNED_DEPRECATION = ( + "BatchProgress.pruned is deprecated and will be removed in oxpull 2.0. " + "Use BatchProgress.unattributed, which returns the same number." +) + + +def test_pruned_warns_names_unattributed_and_points_at_the_caller(): + """ + pruned is the old name for unattributed, and it undersells the + number: LOST and discarded members land there too. Reading it warns + once, names the replacement and the release that removes it, and + blames the line that read it rather than oxpull, so the warning is + actionable where it is shown. + """ + progress = batches.BatchProgress(total=5, pending=1, succeeded=1, failed=1) + + with pytest.warns(DeprecationWarning) as caught: + reading_line = sys._getframe().f_lineno + 1 + value = progress.pruned + + assert value == progress.unattributed == 2 + assert len(caught) == 1 + assert caught[0].category is DeprecationWarning + assert str(caught[0].message) == PRUNED_DEPRECATION + assert (caught[0].filename, caught[0].lineno) == (__file__, reading_line) + + +def test_unattributed_and_the_dataclass_protocol_do_not_warn(): + """ + The warning sits on the one deprecated read. The replacement, the + other derived property and everything the dataclass does with its + fields stay silent, so equality, hashing, repr, asdict, replace, + copying and pickling a progress reading behave as they did. + """ + progress = batches.BatchProgress(total=5, pending=1, succeeded=1, failed=1) + same = batches.BatchProgress(total=5, pending=1, succeeded=1, failed=1) + + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert progress.unattributed == 2 + assert progress.finished == 4 + assert progress == same + assert hash(progress) == hash(same) + assert repr(progress) == ( + "BatchProgress(total=5, pending=1, succeeded=1, failed=1)" + ) + assert dataclasses.asdict(progress) == { + "total": 5, + "pending": 1, + "succeeded": 1, + "failed": 1, + } + assert dataclasses.replace(progress, pending=0) == batches.BatchProgress( + total=5, pending=0, succeeded=1, failed=1 + ) + assert copy.deepcopy(progress) == progress + assert pickle.loads(pickle.dumps(progress)) == progress # noqa: S301 + + +def test_callback_failure_leaves_the_batch_complete(worker): + batch = batches.create("bad-callback", on_complete=tasks.plain_fail) + batch.seal() + assert reconcile() == 1 + callback_id = stored(batch).callback_task_id + + for _ in range(3): + assert worker.run_once() is True + assert OxTask.objects.get(id=callback_id).status == OxTask.Status.FAILED + # Batches do not supervise their callbacks. + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +# -- unique members ---------------------------------------------------- + + +def test_deduplicated_unique_task_can_join_two_batches(): + first = batches.create("one") + second = batches.create("two") + a = first.add(tasks.unique_echo, "x") + b = second.add(tasks.unique_echo, "x") + + assert a.id == b.id + assert OxTask.objects.count() == 1 + assert OxpullBatchMember.objects.count() == 2 + + +def test_adding_one_deduplicated_task_twice_to_a_batch_raises(): + batch = batches.create("dedup") + batch.add(tasks.unique_echo, "x") + + with pytest.raises(BatchMemberExists): + batch.add(tasks.unique_echo, "x") + + assert OxTask.objects.count() == 1 + assert OxpullBatchMember.objects.count() == 1 + # The caller's connection survives the constraint violation. + batch.seal() + assert stored(batch).total_count == 1 + + +# -- routing ----------------------------------------------------------- + + +def test_split_router_refuses_batch_operations(): + with ( + override_settings(DATABASE_ROUTERS=["tests_oxpull.routers.SplitTaskRouter"]), + pytest.raises(ImproperlyConfigured), + ): + batches.create("split") + + +@pytest.mark.django_db(databases=["default", "replica"]) +def test_replica_read_router_cannot_complete_a_pending_batch(): + # The completion join must never run against a lagged replica: an + # empty replica would report nothing pending and complete the batch + # while its member is still queued. Every read is pinned to the + # write database. + batch = batches.create("pinned") + batch.add(tasks.plain_echo, "x") + batch.seal() + + with override_settings(DATABASE_ROUTERS=["tests_oxpull.routers.ReplicaReadRouter"]): + assert reconcile() == 0 + assert batch.progress().pending == 1 + + assert stored(batch).status == OxpullBatch.Status.SEALED + + +@pytest.mark.django_db(transaction=True) +def test_progress_on_an_open_batch_never_invents_a_finished_member(monkeypatch): + """ + Repro: progress() on an OPEN batch reads the status counts and the + membership denominator in two statements. With the denominator read + second, an add() committing between them raises total without + raising pending, and the derived finished and unattributed report + members that neither finished nor were pruned. + """ + batch = batches.create("torn") + batch.add(tasks.plain_echo, "a") + + real_counts = batches._member_task_counts + raced = [] + + def counts_then_a_concurrent_add(*args, **kwargs): + result = real_counts(*args, **kwargs) + if not raced: + raced.append(True) + batch.add(tasks.plain_echo, "b") + return result + + monkeypatch.setattr(batches, "_member_task_counts", counts_then_a_concurrent_add) + progress = batch.progress() + monkeypatch.undo() + + assert raced == [True] + assert progress.unattributed == 0 + assert progress.finished == 0 + assert progress.total == progress.pending + # The batch itself is unharmed: the settled reading is both members. + assert batch.progress() == batches.BatchProgress( + total=2, pending=2, succeeded=0, failed=0 + ) + + +# -- width ------------------------------------------------------------- + + +def statements_during(call): + """Every statement one call executes, as (sql, bound parameter count).""" + alias = oxpull_db_alias() + seen = [] + + def spy(execute, sql, params, many, context): + seen.append((sql, len(params or ()))) + return execute(sql, params, many, context) + + with connections[alias].execute_wrapper(spy): + call() + return seen + + +def test_no_batch_query_binds_one_parameter_per_member(): + """ + The completion join filters task rows with `id__in=`, and that has to stay a SQL subquery rather than becoming + one bound parameter per member. + + A version that inlined the ids would fail on exactly the batches + batches exist for, and fail outright rather than slowly: SQLite + refuses a statement with more than SQLITE_MAX_VARIABLE_NUMBER + parameters and PostgreSQL more than 65,535, so a nightly run of + 10,000 invoices would raise instead of completing. Historical width + measurements found the parameter count stayed at 1 from 1,000 to + 100,000 members on both databases. This test asserts it at two widths: + the property that matters is that the count does not move with + membership, not that it happens to be small in one reading. + """ + + def sealed_batch_of(members): + batch = batches.create(f"width-{members}") + for index in range(members): + batch.add(tasks.plain_echo, index) + batch.seal() + return batch + + def completion_query(batch): + statements = statements_during(batch.progress) + grouped = [(sql, count) for sql, count in statements if "GROUP BY" in sql] + assert len(grouped) == 1, statements + return grouped[0], max(count for _, count in statements) + + (narrow_sql, narrow_params), narrow_widest = completion_query(sealed_batch_of(2)) + (wide_sql, wide_params), wide_widest = completion_query(sealed_batch_of(60)) + + # The member ids never leave the database. + assert "IN (SELECT" in " ".join(narrow_sql.split()) + assert "IN (SELECT" in " ".join(wide_sql.split()) + # One parameter, the batch id, at either width -- and no other + # statement on the path grows either. + assert narrow_params == wide_params == 1 + assert narrow_widest == wide_widest + + +# -- racing reconcilers ------------------------------------------------ + + +@pytest.mark.django_db(transaction=True) +def test_racing_reconcilers_fire_the_callbacks_once(worker): + """ + Exactly-once callbacks come from the compare-and-set rowcount, the + same class of mechanism as the schedule-tick unique constraint. Every + thread sees a finished batch; only the one whose UPDATE matches a + SEALED row may enqueue. + """ + batch = batches.create( + "raced", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + batch.add(tasks.plain_echo, "x") + batch.seal() + assert worker.run_once() is True + + results, errors = race(reconcile) + + assert errors == [] + assert len(results) == THREADS + assert sorted(results) == [0] * (THREADS - 1) + [1] + + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 2 + ) + assert {str(row.callback_task_id), str(row.success_task_id)} == { + str(task_id) + for task_id in OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback" + ).values_list("id", flat=True) + } + + +@pytest.mark.django_db(transaction=True) +def test_add_does_not_deadlock_a_caller_that_already_holds_a_unique_lock(): + """ + Repro for a lock-order inversion. add() must not hold the batch row + lock across enqueue(): a unique member's enqueue takes the + OxpullUniqueLock row for its key, so batch row -> unique lock is the + library's order, while a caller that enqueues a unique task and then + calls add() inside one transaction holds unique lock -> batch row. + That closes a cycle, and PostgreSQL aborts one side with a deadlock + (MySQL with ER_LOCK_DEADLOCK, which add() reissues only when it owns + the transaction, and the caller here does). add() enqueues before it + claims the batch row, so both sides take the two locks in the same + order. + """ + batch = batches.create("lock-order") + holds_the_key = threading.Event() + member_add_started = threading.Event() + + def enqueue_then_add(): + with transaction.atomic(): + tasks.unique_echo.enqueue("k") # holds the unique-lock row + holds_the_key.set() + assert member_add_started.wait(timeout=10) + # Let the other thread reach the batch row first; under the + # inverted order it is holding it and waiting on this + # transaction's unique lock. + time.sleep(0.5) + batch.add(tasks.plain_echo, "x") + return "caller" + + def add_the_unique_member(): + assert holds_the_key.wait(timeout=10) + member_add_started.set() + batch.add(tasks.unique_echo, "k") + return "member" + + results, errors = race_all([enqueue_then_add, add_the_unique_member]) + + assert errors == [] + assert sorted(results) == ["caller", "member"] + assert OxpullBatchMember.objects.filter(batch_id=batch.id).count() == 2 + + +@pytest.mark.django_db(transaction=True) +def test_racing_add_and_seal_never_strands_a_member_outside_the_count(): + """ + seal() freezes membership, so total_count must equal the members that + actually committed. add() takes the batch row lock before it enqueues + anything, so it either commits before seal() counts or finds the + batch already SEALED; a plain status read would let a member land + after the snapshot and make total_count a lie. + """ + batch = batches.create("sealing") + + def add_or_refuse(): + try: + batch.add(tasks.plain_echo, "x") + except BatchSealed: + return "refused" + return "added" + + def seal(): + batch.seal() + return "sealed" + + results, errors = race_all([seal, *[add_or_refuse] * (THREADS - 1)]) + + assert errors == [] + added = results.count("added") + row = stored(batch) + assert row.status == OxpullBatch.Status.SEALED + assert OxpullBatchMember.objects.count() == added + assert OxTask.objects.count() == added + assert row.total_count == added + + +def test_lost_member_withholds_on_success(worker): + """ + Repro: the reaper writes LOST when a claim ages out with no attempts + left, meaning the attempt was never observed. That is an unknown + outcome rather than a clean one, and the straggler holding the lease + may still resolve the row to FAILED afterwards. on_success is the + operator's "the run was clean" signal, so a batch holding an + unobserved member must not fire it. + + The status is written directly rather than driven through the reaper: + what is under test is how completion reads a LOST row, not how the + reaper comes to write one. + """ + batch = batches.create( + "lost-member", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + lost = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + batch.seal() + while worker.run_once(): + pass + + OxTask.objects.filter(id=lost.id).update(status=OxTask.Status.LOST) + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is not None # on_complete still fires + assert row.success_task_id is None, ( + "on_success fired on a batch holding a member whose outcome was " + "never observed; a straggler can still resolve that row to FAILED." + ) + assert row.failed_count == 0 + assert batch.progress().unattributed == 1 + + +def test_lost_member_counts_as_finished(worker): + """ + A LOST row is settled, so it must not hold a batch open indefinitely. + + The reaper deliberately leaves lease_epoch alone when it writes LOST, + so the straggler can still resolve the row. Waiting for one that never + returns would mean a batch that never completes and fires no callback + at all, which is worse than completing with the outcome marked + unknown. + """ + batch = batches.create("lost-finished") + lost = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + batch.seal() + while worker.run_once(): + pass + OxTask.objects.filter(id=lost.id).update(status=OxTask.Status.LOST) + + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 0, 1) + assert progress.unattributed == 1 + assert reconcile() == 1 + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + ("member", "args", "outcome"), + [ + (tasks.plain_echo, ["x"], OxTask.Status.SUCCESSFUL), + (tasks.plain_fail, [], OxTask.Status.FAILED), + ], + ids=["successful", "failed"], +) +def test_a_straggler_resolving_a_lost_member_leaves_the_completed_batch_alone( + worker, member, args, outcome +): + """ + A LOST member completes the batch, and the execution that lost it + writing its outcome afterwards changes nothing the batch decided. + + Driven through a real claim, the reaper and the straggler's own + execute(), rather than by writing statuses: the straggler holds the + lease epoch its claim handed out, which is what lets its write land + on the LOST row. While the lease has only expired the member is + RUNNING and holds the batch open. Once the reaper writes LOST the + member is finished and unattributed, so on_complete is enqueued, + on_success is withheld and failed_count is 0. The straggler then + writes SUCCESSFUL or FAILED onto that row. The next tick leaves the + COMPLETE batch as it was, with no callback enqueued again and + failed_count unchanged. Only progress(), which reads the rows, shows + the new outcome. + """ + batch = batches.create( + "straggler", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + result = batch.add(member, *args) + batch.seal() + OxTask.objects.filter(id=result.id).update(attempts=F("max_attempts") - 1) + straggler = worker.claim_one() + assert straggler is not None and str(straggler.id) == result.id + assert straggler.attempts == straggler.max_attempts + expired = timezone.now() - timedelta(seconds=9999) + OxTask.objects.filter(id=result.id).update( + locked_at=expired, lease_expires_at=expired + ) + + assert reconcile() == 0 + assert batch.progress().pending == 1 + + assert worker.reap() == 1 + assert OxTask.objects.get(id=result.id).status == OxTask.Status.LOST + assert reconcile() == 1 + completed = stored(batch) + assert completed.status == OxpullBatch.Status.COMPLETE + assert completed.callback_task_id is not None + assert completed.success_task_id is None + assert completed.failed_count == 0 + callbacks = OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback") + enqueued = [str(completed.callback_task_id)] + assert [str(pk) for pk in callbacks.values_list("id", flat=True)] == enqueued + + worker.execute(straggler) + assert OxTask.objects.get(id=result.id).status == outcome + + assert reconcile() == 0 + after = stored(batch) + assert after.status == OxpullBatch.Status.COMPLETE + assert after.completed_at == completed.completed_at + assert after.failed_count == 0 + assert after.callback_task_id == completed.callback_task_id + assert after.success_task_id is None + assert [str(pk) for pk in callbacks.values_list("id", flat=True)] == enqueued + progress = batch.progress() + assert progress.unattributed == 0 + succeeded = outcome == OxTask.Status.SUCCESSFUL + assert (progress.succeeded, progress.failed) == (int(succeeded), int(not succeeded)) + + +# -- django-ox 0.3.x operator actions ----------------------------------- + + +def test_discarded_member_withholds_on_success(worker): + """ + Repro against django-ox 0.3.1: django_ox.actions.discard closes a + READY, FAILED or LOST task without running it, and 0.2.1 knew nothing + about that status. A DISCARDED member fell through every bucket, so + the completing transaction counted it as attributed and enqueued + on_success: the batch reported a clean run over work an operator had + just cancelled. progress() already read it the other way, so the two + paths disagreed about the same batch. + + on_complete still fires. The batch is complete; it was not clean. + """ + batch = batches.create( + "discarded-member", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + doomed = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + assert actions.discard(doomed.id) is True + batch.seal() + while worker.run_once(): + pass + + assert OxTask.objects.get(id=doomed.id).status == OxTask.Status.DISCARDED + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is not None # on_complete still fires + assert row.success_task_id is None, ( + "on_success fired on a batch holding a member an operator " + "discarded; the batch is complete, but the work was cancelled " + "rather than done." + ) + assert row.failed_count == 0 # discarded is not failed + + +def test_discarded_member_counts_as_finished_and_unattributed(worker): + """ + DISCARDED is terminal and never claimed, so it must not hold a batch + open. It is not a success and not a recorded failure either, which is + the same standing as a pruned or LOST member, so it lands in the + unattributed shortfall. progress() and the completing transaction + have to agree on that number. + """ + batch = batches.create("discarded-finished") + doomed = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + assert actions.discard(doomed.id) is True + batch.seal() + while worker.run_once(): + pass + + progress = batch.progress() + assert (progress.total, progress.pending, progress.succeeded) == (2, 0, 1) + assert (progress.failed, progress.unattributed) == (0, 1) + assert reconcile() == 1 + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +def test_discard_many_of_every_member_still_completes_the_batch(worker): + """ + A batch an operator cancelled outright completes with no member + attributed. on_complete fires, so the operator's own cleanup still + runs; on_success does not. + """ + batch = batches.create( + "cancelled-run", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + batch.seal() + + changed, skipped = actions.discard_many(OxTask.objects.all()) + assert (changed, skipped) == (2, 0) + + assert reconcile() == 1 + row = stored(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.callback_task_id is not None + assert row.success_task_id is None + assert batch.progress().unattributed == 2 + + +def test_retried_member_holds_the_batch_open(worker): + """ + django_ox.actions.retry puts a FAILED member back on the queue, and a + sealed batch is frozen in membership, not in time. The retry is a + pending member again, so the batch stays SEALED and no callback + fires until it settles. + """ + batch = batches.create( + "retried-member", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + failing = batch.add(tasks.plain_fail) + batch.add(tasks.plain_echo, "b") + batch.seal() + while worker.run_once(): + pass + assert OxTask.objects.get(id=failing.id).status == OxTask.Status.FAILED + + assert actions.retry(failing.id) is True + assert OxTask.objects.get(id=failing.id).status == OxTask.Status.READY + + assert batch.progress().pending == 1 + assert reconcile() == 0 + assert stored(batch).status == OxpullBatch.Status.SEALED + assert stored(batch).callback_task_id is None + + while worker.run_once(): + pass + assert reconcile() == 1 + assert stored(batch).failed_count == 1 + + +def test_retry_landing_inside_the_completing_transaction_is_not_lost( + worker, monkeypatch +): + """ + The narrow window django-ox 0.3.0 opened. The reconciler reads the + member counts, then opens a transaction and compare-and-sets the + batch to COMPLETE. Until 0.3.0 that gap was safe, because nothing + moved a settled row back to READY. actions.retry does exactly that, + so a retry landing in the gap used to complete the batch and fire + its callbacks over a member queued to run again. + + The retry is driven from inside the first count, which is the gap + itself: the counts the reconciler carries into the transaction are + then stale in precisely the way a real retry makes them stale. + """ + batch = batches.create( + "retry-in-the-gap", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + failing = batch.add(tasks.plain_fail) + batch.seal() + while worker.run_once(): + pass + assert OxTask.objects.get(id=failing.id).status == OxTask.Status.FAILED + + real_counts = batches._member_task_counts + calls = [] + + def counts_then_retry(batch_pk, db_alias): + result = real_counts(batch_pk, db_alias) + calls.append(result) + if len(calls) == 1: + # The gap: after the filter has read "nothing pending", before + # the compare-and-set. + assert actions.retry(failing.id) is True + return result + + monkeypatch.setattr(batches, "_member_task_counts", counts_then_retry) + + assert batches.reconcile.call() == 0 + row = stored(batch) + assert row.status == OxpullBatch.Status.SEALED, ( + "the batch completed while a retried member was READY; the " + "compare-and-set has to be re-checked under its own row lock." + ) + assert row.completed_at is None + assert row.failed_count is None + assert row.callback_task_id is None + + monkeypatch.undo() + while worker.run_once(): + pass + assert reconcile() == 1 + assert stored(batch).status == OxpullBatch.Status.COMPLETE + + +# -- the status contract with django-ox -------------------------------- + + +def test_the_reconciler_knows_every_django_ox_status(): + """ + Finished-ness is recognised positively, from an explicit set of + settled statuses, so a status this package has never seen holds a + batch open rather than completing it and attributing an outcome to + it. That is only safe while the sets here cover everything django-ox + can write, and this is the test that fails when a django-ox release + adds a seventh status. + """ + pending = set(batches.PENDING_STATUSES) + settled = set(batches.SETTLED_STATUSES) + assert pending & settled == set() + assert pending | settled == set(OxTask.Status) + assert set(batches.UNATTRIBUTED_STATUSES) <= settled + + +def test_a_status_the_reconciler_does_not_know_holds_the_batch_open(): + """ + A member whose row carries a status this package cannot read must + not count as finished, and must not be attributed to on_success. It + stays pending, and the batch stays SEALED, until a release that knows + the status reads it. + """ + batch = batches.create( + "unknown-status", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + member = batch.add(tasks.plain_echo, "a") + batch.seal() + assert OxTask.objects.filter(pk=member.id).update(status="ZOMBIE") == 1 + + progress = batch.progress() + assert (progress.total, progress.pending, progress.finished) == (1, 1, 0) + assert reconcile() == 0 + row = stored(batch) + assert row.status == OxpullBatch.Status.SEALED + assert row.callback_task_id is None + assert row.success_task_id is None + + +# -- seal: a count, and a lock only where a snapshot could lie ---------- + + +@contextmanager +def session_isolation(level): + """ + This connection's isolation level, MySQL only, for the block. + + Django's MySQL backend sets READ COMMITTED on every connection it + opens; a project that sets OPTIONS["isolation_level"] to "repeatable + read", or to None for the server's default, runs under REPEATABLE + READ, and that is the level whose snapshot the tests below are about. + The connection is closed afterwards so the next use reopens it at the + configured level. + """ + connection = connections["default"] + if connection.vendor != "mysql": + yield + return + with connection.cursor() as cursor: + cursor.execute(f"SET SESSION TRANSACTION ISOLATION LEVEL {level}") + try: + yield + finally: + connection.close() + + +def a_session_waits_on_this_transaction(timeout=20.0): + """ + Whether another session is seen waiting on a lock this thread's open + transaction holds, within the timeout. PostgreSQL only. + + Asked from inside that transaction, so of pg_locks and pg_blocking_pids, + which read the lock manager as it stands and say who blocks whom. + PostgreSQL documents pg_stat_activity as collected once per transaction, + so a poll from here does not rely on it. + """ + connection = connections["default"] + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with connection.cursor() as cursor: + cursor.execute( + "SELECT count(*) FROM pg_locks WHERE NOT granted " + "AND pg_backend_pid() = ANY(pg_blocking_pids(pid))" + ) + (count,) = cursor.fetchone() + if count: + return True + time.sleep(0.02) + return False + + +def member_reads(captured): + table = OxpullBatchMember._meta.db_table + return [ + query["sql"] for query in captured.captured_queries if table in query["sql"] + ] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("ambient", [False, True], ids=["own", "ambient"]) +def test_seal_locks_members_only_inside_a_caller_transaction(ambient): + """ + When seal() owns its transaction the status UPDATE is its first + statement and serialises against every in-flight add(), so the count + that follows is current on every database as a plain COUNT; the row + lock per member it used to take was paid for nothing, 28 s at a + million members on MySQL. Inside a caller's transaction the count + stays a locking read: under MySQL REPEATABLE READ a plain COUNT is + answered from the read view the caller's earlier statements fixed. + """ + batch = batches.create("shape") + batch.add(tasks.plain_echo, "a") + connection = connections[oxpull_db_alias()] + with CaptureQueriesContext(connection) as captured: + if ambient: + with transaction.atomic(): + batch.seal() + else: + batch.seal() + (count_sql,) = member_reads(captured) + locking = ambient and connection.features.has_select_for_update + assert ("FOR UPDATE" in count_sql) is locking + assert ("COUNT(" in count_sql.upper()) is not locking + assert stored(batch).total_count == 1 + + +@pytest.mark.django_db(transaction=True) +def test_ambient_seal_counts_a_member_committed_after_the_callers_snapshot(): + """ + The case the locking read exists for. Under MySQL REPEATABLE READ the + caller's transaction fixes its read view at its first consistent + read, and an add() that commits after that is invisible to a plain + COUNT even though seal()'s UPDATE waited on nothing: the add() had + already committed. Counting with a locking read is a current read + and finds the member. PostgreSQL's READ COMMITTED would find it + either way; the test runs there to show the promise is the same. + """ + if connections["default"].vendor == "sqlite": + pytest.skip( + "a reading SQLite transaction holds its shared lock, so the " + "concurrent add() could not commit under it" + ) + batch = batches.create("snapshot") + + def add_and_commit(): + batch.add(tasks.plain_echo, "late") + return "added" + + with session_isolation("REPEATABLE READ"), transaction.atomic(): + # The caller's first consistent read; the read view starts here. + assert OxpullBatchMember.objects.filter(batch_id=batch.id).count() == 0 + results, errors = race_all([add_and_commit]) + assert (results, errors) == (["added"], []) + batch.seal() + + assert stored(batch).total_count == 1 + + +@pytest.mark.django_db(transaction=True) +def test_seal_owning_its_transaction_counts_the_member_whose_add_it_waited_for(): + """ + Why the plain COUNT is current when seal() owns its transaction, even + under MySQL REPEATABLE READ: the status UPDATE is the transaction's + first statement and waits on an add() still holding the batch row, + and the COUNT after it is the transaction's first consistent read, + so its read view begins after that add() committed. + """ + batch = batches.create("waited") + added = threading.Event() + + def add_slowly(): + with transaction.atomic(): + batch.add(tasks.plain_echo, "slow") + added.set() + # Hold the batch row across seal()'s arrival. + time.sleep(1.0) + return "added" + + def seal_after_the_add_started(): + assert added.wait(timeout=10) + with session_isolation("REPEATABLE READ"): + batch.seal() + return "sealed" + + results, errors = race_all([add_slowly, seal_after_the_add_started]) + + assert errors == [] + assert sorted(results) == ["added", "sealed"] + assert stored(batch).total_count == 1 + + +@pytest.mark.django_db(transaction=True) +def test_an_add_arriving_between_seals_statements_waits_and_is_refused(monkeypatch): + """ + The interleaving a count without locks was feared to open: an add() + committing after seal()'s status UPDATE returned and before its + COUNT ran. It cannot happen, because add() claims the batch row that + the UPDATE holds until seal() commits. Proven by starting the add() + from inside the count, watching it block for a second, and reading + its refusal once seal() has committed. + """ + batch = batches.create("between") + batch.add(tasks.plain_echo, "before") + add_finished = threading.Event() + outcome = [] + + def late_add(): + try: + batch.add(tasks.plain_echo, "late") + outcome.append("added") + except BatchSealed: + outcome.append("refused") + finally: + add_finished.set() + connections.close_all() + + original = batches.Batch._count_members + + def count_after_starting_an_add(self, db_alias, **kwargs): + threading.Thread(target=late_add).start() + assert not add_finished.wait(timeout=1.0), "add() did not wait for seal()" + return original(self, db_alias, **kwargs) + + monkeypatch.setattr(batches.Batch, "_count_members", count_after_starting_an_add) + batch.seal() + + assert add_finished.wait(timeout=10) + assert outcome == ["refused"] + assert stored(batch).total_count == 1 + assert OxpullBatchMember.objects.filter(batch_id=batch.id).count() == 1 + assert OxTask.objects.count() == 1 + + +# -- add(): the OPEN claim without MySQL's CLIENT_FOUND_ROWS ------------- + + +@contextmanager +def without_found_rows(monkeypatch): + """ + A MySQL connection whose UPDATE rowcount is rows changed, not rows + matched. Django's backend asks for CLIENT_FOUND_ROWS on every + connection; a project that sets OPTIONS["client_flag"] replaces it. + The connection is closed on the way in so the next statement + reconnects without the flag, and on the way out so the next test + reconnects with it. + """ + connection = connections["default"] + if connection.vendor != "mysql": + pytest.skip("CLIENT_FOUND_ROWS is a MySQL connection flag") + monkeypatch.setitem(connection.settings_dict["OPTIONS"], "client_flag", 0) + connection.close() + try: + yield + finally: + connection.close() + + +@pytest.mark.django_db(transaction=True) +def test_add_does_not_depend_on_the_mysql_found_rows_flag(monkeypatch): + """ + add() serialises against seal() with a no-op UPDATE under the OPEN + predicate and read its rowcount as "the batch is OPEN". On MySQL + that rowcount counts matched rows only because of CLIENT_FOUND_ROWS; + without the flag a no-op UPDATE reports the rows it changed, none, + and every add() on an OPEN batch raised BatchSealed saying membership + was frozen. A zero rowcount now sends add() to read the status under + the row lock instead, and only a status other than OPEN refuses. + """ + with without_found_rows(monkeypatch): + batch = batches.create("no-found-rows") + result = batch.add(tasks.plain_echo, "x") + assert OxpullBatchMember.objects.filter( + batch_id=batch.id, task_id=result.id + ).exists() + batch.seal() + with pytest.raises(BatchSealed, match="SEALED"): + batch.add(tasks.plain_echo, "y") + assert stored(batch).total_count == 1 + assert OxTask.objects.count() == 1 + + +@pytest.mark.django_db(transaction=True) +def test_add_without_the_flag_reads_the_status_it_locked_not_its_snapshot( + monkeypatch, +): + """ + The fallback read has to be a locking read. Inside a caller's + transaction under REPEATABLE READ, a plain read of the batch's status + answers from the caller's snapshot, which can still say OPEN after a + concurrent seal() committed; the no-op UPDATE saw the current row and + matched nothing, and a snapshot read would then admit a member into a + sealed batch. Reading under the lock sees the seal and refuses. + """ + with without_found_rows(monkeypatch): + batch = batches.create("stale-snapshot") + + def seal_and_commit(): + batch.seal() + return "sealed" + + with session_isolation("REPEATABLE READ"), transaction.atomic(): + # The caller's snapshot: the batch is OPEN in it from here on. + assert batch.status == OxpullBatch.Status.OPEN + assert race_all([seal_and_commit]) == (["sealed"], []) + with pytest.raises(BatchSealed, match="SEALED"): + batch.add(tasks.plain_echo, "late") + + assert OxpullBatchMember.objects.filter(batch_id=batch.id).count() == 0 + assert OxTask.objects.count() == 0 + assert stored(batch).total_count == 0 + + +# -- the reconciler schedule is required, not assumed -------------------- + + +def tasks_without_the_reconciler(**moved_to): + """ + settings.TASKS with every reconciler schedule removed, and optionally + the schedule put back on another alias (moved_to={"limited": True}). + """ + rebuilt = {} + for alias, params in settings.TASKS.items(): + options = { + key: value + for key, value in params.get("OPTIONS", {}).items() + if key != "SCHEDULES" + } + if moved_to.get(alias): + options.update(RECONCILE_SCHEDULE) + rebuilt[alias] = {**params, "OPTIONS": options} + return rebuilt + + +def check_ids(alias): + return [ + message.id + for message in OxpullBackend(alias, settings.TASKS[alias]).check() + if message.id == "oxpull.W002" + ] + + +def test_check_warns_once_when_no_backend_schedules_the_reconciler(): + """ + Batches complete only through oxpull.batches.reconcile on a django-ox + schedule. Nothing checked that any backend declared one; a project + that copied the uniqueness and rate-limit settings and missed the + SCHEDULES block had batches that sat SEALED forever, with no error + and no log line. The check is emitted once, from the first + OxpullBackend alias in sort order, so it can be silenced by one id. + """ + with override_settings(TASKS=tasks_without_the_reconciler()): + messages = [ + message + for message in OxpullBackend("default", settings.TASKS["default"]).check() + if message.id == "oxpull.W002" + ] + assert len(messages) == 1 + assert "oxpull.batches.reconcile" in messages[0].msg + assert "oxpull.batches.reconcile" in messages[0].hint + # Once, not per alias: "limited" is an OxpullBackend too. + assert check_ids("limited") == [] + assert check_ids("default") == [] + + +def test_a_reconciler_schedule_on_any_django_ox_backend_satisfies_the_check(): + # The schedule rebinds the task to the backend it is declared on, so + # any django-ox backend's worker can run the reconciler. + with override_settings(TASKS=tasks_without_the_reconciler(limited=True)): + assert check_ids("default") == [] + assert check_ids("limited") == [] + + +# django-ox 1.2.0 lets a project keep its schedules as admin-editable rows +# instead of settings entries. These cover the shape that has no SCHEDULES +# block at all. It counts only on an engine that reads the rows, so the +# ones that assert it counts run with that engine in place. + + +@pytest.fixture +def engine_reads_schedule_rows(monkeypatch): + """ + An installed django-ox that turns SCHEDULE_SOURCE into a source a + worker dispatches from. + + The loader function is put where the probe looks for it, so the probe + itself runs rather than being replaced by an answer. + """ + import django_ox.schedules + + monkeypatch.setattr( + django_ox.schedules, + "schedule_source_from_options", + lambda options, alias: None, + raising=False, + ) + assert batches._engine_reads_schedule_rows() + + +def tasks_with_a_stored_source( + *, expose=True, source="django_ox.stored.DatabaseScheduleSource" +): + """ + settings.TASKS with no SCHEDULES anywhere, "default" reading its + schedules from rows, and the reconciler exposed as schedulable so a + row is allowed to name it. + """ + rebuilt = tasks_without_the_reconciler() + options = dict(rebuilt["default"]["OPTIONS"]) + options["SCHEDULE_SOURCE"] = source + if expose: + options["SCHEDULABLE_TASKS"] = { + "oxpull-batch-reconcile": "oxpull.batches.reconcile" + } + rebuilt["default"] = {**rebuilt["default"], "OPTIONS": options} + return rebuilt + + +def test_a_stored_schedule_source_satisfies_the_reconciler_requirement( + engine_reads_schedule_rows, +): + """ + A backend that reads its schedules from rows can schedule the + reconciler without a SCHEDULES entry, and does: the row dispatches + and the batch completes. The requirement counted settings entries + only, so this configuration -- the one django-ox 1.2.0 invites -- was + refused out of create() and seal() on a path that had worked. + """ + with override_settings(TASKS=tasks_with_a_stored_source()): + assert batches.reconcile_schedule_aliases() == ["default"] + assert check_ids("default") == [] + batch = batches.create("stored-source") + batch.add(tasks.plain_echo, "x") + batch.seal() + assert stored(batch).status == OxpullBatch.Status.SEALED + + +def test_a_stored_source_that_cannot_run_the_reconciler_is_still_refused( + engine_reads_schedule_rows, +): + """ + SCHEDULE_SOURCE alone proves only that rows are read, not that any + row may name the reconciler. Without the reconciler exposed to the + schedulable registry no row can, so the refusal stands -- and it + names the step that is missing rather than the settings block the + project deliberately left behind. + """ + with override_settings(TASKS=tasks_with_a_stored_source(expose=False)): + assert batches.reconcile_schedule_aliases() == [] + with pytest.raises(ImproperlyConfigured) as caught: + batches.create("orphan") + assert "SCHEDULABLE_TASKS" in str(caught.value) + assert "SCHEDULES" not in str(caught.value) + messages = [ + message + for message in OxpullBackend("default", settings.TASKS["default"]).check() + if message.id == "oxpull.W002" + ] + assert len(messages) == 1 + assert "SCHEDULABLE_TASKS" in messages[0].hint + assert "SCHEDULES" not in messages[0].hint + + +def test_the_settings_block_is_still_the_remedy_when_nothing_reads_rows( + engine_reads_schedule_rows, +): + """ + A project with neither channel is the case the refusal was built for, + and it keeps the hint it has always had. + """ + with override_settings(TASKS=tasks_without_the_reconciler()): + with pytest.raises(ImproperlyConfigured) as caught: + batches.create("orphan") + assert "SCHEDULES" in str(caught.value) + assert "SCHEDULABLE_TASKS" not in str(caught.value) + + +def test_an_unusable_schedule_source_does_not_count(engine_reads_schedule_rows): + """ + The option has to name something. django-ox reports a source it + cannot build as django_ox.E006 and the worker refuses to start, so + the value is not validated twice here; an empty or non-string one + names nothing at all and must not satisfy the requirement. + """ + for source in ("", True, None): + with override_settings(TASKS=tasks_with_a_stored_source(source=source)): + assert batches.reconcile_schedule_aliases() == [] + + +def test_the_reconciler_exposed_on_one_alias_serves_a_source_on_another( + engine_reads_schedule_rows, +): + """ + The schedulable registry is process-wide and rows are not bound to an + alias, so the backend that reads the rows and the backend whose + OPTIONS expose the reconciler need not be the same one. + """ + rebuilt = tasks_without_the_reconciler() + rebuilt["default"] = { + **rebuilt["default"], + "OPTIONS": { + **rebuilt["default"]["OPTIONS"], + "SCHEDULE_SOURCE": "django_ox.stored.DatabaseScheduleSource", + }, + } + rebuilt["limited"] = { + **rebuilt["limited"], + "OPTIONS": { + **rebuilt["limited"]["OPTIONS"], + "SCHEDULABLE_TASKS": {"oxpull-batch-reconcile": "oxpull.batches.reconcile"}, + }, + } + with override_settings(TASKS=rebuilt): + assert batches.reconcile_schedule_aliases() == ["default"] + + +def test_a_schedulable_entry_in_the_long_form_counts_too(engine_reads_schedule_rows): + """SCHEDULABLE_TASKS takes a mapping as well as a bare dotted path.""" + rebuilt = tasks_with_a_stored_source() + rebuilt["default"]["OPTIONS"]["SCHEDULABLE_TASKS"] = { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "permission": "oxpull.run_reconcile", + } + } + with override_settings(TASKS=rebuilt): + assert batches.reconcile_schedule_aliases() == ["default"] + + +def _engine_that_cannot_read_rows(monkeypatch): + """ + Stand in for a django-ox older than the one this release pins, forced + in against the pin. The gate asks the installed engine for its loader + rather than for a version number, so the engine is simulated here too; + depending on whatever django-ox happens to be installed made these tests + true under one pin and false under the next. + """ + import django_ox.schedules + + monkeypatch.delattr( + django_ox.schedules, "schedule_source_from_options", raising=False + ) + + +def test_a_row_shaped_schedule_does_not_count_on_an_engine_that_cannot_read_it( + monkeypatch, +): + """ + SCHEDULE_SOURCE and SCHEDULABLE_TASKS are keys only an engine that reads + rows looks at. On one that reads neither, the two entries are dictionary + keys nothing looks at, no row can dispatch the reconciler, and counting + them would seal a batch with nothing that ever completes it, which is the + condition the requirement exists to prevent. + """ + _engine_that_cannot_read_rows(monkeypatch) + with override_settings(TASKS=tasks_with_a_stored_source()): + assert batches.reconcile_schedule_aliases() == [] + with pytest.raises(ImproperlyConfigured): + batches.create("row-shaped") + + +def test_the_hint_names_the_settings_block_when_the_engine_cannot_read_rows( + monkeypatch, +): + """ + Naming SCHEDULABLE_TASKS here would send the project round a loop: + nothing reads the rows, so exposing the reconciler to them starts + nothing. The settings block is the only remedy on this engine. + """ + _engine_that_cannot_read_rows(monkeypatch) + with override_settings(TASKS=tasks_with_a_stored_source()): + with pytest.raises(ImproperlyConfigured) as caught: + batches.create("row-shaped") + assert "SCHEDULES" in str(caught.value) + assert "SCHEDULABLE_TASKS" not in str(caught.value) + + +def test_the_engine_capability_is_read_from_the_engine_itself(): + """ + Asked of the pinned django-ox's own loader, not of a version number, + so the pair moving to an engine that reads rows needs no edit here + and a fork that carries the feature is not refused for its version + string. + """ + import django_ox.schedules + + exposes = hasattr(django_ox.schedules, "schedule_source_from_options") + assert batches._engine_reads_schedule_rows() is exposes + + +def test_create_and_seal_refuse_when_no_backend_schedules_the_reconciler(): + """ + The check can be silenced and settings can change under a batch that + already exists, so create() and seal() refuse too, from the same + settings the check reads and without a database round trip. A batch + that cannot be sealed cannot sit SEALED forever. + """ + batch = batches.create("before-the-schedule-went") + with override_settings(TASKS=tasks_without_the_reconciler()): + with pytest.raises(ImproperlyConfigured, match=r"oxpull\.batches\.reconcile"): + batches.create("orphan") + with pytest.raises(ImproperlyConfigured, match=r"oxpull\.batches\.reconcile"): + batch.seal() + assert OxpullBatch.objects.count() == 1 + assert stored(batch).status == OxpullBatch.Status.OPEN + with override_settings(TASKS=tasks_without_the_reconciler(limited=True)): + batch.seal() + assert stored(batch).status == OxpullBatch.Status.SEALED + + +def test_seal_on_a_sealed_batch_reports_the_seal_before_the_missing_schedule(): + """ + The refusal for a missing reconciler schedule runs on the line that + seals, and it used to run first: seal() on a batch already SEALED or + COMPLETE raised ImproperlyConfigured about the schedule rather than + BatchSealed about the batch. The status is the thing the caller asked + about, so it is checked first and the settings guard second. + """ + batch = batches.create("sealed-twice") + batch.seal() + with ( + override_settings(TASKS=tasks_without_the_reconciler()), + pytest.raises(BatchSealed), + ): + batch.seal() + assert reconcile() == 1 + with ( + override_settings(TASKS=tasks_without_the_reconciler()), + pytest.raises(BatchSealed), + ): + batch.seal() + + +@pytest.mark.django_db(transaction=True) +def test_seal_under_postgresql_repeatable_read_raises_rather_than_counting_stale(): + """ + PostgreSQL REPEATABLE READ is the one level where "the status write + waits and the count that follows is current" does not describe what + happens. The transaction's snapshot is taken at its first statement, + which is that status UPDATE; when the UPDATE has waited on an add() + holding the batch row, the row it then finds was changed after the + snapshot, and PostgreSQL raises a serialization failure rather than + proceeding. Nothing is counted stale: the batch stays OPEN with no + total, and a fresh seal() counts every member. Django's default is + READ COMMITTED, where the UPDATE re-reads the row and the COUNT that + follows takes a fresh snapshot; this test sets the level on the + sealing session. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("PostgreSQL's REPEATABLE READ takes its snapshot at the UPDATE") + batch = batches.create("rr") + batch.add(tasks.plain_echo, "before") + added = threading.Event() + + def add_slowly(): + with transaction.atomic(): + batch.add(tasks.plain_echo, "slow") + added.set() + # The batch row stays locked until seal()'s UPDATE is waiting on + # it. That UPDATE took the snapshot when it began, so the + # snapshot is older than this commit however late the sealing + # thread got going. + assert a_session_waits_on_this_transaction(), ( + "seal() was never seen waiting on the batch row add() holds" + ) + return "added" + + def seal_at_repeatable_read(): + assert added.wait(timeout=10) + isolation.outlives_the_transaction("REPEATABLE READ") + with connections["default"].cursor() as cursor: + cursor.execute( + "SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL " + "REPEATABLE READ" + ) + try: + batch.seal() + except OperationalError as exc: + return type(exc.__cause__).__name__ + return "sealed" + + results, errors = race_all([add_slowly, seal_at_repeatable_read]) + + assert errors == [] + assert sorted(results) == ["SerializationFailure", "added"] + assert stored(batch).status == OxpullBatch.Status.OPEN + assert stored(batch).total_count is None + batch.seal() + assert stored(batch).total_count == 2 + + +@pytest.mark.django_db(transaction=True) +def test_a_reissued_add_sends_task_enqueued_for_each_attempt(monkeypatch): + """ + task_enqueued fires inside the enqueue transaction, before commit, + unchanged from django-ox. When add() owns its transaction and + reissues it after an InnoDB deadlock, the attempt that lost the + deadlock had already fired the signal for a row the rollback then + discarded, and the reissue fires it again for the row that lands: + one member, two firings, two ids. A receiver that acts on the + signal directly acts twice, once on a task that does not exist; + one deferred with transaction.on_commit acts once, on the row that + committed. `docs/features/batches.md` says so beside the retry. + """ + if connections["default"].vendor != "mysql": + pytest.skip("the reissue is InnoDB's 1213") + fired = [] + committed = [] + task_enqueued.connect( + lambda sender, task_result, **kwargs: fired.append(task_result.id), + weak=False, + dispatch_uid="fired", + ) + task_enqueued.connect( + lambda sender, task_result, **kwargs: transaction.on_commit( + lambda: committed.append(task_result.id) + ), + weak=False, + dispatch_uid="committed", + ) + try: + calls = deadlock_once(monkeypatch) + result = batches.create("victim").add(tasks.plain_echo, "k") + finally: + task_enqueued.disconnect(dispatch_uid="fired") + task_enqueued.disconnect(dispatch_uid="committed") + + assert len(calls) == 2 + assert len(fired) == 2 and len(set(fired)) == 2 + assert committed == [result.id] + assert OxTask.objects.count() == 1 diff --git a/tests_oxpull/test_concurrency.py b/tests_oxpull/test_concurrency.py new file mode 100644 index 0000000..7f4f9cb --- /dev/null +++ b/tests_oxpull/test_concurrency.py @@ -0,0 +1,2090 @@ +""" +Threads racing the same unique key. transaction=True so each thread's +connection sees committed state, mirroring real worker processes. +""" + +import io +import socket +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from datetime import timedelta + +import pytest +from django.core.management import call_command +from django.db import OperationalError, connections, transaction +from django.db.backends.signals import connection_created +from django.db.models import F +from django.tasks.signals import task_enqueued +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.results import task_result_from_db +from oxpull import backend as backend_module +from oxpull.backend import OxpullBackend +from oxpull.exceptions import UniqueEnqueueRace +from oxpull.models import OxpullUniqueLock + +from . import tasks +from .isolation import ( + fresh_read_transactions, + how_new_connections_run, + isolation_for_new_connections, + new_session_settings, + opened_connections, + pg_sqlstate, + record_holder_checks, + result_fields, + session_isolation, + set_session_isolation, + set_transaction_isolation, +) +from .racing import THREADS, race, race_all + + +@pytest.mark.django_db(transaction=True) +def test_racing_enqueues_have_exactly_one_winner(): + results, errors = race(lambda: tasks.unique_echo.enqueue("raced").id) + + assert errors == [] + assert len(results) == THREADS + assert len(set(results)) == 1 + assert OxTask.objects.count() == 1 + lock = OxpullUniqueLock.objects.get() + assert str(lock.task_id) == results[0] + + +@pytest.mark.django_db(transaction=True) +def test_racing_enqueues_over_a_stale_lock_release_it_once(worker): + # Set up a finished holder whose lock is stale, then race the steal + # path: exactly one thread may release-and-reenqueue. + first = tasks.unique_echo.enqueue("raced") + assert worker.run_once() is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.SUCCESSFUL + + results, errors = race(lambda: tasks.unique_echo.enqueue("raced").id) + + assert errors == [] + assert len(set(results)) == 1 + assert results[0] != first.id + # The finished task plus exactly one new one; one live lock. + assert OxTask.objects.count() == 2 + lock = OxpullUniqueLock.objects.get() + assert str(lock.task_id) == results[0] + + +def shorten_lock_wait(monkeypatch): + """ + One second, so the loser below gives up in a second rather than in + the database's default: SQLite's busy timeout is the test settings' + 20 s, MySQL's innodb_lock_wait_timeout is 50 s, and PostgreSQL waits + forever unless a project sets lock_timeout. SQLite's is a connection + parameter, applied through the dict every new connection is built + from; the other two are set on the loser's own session. + """ + connection = connections["default"] + if connection.vendor == "sqlite": + monkeypatch.setitem(connection.settings_dict["OPTIONS"], "timeout", 1) + + def on_this_session() -> None: + session = connections["default"] + if session.vendor == "mysql": + with session.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 1") + elif session.vendor == "postgresql": + with session.cursor() as cursor: + cursor.execute("SET lock_timeout = '1s'") + + return on_this_session + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("ambient", [False, True], ids=["own", "ambient"]) +def test_a_holder_that_outlives_the_lock_wait_raises_the_named_race( + monkeypatch, ambient +): + """ + The winner's transaction can outlive the database's lock wait: the + enqueue joins the caller's transaction, so a slow request holds the + key for as long as it runs. SQLite then raises busy after its + timeout, MySQL 1205 after innodb_lock_wait_timeout, PostgreSQL + lock_not_available if a lock_timeout is set. The loser used to get + that raw OperationalError where the API promises a TaskResult. It + now gets UniqueEnqueueRace, chained to the original, which already + means "nothing was written, safe to retry". Inside a caller's + transaction the same exception says to retry the whole transaction, + and the caller's transaction is still usable after it, on a server + that rolls back the statement and not the transaction (MySQL's + default; the test below covers the other setting). + """ + on_this_session = shorten_lock_wait(monkeypatch) + + def loser(): + on_this_session() + caught = None + if ambient: + with transaction.atomic(): + try: + tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + caught = exc + # The enqueue rolled back to its savepoint; the caller's + # transaction is still open and still usable. + assert OxpullUniqueLock.objects.count() == 0 + else: + try: + tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + caught = exc + return caught + + with transaction.atomic(): + holder = tasks.unique_echo.enqueue("held") + results, errors = race_all([loser]) + + assert errors == [] + (caught,) = results + assert isinstance(caught, UniqueEnqueueRace) + assert isinstance(caught.__cause__, OperationalError) + assert "lock wait" in str(caught) + assert ("whole transaction" in str(caught)) is ambient + # Nothing was written for the loser, and the key still deduplicates. + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + assert tasks.unique_echo.enqueue("held").id == holder.id + + +def touch_elsewhere(task_id, status=None, other=None): + """ + Commit a write on the task row from a connection of the test's own, or on + `other`, an autocommit connection the caller opened and closes. With no + status the write sets a column to its own value, which leaves the task as + it was. PostgreSQL still writes a new version of the row, as it does for a + lease renewal. + """ + own = other is None + if own: + other = connections.create_connection("default") + table = other.ops.quote_name(OxTask._meta.db_table) + pk = OxTask._meta.pk.get_db_prep_value(uuid.UUID(str(task_id)), other) + if status is None: + sql, params = f"UPDATE {table} SET priority = priority WHERE id = %s", [pk] # noqa: S608 + else: + sql, params = f"UPDATE {table} SET status = %s WHERE id = %s", [status, pk] # noqa: S608 + try: + with other.cursor() as cursor: + cursor.execute(sql, params) + assert cursor.rowcount == 1 + finally: + if own: + other.close() + + +def other_sessions(): + """PostgreSQL sessions on the test database other than this thread's own.""" + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT count(*) FROM pg_stat_activity " + "WHERE datname = current_database() AND pid <> pg_backend_pid()" + ) + (count,) = cursor.fetchone() + return count + + +def warmed_pool(): + """ + When the alias is pooled, give this thread's connection back to Django's + pool and wait until the pool holds its minimum of idle connections. The + pool opens its connections in the background, so a session count taken + before then can miss one that is still connecting and see it afterwards. + A pool that is not ready within the timeout fails the test. + """ + connection = connections["default"] + pool = getattr(connection, "pool", None) + if pool is None: + return + connection.close() + pool.open() # as Django does before every checkout; nothing once open + pool.wait(timeout=30) + + +def settle_sessions(expected): + """ + The session count once it reaches `expected`, or what it is after two + seconds. A closed session leaves pg_stat_activity a moment after the + client closes it. + """ + deadline = time.monotonic() + 2 + count = other_sessions() + while count != expected and time.monotonic() < deadline: + time.sleep(0.05) + count = other_sessions() + return count + + +def locked_elsewhere(task_id): + """ + Whether another connection is refused an exclusive lock on the task row + at once. A lock of any strength that another transaction keeps on the + row refuses it. + """ + probe = connections.create_connection("default") + table = probe.ops.quote_name(OxTask._meta.db_table) + try: + with probe.cursor() as cursor: + cursor.execute( + f"SELECT 1 FROM {table} WHERE id = %s FOR UPDATE NOWAIT", # noqa: S608 + [OxTask._meta.pk.get_db_prep_value(uuid.UUID(str(task_id)), probe)], + ) + cursor.fetchall() + except OperationalError: + return True + finally: + probe.close() + return False + + +def blocking_locks_held_here(): + """ + The locks this thread's open transaction holds on the task table that + could make another session wait for it. + + PostgreSQL records a locking read as a RowShareLock on the table, next to + the AccessShareLock every read takes. Two modes are left out, because + neither can hold another session off the task table: + + - AccessShareLock, which every read takes and which conflicts only with + the AccessExclusiveLock a DDL statement asks for. Nothing under test + here runs DDL. + - SIReadLock, the predicate lock SERIALIZABLE records for a read. It + conflicts with no lock mode at all: nothing waits on it, and no claim + skips a row because of it. It exists so that SSI can spot a + serialization anomaly and abort a transaction, never to make one wait. + Its granularity follows the plan, tuple for an index scan and relation + for a sequential scan, so a plain read of one row records a + relation-wide one as soon as the planner has statistics that make the + sequential scan cheaper. The read that fixes this transaction's + snapshot, above, is enough to record it. + + Every other mode PostgreSQL can record on a table conflicts with + something, so leaving the filter negative keeps an unexpected one + visible. InnoDB lists each record lock, and each of those can block. + """ + connection = connections["default"] + table = OxTask._meta.db_table + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute( + "SELECT mode FROM pg_locks WHERE pid = pg_backend_pid() " + "AND locktype = 'relation' AND relation = %s::regclass " + "AND mode <> 'AccessShareLock' AND mode <> 'SIReadLock'", + [table], + ) + else: + cursor.execute( + "SELECT LOCK_MODE FROM performance_schema.data_locks " + "WHERE OBJECT_NAME = %s AND LOCK_TYPE = 'RECORD' " + "AND THREAD_ID = PS_CURRENT_THREAD_ID()", + [table], + ) + return sorted(row[0] for row in cursor.fetchall()) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "write", ["renew", "finish", "requeue", "lose", "claim", "prune"] +) +@pytest.mark.parametrize("path", ["unchanged", "changed"]) +@pytest.mark.parametrize( + "level", + [None, "REPEATABLE READ", "SERIALIZABLE"], + ids=["read-committed", "repeatable-read", "serializable"], +) +def test_a_duplicate_inside_a_callers_transaction_locks_no_live_holder( + monkeypatch, worker, level, path, write +): + """ + Deduplicating against a READY or RUNNING holder keeps no row lock. + + A duplicate enqueue inside a caller's transaction (ATOMIC_REQUESTS, + any atomic() block) used to read the holder with SELECT ... FOR + UPDATE, and a row lock taken inside a savepoint lives until the + outer transaction commits. For as long as the request ran, the + holder's own worker could not renew its lease or write its outcome, + and the reaper could not take the row. A live holder has nothing to + release, so its row is read without a lock. + + On PostgreSQL at REPEATABLE READ and SERIALIZABLE the enqueue then asks + for a share lock on the row without waiting, in a savepoint it rolls + back at once. On the path "changed", a write committed on the holder's + row after the request's snapshot began and left the task pending. The + request is refused (40001), reads the holder again on a connection of + its own, and closes that connection before returning. Either way no + lock is left when the enqueue returns. With the request still open, the + worker renews the lease, writes the outcome, is requeued or recorded + LOST by the reaper, claims a READY holder, or ox_prune deletes the + finished row. Each runs with a one-second lock wait, and the request + stays open until it has run, so a statement that waited on the request + would fail with the lock wait's error. A share lock kept to the request's + commit would make the claim skip the holder and every other statement + wait. At SERIALIZABLE every read also records a predicate lock, which + makes nothing wait and is not counted; the helper says why. MySQL at + REPEATABLE READ opens a connection on both paths, as in 1.2.0. SQLite + has no row locks, and InnoDB's SERIALIZABLE is pinned by the test below. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite has no row locks for a transaction to hold") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE locks the holder, pinned below") + if level is None and path == "changed": + pytest.skip("READ COMMITTED reads the current row and runs no check") + on_this_session = shorten_lock_wait(monkeypatch) + holder = tasks.unique_echo.enqueue("held") + db_task = None + if write != "claim": + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + if write in ("requeue", "lose"): + expired = timezone.now() - timedelta(hours=1) + updates = {"locked_at": expired, "lease_expires_at": expired} + if write == "lose": + updates["attempts"] = db_task.max_attempts + OxTask.objects.filter(pk=db_task.pk).update(**updates) + if level is not None: + # Connections opened from here on, the request's among them. + isolation_for_new_connections(monkeypatch, level) + checks = record_holder_checks(monkeypatch) + + snapshot_taken = threading.Event() + changed = threading.Event() + deduplicated = threading.Event() + request_done = threading.Event() + outcome = {} + + def web_request(): + try: + with transaction.atomic(): + outcome["level"] = session_isolation() + # The first read fixes the snapshot. + OxTask.objects.get(id=holder.id) + snapshot_taken.set() + assert changed.wait(timeout=10) + with opened_connections() as opened: + outcome["result"] = tasks.unique_echo.enqueue("held") + outcome["opened"] = [wrapper.alias for wrapper in opened] + outcome["closed"] = all( + wrapper.connection is None for wrapper in opened + ) + outcome["blocking_locks"] = blocking_locks_held_here() + deduplicated.set() + assert request_done.wait(timeout=30) + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + snapshot_taken.set() + deduplicated.set() + finally: + connections.close_all() + + request = threading.Thread(target=web_request) + request.start() + try: + assert snapshot_taken.wait(timeout=10) + if path == "changed": + # A write that leaves the holder pending: its lease renewal where + # the write under test allows one, otherwise a column set to its + # own value. + if write in ("renew", "finish", "prune"): + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + else: + rows = OxTask.objects.filter(pk=holder.id) + assert rows.update(priority=F("priority")) == 1 + changed.set() + assert deduplicated.wait(timeout=10) + assert "error" not in outcome, outcome["error"] + assert outcome["level"] == (level or "READ COMMITTED") + assert outcome["result"].id == holder.id + expected = task_result_from_db(OxTask.objects.get(pk=holder.id)) + assert result_fields(outcome["result"]) == result_fields(expected) + refused = vendor == "postgresql" and path == "changed" + checked = vendor == "postgresql" and level is not None + assert checks == ((["40001"] if refused else ["granted"]) if checked else []) + reread = refused or (vendor == "mysql" and level == "REPEATABLE READ") + assert outcome["opened"] == (["default"] if reread else []) + assert outcome["closed"] + assert outcome["blocking_locks"] == [], ( + "the request kept a lock on the task table" + ) + assert not locked_elsewhere(holder.id), "the request kept a lock on the row" + on_this_session() + if write == "renew": + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + elif write == "finish": + worker.execute(db_task) + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.SUCCESSFUL + elif write == "requeue": + assert worker.reap() == 1 + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.READY + elif write == "lose": + assert worker.reap() == 1 + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.LOST + elif write == "claim": + claimed = worker.claim_one() + assert claimed is not None, "the claim skipped the holder" + assert str(claimed.id) == holder.id + else: + worker.execute(db_task) + call_command("ox_prune", older_than="0s", stdout=io.StringIO()) + assert not OxTask.objects.filter(pk=db_task.pk).exists() + finally: + changed.set() + request_done.set() + request.join(timeout=30) + assert not request.is_alive() + assert "error" not in outcome, outcome["error"] + + +@pytest.mark.django_db(transaction=True) +def test_a_duplicate_at_mysql_serializable_holds_off_the_lease_renewal( + monkeypatch, worker +): + """ + A known limit, pinned so it cannot change unnoticed. + + At MySQL's SERIALIZABLE, InnoDB turns every plain read inside a + transaction into a locking read. A duplicate enqueued there reads the + holder's row and keeps a shared lock on it until the caller's + transaction ends. The holder's lease renewal waits on that lock and + fails with 1205 once innodb_lock_wait_timeout passes, one second here. + After the caller commits, the renewal lands. 1.2.0 behaves the same. + """ + if connections["default"].vendor != "mysql": + pytest.skip("only InnoDB's SERIALIZABLE makes a plain read a locking read") + on_this_session = shorten_lock_wait(monkeypatch) + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + isolation_for_new_connections(monkeypatch, "SERIALIZABLE") + + deduplicated = threading.Event() + request_done = threading.Event() + outcome = {} + + def web_request(): + try: + with transaction.atomic(): + outcome["level"] = session_isolation() + outcome["result"] = tasks.unique_echo.enqueue("held") + deduplicated.set() + assert request_done.wait(timeout=30) + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + deduplicated.set() + finally: + connections.close_all() + + request = threading.Thread(target=web_request) + request.start() + try: + assert deduplicated.wait(timeout=10) + assert "error" not in outcome, outcome["error"] + assert outcome["level"] == "SERIALIZABLE" + assert outcome["result"].id == holder.id + assert locked_elsewhere(holder.id) + on_this_session() + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + with pytest.raises(OperationalError) as excinfo: + worker.renew_leases() + assert excinfo.value.args[0] == 1205 + finally: + request_done.set() + request.join(timeout=30) + assert not request.is_alive() + assert "error" not in outcome, outcome["error"] + assert worker.renew_leases() == 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("order", ["snapshot-first", "change-first"]) +@pytest.mark.parametrize("other", ["outcome", "for-update", "delete"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_holder_another_transaction_is_changing_is_deduplicated_at_once( + monkeypatch, worker, level, other, order +): + """ + Another transaction is writing the holder's row, locking it FOR UPDATE + (as a release path does), or deleting it, and has not committed. The + row the caller's snapshot holds is still the last committed one. The + share lock request is refused at once with lock_not_available (55P03), + and the enqueue deduplicates against that row without waiting, as READ + COMMITTED would. It must not report the refusal as a lock wait. The + other transaction's change comes before or after the snapshot, and it + is rolled back once the enqueue returns. No connection is opened. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + checks = record_holder_checks(monkeypatch) + + writer = connections.create_connection("default") + writer.set_autocommit(False) + table = writer.ops.quote_name(OxTask._meta.db_table) + pk = OxTask._meta.pk.get_db_prep_value(db_task.pk, writer) + statement, params = { + "outcome": ( + f"UPDATE {table} SET status = %s, finished_at = now() WHERE id = %s", # noqa: S608 + [OxTask.Status.SUCCESSFUL, pk], + ), + "for-update": (f"SELECT 1 FROM {table} WHERE id = %s FOR UPDATE", [pk]), # noqa: S608 + "delete": (f"DELETE FROM {table} WHERE id = %s", [pk]), # noqa: S608 + }[other] + + def change(): + with writer.cursor() as cursor: + cursor.execute(statement, params) + + isolation_for_new_connections(monkeypatch, level) + taken = threading.Event() + go = threading.Event() + outcome = {} + + def request(): + try: + with transaction.atomic(): + outcome["level"] = session_isolation() + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.RUNNING + taken.set() + assert go.wait(timeout=10) + with opened_connections() as opened: + outcome["result"] = tasks.unique_echo.enqueue("held") + outcome["opened"] = [wrapper.alias for wrapper in opened] + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + finally: + taken.set() + connections.close_all() + + try: + if order == "change-first": + change() + thread = threading.Thread(target=request) + thread.start() + assert taken.wait(timeout=10) + if order == "snapshot-first": + change() + go.set() + # The other transaction is still open, so an enqueue that waited for + # it would still be running when the join gives up. + thread.join(timeout=30) + assert not thread.is_alive() + finally: + writer.rollback() + writer.close() + + assert "error" not in outcome, outcome["error"] + assert outcome["level"] == level + assert outcome["result"].id == holder.id + assert outcome["result"].status == "RUNNING" + assert checks == ["55P03"] + assert outcome["opened"] == [] + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.RUNNING + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_enqueues_of_two_keys_in_open_transactions_do_not_wait_on_each_other( + monkeypatch, level +): + """ + A request that enqueued one unique key and keeps its transaction open + does not hold up an enqueue of another key. The lock UPDATE that names + the new task matches the row's key as well as its unset task. The first + request stays open until the second enqueue is done, and the lock wait + here is one second, so an enqueue that waited on the first would raise + UniqueEnqueueRace. + """ + if connections["default"].vendor == "sqlite": + pytest.skip("SQLite has one writer at a time") + on_this_session = shorten_lock_wait(monkeypatch) + tasks.unique_echo.enqueue("seed") + connections["default"].close() + isolation_for_new_connections(monkeypatch, level) + enqueued = threading.Event() + release = threading.Event() + outcome = {} + + def open_request(): + try: + with transaction.atomic(): + outcome["a"] = tasks.unique_echo.enqueue("a") + enqueued.set() + assert release.wait(timeout=30) + except BaseException as exc: # pragma: no cover - reported below + outcome["a_error"] = exc + enqueued.set() + finally: + connections.close_all() + + request = threading.Thread(target=open_request) + request.start() + try: + assert enqueued.wait(timeout=10) + on_this_session() + try: + outcome["b"] = tasks.unique_echo.enqueue("b") + except BaseException as exc: # pragma: no cover - reported below + outcome["b_error"] = exc + finally: + release.set() + request.join(timeout=30) + connections["default"].close() + assert not request.is_alive() + + assert "a_error" not in outcome, outcome["a_error"] + assert "b_error" not in outcome, outcome["b_error"] + assert OxTask.objects.count() == 3 + + +def session_id(): + """This thread's server session, as the lock views name it.""" + connection = connections["default"] + query = ( + "SELECT pg_backend_pid()" + if connection.vendor == "postgresql" + else "SELECT CONNECTION_ID()" + ) + with connection.cursor() as cursor: + cursor.execute(query) + (value,) = cursor.fetchone() + return value + + +def waits_on_a_lock(session, timeout=10.0): + """ + Whether that session is seen waiting for a row lock within the timeout. + + MySQL is asked through performance_schema, which reports each wait as + it stands. information_schema.innodb_trx serves a cache that InnoDB + refreshes only after 0.1 s without a read, so polling it faster keeps + returning what it held when polling began. + """ + connection = connections["default"] + if connection.vendor == "postgresql": + query = ( + "SELECT count(*) FROM pg_stat_activity " + "WHERE pid = %s AND wait_event_type = 'Lock'" + ) + else: + query = ( + "SELECT count(*) FROM performance_schema.data_lock_waits waits " + "JOIN performance_schema.threads threads " + "ON threads.THREAD_ID = waits.REQUESTING_THREAD_ID " + "WHERE threads.PROCESSLIST_ID = %s" + ) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with connection.cursor() as cursor: + cursor.execute(query, [session]) + (count,) = cursor.fetchone() + if count: + return True + time.sleep(0.02) + return False + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"]) +def test_a_release_committing_while_the_lock_insert_waits_does_not_fail_the_enqueue( + monkeypatch, level +): + """ + An enqueue that owns its transaction queues its task when another + process's release of the key commits while the lock INSERT waits on it. + + PostgreSQL at REPEATABLE READ or SERIALIZABLE takes the snapshot at the + lock INSERT. The INSERT waits on the release's DELETE of the old lock + row and goes ahead once it commits, but the snapshot still shows that + row. The UPDATE that names the new task on the lock matched the old row + by key, and PostgreSQL refused it as a concurrent delete with a + serialization failure (SQLSTATE 40001), which reached the caller raw. + The UPDATE now also requires the lock's task to be unset. Only the row + this transaction inserted is, because a committed lock row always + names its task. + + The enqueue takes one pass and sends task_enqueued once, for the task + that commits. The release runs on another connection at READ COMMITTED, + and it commits once the enqueue is seen waiting on it. + """ + if connections["default"].vendor == "sqlite": + pytest.skip("SQLite's single writer leaves no lock INSERT waiting on a row") + first = tasks.unique_echo.enqueue("x") + OxTask.objects.filter(id=first.id).update(status=OxTask.Status.FAILED) + lock = OxpullUniqueLock.objects.get() + + releaser = connections.create_connection("default") + releaser.set_autocommit(False) + table = releaser.ops.quote_name(OxpullUniqueLock._meta.db_table) + key = releaser.ops.quote_name(OxpullUniqueLock._meta.pk.column) + with releaser.cursor() as cursor: + cursor.execute(f"DELETE FROM {table} WHERE {key} = %s", [lock.pk]) # noqa: S608 + assert cursor.rowcount == 1 + connections["default"].close() + isolation_for_new_connections(monkeypatch, level) + + started = threading.Event() + outcome = {} + + def enqueue(): + try: + outcome["session"] = session_id() + with transaction.atomic(): + # PostgreSQL applies a level set in OPTIONS as each + # transaction begins, so the level is asked inside one. + outcome["level"] = session_isolation() + started.set() + outcome["result"] = tasks.unique_echo.enqueue("x") + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + finally: + started.set() + connections.close_all() + + signalled = [] + + def record(sender, task_result, **kwargs): + signalled.append(task_result.id) + + task_enqueued.connect(record) + thread = threading.Thread(target=enqueue) + thread.start() + try: + assert started.wait(timeout=10) + assert waits_on_a_lock(outcome.get("session")), "the INSERT never waited" + releaser.commit() + finally: + releaser.close() + thread.join(timeout=30) + task_enqueued.disconnect(record) + assert not thread.is_alive() + + assert "error" not in outcome, outcome["error"] + assert outcome["level"] == level + second = outcome["result"] + assert second.id != first.id + ready = OxTask.objects.filter(status=OxTask.Status.READY) + assert [str(task.id) for task in ready] == [second.id] + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + assert signalled == [second.id], "a pass was refused and reissued" + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_serialization_failure_in_the_enqueues_own_transaction_is_reissued( + monkeypatch, worker, level +): + """ + PostgreSQL at REPEATABLE READ or SERIALIZABLE refuses to lock a row + that changed after the transaction's snapshot began. It raises a + serialization failure (SQLSTATE 40001), and the transaction it + happened in is lost. With no caller's transaction around the enqueue, + the snapshot is the resolution's own, taken at its read of the lock. An + operator retry committing between that read and the locking read of a + FAILED holder raised the failure raw where the API promises a + TaskResult. The enqueue owned the transaction the failure ended, so + nothing of it survived, and the pass is now reissued as an InnoDB + deadlock's is. The next pass reads the retried holder as live. + """ + from django_ox import actions + + if connections["default"].vendor != "postgresql": + pytest.skip("only PostgreSQL refuses a locking read behind the snapshot") + first = tasks.unique_fail_always.enqueue() + while worker.run_once(): + pass + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + + retried = {} + + def retry_on_another_connection(): + try: + retried["ok"] = actions.retry(first.id) + finally: + connections.close_all() + + holder = OxpullBackend._holder + staged = [] + refused = [] + + def retry_before_the_locking_read(task_id, db_alias, *, locking, **kwargs): + if not locking or staged: + return holder(task_id, db_alias, locking=locking, **kwargs) + staged.append(task_id) + thread = threading.Thread(target=retry_on_another_connection) + thread.start() + thread.join(timeout=30) + assert not thread.is_alive() + try: + return holder(task_id, db_alias, locking=True, **kwargs) + except OperationalError as exc: + refused.append(pg_sqlstate(exc)) + raise + + monkeypatch.setattr( + OxpullBackend, "_holder", staticmethod(retry_before_the_locking_read) + ) + set_session_isolation(level) + try: + second = tasks.unique_fail_always.enqueue() + finally: + connections["default"].close() + + assert retried.get("ok") is True + assert refused == ["40001"] + assert second.id == first.id + ready = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_fail_always", status=OxTask.Status.READY + ) + assert ready.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("path", ["unchanged", "changed"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_live_thread_that_resolved_a_duplicate_leaves_no_session_behind( + monkeypatch, level, path +): + """ + PostgreSQL refuses DROP DATABASE while any session is connected, so a + test database cannot be dropped at teardown while a thread that is still + alive keeps a connection open. A thread from a pool resolves a duplicate + inside a transaction at a snapshot level and closes its own connections, + as a request thread does. On the path "changed", the holder's row changed + after the thread's snapshot began, so the enqueue also read the holder on + a connection of its own. The pool stays alive, and the database must have + no more sessions than before. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("only PostgreSQL refuses to drop a database with sessions") + holder = tasks.unique_echo.enqueue("held") + connections["default"].close() + isolation_for_new_connections(monkeypatch, level) + checks = record_holder_checks(monkeypatch) + + def work(): + try: + with transaction.atomic(): + OxTask.objects.get(id=holder.id) + if path == "changed": + touch_elsewhere(holder.id) + return tasks.unique_echo.enqueue("held").id + finally: + connections.close_all() + + pool = ThreadPoolExecutor(max_workers=1) + try: + warmed_pool() + before = other_sessions() + assert pool.submit(work).result(timeout=30) == holder.id + assert settle_sessions(before) <= before, "a live thread left a session open" + assert checks == (["40001"] if path == "changed" else ["granted"]) + finally: + pool.shutdown(wait=True) + connections["default"].close() + + +@pytest.mark.django_db(transaction=True) +def test_a_retry_landing_between_the_read_and_the_release_keeps_its_lock( + monkeypatch, worker +): + """ + SQLite: the release is conditional on the holder in the DELETE itself. + + SQLite has no row locks, and a transaction that reads and then + writes fails "database is locked" the moment two of them race, so + the resolver runs statement by statement there. Between its read of + a FAILED holder and its DELETE of the lock, an operator retry could + commit READY; the DELETE then removed a live lock and the next + attempt inserted a second READY row beside the retried one. The + DELETE now carries the holder's status in its own WHERE clause, and + a statement is atomic against SQLite's single writer, so a retry + that landed first leaves the lock where it is. + + PostgreSQL and MySQL take the holder's row lock on this path, so a + retry waits behind it and lands after the release, as the documented + operator bypass; the interleaving here cannot be staged there. + """ + from django.db.models.query import QuerySet + + from django_ox import actions + from oxpull.models import OxpullUniqueLock as Lock + + if connections["default"].vendor != "sqlite": + pytest.skip("the row lock makes a retry wait behind the release here") + first = tasks.unique_fail_always.enqueue() + while worker.run_once(): + pass + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + + retried = {} + + def retry_on_another_connection(): + try: + retried["ok"] = actions.retry(first.id) + finally: + connections.close_all() + + original_delete = QuerySet.delete + + def delete_after_a_retry(self, *args, **kwargs): + if self.model is Lock and "ok" not in retried: + thread = threading.Thread(target=retry_on_another_connection) + thread.start() + thread.join(timeout=30) + assert retried.get("ok") is True + return original_delete(self, *args, **kwargs) + + monkeypatch.setattr(QuerySet, "delete", delete_after_a_retry) + second = tasks.unique_fail_always.enqueue() + + assert retried.get("ok") is True + assert second.id == first.id + ready = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_fail_always", status=OxTask.Status.READY + ) + assert ready.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + + +@pytest.mark.django_db(transaction=True) +def test_the_message_says_when_the_server_rolled_the_whole_transaction_back( + monkeypatch, +): + """ + MySQL with innodb_rollback_on_timeout=ON rolls the whole transaction + back on 1205, not the statement. Django's rollback to the enqueue's + savepoint then fails, the connection is marked for rollback, and + nothing the caller did in that transaction survives; the message + used to promise a savepoint rollback and a usable transaction there. + The server setting is not dynamic, so this stages what the driver + reports on such a server: the savepoint rollback raising once the + holder is in place. Against a server with the setting on, the real + exchange produces the same message. + """ + from django.db.backends.base.base import BaseDatabaseWrapper + + on_this_session = shorten_lock_wait(monkeypatch) + armed = {"fail": False} + real_rollback = BaseDatabaseWrapper.savepoint_rollback + + def rollback_as_if_the_transaction_were_gone(self, sid): + if armed["fail"]: + raise OperationalError(1305, f"SAVEPOINT {sid} does not exist") + return real_rollback(self, sid) + + monkeypatch.setattr( + BaseDatabaseWrapper, + "savepoint_rollback", + rollback_as_if_the_transaction_were_gone, + ) + + def loser(): + on_this_session() + armed["fail"] = True + caught = None + try: + with transaction.atomic(): + try: + tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + caught = exc + assert connections["default"].needs_rollback + except OperationalError: + # The atomic block's own exit rolls the marked transaction + # back; the driver's error is not what this test is about. + pass + return caught + + with transaction.atomic(): + tasks.unique_echo.enqueue("held") + results, errors = race_all([loser]) + + assert errors == [] + (caught,) = results + assert isinstance(caught, UniqueEnqueueRace) + assert "rolled that whole transaction back" in str(caught) + assert "innodb_rollback_on_timeout" in str(caught) + assert "still usable" not in str(caught) + + +def test_the_message_names_a_refused_snapshot_upgrade_on_sqlite(): + """ + SQLITE_BUSY_SNAPSHOT is not a lock wait: SQLite in WAL mode refuses + at once to make a writer of a reader whose snapshot is behind another + connection's commit. The exception class is the same, the sentence + is not. The extended code is staged on the cause; the primary code + (5) is what the classifier keys on either way. + """ + import sqlite3 + + from django.tasks import default_task_backend + + from oxpull.backend import SQLITE_BUSY_SNAPSHOT, _is_lock_wait_timeout + + if connections["default"].vendor != "sqlite": + pytest.skip("the extended code is SQLite's") + + class Refused(sqlite3.OperationalError): + # The module sets sqlite_errorcode on the errors it raises; a + # constructed one carries it as a class attribute instead. + sqlite_errorcode = SQLITE_BUSY_SNAPSHOT + + class Busy(sqlite3.OperationalError): + sqlite_errorcode = 5 + + backend = default_task_backend + exc = OperationalError("database is locked") + exc.__cause__ = Refused("database is locked") + assert _is_lock_wait_timeout(exc, "default") + message = backend._lock_wait_message( + tasks.unique_echo, exc, "default", ambient_atomic=True + ) + assert "SQLITE_BUSY_SNAPSHOT" in message + assert "did not wait" in message + assert "whole transaction" in message + plain = OperationalError("database is locked") + plain.__cause__ = Busy("database is locked") + assert "SQLITE_BUSY_SNAPSHOT" not in backend._lock_wait_message( + tasks.unique_echo, plain, "default", ambient_atomic=False + ) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("project", ["plain", "session-defaults"]) +@pytest.mark.parametrize("path", ["deduplicated", "race", "error", "refused"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_fresh_read_connection_is_closed_on_every_path( + monkeypatch, worker, level, path, project +): + """ + The read on a connection of its own opens one session and closes it + before the enqueue returns. That holds when the enqueue deduplicates, + when it raises UniqueEnqueueRace, when the read fails once connected, and + when the connection is refused. + + The holder's row changes after the request's snapshot began. A lease + renewal leaves it RUNNING, or the outcome write finishes it. For "error" + the statement on the new connection fails once it has connected, and for + "refused" the connection goes to a port nothing listens on. A failed read + raises UniqueEnqueueRace chained to the failure, with its first line in + the message. While the request is still open, pg_stat_activity must show + no session beyond the ones there before the enqueue, and each connection + the enqueue created must be closed. + + The project's OPTIONS name the caller's level. For "session-defaults" + they also start every new session at SERIALIZABLE with a 700 ms + lock_timeout and statement_timeout, as a project's OPTIONS["options"] or + a role's settings can. The new connection is built without the level or a + pool, and its read runs in a transaction of its own at READ COMMITTED + with lock_timeout 100ms and statement_timeout 2s. Those last for that + transaction alone: after its COMMIT the session is back at its own + settings. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + connections["default"].close() + isolation_for_new_connections(monkeypatch, level) + options = connections["default"].settings_dict["OPTIONS"] + injected = OperationalError("injected once connected") + + snapshot_taken = threading.Event() + changed = threading.Event() + returned = threading.Event() + request_done = threading.Event() + outcome = {} + + def failing(create): + def create_connection(alias): + wrapper = create(alias) + if path == "refused": + wrapper.settings_dict = {**wrapper.settings_dict, "PORT": "1"} + return wrapper + connect_and_open = wrapper.cursor + + def cursor(*args, **kwargs): + connect_and_open(*args, **kwargs) + raise injected + + wrapper.cursor = cursor + return wrapper + + return create_connection + + def web_request(): + try: + with transaction.atomic(): + OxTask.objects.get(id=holder.id) + snapshot_taken.set() + assert changed.wait(timeout=10) + # Set here, after every other connection in the test is open, + # so only the sessions opened from now on start with them. + if project == "session-defaults": + options["options"] = ( + "-c default_transaction_isolation=serializable " + "-c lock_timeout=700ms -c statement_timeout=700ms" + ) + try: + outcome["session"] = new_session_settings() + with ( + how_new_connections_run() as runs, + opened_connections() as opened, + fresh_read_transactions() as reads, + ): + if path in ("error", "refused"): + connections.create_connection = failing( + connections.create_connection + ) + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except (UniqueEnqueueRace, OperationalError) as exc: + outcome["raised"] = exc + finally: + options.pop("options", None) + outcome["opened"] = [wrapper.alias for wrapper in opened] + outcome["closed"] = all( + wrapper.connection is None for wrapper in opened + ) + outcome["runs"] = runs + outcome["reads"] = reads + outcome["usable"] = OxTask.objects.filter(id=holder.id).count() + returned.set() + assert request_done.wait(timeout=30) + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + snapshot_taken.set() + returned.set() + finally: + connections.close_all() + + warmed_pool() + request = threading.Thread(target=web_request) + request.start() + try: + assert snapshot_taken.wait(timeout=10) + if path == "race": + worker.execute(db_task) + else: + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + with_request = settle_sessions(other_sessions()) + changed.set() + assert returned.wait(timeout=10) + assert "error" not in outcome, outcome["error"] + while_open = settle_sessions(with_request) + finally: + changed.set() + request_done.set() + request.join(timeout=30) + assert not request.is_alive() + # The request's own session ends with its thread's close_all(), unless + # Django's pool took the connection back, where it stays open and idle. + ended = 0 if connections["default"].settings_dict["OPTIONS"].get("pool") else 1 + after = settle_sessions(with_request - ended) + + assert "error" not in outcome, outcome["error"] + assert while_open == with_request, "the enqueue left its connection open" + assert after == with_request - ended + assert outcome["opened"] == ["default"] + assert outcome["closed"] + assert outcome["usable"] == 1 + session = outcome["session"] + if project == "session-defaults": + assert session == ("serializable", "700ms", "700ms") + (read,) = outcome["reads"] + if path == "refused": + assert outcome["runs"] == [] + else: + assert [(run["level"], run["autocommit"]) for run in outcome["runs"]] == [ + (session[0].upper(), True) + ] + assert "isolation_level" not in outcome["runs"][0]["options"] + assert "pool" not in outcome["runs"][0]["options"] + if path in ("deduplicated", "race"): + assert read["statements"] == ["BEGIN", "SELECT", "COMMIT"] + assert read["at_read"] == ("read committed", "100ms", "2s") + assert read["after"] == session + else: + assert read["statements"] == [] + if path == "deduplicated": + assert "raised" not in outcome, outcome["raised"] + assert outcome["result"].id == holder.id + elif path == "race": + assert isinstance(outcome.get("raised"), UniqueEnqueueRace) + assert "A fresh read found the task SUCCESSFUL." in str(outcome["raised"]) + else: + raised = outcome.get("raised") + assert isinstance(raised, UniqueEnqueueRace), raised + if path == "error": + assert raised.__cause__ is injected + else: + assert isinstance(raised.__cause__, OperationalError) + first_line = str(raised.__cause__).strip().splitlines()[0] + message = str(raised) + assert f"A fresh read on a new connection failed ({first_line})." in message + assert "on database 'default'" in message + assert f"inside a transaction at {level}," in message + assert "The enqueue created nothing." in message + + +def cancel_lock_waits(besides): + """ + Cancel every session on the test database that waits for a lock, other + than this thread's and `besides`. Returns the sessions it cancelled. + """ + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT pid, pg_cancel_backend(pid) FROM pg_stat_activity " + "WHERE datname = current_database() AND wait_event_type = 'Lock' " + "AND pid <> pg_backend_pid() AND pid IS DISTINCT FROM %s", + [besides], + ) + return cursor.fetchall() + + +def a_duplicate_behind_a_table_lock(monkeypatch, worker, level, table, blocker): + """ + Enqueue a duplicate whose fresh read meets a lock on the task table or the + lock table that can't be granted while the caller's transaction is open. + + The caller's transaction reads the key's lock and the holder, which holds + ACCESS SHARE on both tables, and the holder's lease is then renewed, so + the holder check is refused and the enqueue reads again on a new + connection. Before the enqueue, another session asks for ACCESS EXCLUSIVE + on the table and waits on the caller ("queued"), or the caller's own + transaction takes it ("caller"). After the enqueue the caller checks its + transaction still answers, commits, and retries at READ COMMITTED. + + A read with no bound would wait for good, so if the enqueue hasn't + returned after five seconds the waiting read is cancelled and that is + recorded as "watchdog". + """ + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + checks = record_holder_checks(monkeypatch) + model = OxTask if table == "task" else OxpullUniqueLock + quoted = connections["default"].ops.quote_name(model._meta.db_table) + take_the_table = f"LOCK TABLE {quoted} IN ACCESS EXCLUSIVE MODE" + snapshot_taken = threading.Event() + go = threading.Event() + outcome = {"checks": checks, "holder": holder} + locker = {} + + def web_request(): + try: + set_session_isolation(level) + with transaction.atomic(): + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.RUNNING + snapshot_taken.set() + assert go.wait(timeout=10) + if blocker == "caller": + with connections["default"].cursor() as cursor: + cursor.execute(take_the_table) + with opened_connections() as opened: + started = time.monotonic() + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except (UniqueEnqueueRace, OperationalError) as exc: + outcome["raised"] = exc + outcome["seconds"] = time.monotonic() - started + outcome["opened"] = [wrapper.alias for wrapper in opened] + outcome["closed"] = all( + wrapper.connection is None for wrapper in opened + ) + outcome["usable"] = OxTask.objects.filter(id=holder.id).count() + outcome["before_commit"] = time.monotonic() + with transaction.atomic(): + set_transaction_isolation("READ COMMITTED") + outcome["retried"] = tasks.unique_echo.enqueue("held") + except BaseException as exc: # pragma: no cover - reported by the test + outcome["error"] = exc + finally: + snapshot_taken.set() + connections.close_all() + + def lock_the_table(): + other = connections.create_connection("default") + try: + other.set_autocommit(False) + with other.cursor() as cursor: + cursor.execute("SET LOCAL lock_timeout = '20s'") + cursor.execute("SELECT pg_backend_pid()") + (locker["pid"],) = cursor.fetchone() + try: + cursor.execute(take_the_table) + locker["granted_at"] = time.monotonic() + except OperationalError as exc: + locker["error"] = exc + other.rollback() + finally: + other.close() + + request = threading.Thread(target=web_request) + request.start() + lock_taker = None + try: + assert snapshot_taken.wait(timeout=10) + assert "error" not in outcome, outcome["error"] + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + if blocker == "queued": + lock_taker = threading.Thread(target=lock_the_table) + lock_taker.start() + deadline = time.monotonic() + 10 + while "pid" not in locker and time.monotonic() < deadline: + time.sleep(0.01) + assert waits_on_a_lock(locker["pid"]), "the table lock is not waiting" + go.set() + request.join(timeout=5) + if request.is_alive(): + outcome["watchdog"] = cancel_lock_waits(locker.get("pid")) + finally: + go.set() + request.join(timeout=30) + if lock_taker is not None: + lock_taker.join(timeout=30) + assert not request.is_alive() + outcome["locker"] = locker + return outcome + + +def assert_the_read_gave_up(outcome, level, sqlstate, reason): + """The enqueue raised UniqueEnqueueRace for the read's failure, in time.""" + assert "error" not in outcome, outcome["error"] + assert "watchdog" not in outcome, ( + f"the read waited until it was cancelled: {outcome['watchdog']}" + ) + raised = outcome.get("raised") + assert isinstance(raised, UniqueEnqueueRace), raised + assert pg_sqlstate(raised.__cause__) == sqlstate + message = str(raised) + assert f"A fresh read on a new connection failed ({reason})." in message + assert "on database 'default'" in message + assert f"inside a transaction at {level}," in message + assert "The enqueue created nothing." in message + assert outcome["seconds"] < 2 + assert outcome["checks"] == ["40001"] + assert outcome["opened"] == ["default"] + assert outcome["closed"] + assert outcome["usable"] == 1 + locker = outcome["locker"] + if locker: + assert "error" not in locker, locker["error"] + assert locker["granted_at"] > outcome["before_commit"] + holder = outcome["holder"] + assert outcome["retried"].id == holder.id + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + return message + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("blocker", ["queued", "caller"]) +@pytest.mark.parametrize("table", ["task", "lock"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_fresh_read_stops_waiting_for_a_table_lock_held_off_by_the_caller( + monkeypatch, worker, level, table, blocker +): + """ + The read on a connection of its own can meet a lock cycle PostgreSQL + can't see, and it stops waiting instead of waiting for good. + + A request for ACCESS EXCLUSIVE on the task table or the lock table, such + as a migration's ALTER TABLE, waits on the caller's transaction, or that + transaction holds the lock itself. The fresh read queues behind it, and + the caller waits on the read outside the database, so no deadlock is + detected and nothing times out by default. The read stops after its + 100ms lock timeout. The enqueue raises UniqueEnqueueRace chained to + lock_not_available (55P03), says the read waits at most 100ms for a lock, + and closes the connection. The caller's transaction still answers and + commits, the queued request gets its lock only then, and the retry at + READ COMMITTED returns the holder. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + outcome = a_duplicate_behind_a_table_lock( + monkeypatch, worker, level, table, blocker + ) + + message = assert_the_read_gave_up( + outcome, level, "55P03", "canceling statement due to lock timeout" + ) + assert "That read waits at most 100ms for a lock." in message + assert bool(outcome["locker"]) is (blocker == "queued") + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_fresh_read_stops_at_its_statement_timeout(monkeypatch, worker, level): + """ + The statement timeout bounds the read too. With the read's lock timeout + set above it, a read queued behind a table lock is ended by the statement + timeout instead (57014). The enqueue raises UniqueEnqueueRace chained to + that, without the sentence about lock waits. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + monkeypatch.setattr(backend_module, "FRESH_READ_LOCK_TIMEOUT", "10s", raising=False) + monkeypatch.setattr( + backend_module, "FRESH_READ_STATEMENT_TIMEOUT", "200ms", raising=False + ) + outcome = a_duplicate_behind_a_table_lock( + monkeypatch, worker, level, "task", "queued" + ) + + message = assert_the_read_gave_up( + outcome, level, "57014", "canceling statement due to statement timeout" + ) + assert "waits at most" not in message + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("released", ["replaced", "pruned"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_lock_released_behind_a_snapshot_never_leaves_two_pending_tasks( + monkeypatch, worker, level, released, request +): + """ + A release that races a replaced lock, run through the enqueue itself. + + The first task ran and finished, so its lock is stale. The caller's + snapshot reads the key's lock and the finished task. Then, at READ + COMMITTED, another enqueue of the key releases that lock and queues a + second task ("replaced"), or oxpull_prune clears the stale lock + ("pruned"). Both commit. The caller then enqueues the key inside its + transaction. Its snapshot still shows the old lock naming the finished + task, so the enqueue takes the release path. Whatever that path does, it + must never leave two pending tasks for the key, and after the caller + commits, the key's lock must name the one pending task. On PostgreSQL + the release is refused (40001) when the lock was replaced, and the + enqueue raises UniqueEnqueueRace. MySQL cannot see the new lock and + raises after five attempts. When the lock was only pruned, the enqueue + queues the task. What the caller got is added to the test's report. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and the release would wait") + first = tasks.unique_echo.enqueue("key") + assert worker.run_once() is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.SUCCESSFUL + checks = record_holder_checks(monkeypatch) + + snapshot_taken = threading.Event() + lock_released = threading.Event() + outcome = {} + + def caller(): + try: + set_session_isolation(level) + with transaction.atomic(): + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + assert ( + OxTask.objects.get(id=first.id).status == OxTask.Status.SUCCESSFUL + ) + snapshot_taken.set() + assert lock_released.wait(timeout=10) + try: + outcome["result"] = tasks.unique_echo.enqueue("key") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + except OperationalError as exc: + outcome["raw"] = exc + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + finally: + snapshot_taken.set() + connections.close_all() + + thread = threading.Thread(target=caller) + thread.start() + assert snapshot_taken.wait(timeout=10) + if released == "replaced": + second = tasks.unique_echo.enqueue("key") + assert second.id != first.id + else: + call_command("oxpull_prune", stdout=io.StringIO()) + assert OxpullUniqueLock.objects.count() == 0 + lock_released.set() + thread.join(timeout=30) + assert not thread.is_alive() + assert "error" not in outcome, outcome["error"] + + pending = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_echo", + status__in=(OxTask.Status.READY, OxTask.Status.RUNNING), + ) + got = next(key for key in ("result", "race", "raw") if key in outcome) + if got == "result": + detail = outcome["result"].id + else: + cause = outcome[got].__cause__ if got == "race" else outcome[got] + detail = f"{outcome[got]} | cause: {cause!r}" + request.node.add_report_section( + "call", + "released behind a snapshot", + f"{vendor} {level} {released}: {got} [{detail}] checks={checks} " + f"pending={pending.count()}", + ) + assert "raw" not in outcome, outcome["raw"] + if released == "pruned": + assert got == "result" + assert outcome["result"].id != first.id + elif vendor == "postgresql": + assert got == "race" + assert "PostgreSQL refused a row" in str(outcome["race"]) + assert pg_sqlstate(outcome["race"].__cause__) == "40001" + else: + assert got == "race" + assert "after 5 attempts" in str(outcome["race"]) + assert pending.count() == 1, [(str(t.id), t.status) for t in pending] + assert str(OxpullUniqueLock.objects.get().task_id) == str(pending.get().id) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("writer", ["outcome", "delete", "release"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_writer_committing_after_the_refusal_is_the_read_committed_window( + monkeypatch, worker, level, writer +): + """ + lock_not_available (55P03) is contention compatible with deduplicating, + not a fresh read. + + The holder is RUNNING in the caller's snapshot. Another transaction holds + the row: it wrote SUCCESSFUL ("outcome"), deleted the row ("delete"), or + locked it FOR UPDATE to release the key ("release"). The check is refused + with 55P03. That transaction commits after the refusal and before the + enqueue returns, and for "release" it writes SUCCESSFUL and deletes the + key's lock first. The enqueue returns the RUNNING row it read, as a READ + COMMITTED read made just before that commit would have. It opens no + connection and does not wait. After the caller commits, the next enqueue + of the key queues a new task, and exactly one task is pending. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + lock_key = OxpullUniqueLock.objects.get().pk + checks = record_holder_checks(monkeypatch) + + other = connections.create_connection("default") + other.set_autocommit(False) + task_table = other.ops.quote_name(OxTask._meta.db_table) + lock_table = other.ops.quote_name(OxpullUniqueLock._meta.db_table) + key_column = other.ops.quote_name(OxpullUniqueLock._meta.pk.column) + pk = OxTask._meta.pk.get_db_prep_value(db_task.pk, other) + + def run(sql, params): + with other.cursor() as cursor: + cursor.execute(sql, params) + assert cursor.rowcount == 1 + + finish = ( + f"UPDATE {task_table} SET status = %s WHERE id = %s", # noqa: S608 + [OxTask.Status.SUCCESSFUL, pk], + ) + begin = { + "outcome": finish, + "delete": (f"DELETE FROM {task_table} WHERE id = %s", [pk]), # noqa: S608 + "release": (f"SELECT 1 FROM {task_table} WHERE id = %s FOR UPDATE", [pk]), # noqa: S608 + }[writer] + check_holder_row = OxpullBackend._check_holder_row + committed = [] + + def check_then_commit(cls, task_pk, db_alias): + answer = check_holder_row(task_pk, db_alias) + if writer == "release": + run(*finish) + run(f"DELETE FROM {lock_table} WHERE {key_column} = %s", [lock_key]) # noqa: S608 + other.commit() + committed.append(answer) + return answer + + monkeypatch.setattr( + OxpullBackend, "_check_holder_row", classmethod(check_then_commit) + ) + set_session_isolation(level) + try: + with transaction.atomic(): + assert session_isolation() == level + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.RUNNING + run(*begin) + # The writer commits only once the check has answered, so a check + # that waited for the writer would wait for good. The statement + # timeout ends such a wait with query_canceled (57014), which the + # enqueue raises as it is. + with connections["default"].cursor() as cursor: + cursor.execute("SET LOCAL statement_timeout = '30s'") + with opened_connections() as opened: + result = tasks.unique_echo.enqueue("held") + finally: + other.rollback() + other.close() + connections["default"].close() + + assert checks == ["55P03"] + assert committed == [None], "the writer did not commit before the return" + assert result.id == holder.id + assert result.status == "RUNNING" + assert opened == [] + after = tasks.unique_echo.enqueue("held") + assert after.id != holder.id + pending = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_echo", + status__in=(OxTask.Status.READY, OxTask.Status.RUNNING), + ) + assert [str(task.id) for task in pending] == [after.id] + assert str(OxpullUniqueLock.objects.get().task_id) == after.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("path", ["unchanged", "changed", "finished"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_holder_check_inside_nested_atomic_blocks(monkeypatch, worker, level, path): + """ + The check's savepoint sits inside Django's own savepoints. The enqueue + runs two atomic blocks deep inside the caller's transaction, after the + holder's row stayed as it was, changed while the task stayed RUNNING, or + changed as the task finished. Each block exits as it would around any + call: normally after a deduplicated enqueue, and through the exception + after UniqueEnqueueRace. The caller's transaction then still answers a + query, writes a task, and commits. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + checks = record_holder_checks(monkeypatch) + set_session_isolation(level) + outcome = {} + try: + with transaction.atomic(): + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.RUNNING + if path == "changed": + touch_elsewhere(holder.id) + elif path == "finished": + touch_elsewhere(holder.id, OxTask.Status.SUCCESSFUL) + try: + with transaction.atomic(), transaction.atomic(): + outcome["result"] = tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + outcome["count"] = OxTask.objects.filter(id=holder.id).count() + outcome["plain"] = tasks.plain_echo.enqueue("after").id + finally: + connections["default"].close() + + assert OxTask.objects.filter(id=outcome["plain"]).exists() + assert outcome["count"] == 1 + if path == "finished": + assert isinstance(outcome.get("race"), UniqueEnqueueRace) + assert checks == ["40001"] + else: + assert "race" not in outcome, outcome["race"] + assert outcome["result"].id == holder.id + assert checks == (["40001"] if path == "changed" else ["granted"]) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("earlier", ["for-update", "update"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_lock_the_caller_already_holds_on_the_holder_outlives_the_check( + monkeypatch, level, earlier +): + """ + Rolling back the check's savepoint releases only what the check took. + + The caller's transaction locked the holder's row FOR UPDATE, or updated + it, before it enqueued a duplicate. The check's share lock is granted on + a row the transaction already holds, and the enqueue deduplicates without + opening a connection. The caller's own row lock is still there after the + savepoint is rolled back. Another connection is refused the row until the + caller commits, and gets it right after. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + checks = record_holder_checks(monkeypatch) + set_session_isolation(level) + try: + with transaction.atomic(): + if earlier == "for-update": + OxTask.objects.select_for_update().get(id=holder.id) + else: + assert OxTask.objects.filter(id=holder.id).update(priority=5) == 1 + assert locked_elsewhere(holder.id), "the caller holds no lock to keep" + with opened_connections() as opened: + result = tasks.unique_echo.enqueue("held") + held_after = locked_elsewhere(holder.id) + finally: + connections["default"].close() + + assert result.id == holder.id + assert checks == ["granted"] + assert opened == [] + assert held_after, "the savepoint rollback released the caller's own lock" + assert not locked_elsewhere(holder.id) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("change", ["finished", "renewed"]) +@pytest.mark.parametrize("how", ["set_autocommit", "settings"]) +@pytest.mark.parametrize("level", ["READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"]) +def test_a_connection_with_autocommit_off_is_a_callers_transaction( + monkeypatch, level, how, change +): + """ + With autocommit turned off every statement joins a transaction that only + the caller ends, as inside an atomic block, though no atomic block is open. + A project turns it off with transaction.set_autocommit(False), or with + AUTOCOMMIT False in the database settings. + + The caller reads the holder, which fixes the snapshot at REPEATABLE READ + and SERIALIZABLE. Another session then finishes the task, or writes its + row and leaves it READY, as a lease renewal does. The caller enqueues the + key. On PostgreSQL at those levels the holder check runs: a finished task + raises UniqueEnqueueRace, and a task still pending is returned from the + read on a new connection. On MySQL at REPEATABLE READ the committed status + is read on a connection of its own, so a finished task's lock is released + and a new task queued. At READ COMMITTED the read is current. Each time the + caller's transaction still answers and commits. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and the write would wait") + holder = tasks.unique_echo.enqueue("held") + checks = record_holder_checks(monkeypatch) + connection = connections["default"] + connection.close() + if level != "READ COMMITTED": + isolation_for_new_connections(monkeypatch, level) + # Connected before AUTOCOMMIT changes, so its write commits at once. + other = connections.create_connection("default") + other.ensure_connection() + settings_dict = connection.settings_dict + outcome = {} + try: + if how == "settings": + settings_dict["AUTOCOMMIT"] = False + connection.ensure_connection() + else: + transaction.set_autocommit(False) + outcome["autocommit"] = transaction.get_autocommit() + outcome["atomic"] = connection.in_atomic_block + outcome["level"] = session_isolation() + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.READY + finished = OxTask.Status.SUCCESSFUL if change == "finished" else None + touch_elsewhere(holder.id, finished, other=other) + with opened_connections() as opened: + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + outcome["usable"] = OxTask.objects.filter(id=holder.id).count() + transaction.commit() + finally: + if connection.connection is not None and not connection.get_autocommit(): + connection.rollback() + connection.set_autocommit(True) + settings_dict["AUTOCOMMIT"] = True + other.close() + connection.close() + + assert outcome["autocommit"] is False + assert outcome["atomic"] is False + assert outcome["level"] == level + snapshot = level != "READ COMMITTED" + checked = vendor == "postgresql" and snapshot + ready = [str(task.id) for task in OxTask.objects.filter(status=OxTask.Status.READY)] + lock = str(OxpullUniqueLock.objects.get().task_id) + if change == "finished" and checked: + race = outcome.get("race") + assert isinstance(race, UniqueEnqueueRace), ( + "deduplicated against a task that had settled" + ) + assert pg_sqlstate(race.__cause__) == "40001" + assert "A fresh read found the task SUCCESSFUL." in str(race) + assert f"inside a transaction at {level}," in str(race) + assert ready == [] + assert lock == holder.id + elif change == "finished": + assert "race" not in outcome, outcome["race"] + second = outcome["result"].id + assert second != holder.id, "deduplicated against a task that had settled" + assert ready == [second] + assert lock == second + else: + assert "race" not in outcome, outcome["race"] + assert outcome["result"].id == holder.id + if checked: + committed = task_result_from_db(OxTask.objects.get(id=holder.id)) + assert result_fields(outcome["result"]) == result_fields(committed) + assert ready == [holder.id] + assert lock == holder.id + assert checks == (["40001"] if checked else []) + assert [wrapper.alias for wrapper in opened] == (["default"] if snapshot else []) + assert all(wrapper.connection is None for wrapper in opened) + assert outcome["usable"] == 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_pending_holder_whose_lock_is_gone_is_not_deduplicated_against( + monkeypatch, level +): + """ + The read on a connection of its own deduplicates only while the key's lock + is there. + + The caller's snapshot sees the task READY and holding the key. The task's + row then changes while it stays READY, as a lease renewal changes it. The + caller enqueues the key. The lock insert meets the key's lock, and the + holder check is refused (40001). As the read's new connection opens, + another session deletes the key's lock and commits, as an operator + clearing the key by hand would. The read finds the task pending and no + lock. The key is free, and a READ COMMITTED enqueue would now queue a new + task, so the enqueue raises UniqueEnqueueRace saying it found the lock + gone, and creates nothing. The retry at READ COMMITTED queues a new task, + which takes the key. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + lock_key = OxpullUniqueLock.objects.get().pk + checks = record_holder_checks(monkeypatch) + other = connections.create_connection("default") + other.ensure_connection() + lock_table = other.ops.quote_name(OxpullUniqueLock._meta.db_table) + key_column = other.ops.quote_name(OxpullUniqueLock._meta.pk.column) + deleted = [] + + def delete_the_lock(sender, connection, **kwargs): + if deleted or connection is other: + return + with other.cursor() as cursor: + cursor.execute( + f"DELETE FROM {lock_table} WHERE {key_column} = %s", # noqa: S608 + [lock_key], + ) + deleted.append(cursor.rowcount) + + set_session_isolation(level) + outcome = {} + try: + with transaction.atomic(): + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.READY + touch_elsewhere(holder.id) + connection_created.connect(delete_the_lock) + try: + with opened_connections() as opened: + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + finally: + connection_created.disconnect(delete_the_lock) + outcome["usable"] = OxTask.objects.filter(id=holder.id).count() + with transaction.atomic(): + set_transaction_isolation("READ COMMITTED") + outcome["retried"] = tasks.unique_echo.enqueue("held") + finally: + other.close() + connections["default"].close() + + assert deleted == [1], "the key's lock was not deleted before the read" + race = outcome.get("race") + assert isinstance(race, UniqueEnqueueRace), ( + "deduplicated against a task whose lock was gone" + ) + assert pg_sqlstate(race.__cause__) == "40001" + message = str(race) + assert "A fresh read found the key's lock gone." in message + assert f"inside a transaction at {level}," in message + assert "The enqueue created nothing." in message + assert checks == ["40001"] + assert [wrapper.alias for wrapper in opened] == ["default"] + assert all(wrapper.connection is None for wrapper in opened) + assert outcome["usable"] == 1 + retried = outcome["retried"].id + assert retried != holder.id + assert str(OxpullUniqueLock.objects.get().task_id) == retried + ready = OxTask.objects.filter(status=OxTask.Status.READY) + assert sorted(str(task.id) for task in ready) == sorted([holder.id, retried]) + + +@contextmanager +def a_server_that_never_answers(hold): + """ + A port on this host that accepts connections and never answers them, so a + client waits for the server's first reply. The connections it accepted are + closed after `hold` seconds, or when the block ends, so a client with no + connect timeout fails then instead of waiting for good. Yields the port and + the list of accepted connections. + """ + listener = socket.create_server(("127.0.0.1", 0)) + listener.settimeout(0.05) + port = listener.getsockname()[1] + accepted = [] + done = threading.Event() + + def serve(): + deadline = time.monotonic() + hold + while not done.is_set() and time.monotonic() < deadline: + try: + accepted.append(listener.accept()[0]) + except TimeoutError: + continue + for client in accepted: + client.close() + listener.close() + + server = threading.Thread(target=serve) + server.start() + try: + yield port, accepted + finally: + done.set() + server.join(timeout=10) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_fresh_read_stops_waiting_to_connect(monkeypatch, level): + """ + Connecting to the server is bounded too. The read's connection goes to a + server that accepts it and never answers. The project sets no connect + timeout, so the read gives libpq a connect_timeout of 2 seconds, and the + enqueue raises UniqueEnqueueRace chained to the timeout. The server holds + the connection for 8 seconds, so a read with no bound would fail only + then. The caller's transaction still answers, the connection is closed, + and nothing was created. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + checks = record_holder_checks(monkeypatch) + set_session_isolation(level) + outcome = {} + try: + with a_server_that_never_answers(hold=8) as (port, accepted): + with transaction.atomic(): + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.READY + touch_elsewhere(holder.id) + with opened_connections() as opened: + create = connections.create_connection + + def silent(alias): + wrapper = create(alias) + wrapper.settings_dict = { + **wrapper.settings_dict, + "HOST": "127.0.0.1", + "PORT": str(port), + } + return wrapper + + connections.create_connection = silent + started = time.monotonic() + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except (UniqueEnqueueRace, OperationalError) as exc: + outcome["raised"] = exc + outcome["seconds"] = time.monotonic() - started + outcome["usable"] = OxTask.objects.filter(id=holder.id).count() + outcome["accepted"] = len(accepted) + finally: + connections["default"].close() + + raised = outcome.get("raised") + assert isinstance(raised, UniqueEnqueueRace), raised + assert isinstance(raised.__cause__, OperationalError) + message = str(raised) + assert "A fresh read on a new connection failed (" in message + assert "timeout expired" in message + assert f"inside a transaction at {level}," in message + assert "The enqueue created nothing." in message + assert outcome["accepted"] >= 1, "the read never reached the silent server" + assert outcome["seconds"] < 5, f"the read waited {outcome['seconds']:.1f} s" + assert checks == ["40001"] + assert [wrapper.alias for wrapper in opened] == ["default"] + assert all(wrapper.connection is None for wrapper in opened) + assert outcome["usable"] == 1 + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "project", ["none", "options", "environment", "service", "service-environment"] +) +def test_the_fresh_read_keeps_a_connect_timeout_the_project_set(monkeypatch, project): + """ + The read sets its connect timeout only where the project sets none. A + timeout in OPTIONS["connect_timeout"] or in PGCONNECT_TIMEOUT, or a + connection service, named in OPTIONS["service"] or PGSERVICE, whose entry + can set one, is left for libpq to apply. A value the read passed would + override each of them. Checked on the parameters the driver is handed. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the read on a connection of its own is PostgreSQL's") + options = connections["default"].settings_dict["OPTIONS"] + for name in ("PGCONNECT_TIMEOUT", "PGSERVICE"): + monkeypatch.delenv(name, raising=False) + if project == "options": + monkeypatch.setitem(options, "connect_timeout", 9) + elif project == "environment": + monkeypatch.setenv("PGCONNECT_TIMEOUT", "9") + elif project == "service": + monkeypatch.setitem(options, "service", "oxpull-test") + elif project == "service-environment": + monkeypatch.setenv("PGSERVICE", "oxpull-test") + handed = [] + create = connections.create_connection + + def recording(alias): + wrapper = create(alias) + + def get_new_connection(conn_params): + handed.append(dict(conn_params)) + raise OperationalError("stopped before connecting") + + wrapper.get_new_connection = get_new_connection + return wrapper + + monkeypatch.setattr(connections, "create_connection", recording) + with pytest.raises(OperationalError, match="stopped before connecting"): + OxpullBackend._read_holder_fresh(uuid.uuid4(), "key", "default") + + (params,) = handed + assert params.get("connect_timeout") == {"none": 2, "options": 9}.get(project) + assert ("service" in params) is (project == "service") diff --git a/tests_oxpull/test_heartbeat_file.py b/tests_oxpull/test_heartbeat_file.py new file mode 100644 index 0000000..8d748fa --- /dev/null +++ b/tests_oxpull/test_heartbeat_file.py @@ -0,0 +1,548 @@ +""" +The `oxpull.worker.OxpullWorker` under `ox_worker --heartbeat-file` and `ox_health +--heartbeat-file`, through the real commands, for the backend and worker +shipped together in django-ox. + +django-ox owns the heartbeat. Worker.run() touches the file at the head of +every poll pass and every drain pass, on the thread that runs the loop. Under +--processes the supervisor touches PATH.supervisor from its own loop and +removes a slot's PATH.i when that slot starts or exits. oxpull changes none +of that, and these tests hold what oxpull puts around the loop: + + * OxpullWorker takes heartbeat_file through **kwargs and hands it on, so + every `oxpull.worker.OxpullWorker` process, each supervised slot included, + writes its own file. The flag reaches the processes that actually run. + * OxpullWorker.run() checks the workflow schema before it enters the + django-ox loop, so a workflow worker's first heartbeat comes after that + check. When the database refuses connections the check gives up with a + warning and the loop starts and turns anyway: a heartbeat says the loop + is turning, not that the database answers. + * The batch and workflow reconcilers are ordinary scheduled tasks. A batch + and a workflow complete only through them, and here they complete on a + fleet whose every heartbeat file stays fresh the whole time. + * Nothing runs twice. Three worker processes race for one queue with the + flag on, and every row is claimed once, by one worker. + * `ox_health --heartbeat-file` reads file metadata only. With oxpull + installed and the database refusing connections, it answers from the + files, while the database mode in the same environment cannot answer. + +A fresh file means one thing: the loops it belongs to have advanced +recently. Not that a task is making progress, and not that the database +answers. + +Every wait ends on something observed (a file's modification time moving, a +row's status) and fails only at a deadline that a passing run never reaches. +Age is never waited out: a file whose writer is stopped is aged by setting its +modification time, which the stopped writer cannot move back. + +Not covered here, and covered by django-ox's own suite: the exact age +boundaries, symlinks and other files that are not regular files, a loop wedged +in a statement, a stopped or killed child and its restart, and the drain. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import signal +import socket +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path + +import pytest +from django.conf import settings +from django.db import connection + +from django_ox.models import OxTask +from oxpull import batches +from oxpull.models import OxpullBatch, OxpullWorkflow + +from . import tasks +from .subprocesses import django_ox_path +from .test_workflows import diamond + +REPO_ROOT = Path(__file__).resolve().parent.parent +SRC = REPO_ROOT / "src" + +# A safety bound on every wait, not an expected time. +DEADLINE = 120.0 + +# The suite's settings, run as an operator would run them, before each test +# adds its own part. "flows_ungated" is the suite's deliberate oxpull.E007 +# fixture, which would stop ox_worker at start-up; its alias stays because +# tests_oxpull.tasks declares a task on it. +BASE_SETTINGS = ( + "import copy\n" + "from {module} import * # noqa: F403\n" + "TASKS = copy.deepcopy(TASKS) # noqa: F405\n" + "TASKS['flows_ungated']['OPTIONS']['WORKFLOWS'] = False\n" +) + +# Both OxpullBackend aliases correctly configured for the same fleet: the +# worker on "default" is an OxpullWorker too, because it serves the queue the +# workflow nodes are on. The reconcilers keep the schedule entries the suite +# gives them, run every second instead of every minute, so the batch and the +# workflow complete within seconds of their last member rather than at the next +# minute. DEBUG django_ox logs, whose "Claimed task" lines are the claim record. +FLEET_SETTINGS = ( + "TASKS['default']['OPTIONS']['WORKER_CLASS'] = 'oxpull.worker.OxpullWorker'\n" + "for _name in ('oxpull-batch-reconcile', 'oxpull-workflow-reconcile'):\n" + " _schedule = TASKS['default']['OPTIONS']['SCHEDULES'][_name]\n" + " del _schedule['cron']\n" + " _schedule['every'] = 1\n" + "LOGGING = {\n" + " 'version': 1,\n" + " 'disable_existing_loggers': False,\n" + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},\n" + " 'loggers': {'django_ox': {'handlers': ['console'], 'level': 'DEBUG'}},\n" + "}\n" +) + +# Every alias on a database that refuses connections: a directory that does +# not exist for SQLite, a port nothing listens on for a server. +REFUSED_SETTINGS = ( + "DATABASES = copy.deepcopy(DATABASES) # noqa: F405\n" + "for _alias, _database in DATABASES.items():\n" + " if _database['ENGINE'].endswith('sqlite3'):\n" + " _database['NAME'] = {absent!r} + '/' + _alias + '.sqlite3'\n" + " else:\n" + " _database['HOST'] = '127.0.0.1'\n" + " _database['PORT'] = {port!r}\n" +) + +CLAIMED = re.compile(r"Claimed task id=([0-9a-f-]{36}) ") +PASS_FAILED = "could not reach the database this pass" +SCHEMA_CHECK_FAILED = "Could not check the workflow schema before starting" + + +def process_env(tmp_path, extra): + """ + A separate process on this session's test database, with the suite's + settings module underneath the test's own part. + """ + (tmp_path / "heartbeat_settings.py").write_text( + BASE_SETTINGS.format(module=settings.SETTINGS_MODULE) + extra + ) + name = connection.settings_dict["NAME"] + if connection.vendor == "sqlite": + name = Path(name).resolve() + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "heartbeat_settings", + "PYTHONPATH": os.pathsep.join( + [str(SRC), django_ox_path(), str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(name), + } + + +def run_python(env, *args): + return subprocess.run( # noqa: S603 + [sys.executable, *args], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +class WorkerProcess: + """One `ox_worker` process and the file it logs to.""" + + def __init__(self, env, log_path, *args): + self.args = args + self.log_path = log_path + with log_path.open("w") as log: + self.process = subprocess.Popen( # noqa: S603 + [sys.executable, "-m", "django", "ox_worker", *args], + env=env, + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + + @property + def pid(self): + return self.process.pid + + def log(self): + return self.log_path.read_text() + + def assert_running(self): + code = self.process.poll() + if code is not None: + pytest.fail( + f"ox_worker {' '.join(self.args)} exited {code}:\n{self.log()[-3000:]}" + ) + + def stop(self): + """SIGTERM, as a deploy does, after a SIGCONT in case a test stopped it.""" + if self.process.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.kill(self.pid, signal.SIGCONT) + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=60) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + return self.process.returncode + + +def probe(env, path, *extra): + """`ox_health --heartbeat-file`, as a container probe runs it.""" + result = run_python( + env, + "-m", + "django", + "ox_health", + "--heartbeat-file", + str(path), + "--format", + "json", + *extra, + ) + try: + report = json.loads(result.stdout) + except ValueError: + report = None + return result.returncode, report, result.stdout + result.stderr + + +def failing(report): + return [entry["path"] for entry in report["files"] if not entry["ok"]] + + +def mtime(path): + try: + return path.stat().st_mtime_ns + except FileNotFoundError: + return None + + +def watch(paths, times, *, still=(), running=()): + """ + Sample the files until each of `paths` has shown `times` new modification + times; a file appearing counts as one. Every path in `still` must keep the + modification time it had when the watch began, and every process in + `running` must still be running. + """ + last = {path: mtime(path) for path in [*paths, *still]} + seen = Counter() + give_up = time.monotonic() + DEADLINE + while any(seen[path] < times for path in paths): + for worker in running: + worker.assert_running() + if time.monotonic() > give_up: + pytest.fail( + f"after {DEADLINE}s, updates seen: " + f"{ {path.name: seen[path] for path in paths} }, wanted {times} each" + ) + time.sleep(0.02) + for path in paths: + now = mtime(path) + if now is not None and now != last[path]: + seen[path] += 1 + last[path] = now + for path in still: + assert mtime(path) == last[path], ( + f"{path.name} moved while its writer was stopped" + ) + + +def wait_until(condition, what, *, running=()): + give_up = time.monotonic() + DEADLINE + while not condition(): + for worker in running: + worker.assert_running() + if time.monotonic() > give_up: + pytest.fail(f"after {DEADLINE}s: {what}") + time.sleep(0.1) + + +def age(path, seconds): + """Put the file's modification time `seconds` in the past.""" + past = time.time() - seconds + os.utime(path, (past, past)) + + +def children_of(pid): + found = subprocess.run( # noqa: S603 + ["pgrep", "-P", str(pid)], # noqa: S607 + capture_output=True, + text=True, + check=False, + ) + return sorted(int(line) for line in found.stdout.split()) + + +def argv_of(pid): + found = subprocess.run( # noqa: S603 + ["ps", "-ww", "-o", "args=", "-p", str(pid)], # noqa: S607 + capture_output=True, + text=True, + check=False, + ) + return found.stdout.split() + + +def flag_value(argv, flag): + # Read both "--flag value" and "--flag=value"; the supervisor passes each + # child the second form. + for i, arg in enumerate(argv): + if arg == flag: + return argv[i + 1] if i + 1 < len(argv) else None + if arg.startswith(flag + "="): + return arg[len(flag) + 1 :] + return None + + +@pytest.mark.django_db(transaction=True) +def test_a_heartbeat_fleet_completes_workflows_and_batches_and_runs_each_task_once( + tmp_path, +): + env = process_env(tmp_path, FLEET_SETTINGS) + # Which oxpull the processes import, and which worker ox_worker builds for + # each alias, before believing anything they do. A failure here is the + # set-up, not the heartbeat. + setup = run_python( + env, + "-c", + "import django; django.setup(); import oxpull; " + "from django_ox.worker import worker_class; print(oxpull.__file__); " + "print(*(cls.__module__ + '.' + cls.__qualname__ for cls in" + " map(worker_class, ('flows', 'default'))))", + ) + lines = setup.stdout.split() + if lines[:1] != [str(SRC / "oxpull" / "__init__.py")] or lines[1:] != [ + "oxpull.worker.OxpullWorker", + "oxpull.worker.OxpullWorker", + ]: + pytest.fail(f"unexpected worker set-up:\n{setup.stdout}{setup.stderr}") + + beats = tmp_path / "run" + beats.mkdir() + fleet_file = beats / "flows" + supervisor_file = beats / "flows.supervisor" + slot_files = [beats / "flows.0", beats / "flows.1"] + single_file = beats / "default" + every_file = [supervisor_file, *slot_files, single_file] + + fleet = WorkerProcess( + env, + tmp_path / "flows.log", + "--backend", + "flows", + "--processes", + "2", + "--interval", + "0.1", + "--heartbeat-file", + str(fleet_file), + ) + single = WorkerProcess( + env, + tmp_path / "default.log", + "--backend", + "default", + "--interval", + "0.1", + "--heartbeat-file", + str(single_file), + ) + workers = [fleet, single] + try: + watch(every_file, 3, running=workers) + # Exactly the expected set: no PATH for the fleet, no file per pid. + assert sorted(path.name for path in beats.iterdir()) == [ + "default", + "flows.0", + "flows.1", + "flows.supervisor", + ] + # The supervisor hands each slot the base path and its index, and + # each slot writes PATH.: the flag reached the workers that run. + slots = { + flag_value(argv, "--worker-index"): flag_value(argv, "--heartbeat-file") + for argv in map(argv_of, children_of(fleet.pid)) + } + assert slots == {"0": str(fleet_file), "1": str(fleet_file)}, slots + + code, report, text = probe(env, fleet_file, "--processes", "2") + assert code == 0 and report["ok"], text + assert [entry["path"] for entry in report["files"]] == [ + str(path) for path in [supervisor_file, *slot_files] + ], text + code, report, text = probe(env, single_file) + assert code == 0 and report["ok"], text + + # The work: a workflow and a batch, which only the scheduled + # reconcilers complete, and plain tasks for all three workers to race. + flow = diamond("heartbeat-fleet") + batch = batches.create( + "heartbeat-fleet", + on_complete=tasks.record_callback, + on_complete_args=["heartbeat-fleet"], + ) + for number in range(5): + batch.add(tasks.plain_echo, number) + batch.seal() + plain = [tasks.plain_echo.enqueue(number).id for number in range(30)] + + def done(): + return ( + OxpullWorkflow.objects.get(pk=flow.id).status + == OxpullWorkflow.Status.COMPLETE + and OxpullBatch.objects.get(pk=batch.id).status + == OxpullBatch.Status.COMPLETE + and OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback", + status=OxTask.Status.SUCCESSFUL, + ).exists() + and not OxTask.objects.filter(pk__in=plain) + .exclude(status=OxTask.Status.SUCCESSFUL) + .exists() + ) + + wait_until( + done, + "the workflow, the batch and the plain tasks to finish", + running=workers, + ) + + # Still turning after the work, and both probes still pass. + watch(every_file, 3, running=workers) + code, report, text = probe(env, fleet_file, "--processes", "2") + assert code == 0 and report["ok"], text + code, report, text = probe(env, single_file) + assert code == 0 and report["ok"], text + + # A stopped supervisor fails the fleet's probe while its children, + # still turning, cannot hide it. + os.kill(fleet.pid, signal.SIGSTOP) + try: + watch([*slot_files, single_file], 1, running=workers) + watch( + [*slot_files, single_file], 3, still=[supervisor_file], running=workers + ) + age(supervisor_file, 120) + code, report, text = probe(env, fleet_file, "--processes", "2") + assert code == 1 and not report["ok"], text + assert failing(report) == [str(supervisor_file)], text + assert "old" in report["problems"][0], text + finally: + os.kill(fleet.pid, signal.SIGCONT) + watch([supervisor_file], 1, running=workers) + code, report, text = probe(env, fleet_file, "--processes", "2") + assert code == 0 and report["ok"], text + finally: + codes = [worker.stop() for worker in workers] + text = "".join(worker.log() for worker in workers) + + assert codes == [0, 0], f"exit codes {codes}\n{text[-3000:]}" + # The supervisor removes a slot's file once it sees the slot exit, so a + # stopped fleet fails its probe at once rather than when the files age. + assert not any(path.exists() for path in slot_files) + code, report, _ = probe(env, fleet_file, "--processes", "2") + assert code == 1 and failing(report) == [str(path) for path in slot_files] + + # Once each. attempts is the number of claims (django-ox keeps it equal to + # len(worker_ids)), and the logs of all three workers name each claim. + claims = Counter(CLAIMED.findall(text)) + twice = {task_id: count for task_id, count in claims.items() if count > 1} + assert twice == {}, f"claimed more than once: {twice}" + rows = list(OxTask.objects.values_list("id", "task_path", "status", "attempts")) + finished = { + str(pk) for pk, _, status, _ in rows if status == OxTask.Status.SUCCESSFUL + } + expected = { + *map(str, flow.node_ids().values()), + *map(str, plain), + *( + str(pk) + for pk, path, _, _ in rows + if path + in ("tests_oxpull.tasks.plain_echo", "tests_oxpull.tasks.record_callback") + ), + } + assert expected <= finished + assert len(expected) == 4 + 30 + 5 + 1 + over = [(str(pk), path, attempts) for pk, path, _, attempts in rows if attempts > 1] + assert over == [] + assert all(claims[task_id] == 1 for task_id in finished), { + task_id: claims[task_id] for task_id in finished if claims[task_id] != 1 + } + # Both reconcilers ran from the schedule, through the workers. + ran = {path for pk, path, status, _ in rows if str(pk) in finished} + assert {"oxpull.batches.reconcile", "oxpull.workflows.reconcile"} <= ran + + +def refused_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return str(sock.getsockname()[1]) + + +def test_a_workflow_worker_whose_database_refuses_connections_keeps_turning(tmp_path): + """ + Started while the database refuses every connection. The workflow schema + check gives up (on MySQL, the one database where it asks the server + anything), the loop starts anyway, every pass fails, and the heartbeat + stays fresh: the file says the loop turns, which it does. The probe + answers from the files alone in that same environment, with oxpull + installed, while the database mode there cannot answer at all. + """ + env = process_env( + tmp_path, + REFUSED_SETTINGS.format(absent=str(tmp_path / "absent"), port=refused_port()), + ) + beats = tmp_path / "run" + beats.mkdir() + path = beats / "flows" + worker = WorkerProcess( + env, + tmp_path / "flows.log", + "--backend", + "flows", + "--interval", + "0.1", + "--heartbeat-file", + str(path), + ) + try: + watch([path], 3, running=[worker]) + wait_until( + lambda: PASS_FAILED in worker.log(), + "a failed pass in the log", + running=[worker], + ) + watch([path], 3, running=[worker]) + code, report, text = probe(env, path) + assert code == 0 and report["ok"], text + + stale = beats / "stale" + stale.touch() + age(stale, 120) + code, report, text = probe(env, stale) + assert code == 1 and failing(report) == [str(stale)], text + assert "old" in report["problems"][0], text + code, report, text = probe(env, beats / "missing") + assert code == 1 and "missing" in report["problems"][0], text + + # The control: the database mode, in the same environment. + control = run_python(env, "-m", "django", "ox_health") + assert control.returncode == 1, control.stdout + control.stderr + assert "Database unreachable" in control.stderr, control.stderr + finally: + code = worker.stop() + text = worker.log() + + assert code == 0, text[-3000:] + if connection.vendor == "mysql": + assert SCHEMA_CHECK_FAILED in text, text[-3000:] diff --git a/tests_oxpull/test_keys.py b/tests_oxpull/test_keys.py new file mode 100644 index 0000000..77a0040 --- /dev/null +++ b/tests_oxpull/test_keys.py @@ -0,0 +1,78 @@ +"""Key derivation is pure; no database involved.""" + +from oxpull.unique import unique_key + +ALIAS = "default" +PATH = "tests_oxpull.tasks.unique_echo" + + +def test_key_is_stable(): + a = unique_key(ALIAS, PATH, "default", [1, "x"], {"a": 1}) + b = unique_key(ALIAS, PATH, "default", [1, "x"], {"a": 1}) + assert a == b + + +def test_key_is_sha256_hex(): + key = unique_key(ALIAS, PATH, "default", [], {}) + assert len(key) == 64 + assert set(key) <= set("0123456789abcdef") + + +def test_kwarg_order_is_irrelevant(): + a = unique_key(ALIAS, PATH, "default", [], {"a": 1, "b": 2}) + b = unique_key(ALIAS, PATH, "default", [], {"b": 2, "a": 1}) + assert a == b + + +def test_nested_dict_order_is_irrelevant(): + a = unique_key(ALIAS, PATH, "default", [], {"outer": {"a": 1, "b": 2}}) + b = unique_key(ALIAS, PATH, "default", [], {"outer": {"b": 2, "a": 1}}) + assert a == b + + +def test_tuples_equal_lists(): + # normalize_json stores tuples as JSON arrays, so the key does too. + assert unique_key(ALIAS, PATH, "default", (1, (2, 3)), {}) == unique_key( + ALIAS, PATH, "default", [1, [2, 3]], {} + ) + + +def test_positional_order_matters(): + assert unique_key(ALIAS, PATH, "default", [1, 2], {}) != unique_key( + ALIAS, PATH, "default", [2, 1], {} + ) + + +def test_queue_is_part_of_the_key(): + assert unique_key(ALIAS, PATH, "default", [1], {}) != unique_key( + ALIAS, PATH, "emails", [1], {} + ) + + +def test_task_path_is_part_of_the_key(): + assert unique_key(ALIAS, PATH, "default", [1], {}) != unique_key( + ALIAS, "tests_oxpull.tasks.other", "default", [1], {} + ) + + +def test_backend_alias_is_part_of_the_key(): + assert unique_key("default", PATH, "default", [1], {}) != unique_key( + "other", PATH, "default", [1], {} + ) + + +def test_int_and_float_differ_like_their_json(): + # 1 and 1.0 store differently in the task row's JSON, so they key + # differently too. Key equality is JSON equality, nothing smarter. + assert unique_key(ALIAS, PATH, "default", [1], {}) != unique_key( + ALIAS, PATH, "default", [1.0], {} + ) + + +def test_mixed_int_and_str_dict_keys_do_not_raise(): + # JSON coerces int keys to strings when the row is stored; the key + # derivation applies the same coercion instead of raising over an + # unsortable mixed-key dict. + a = unique_key(ALIAS, PATH, "default", [], {"m": {1: "a", "b": 2}}) + b = unique_key(ALIAS, PATH, "default", [], {"m": {"1": "a", "b": 2}}) + assert a == b diff --git a/tests_oxpull/test_policy.py b/tests_oxpull/test_policy.py new file mode 100644 index 0000000..5b326dc --- /dev/null +++ b/tests_oxpull/test_policy.py @@ -0,0 +1,651 @@ +""" +django-ox's per-task policy, through every `oxpull` app path. + +django-ox's PolicyTask gives a task max_attempts, backoff and timeout of its +own. The budget is written onto the row at enqueue, by the OxBackend._row that +every enqueue in this package reaches, and from then on the stored value is +the one the worker and the reaper decide on. Backoff and timeout are not +stored: the worker reads them from the code it imports for each attempt. +None of that is `oxpull` app code, and the `oxpull` app changes none of it. +What is pinned here is that it holds on the `oxpull` app's own paths, where a +row is written some other way or a failure is recorded through the `oxpull` +app's outcome hook: + +- the budget reaches the row through unique enqueue, bulk enqueue, batch + members and callbacks, workflow roots and WAITING nodes, and survives + using() routing and rebinding; +- a unique duplicate returns the holder, whose row keeps its stored budget, + because policy is not part of the key; +- a throttled claim spends nothing, and every admitted retry spends a claim + and an admission; +- a workflow node's retry is not an outcome, and a backoff that answers None, + a timeout and a spent budget each record FAILED through the outcome hook, + while a release never touches a retry's backoff; +- a task's own timeout reaches the remaining() the reconcilers read, and the + reconcilers, built lazily as plain tasks, keep the backend's defaults. + +On Django 6.0 the tasks in tests_oxpull/policy_tasks.py are built without the +decorator spelling, which 6.0 does not have; everything below still runs. +""" + +import dataclasses +import logging +from contextlib import contextmanager +from datetime import timedelta + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.tasks import default_task_backend, task_backends +from django.tasks.exceptions import InvalidTask +from django.tasks.signals import task_finished +from django.utils import timezone + +from django_ox.bulk import enqueue_many +from django_ox.models import OxTask +from django_ox.tasks import PolicyTask +from django_ox.worker import Worker +from oxpull import batches, workflows +from oxpull.backend import OxpullBackend +from oxpull.models import OxpullBatch, OxpullWorkflow, OxpullWorkflowNode + +from . import policy_tasks, tasks +from .test_rate_limits import oxpull_worker, rate_limits, window +from .test_workflows import drain, flow_worker +from .test_workflows_hazard_the_reconciler_s_per_workflow_except_exception import ( + interruptible_attempts, # noqa: F401 - a fixture +) +from .test_workflows_waiting import edge_marks + +pytestmark = pytest.mark.django_db + +READY = OxTask.Status.READY +RUNNING = OxTask.Status.RUNNING +WAITING = OxTask.Status.WAITING +FAILED = OxTask.Status.FAILED +LOST = OxTask.Status.LOST +DISCARDED = OxTask.Status.DISCARDED +SUCCESSFUL = OxTask.Status.SUCCESSFUL + +TIMEOUT_PATH = "django_ox.exceptions.TaskTimeout" + +# The backend-wide budget every test backend in tests_oxpull/settings.py declares. +BACKEND_BUDGET = 3 + + +def row(result): + return OxTask.objects.get(pk=result.id) + + +def budget(result): + return row(result).max_attempts + + +def record_of(result): + return OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch", "succeeded" + ).get(task_id=result.id) + + +def reconcile_batches(): + return batches.reconcile.call() + + +def reconcile_workflows(): + return workflows._reconcile() + + +def notes(): + return list(tasks.STATE.get("notes", [])) + + +def events(caplog, name): + return [r for r in caplog.records if getattr(r, "event", None) == name] + + +@contextmanager +def finished_signals(): + sent = [] + + def receiver(sender, task_result, **kwargs): + sent.append(task_result) + + task_finished.connect(receiver) + try: + yield sent + finally: + task_finished.disconnect(receiver) + + +@contextmanager +def remaining_as_the_reconcilers_read_it(monkeypatch): + """ + Every value django-ox's remaining() gave the two reconcilers, through the + names they read it by. The real function answers; this only listens. + """ + seen = [] + for module in (batches, workflows): + real = module.remaining + + def listening(real=real, module=module): + value = real() + seen.append((module.__name__, value)) + return value + + monkeypatch.setattr(module, "remaining", listening) + yield seen + + +# -- the budget reaches the row on every `oxpull` app path ---------------- + + +def test_a_unique_task_stores_its_declared_budget(): + declared = policy_tasks.unique_with_budget.enqueue("a") + inherited = tasks.unique_echo.enqueue("a") + + assert budget(declared) == 5 + assert declared.task == policy_tasks.unique_with_budget + assert budget(inherited) == BACKEND_BUDGET + # The result carries the task as declared; the budget is the row's. + assert inherited.task == tasks.unique_echo + assert inherited.task.max_attempts is None + + +def test_a_unique_duplicate_returns_the_holder_with_its_stored_budget(): + """ + A duplicate returns the holder's result, and the holder's row keeps its + stored budget, which is the one the worker and the reaper decide on. A + row can hold another budget than the code now declares: an operator + retry rewrites it, and a row enqueued before a deploy keeps the one it + was written with. The result's task is the task as the code declares + it, not that budget. Policy is not part of the key, so a copy that + declares yet another budget coalesces onto the same holder too. + """ + holder = policy_tasks.unique_with_budget.enqueue("a") + OxTask.objects.filter(pk=holder.id).update(max_attempts=9) + + duplicate = policy_tasks.unique_with_budget.enqueue("a") + replaced = dataclasses.replace(policy_tasks.unique_with_budget, max_attempts=2) + other_policy = replaced.enqueue("a") + + assert duplicate.id == other_policy.id == holder.id + assert OxTask.objects.count() == 1 + assert budget(holder) == budget(duplicate) == budget(other_policy) == 9 + assert duplicate.task == policy_tasks.unique_with_budget + assert other_policy.task == policy_tasks.unique_with_budget + fetched = default_task_backend.get_result(holder.id).task + assert fetched == policy_tasks.unique_with_budget + + +def test_the_reaper_decides_a_unique_task_on_its_stored_budget(worker): + """ + The reaper's requeue-or-LOST is SQL on the row's max_attempts. With three + attempts spent, the backend's budget of 3 would make the row LOST and + free the key; the task's own budget of 5 puts it back, and the key stays + with it. + """ + first = policy_tasks.unique_with_budget.enqueue("a") + stale = timezone.now() - timedelta(seconds=9999) + OxTask.objects.filter(pk=first.id).update( + status=RUNNING, attempts=3, locked_by="dead-worker", locked_at=stale + ) + + assert worker.reap() == 1 + assert row(first).status == READY + assert policy_tasks.unique_with_budget.enqueue("a").id == first.id + + OxTask.objects.filter(pk=first.id).update( + status=RUNNING, attempts=5, locked_by="dead-worker", locked_at=stale + ) + assert worker.reap() == 1 + assert row(first).status == LOST + second = policy_tasks.unique_with_budget.enqueue("a") + assert second.id != first.id + assert budget(second) == 5 + + +def test_bulk_enqueue_stores_the_declared_budget_on_every_row(): + calls = [((value,), {}) for value in range(3)] + + declared = enqueue_many(policy_tasks.declares_everything, calls) + inherited = enqueue_many(tasks.plain_echo, calls) + + assert isinstance(policy_tasks.declares_everything.get_backend(), OxpullBackend) + assert [budget(result) for result in declared] == [5, 5, 5] + assert [result.task for result in declared] == [ + policy_tasks.declares_everything + ] * 3 + assert [budget(result) for result in inherited] == [BACKEND_BUDGET] * 3 + assert [result.task for result in inherited] == [tasks.plain_echo] * 3 + + +def test_bulk_enqueue_still_refuses_a_unique_task_that_declares_policy(): + with pytest.raises(InvalidTask, match="is unique"): + enqueue_many(policy_tasks.unique_with_budget, [(("a",), {})]) + assert OxTask.objects.count() == 0 + + +def test_batch_members_store_their_own_budgets(): + batch = batches.create("budgets") + declared = batch.add(policy_tasks.declares_everything, 1) + routed = batch.add( + policy_tasks.declares_everything.using(queue_name="emails", priority=3), 2 + ) + inherited = batch.add(tasks.plain_echo, 3) + + assert [budget(declared), budget(routed), budget(inherited)] == [ + 5, + 5, + BACKEND_BUDGET, + ] + assert (row(routed).queue_name, row(routed).priority) == ("emails", 3) + assert declared.task.max_attempts == routed.task.max_attempts == 5 + + +def test_batch_callbacks_are_enqueued_with_their_declared_budget(worker): + batch = batches.create( + "callbacks", + on_complete=policy_tasks.callback_with_budget, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + batch.add(tasks.plain_echo, 1) + batch.seal() + assert worker.run_once() is True + assert reconcile_batches() == 1 + + stored = OxpullBatch.objects.get(pk=batch.id) + complete = OxTask.objects.get(pk=stored.callback_task_id) + success = OxTask.objects.get(pk=stored.success_task_id) + assert complete.task_path == "tests_oxpull.policy_tasks.callback_with_budget" + assert complete.max_attempts == 6 + assert success.max_attempts == BACKEND_BUDGET + + +@pytest.mark.parametrize("module", [batches, workflows], ids=["batch", "workflow"]) +def test_a_callback_whose_policy_was_replaced_per_call_is_refused(module): + """ + A callback is stored as its dotted path, and the path imports the + declared task, so a per-call replace() of its policy would be dropped in + silence at completion. It is refused on the create() line, as a using() + override is. + """ + replaced = dataclasses.replace(policy_tasks.callback_with_budget, max_attempts=2) + with pytest.raises(ImproperlyConfigured, match="does not import back to itself"): + module.create("replaced", on_complete=replaced) + assert not OxpullBatch.objects.exists() + assert not OxpullWorkflow.objects.exists() + + +def test_workflow_roots_waiting_nodes_and_callbacks_store_their_own_budgets(): + flow = workflows.create( + "budgets", + on_complete=policy_tasks.callback_with_budget, + on_complete_args=["complete"], + ) + root = flow.add("root", policy_tasks.node_with_budget, "root") + waiting = flow.add( + "waiting", policy_tasks.node_with_budget, "waiting", depends_on="root" + ) + inherited = flow.add("inherited", tasks.step, "inherited", depends_on="root") + flow.seal() + + assert (row(root).status, budget(root)) == (READY, 5) + assert (row(waiting).status, budget(waiting)) == (WAITING, 5) + assert (row(inherited).status, budget(inherited)) == (WAITING, BACKEND_BUDGET) + assert root.task == waiting.task == policy_tasks.node_with_budget + assert inherited.task == tasks.step + + assert drain(flow_worker()) == 3 + # The release moves the status and nothing else on the row. + assert [budget(r) for r in (root, waiting, inherited)] == [5, 5, BACKEND_BUDGET] + assert {row(r).status for r in (root, waiting, inherited)} == {SUCCESSFUL} + + reconcile_workflows() + stored = OxpullWorkflow.objects.get(pk=flow.id) + assert stored.status == OxpullWorkflow.Status.COMPLETE + assert OxTask.objects.get(pk=stored.callback_task_id).max_attempts == 6 + + +# -- routing and rebinding keep the policy ----------------------------------- + + +def test_using_keeps_the_policy_and_the_row_keeps_the_routing(): + declared = policy_tasks.declares_everything + routed = declared.using(queue_name="emails", priority=4) + rebound = policy_tasks.node_with_budget.using(backend="flows_deferred") + + assert (routed.max_attempts, routed.backoff, routed.timeout) == ( + 5, + policy_tasks.retry_now, + 30, + ) + assert isinstance(rebound, PolicyTask) + assert rebound.max_attempts == 5 + + first = routed.enqueue(1) + second = rebound.enqueue("x") + third = policy_tasks.unique_with_budget.using(queue_name="emails").enqueue("a") + + assert (row(first).queue_name, row(first).priority, budget(first)) == ( + "emails", + 4, + 5, + ) + assert (row(second).backend_name, budget(second)) == ("flows_deferred", 5) + # A unique task routed elsewhere is another key, with the same budget. + assert (row(third).queue_name, budget(third)) == ("emails", 5) + assert third.id != policy_tasks.unique_with_budget.enqueue("a").id + + # Read back, the result carries the row's routing and the live policy. + back = default_task_backend.get_result(first.id).task + assert (back.queue_name, back.priority) == ("emails", 4) + assert (back.max_attempts, back.backoff, back.timeout) == ( + 5, + policy_tasks.retry_now, + 30, + ) + back = task_backends["flows_deferred"].get_result(second.id).task + assert (back.backend, back.max_attempts) == ("flows_deferred", 5) + + +def test_a_plain_task_rebound_onto_an_oxpull_backend_inherits_the_backend_budget(): + """ + A task declared on a backend whose tasks carry no policy keeps its own + class when using() rebinds it here, and declares nothing, so it takes + the backend's budget. Its result carries that task, still plain; the + budget is on the row. + """ + rebound = tasks.immediate_echo.using(backend="default") + assert not isinstance(rebound, PolicyTask) + + result = rebound.enqueue("x") + + assert budget(result) == BACKEND_BUDGET + assert not isinstance(result.task, PolicyTask) + assert result.task == rebound + fetched = default_task_backend.get_result(result.id).task + assert not isinstance(fetched, PolicyTask) + assert fetched == rebound + + +# -- rate limits ------------------------------------------------------------- + + +def test_a_throttled_claim_spends_nothing_and_every_admitted_retry_spends_one(): + """ + The limiter decides before the claim, so a throttled task keeps its + attempts and its lease epoch. Each retry the limiter admits is a claim + like the first, and it spends one attempt of the task's own budget and + one admission of the limit's window. + """ + path = "tests_oxpull.policy_tasks.fails_with_budget_four" + result = policy_tasks.fails_with_budget_four.enqueue() + + with rate_limits({"budget": {"limit": 2, "per": 600, "tasks": [path]}}): + worker = oxpull_worker() + for attempt in (1, 2): + assert worker.run_once() is True # fails, and its backoff retries at once + assert (row(result).status, row(result).attempts) == (READY, attempt) + assert window("budget").used == attempt + + for _ in range(3): + assert worker.run_once() is False + throttled = row(result) + assert throttled.status == READY + assert (throttled.attempts, throttled.lease_epoch) == (2, 2) + assert throttled.locked_by is None + assert window("budget").used == 2 + + with rate_limits({"budget": {"limit": 10, "per": 600, "tasks": [path]}}): + worker = oxpull_worker() + assert worker.run_once() is True # attempt 3: the backend's budget ends here + assert (row(result).status, row(result).attempts) == (READY, 3) + assert worker.run_once() is True # attempt 4, the task's last + assert worker.run_once() is False + + spent = row(result) + assert spent.status == FAILED + assert (spent.attempts, spent.max_attempts, spent.lease_epoch) == (4, 4, 4) + assert window("budget").used == 4 + + +# -- workflow nodes: a retry is not an outcome -------------------------------- + + +def test_a_node_retry_is_not_an_outcome_until_its_own_budget_is_spent(): + """ + A retry is django-ox's write to READY, which the `oxpull` app's hook passes through + and records nothing for. The node's budget of 5 outlives the backend's + 3, so attempts 3 and 4 are still retries, and only the fifth failure is + the node's outcome. + """ + flow = workflows.create("retrying") + root = flow.add("root", policy_tasks.failing_node_with_budget_five, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + + for attempt in range(1, 5): + assert worker.run_once() is True + assert (row(root).status, row(root).attempts) == (READY, attempt) + assert record_of(root) == (None, None, False) + reconcile_workflows() + assert row(child).status == WAITING + assert ( + OxpullWorkflow.objects.get(pk=flow.id).status + == OxpullWorkflow.Status.SEALED + ) + + with finished_signals() as finished: + assert worker.run_once() is True + + spent = row(root) + assert (spent.status, spent.attempts, spent.max_attempts) == (FAILED, 5, 5) + assert record_of(root) == (FAILED, spent.lease_epoch, False) + assert edge_marks(child) == [False] + assert [(str(r.id), r.task.max_attempts) for r in finished] == [(root.id, 5)] + reconcile_workflows() + assert row(child).status == DISCARDED + + +def test_a_backoff_that_answers_none_fails_the_node_through_the_outcome_hook(caplog): + flow = workflows.create("declined") + root = flow.add("root", policy_tasks.failing_node_that_stops, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + + with ( + caplog.at_level(logging.ERROR, logger="django_ox"), + finished_signals() as finished, + ): + assert flow_worker().run_once() is True + + failed = row(root) + assert (failed.status, failed.attempts, failed.max_attempts) == (FAILED, 1, 5) + assert record_of(root) == (FAILED, failed.lease_epoch, False) + assert notes() == [{"callback": "stop", "attempts": 1, "max_attempts": 5}] + assert [(str(r.id), r.task.max_attempts) for r in finished] == [(root.id, 5)] + (declined,) = events(caplog, "task_failed") + assert declined.reason == "backoff_declined" + assert edge_marks(child) == [False] + reconcile_workflows() + assert row(child).status == DISCARDED + + +@pytest.mark.django_db(transaction=True) +def test_a_timeout_retries_the_node_and_the_last_one_fails_it_through_the_hook( + interruptible_attempts, # noqa: F811 +): + """ + The node declares a timeout of 1 s and a budget of 2, and nothing in the + options sets a timeout. Its first timed-out attempt is a retry, which + records nothing; its second spends the budget and records FAILED. + """ + flow = workflows.create("timed-out") + root = flow.add("root", policy_tasks.node_past_its_timeout, "root", 30) + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + assert not worker.timeouts.enabled + + assert worker.run_once() is True + first = row(root) + assert (first.status, first.attempts) == (READY, 1) + assert [e["exception_class_path"] for e in first.errors] == [TIMEOUT_PATH] + assert record_of(root) == (None, None, False) + + assert worker.run_once() is True + spent = row(root) + assert (spent.status, spent.attempts) == (FAILED, 2) + assert [e["exception_class_path"] for e in spent.errors] == [TIMEOUT_PATH] * 2 + assert record_of(root) == (FAILED, spent.lease_epoch, False) + assert not worker.recycling + assert row(child).status == WAITING + + +def test_a_release_leaves_a_retrying_nodes_own_backoff_alone(): + """ + The worker's own backoff here is zero, so a run_after an hour out can + only be the node's backoff answering. The release moves only WAITING + rows, so a second success of the upstream, and the reconciler, leave the + retrying node's hour where it is. + """ + flow = workflows.create("backoff") + parent = flow.add("parent", tasks.step, "parent") + child = flow.add( + "child", + policy_tasks.failing_node_that_waits_an_hour, + "child", + depends_on="parent", + ) + flow.seal() + worker = flow_worker() + assert worker.backoff_initial == 0 + + assert worker.run_once() is True # the parent succeeds and releases the child + assert row(child).status == READY + before = timezone.now() + assert worker.run_once() is True # the child fails; its backoff asks for an hour + backoff = row(child).run_after + assert backoff is not None + assert backoff >= before + timedelta(minutes=59) + assert notes() == [{"callback": "hour", "attempts": 1}] + + OxTask.objects.filter(pk=parent.id).update(status=READY) + assert worker.run_once() is True # the parent succeeds again + reconcile_workflows() + + assert (row(child).status, row(child).run_after) == (READY, backoff) + assert record_of(child) == (None, None, False) + assert worker.run_once() is False + + +def test_a_direct_failure_record_on_a_node_keeps_its_contract_and_asks_no_backoff(): + """ + _handle_failure(db_task, exc, duration_ms, *, release=True) is called + directly, by the stuck-thread path and by this package's tests. Neither + call runs inside an attempt, so neither asks the node's backoff: the + budget on the row and the worker's backoff decide, and a node with + attempts left is a retry that records nothing. + """ + flow = workflows.create("direct") + root = flow.add("root", policy_tasks.failing_node_that_stops, "root") + flow.seal() + worker = flow_worker() + + claimed = worker.claim_one() + assert ( + worker._handle_failure(claimed, RuntimeError("stuck"), 0, release=True) is True + ) + assert (row(root).status, row(root).attempts) == (READY, 1) + + claimed = worker.claim_one() + assert worker._handle_failure(claimed, ValueError("again"), 0) is True + assert (row(root).status, row(root).attempts) == (READY, 2) + + assert notes() == [] + assert record_of(root) == (None, None, False) + + +# -- timeouts and the reconcilers --------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_a_tasks_own_timeout_reaches_the_remaining_the_reconcilers_read( + monkeypatch, worker +): + """ + The reconcilers budget a tick from django-ox's remaining(), which reads + the deadline the worker publishes for the attempt. A task that declares + its own timeout publishes one with nothing in the options, and both + reconcilers, run inside that attempt, see it. + """ + batch = batches.create("swept") + batch.add(tasks.plain_echo, 1) + batch.seal() + assert worker.run_once() is True # the member + assert not worker.timeouts.enabled + policy_tasks.reconciles_under_its_own_timeout.enqueue() + + with remaining_as_the_reconcilers_read_it(monkeypatch) as seen: + assert worker.run_once() is True + + readers = {module for module, _ in seen} + assert readers == {"oxpull.batches", "oxpull.workflows"}, seen + assert all(value is not None and 0 < value <= 8 for _, value in seen), seen + # A quarter of what is left, under the five seconds a tick gets without one. + assert 0 < tasks.STATE["backstop_budget"] <= 2, tasks.STATE + assert tasks.STATE["reconciled"][0] == 1 + assert OxpullBatch.objects.get(pk=batch.id).status == OxpullBatch.Status.COMPLETE + + +def a_sealed_batch_to_sweep(name): + """ + A sealed batch with no members, so the next tick's sweep visits it and + reads remaining() on the way in. The workflow tick reads it whatever is + open, for its backstop budget. + """ + batches.create(name).seal() + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("module", [batches, workflows], ids=["batch", "workflow"]) +def test_the_lazily_built_reconcilers_keep_the_backend_defaults(monkeypatch, module): + """ + The reconcilers are plain tasks built on access, so they declare no + policy of their own: the backend's budget goes on their row, and their + attempt runs under the worker's timeout, or none. A task with a timeout + of its own that ran just before, on the same worker and thread, leaves + nothing behind for them. + """ + reconciler = module.reconcile + assert isinstance(reconciler, PolicyTask) + assert (reconciler.max_attempts, reconciler.backoff, reconciler.timeout) == ( + None, + None, + None, + ) + + unlimited = Worker(backoff_initial=0, poll_interval=0.01) + a_sealed_batch_to_sweep("unlimited") + result = reconciler.enqueue() + assert budget(result) == BACKEND_BUDGET + assert result.task == reconciler + with remaining_as_the_reconcilers_read_it(monkeypatch) as seen: + assert unlimited.run_once() is True + assert row(result).status == SUCCESSFUL + assert seen + assert {value for _, value in seen} == {None}, seen + + limited = Worker(backoff_initial=0, poll_interval=0.01, task_timeout=60) + policy_tasks.reconciles_under_its_own_timeout.enqueue() + assert limited.run_once() is True + assert 0 < tasks.STATE["remaining_after"] <= 8 + a_sealed_batch_to_sweep("limited") + result = module.reconcile.enqueue() + with remaining_as_the_reconcilers_read_it(monkeypatch) as seen: + assert limited.run_once() is True + assert row(result).status == SUCCESSFUL + assert seen + assert all(value is not None and 8 < value <= 60 for _, value in seen), seen diff --git a/tests_oxpull/test_prune.py b/tests_oxpull/test_prune.py new file mode 100644 index 0000000..09034d3 --- /dev/null +++ b/tests_oxpull/test_prune.py @@ -0,0 +1,1298 @@ +import copy +import json +import re +import threading +import time +import uuid +from contextlib import contextmanager +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import CommandError, ManagementUtility, call_command +from django.db import connection, connections, transaction +from django.db.models import F +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox import _waiting, actions +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.models import ( + OxpullBatch, + OxpullBatchMember, + OxpullUniqueLock, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, + OxpullWorkflowScan, +) +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile +from .test_workflows_reconciler import ( + EDGE_TABLE, + LEVELS, + NODE_TABLE, + WORKFLOW_TABLE, + events_named, + every_connection_at, + every_statement, + flagged_step, + lose_a_deadlock_once, + ordered_lock_of, + record_of, + serialization_failures_at, +) + +pytestmark = pytest.mark.django_db + + +def _prune(*args): + out = StringIO() + call_command("oxpull_prune", *args, stdout=out) + return out.getvalue() + + +def test_prune_deletes_stale_locks_only(worker): + finished = tasks.unique_echo.enqueue("finished") + assert worker.run_once() is True + assert OxTask.objects.get(id=finished.id).status == OxTask.Status.SUCCESSFUL + + pruned_away = tasks.unique_echo.enqueue("row-pruned") + OxTask.objects.filter(id=pruned_away.id).delete() + + live = tasks.unique_echo.enqueue("live") + + assert OxpullUniqueLock.objects.count() == 3 + # batch size 1 drives the bounded-delete loop through several + # passes; the result must not change. + output = _prune("--batch-size", "1") + assert "Deleted 2 stale unique lock(s)." in output + lock = OxpullUniqueLock.objects.get() + assert str(lock.task_id) == live.id + + +def test_prune_dry_run_deletes_nothing(worker): + tasks.unique_echo.enqueue("finished") + assert worker.run_once() is True + + output = _prune("--dry-run") + assert "Would delete 1 stale unique lock(s)." in output + assert OxpullUniqueLock.objects.count() == 1 + + +def test_prune_rejects_non_positive_batch_size(): + with pytest.raises(CommandError): + call_command("oxpull_prune", "--batch-size", "0") + + +# -- workflow and batch rows ------------------------------------------------ +# +# Node and edge rows outlive the task rows they name, and the edge table is +# read inside every claim on a gating backend, so a long-lived install would +# grow it without bound if nothing reaped it. oxpull_prune deletes a node and +# the edges into it only once no retry can move the node again, and it never +# deletes a batch member. +# +# Shortcuts, each named where it is taken: completed_at moved back stands in +# for waiting past --older-than, and ox_prune's finished_at moved back for +# waiting past its own cutoff. + +LONG_AGO = timedelta(days=30) + + +def _complete(row, ago): + """Mark a batch COMPLETE as of `ago` before now (shortcut).""" + type(row).objects.filter(pk=row.pk).update( + status=type(row).Status.COMPLETE, completed_at=timezone.now() - ago + ) + + +def completed_long_ago(flow, ago=LONG_AGO): + """Shortcut: move a COMPLETE workflow's completed_at back.""" + row = OxpullWorkflow.objects.get(pk=flow.id) + assert row.status == OxpullWorkflow.Status.COMPLETE + OxpullWorkflow.objects.filter(pk=flow.id).update( + completed_at=row.completed_at - ago + ) + + +def run_to_completion(flow): + """A worker drains it and the reconciler ticks until it completes.""" + worker = flow_worker() + for _ in range(4): + drain(worker) + reconcile() + if OxpullWorkflow.objects.get(pk=flow.id).status == "COMPLETE": + return + raise AssertionError("the workflow did not complete") + + +def rows_of(flow): + return ( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count(), + OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count(), + ) + + +def status_of(result): + return OxTask.objects.values_list("status", flat=True).get(pk=result.id) + + +def two_nodes(name, second=tasks.step): + flow = workflows.create(name) + a = flow.add("a", tasks.step, f"{name}-a") + b = flow.add("b", second, f"{name}-b", depends_on="a") + flow.seal() + return flow, a, b + + +def test_prune_reaps_the_nodes_and_edges_of_a_long_completed_workflow(): + flow, _, _ = two_nodes("old") + run_to_completion(flow) + completed_long_ago(flow) + + output = _prune() + + assert rows_of(flow) == (0, 0) + assert "Deleted 2 workflow node row(s)." in output + assert "Deleted 1 workflow edge row(s)." in output + # The workflow row itself stays: one row per workflow is bounded by + # how many an operator creates, and it is the record of what ran. + row = OxpullWorkflow.objects.get(pk=flow.id) + assert row.metadata_retired_at is not None + assert row.completions["0"]["succeeded"] == 2 + + +def test_prune_leaves_a_sealed_workflow_alone(): + """ + A SEALED workflow's edges are read by every claim on a gating + backend. Reaping them would un-gate live nodes. + """ + flow, _, _ = two_nodes("live") + _prune("--older-than", "0") + assert rows_of(flow) == (2, 1) + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is None + + +def test_prune_respects_older_than_for_workflows(): + flow, _, _ = two_nodes("recent") + run_to_completion(flow) + completed_long_ago(flow, timedelta(hours=1)) + + _prune() # default 7d + assert rows_of(flow) == (2, 1) + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is None + _prune("--older-than", "30m") + assert rows_of(flow) == (0, 0) + + +def test_prune_keeps_batch_members(): + batch = batches.create("old-batch") + batch.add(tasks.plain_echo, "one") + batch.seal() + assert OxpullBatchMember.objects.count() == 1 + _complete(OxpullBatch.objects.get(pk=batch.id), LONG_AGO) + output = _prune("--older-than", "0") + assert OxpullBatchMember.objects.count() == 1 + assert "batch" not in output.lower() + assert OxpullBatch.objects.filter(pk=batch.id).exists() + + +def test_dry_run_counts_what_would_go_without_deleting_or_stamping(): + flow, _, _ = two_nodes("counted") + run_to_completion(flow) + completed_long_ago(flow) + + out = _prune("--dry-run") + assert "Would delete 2 workflow node row(s)." in out + assert "Would delete 1 workflow edge row(s)." in out + assert rows_of(flow) == (2, 1) + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is None + + +@pytest.mark.parametrize("dead", ["failed", "lost"]) +def test_a_failed_or_lost_node_stays_gated_until_ox_prune_removes_its_row(dead): + """ + b settled without succeeding. Its node and the edge into it stay while its + row is there, so an operator's retry of b is still gated. a succeeded, so + its node goes, and the edge a -> b stays marked satisfied. Once ox_prune + --include-failed deletes b's row, b's node and edge go too. + """ + flow = workflows.create(f"dead-{dead}") + flow.add("a", tasks.step, "a") + if dead == "failed": + b = flow.add("b", tasks.failing_step, "b", depends_on="a") + flow.seal() + run_to_completion(flow) + else: + b = flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + assert drain(flow_worker(), limit=1) == 1 # a + reaper = flow_worker() + for _ in range(3): + claimed = flow_worker().claim_one() + assert claimed is not None and str(claimed.id) == str(b.id) + assert actions.expire_lease(b.id) is True + assert reaper.reap() == 1 + run_to_completion(flow) + assert status_of(b) == dead.upper() + completed_long_ago(flow) + + output = _prune() + + assert rows_of(flow) == (1, 1), output + assert list( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values_list( + "name", flat=True + ) + ) == ["b"] + assert OxpullWorkflowEdge.objects.get(workflow_id=flow.id).satisfied is True + assert "Kept 1 workflow node row(s) and 1 workflow edge row(s)" in output + + OxTask.objects.filter(pk=b.id).update( + finished_at=timezone.now() - timedelta(days=8) + ) + call_command("ox_prune", "--include-failed", stdout=StringIO()) + assert not OxTask.objects.filter(pk=b.id).exists() + _prune() + assert rows_of(flow) == (0, 0) + + +def test_a_node_whose_row_went_before_anything_recorded_it_is_kept(): + """ + b's row is gone and nothing recorded how b finished. Its node is never + deleted, and neither is the edge into it. + """ + flow, _, b = two_nodes("unrecorded") + drain(flow_worker()) + OxTask.objects.filter(pk=b.id).delete() # shortcut: a prune by hand + OxpullWorkflowNode.objects.filter(task_id=b.id).update( + outcome=None, outcome_epoch=None, succeeded=False + ) + run_to_completion(flow) + completed_long_ago(flow) + + _prune() + + assert list( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values_list( + "name", flat=True + ) + ) == ["b"] + assert rows_of(flow) == (1, 1) + + +def test_a_workflow_with_a_node_retried_after_completion_keeps_every_row(): + """ + An operator retried b after the workflow completed, and b has not run + again. b is pending, so nothing of the workflow is deleted, the stamp is + set, and the next run tries again. + """ + flow, _, b = two_nodes("retried", second=tasks.failing_step) + run_to_completion(flow) + assert actions.retry(b.id) is True + completed_long_ago(flow) + + output = _prune() + + assert rows_of(flow) == (2, 1) + assert "Skipped 1 workflow(s) with a node still pending." in output + stamped = OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at + assert stamped is not None + + drain(flow_worker()) + assert status_of(b) == OxTask.Status.FAILED + _prune() + assert rows_of(flow) == (1, 1) + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at == stamped + + +def test_prune_repairs_a_record_no_worker_wrote_before_it_deletes(): + """ + b succeeded under a writer that recorded nothing, and marked nothing. The + prune records it first, and b's node goes with the edge into it. + """ + flow, _, b = two_nodes("repair-first") + run_to_completion(flow) + OxpullWorkflowNode.objects.filter(task_id=b.id).update( + outcome=None, outcome_epoch=None, succeeded=False + ) + completed_long_ago(flow) + _prune() + assert rows_of(flow) == (0, 0) + + +def test_an_unmarked_edge_into_a_kept_node_is_marked_before_its_upstream_goes(): + """ + a succeeded, b failed, and the edge a -> b was never marked (shortcut: the + mark cleared by hand, as a writer of another version leaves it). a's node is + the record of a's success once ox_prune deletes a's row. The prune marks the + edge in the transaction that deletes that node, so an operator's retry of b + still runs after a's row is gone. + """ + flow = workflows.create("mark-first") + a = flow.add("a", tasks.step, "a") + b = flow.add("b", flagged_step, "b", depends_on="a") + flow.seal() + tasks.STATE["fail:b"] = True + run_to_completion(flow) + assert status_of(b) == OxTask.Status.FAILED + OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).update(satisfied=False) + completed_long_ago(flow) + + _prune() + assert OxpullWorkflowEdge.objects.get(workflow_id=flow.id).satisfied is True + + OxTask.objects.filter(pk=a.id).update( + finished_at=timezone.now() - timedelta(days=8) + ) + call_command("ox_prune", stdout=StringIO()) + assert not OxTask.objects.filter(pk=a.id).exists() + tasks.STATE["fail:b"] = False + assert actions.retry(b.id) is True + assert drain(flow_worker()) == 1 + assert status_of(b) == OxTask.Status.SUCCESSFUL + + +def test_a_delete_marks_the_edges_out_of_a_node_that_succeeded(): + """ + The delete transaction's mark, called on its own: the edges out of a + successful node that goes, into a node that stays, are marked in the + transaction that deletes the node. + """ + flow = workflows.create("mark-in-delete") + flow.add("a", tasks.step, "a") + b = flow.add("b", tasks.failing_step, "b", depends_on="a") + flow.seal() + run_to_completion(flow) + OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).update(satisfied=False) + alias = oxpull_db_alias() + workflows._stamp_retired(flow.id, alias) + chunk = list( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id) + .order_by("pk") + .values_list("pk", "task_id", "outcome") + ) + nodes, edges, marked = workflows._retire_chunk(flow.id, chunk, alias) + assert (nodes, edges) == (1, 0) + assert marked == {uuid.UUID(str(b.id))} + assert OxpullWorkflowEdge.objects.get(workflow_id=flow.id).satisfied is True + + +def test_a_node_whose_record_is_gone_keeps_the_edges_into_it(): + """ + A node the delete cannot take keeps whatever gates it. + + The delete takes recorded nodes only. If it took the edges into an + unrecorded node anyway, that node would be left with nothing gating it, + which is the one thing the claim gate exists to prevent. Nothing in this + release clears a record, so this plants one: b's record is cleared, its + task row still reads SUCCESSFUL, and b's node and the edge into it both + have to survive the chunk that deletes a. + """ + flow = workflows.create("record-cleared") + flow.add("a", tasks.step, "a") + b = flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + run_to_completion(flow) + alias = oxpull_db_alias() + workflows._stamp_retired(flow.id, alias) + OxpullWorkflowNode.objects.filter(workflow_id=flow.id, task_id=b.id).update( + outcome=None, outcome_epoch=None + ) + chunk = list( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id) + .order_by("pk") + .values_list("pk", "task_id", "outcome") + ) + nodes, edges, _ = workflows._retire_chunk(flow.id, chunk, alias) + + assert (nodes, edges) == (1, 0) + kept = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values_list( + "task_id", flat=True + ) + assert [uuid.UUID(str(pk)) for pk in kept] == [uuid.UUID(str(b.id))] + gating = OxpullWorkflowEdge.objects.filter( + workflow_id=flow.id, downstream_task_id=b.id + ) + assert gating.count() == 1, "b was left with nothing gating it" + + +@pytest.mark.skipif( + connection.vendor == "sqlite", + reason="SQLite serialises writers, so nothing lands inside the delete", +) +@pytest.mark.django_db(transaction=True) +def test_a_node_revived_before_the_delete_locks_its_row_is_kept(): + """ + b was cancelled, and its node is recorded DISCARDED. The delete reads the + chunk's rows, and before it locks them b is revived to WAITING. The delete + reads b's row again under its lock, finds it WAITING, and keeps its node + and the edge into it. + """ + flow = workflows.create("revived-inside") + flow.add("a", tasks.failing_step, "a") + b = flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + run_to_completion(flow) + assert status_of(b) == OxTask.Status.DISCARDED + epoch = OxTask.objects.get(pk=b.id).lease_epoch + completed_long_ago(flow) + fired = [] + task_table = OxTask._meta.db_table.upper() + + def revive_on_another_connection(): + try: + fired.append(_waiting.revive_many([(b.id, epoch)], using=oxpull_db_alias())) + finally: + connections.close_all() + + def pause(execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + if ( + not fired + and text.startswith("SELECT") + and task_table in text + and "FOR UPDATE" in text + ): + thread = threading.Thread(target=revive_on_another_connection) + thread.start() + thread.join(30) + return execute(sql, params, many, context) + + with connection.execute_wrapper(pause): + _prune() + + assert fired and list(fired[0].values()) == [_waiting.Revival.REVIVED] + assert status_of(b) == OxTask.Status.WAITING + names = set( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values_list( + "name", flat=True + ) + ) + assert "b" in names, names + assert OxpullWorkflowEdge.objects.filter(downstream_task_id=b.id).count() == 1 + + +def test_prune_cleans_the_scan_rows_of_complete_workflows(): + """ + The backstop visits OPEN and SEALED workflows, and their scan rows hold its + skip queue and pass position. Only a COMPLETE workflow's scan row goes. + """ + done, _, _ = two_nodes("scan-done") + run_to_completion(done) + sealed, _, _ = two_nodes("scan-sealed") + still_open = workflows.create("scan-open") + still_open.add("x", tasks.step, "scan-open-x") + OxpullWorkflowScan.objects.create(workflow_id=done.id, skipped=["x"]) + OxpullWorkflowScan.objects.create(workflow_id=sealed.id, skipped=["y"]) + OxpullWorkflowScan.objects.create(workflow_id=still_open.id, skipped=["z"]) + + output = _prune() + + assert "Deleted 1 workflow scan row(s)." in output + kept = OxpullWorkflowScan.objects.values_list("workflow_id", flat=True) + assert {uuid.UUID(str(pk)) for pk in kept} == { + uuid.UUID(str(sealed.id)), + uuid.UUID(str(still_open.id)), + } + + +def test_retained_rows_count_what_prune_keeps(): + flow, _, _ = two_nodes("retained", second=tasks.failing_step) + run_to_completion(flow) + completed_long_ago(flow) + assert workflows.retained_rows() == workflows.RetainedRows(0, 0, 0) + _prune() + assert workflows.retained_rows() == workflows.RetainedRows(1, 1, 1) + + +@pytest.mark.skipif( + connection.vendor == "sqlite", + reason="SQLite serialises writers, so no stamp waits on another", +) +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_stamp_that_waited_on_another_prune_runs_again(caplog, level): + """ + Two oxpull_prune runs reach the same workflow. The other one has stamped it + and not yet committed, so this run's stamp waits for the workflow row. At + PostgreSQL REPEATABLE READ the row it finds after that commit changed after + its snapshot, and the stamp raises a serialization failure. The stamp owns + its transaction, so it runs it again, finds the workflow stamped, keeps that + stamp, and the run deletes the rows as usual. At the other levels the stamp + reads the row as committed and nothing runs again. + """ + flow, _, _ = two_nodes(f"stamp-waits-{level}") + run_to_completion(flow) + completed_long_ago(flow) + other_stamp = timezone.now() - timedelta(minutes=5) + held = threading.Event() + release = threading.Event() + errors: list[BaseException] = [] + + def stamp_on_another_connection(): + try: + with transaction.atomic(): + stamped = OxpullWorkflow.objects.filter( + pk=flow.id, metadata_retired_at__isnull=True + ).update(metadata_retired_at=other_stamp) + assert stamped == 1 + held.set() + release.wait(60) + except BaseException as exc: + errors.append(exc) + held.set() + finally: + connections.close_all() + + def commit_the_other_while_the_stamp_waits(execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + if ( + not release.is_set() + and text.startswith("UPDATE") + and WORKFLOW_TABLE in text + and "METADATA_RETIRED_AT" in text + ): + threading.Timer(0.5, release.set).start() + return execute(sql, params, many, context) + + with every_connection_at(level): + other = threading.Thread(target=stamp_on_another_connection) + other.start() + assert held.wait(30) and not errors, errors + try: + with ( + caplog.at_level("INFO", logger="oxpull"), + connection.execute_wrapper(commit_the_other_while_the_stamp_waits), + ): + output = _prune() + finally: + release.set() + other.join(60) + + reissued = events_named(caplog, "workflow_reconcile_conflict_retry") + facts = f"level={level} reissued={reissued} output={output!r}" + assert not errors, errors + if serialization_failures_at(level): + assert len(reissued) == 1, facts + else: + assert reissued == [], facts + assert rows_of(flow) == (0, 0), facts + row = OxpullWorkflow.objects.get(pk=flow.id) + assert row.metadata_retired_at == other_stamp, facts + + +# -- what a run costs once workflows are retired -------------------------------- + + +def prune_statements(*args): + """How many statements one oxpull_prune run sends, and what it prints.""" + with CaptureQueriesContext(connection) as captured: + output = _prune(*args) + return len(captured.captured_queries), output + + +def deletes_from(sql, table): + """Whether a statement is a DELETE of `table`.""" + words = sql.split() + return words[:2] == ["DELETE", "FROM"] and words[2] == connection.ops.quote_name( + table + ) + + +def emptied_and_kept(label): + """ + Two workflows past the cutoff. The first run empties one. It keeps the + other's failed node and the edge into it. + """ + done, _, _ = two_nodes(f"{label}-done") + kept, _, failed = two_nodes(f"{label}-kept", second=tasks.failing_step) + for flow in (done, kept): + run_to_completion(flow) + completed_long_ago(flow) + return done, kept, failed + + +def test_a_run_costs_the_same_however_many_workflows_are_already_retired(): + """ + A retired workflow whose node rows are all gone gives a later run nothing to + do, and neither does one that keeps only a recorded FAILED node. oxpull_prune + runs on a schedule and never deletes a workflow row, so a run that visited + every one of them would grow slower with every workflow that ever completed. + A run over four of each sends what a run over one of each sends, and so does + the dry run. + """ + done, kept, _ = emptied_and_kept("first") + _prune() + assert (rows_of(done), rows_of(kept)) == ((0, 0), (1, 1)) + once, _ = prune_statements() + dry_once, _ = prune_statements("--dry-run") + + for i in range(3): + emptied_and_kept(f"more-{i}") + _prune() + assert workflows.retained_rows() == workflows.RetainedRows(8, 4, 4) + again, output = prune_statements() + dry_again, _ = prune_statements("--dry-run") + + counts = f"statements: run {once} then {again}, dry run {dry_once} then {dry_again}" + assert (again, dry_again) == (once, dry_once), counts + assert "Deleted 0 workflow node row(s)." in output + assert "Kept 4 workflow node row(s) and 4 workflow edge row(s)" in output + + +WORKFLOW_TABLES = (OxpullWorkflow._meta.db_table, NODE_TABLE, EDGE_TABLE) +PLAN_TABLES = (OxTask._meta.db_table, *WORKFLOW_TABLES) + + +def _tables_by_alias(sql, quote): + """Each workflow or task table in the SQL, and each alias of one, to the table.""" + found = {table: table for table in PLAN_TABLES} + for table, alias in re.findall( + rf"{quote}(\w+){quote}(?: AS)? ({quote}?[A-Za-z_]\w*{quote}?)", sql + ): + if table in PLAN_TABLES: + found[alias.strip(quote)] = table + return found + + +def _postgresql_rereads(plan, *, per_row=False): + """ + Whole reads of a table under a SubPlan that is not hashed, or on the inner + side of a nested loop. PostgreSQL runs those again for each outer row. + """ + kind = plan.get("Node Type", "") + whole = kind == "Seq Scan" or ( + kind in ("Index Scan", "Index Only Scan") and "Index Cond" not in plan + ) + found = [] + if per_row and whole and plan.get("Relation Name") in PLAN_TABLES: + found.append(f"{kind} on {plan['Relation Name']}") + for child in plan.get("Plans", []): + relation = child.get("Parent Relationship") + again = ( + per_row + or ( + relation == "SubPlan" + and not child.get("Subplan Name", "").startswith("hashed") + ) + or (relation == "Inner" and kind == "Nested Loop") + ) + found.extend(_postgresql_rereads(child, per_row=again)) + return found + + +MYSQL_WHOLE_SCAN = r"-> (?:Table scan|Covering index scan|Index scan) on (\w+)" + + +def _mysql_rereads(text, sql): + """ + Whole reads of a table inside a dependent subquery or a materialisation + redone per row, or on a nested loop's inner side with no materialisation or + hash between. MySQL runs those again for each outer row. + """ + tables = _tables_by_alias(sql, "`") + lines = text.splitlines() + found = [] + for index, line in enumerate(lines): + match = re.search(MYSQL_WHOLE_SCAN, line) + if not match or tables.get(match.group(1)) not in PLAN_TABLES: + continue + depth = len(line) - len(line.lstrip()) + child = index + buffered = False + per_row = False + for above in range(index - 1, -1, -1): + text_above = lines[above] + above_depth = len(text_above) - len(text_above.lstrip()) + if above_depth >= depth: + continue + if "dependent)" in text_above or "invalidate on row" in text_above: + per_row = True + if "Materialize" in text_above or "-> Hash" in text_above: + buffered = True + if "Nested loop" in text_above and not buffered and child != above + 1: + per_row = True + depth, child = above_depth, above + if per_row: + found.append(line.strip()) + return found + + +def _sqlite_rereads(rows, sql): + """ + SCAN steps under a CORRELATED subquery, or after an earlier SCAN or SEARCH + of the same parent, which makes them a join's inner loop. SQLite runs those + again for each outer row. + """ + tables = _tables_by_alias(sql, '"') + parents = {row[0]: row[1] for row in rows} + details = {row[0]: row[3] for row in rows} + found = [] + for node, parent, _, detail in rows: + match = re.match(r"SCAN (\w+)", detail) + if not match or tables.get(match.group(1)) not in PLAN_TABLES: + continue + per_row = any( + re.match(r"(SCAN|SEARCH) ", other[3]) + for other in rows + if other[1] == parent and other[0] < node + ) + up = parent + while up in details: + per_row = per_row or details[up].startswith("CORRELATED") + up = parents[up] + if per_row: + found.append(detail) + return found + + +def rereads(sql): + """ + The steps of a SELECT's plan that read a whole workflow or task table again + for each row of another table, and the plan as text. + """ + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("EXPLAIN (FORMAT JSON) " + sql) + plan = cursor.fetchone()[0] + if isinstance(plan, str): + plan = json.loads(plan) + return _postgresql_rereads(plan[0]["Plan"]), json.dumps(plan, indent=1) + if connection.vendor == "mysql": + cursor.execute("EXPLAIN FORMAT=TREE " + sql) + text = cursor.fetchone()[0] + return _mysql_rereads(text, sql), text + cursor.execute("EXPLAIN QUERY PLAN " + sql) + rows = cursor.fetchall() + return _sqlite_rereads(rows, sql), "\n".join(str(row) for row in rows) + + +@contextmanager +def little_memory_for_the_planner(): + """ + On PostgreSQL, give this connection's planner the least memory it takes, so + that a hash of thousands of rows does not fit, as a hash of a node table of + millions of rows would not fit the default. Elsewhere, nothing. + """ + if connection.vendor != "postgresql": + yield + return + with connection.cursor() as cursor: + cursor.execute("SET work_mem = '64kB'") + cursor.execute("SET hash_mem_multiplier = 1") + try: + yield + finally: + with connection.cursor() as cursor: + cursor.execute("RESET work_mem") + cursor.execute("RESET hash_mem_multiplier") + + +@pytest.mark.django_db(transaction=True) +def test_a_run_reads_no_table_again_for_each_retired_workflow(): + """ + Hundreds of retired workflows are already emptied, one keeps a failed node, + and an open workflow has 20,000 waiting nodes. On PostgreSQL the planner has + too little memory to hash the node table, as on a large database. Every + SELECT of a workflow table that a run and a dry run send is explained with + the database's EXPLAIN. No plan may read a whole workflow, node, edge or + task table again for each row of another. A read of the node table for each + retired workflow made a run's cost the number of retired workflows times the + number of node rows. + """ + _, kept, failed = emptied_and_kept("plans") + _prune() + assert rows_of(kept) == (1, 1) + long_ago = timezone.now() - LONG_AGO + OxpullWorkflow.objects.bulk_create( + [ + OxpullWorkflow( + name=f"emptied-{i}", + status=OxpullWorkflow.Status.COMPLETE, + created_at=long_ago, + sealed_at=long_ago, + completed_at=long_ago, + metadata_retired_at=long_ago, + ) + for i in range(300) + ], + batch_size=500, + ) + open_flow = workflows.create("plans-open") + template = OxTask.objects.get(pk=failed.id) + rows = [] + for _ in range(20000): + row = copy.copy(template) + row.pk = uuid.uuid4() + row.status = OxTask.Status.WAITING + rows.append(row) + OxTask.objects.bulk_create(rows, batch_size=1000) + OxpullWorkflowNode.objects.bulk_create( + [ + OxpullWorkflowNode( + workflow_id=open_flow.id, + name=f"n{i}", + name_key=workflows.name_key(f"n{i}"), + task_id=row.pk, + created_at=timezone.now(), + ) + for i, row in enumerate(rows) + ], + batch_size=1000, + ) + with connection.cursor() as cursor: + if connection.vendor == "mysql": + for table in PLAN_TABLES: + cursor.execute(f"ANALYZE TABLE {connection.ops.quote_name(table)}") + cursor.fetchall() + else: + cursor.execute("ANALYZE") + + bad = [] + shown = [] + with little_memory_for_the_planner(): + for args in ((), ("--dry-run",)): + with CaptureQueriesContext(connection) as captured: + output = _prune(*args) + for query in captured.captured_queries: + sql = query["sql"] + if not sql.lstrip().upper().startswith("SELECT"): + continue + if not any(table in sql for table in WORKFLOW_TABLES): + continue + found, plan = rereads(sql) + shown.append(f"{' '.join(sql.split())[:300]}\n{plan[:3000]}") + bad.extend(f"{step} in {' '.join(sql.split())[:300]}" for step in found) + assert "Kept 1 workflow node row(s) and 1 workflow edge row(s)" in output + + assert bad == [], "\n".join(bad) + "\n\n" + "\n\n".join(shown) + + +def test_a_run_passes_over_a_workflow_that_keeps_only_a_node_gone_unrecorded(): + """ + b's task row is gone and nothing recorded its outcome, so nothing ever can. + Shortcut: b ran, then its row was deleted and its record cleared, as when a + writer outside this version settles it and ox_prune deletes the row before + a repair. The first run keeps b's node and the edge into it. After that the + workflow is not one a run acts on. + """ + flow, _, b = two_nodes("gone-unrecorded") + drain(flow_worker()) + OxTask.objects.filter(pk=b.id).delete() + OxpullWorkflowNode.objects.filter(task_id=b.id).update( + outcome=None, outcome_epoch=None, succeeded=False + ) + run_to_completion(flow) + completed_long_ago(flow) + + _prune() + + assert rows_of(flow) == (1, 1) + alias = oxpull_db_alias() + assert workflows.retirable_workflows(timezone.now(), alias) == [] + + +def test_a_workflow_another_run_stamps_mid_read_is_counted_once(): + """ + A run reads the workflows past the cutoff that are not yet stamped, then the + stamped ones that still have nodes. Another run stamps the workflow between + the two reads, so both find it. The dry run still counts its two nodes and + one edge once. + """ + flow, _, _ = two_nodes("stamped-mid-read") + run_to_completion(flow) + completed_long_ago(flow) + stamped: list[str] = [] + + def stamp_before_the_stamped_read(execute, sql, params, many, context): + if not stamped and "IS NOT NULL" in sql and "EXISTS" in sql: + stamped.append(sql) + with connections["default"].cursor() as cursor: + cursor.execute( + f"UPDATE {connection.ops.quote_name(OxpullWorkflow._meta.db_table)}" # noqa: S608 + " SET metadata_retired_at = completed_at WHERE id = %s", + [OxpullWorkflow._meta.pk.get_db_prep_value(flow.id, connection)], + ) + return execute(sql, params, many, context) + + with connection.execute_wrapper(stamp_before_the_stamped_read): + output = _prune("--dry-run") + + assert stamped, output + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is not None + assert "Would delete 2 workflow node row(s)." in output, output + assert "Would delete 1 workflow edge row(s)." in output, output + + +def test_a_retained_workflow_counts_again_once_a_retry_moves_its_node(): + """ + The first run keeps b's failed node. An operator retries b: the next run + finds it pending and deletes nothing. b then succeeds, and the run after + that deletes b's node and the edge into it. + """ + flow = workflows.create("retried-after-retirement") + flow.add("a", tasks.step, "a") + b = flow.add("b", flagged_step, "b", depends_on="a") + flow.seal() + tasks.STATE["fail:b"] = True + run_to_completion(flow) + completed_long_ago(flow) + _prune() + assert rows_of(flow) == (1, 1) + + tasks.STATE["fail:b"] = False + assert actions.retry(b.id) is True + output = _prune() + assert "Skipped 1 workflow(s) with a node still pending." in output + assert rows_of(flow) == (1, 1) + + assert drain(flow_worker()) == 1 + assert status_of(b) == OxTask.Status.SUCCESSFUL + _prune() + assert rows_of(flow) == (0, 0) + + +def test_a_retained_node_whose_record_fell_behind_its_row_is_repaired(): + """ + The first run keeps b's failed node, recorded FAILED. An operator retries b, + which grants one more attempt, and its worker is lost, so the reaper writes + LOST at a later epoch. The reaper records nothing on the node. The next run + repairs the record to LOST, as completion would, and keeps the node. + """ + _, flow, b = emptied_and_kept("record-behind") + _prune() + assert record_of(b.id)[0] == OxTask.Status.FAILED + assert actions.retry(b.id) is True + claimed = flow_worker().claim_one() + assert claimed is not None and str(claimed.id) == str(b.id) + assert actions.expire_lease(b.id) is True + assert flow_worker().reap() == 1 + row = OxTask.objects.values_list("status", "lease_epoch").get(pk=b.id) + assert row[0] == OxTask.Status.LOST + assert record_of(b.id) != row + + _prune() + assert record_of(b.id) == row + assert rows_of(flow) == (1, 1) + + +def test_a_completed_workflow_with_no_nodes_is_stamped(): + """A workflow sealed empty completes, and a run past its cutoff stamps it.""" + flow = workflows.create("empty") + flow.seal() + reconcile() + assert OxpullWorkflow.objects.get(pk=flow.id).status == "COMPLETE" + completed_long_ago(flow) + _prune() + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is not None + + +def test_batch_size_bounds_the_rows_each_workflow_delete_names(): + """ + d depends on a, b and c. With --batch-size 2 the nodes go two at a time, and + the three edges into d go in DELETEs of at most two rows. So do the scan + rows of three completed workflows. + """ + flow = workflows.create("batch-size") + for name in ("a", "b", "c"): + flow.add(name, tasks.step, name) + flow.add("d", tasks.step, "d", depends_on=["a", "b", "c"]) + flow.seal() + run_to_completion(flow) + completed_long_ago(flow) + done = [flow] + for i in range(2): + empty = workflows.create(f"batch-size-empty-{i}") + empty.seal() + done.append(empty) + reconcile() + for row in done: + OxpullWorkflowScan.objects.create(workflow_id=row.id, skipped=["x"]) + scan_table = OxpullWorkflowScan._meta.db_table + deletes: dict[str, list[int]] = {NODE_TABLE: [], EDGE_TABLE: [], scan_table: []} + + def count_ids(execute, sql, params, many, context): + for table, sizes in deletes.items(): + if deletes_from(sql, table): + sizes.append(len(params or ())) + return execute(sql, params, many, context) + + with connection.execute_wrapper(count_ids): + output = _prune("--batch-size", "2") + + assert rows_of(flow) == (0, 0), output + assert "Deleted 3 workflow scan row(s)." in output, output + assert sum(deletes[NODE_TABLE]) == 4 and sum(deletes[EDGE_TABLE]) == 3, deletes + assert sum(deletes[scan_table]) == 3, deletes + assert max(size for sizes in deletes.values() for size in sizes) <= 2, deletes + + +@pytest.mark.skipif(connection.vendor == "sqlite", reason="SQLite has no locking reads") +@pytest.mark.django_db(transaction=True) +def test_the_delete_locks_task_node_and_edge_rows_in_key_order(): + """ + Every locking read of task, node or edge rows a prune run sends is ordered + by key, as every multi-row writer of those tables locks, and every DELETE of + several node or edge rows by key follows such a read of its table. a, b, c + and e go with the two edges into e. d failed and stays with its three. + """ + flow = workflows.create("prune-lock-order") + for name in ("a", "b", "c"): + flow.add(name, tasks.step, name) + flow.add("e", tasks.step, "e", depends_on=["a", "b"]) + flow.add("d", tasks.failing_step, "d", depends_on=["a", "b", "c"]) + flow.seal() + run_to_completion(flow) + completed_long_ago(flow) + quoted = connection.ops.quote_name + tables = (OxTask._meta.db_table, NODE_TABLE, EDGE_TABLE) + + with every_statement() as log: + _prune() + + assert rows_of(flow) == (1, 3), log + locks = [ + (table, sql) + for sql in log + for table in tables + if sql.startswith("SELECT") + and f"FROM {quoted(table)}" in sql + and sql.endswith("FOR UPDATE") + ] + assert {table for table, _ in locks} == set(tables), log + assert all(ordered_lock_of(sql, table) for table, sql in locks), locks + for at, sql in enumerate(log): + for table in (NODE_TABLE, EDGE_TABLE): + if deletes_from(sql, table): + assert any(ordered_lock_of(s, table) for s in log[:at]), sql + + +# -- conflicts ------------------------------------------------------------------ + + +@pytest.mark.skipif(connection.vendor == "sqlite", reason="SQLite raises no deadlock") +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "attempt", ["_stamp_retired_once", "_retire_chunk", "_clean_scan_chunk"] +) +def test_each_prune_transaction_runs_again_after_a_deadlock( + monkeypatch, caplog, attempt +): + """ + The stamp, a delete and the scan row cleanup each own their transaction in + oxpull_prune. One that loses a deadlock runs again, and the run finishes as + if nothing had happened. + """ + flow, _, _ = two_nodes(f"deadlock-{attempt}") + run_to_completion(flow) + completed_long_ago(flow) + OxpullWorkflowScan.objects.create(workflow_id=flow.id, skipped=["x"]) + calls = lose_a_deadlock_once(monkeypatch, attempt) + + with caplog.at_level("INFO", logger="oxpull"): + output = _prune() + + reissued = [ + record.conflict + for record in caplog.records + if getattr(record, "event", "") == "workflow_reconcile_conflict_retry" + ] + assert len(calls) == 2, output + assert reissued == ["deadlock"], output + assert rows_of(flow) == (0, 0), output + assert not OxpullWorkflowScan.objects.filter(workflow_id=flow.id).exists() + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is not None + + +@pytest.mark.skipif( + connection.vendor == "sqlite", + reason="SQLite has no row locks. The next test covers its write lock", +) +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_scan_row_cleanup_that_waited_on_the_skip_queue_runs_again(caplog, level): + """ + A COMPLETE workflow still has a scan row, and the backstop's skip queue is + taking ids out of it on another connection by compare-and-set. The cleanup's + DELETE waits for that row. At PostgreSQL REPEATABLE READ the row it finds + after the commit changed after its snapshot, and the DELETE raises a + serialization failure. The cleanup runs in autocommit, so it owns what the + database rolled back. It reads the statuses again, runs again and deletes + the row. At the other levels nothing runs again. + """ + flow, _, _ = two_nodes(f"scan-waits-{level}") + run_to_completion(flow) + OxpullWorkflowScan.objects.create(workflow_id=flow.id, skipped=["x"]) + scan_table = OxpullWorkflowScan._meta.db_table.upper() + held = threading.Event() + release = threading.Event() + errors: list[BaseException] = [] + + def take_the_queue_on_another_connection(): + try: + with transaction.atomic(): + taken = OxpullWorkflowScan.objects.filter(workflow_id=flow.id).update( + skipped=[], version=F("version") + 1 + ) + assert taken == 1 + held.set() + release.wait(60) + except BaseException as exc: + errors.append(exc) + held.set() + finally: + connections.close_all() + + def commit_the_queue_while_the_delete_waits(execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + if not release.is_set() and text.startswith("DELETE") and scan_table in text: + threading.Timer(0.5, release.set).start() + return execute(sql, params, many, context) + + output = "" + with every_connection_at(level): + other = threading.Thread(target=take_the_queue_on_another_connection) + other.start() + assert held.wait(30) and not errors, errors + try: + with ( + caplog.at_level("INFO", logger="oxpull"), + connection.execute_wrapper(commit_the_queue_while_the_delete_waits), + ): + output = _prune() + finally: + release.set() + other.join(60) + + reissued = events_named(caplog, "workflow_reconcile_conflict_retry") + facts = f"level={level} reissued={reissued} output={output!r}" + assert not errors, errors + if serialization_failures_at(level): + assert len(reissued) == 1, facts + else: + assert reissued == [], facts + assert "Deleted 1 workflow scan row(s)." in output, facts + assert not OxpullWorkflowScan.objects.filter(workflow_id=flow.id).exists() + + +@pytest.mark.skipif(connection.vendor != "sqlite", reason="SQLite's write lock") +@pytest.mark.django_db(transaction=True) +def test_a_scan_row_cleanup_beside_another_write_finishes_on_sqlite(): + """ + A COMPLETE workflow still has a scan row. As the cleanup sends its DELETE, + another connection has just started an ordinary write, as a worker's claim + or outcome does. SQLite lets one connection write at a time. A cleanup that + had read the statuses inside its own transaction could not then take the + write lock, and SQLite refused it at once with "database is locked", after + the run's node and edge deletes had committed. The cleanup reads and deletes + in autocommit, so its DELETE waits for the other write and the run finishes. + """ + flow, a, _ = two_nodes("scan-beside-a-write") + run_to_completion(flow) + OxpullWorkflowScan.objects.create(workflow_id=flow.id, skipped=["x"]) + scan_table = OxpullWorkflowScan._meta.db_table.upper() + writers: list[threading.Thread] = [] + other: dict[str, object] = {} + + def write_on_another_connection(): + try: + other["updated"] = OxTask.objects.filter(pk=a.id).update(priority=1) + except BaseException as exc: + other["error"] = exc + finally: + connections.close_all() + + def start_a_write_before_the_delete(execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + if not writers and text.startswith("DELETE") and scan_table in text: + writer = threading.Thread(target=write_on_another_connection) + writers.append(writer) + writer.start() + time.sleep(0.5) + return execute(sql, params, many, context) + + try: + with connection.execute_wrapper(start_a_write_before_the_delete): + output = _prune() + finally: + for writer in writers: + writer.join(60) + + facts = f"writers={len(writers)} other={other}" + assert writers and other == {"updated": 1}, facts + assert "Deleted 1 workflow scan row(s)." in output, facts + assert "Kept 0 workflow node row(s)" in output, facts + assert not OxpullWorkflowScan.objects.filter(workflow_id=flow.id).exists() + + +# -- out-of-range --older-than ---------------------------------------------- + +OUT_OF_RANGE = ["3000000d", "1000000000d", "99999999999999d", "9" * 5000 + "d"] + + +def prunable_rows(worker): + """A stale unique lock and a long-completed workflow: rows a prune deletes.""" + flow, _, _ = two_nodes("out-of-range") + run_to_completion(flow) + completed_long_ago(flow) + tasks.unique_echo.enqueue("finished") + assert worker.run_once() is True + assert OxpullUniqueLock.objects.count() == 1 + return flow + + +def assert_nothing_pruned(flow): + assert OxpullUniqueLock.objects.count() == 1 + assert rows_of(flow) == (2, 1) + assert OxpullWorkflow.objects.get(pk=flow.id).metadata_retired_at is None + + +@pytest.mark.django_db(transaction=True) +class TestOutOfRangeDuration: + """A retention too large to convert or to subtract is an argument error.""" + + @pytest.mark.parametrize("dry_run", [False, True]) + @pytest.mark.parametrize("value", OUT_OF_RANGE) + def test_is_a_command_error_and_deletes_nothing(self, worker, value, dry_run): + flow = prunable_rows(worker) + args = [f"--older-than={value}", *(["--dry-run"] if dry_run else [])] + + with ( + CaptureQueriesContext(connection) as queries, + pytest.raises(CommandError) as info, + ): + _prune(*args) + + assert queries.captured_queries == [] + assert str(info.value) == f"Invalid duration {value!r}; it is out of range." + assert_nothing_pruned(flow) + + def test_exits_1_without_a_traceback_from_the_command_line(self, worker, capsys): + flow = prunable_rows(worker) + argv = ["manage.py", "oxpull_prune", "--older-than=3000000d", "--skip-checks"] + + with pytest.raises(SystemExit) as info: + ManagementUtility(argv).execute() + + assert info.value.code == 1 + err = capsys.readouterr().err + assert "Invalid duration '3000000d'; it is out of range." in err + assert "Traceback" not in err + assert_nothing_pruned(flow) + + def test_traceback_flag_still_raises(self): + argv = [ + "manage.py", + "oxpull_prune", + "--older-than=3000000d", + "--skip-checks", + "--traceback", + ] + with pytest.raises(CommandError) as info: + ManagementUtility(argv).execute() + assert str(info.value) == "Invalid duration '3000000d'; it is out of range." diff --git a/tests_oxpull/test_rate_limit_names.py b/tests_oxpull/test_rate_limit_names.py new file mode 100644 index 0000000..93f0354 --- /dev/null +++ b/tests_oxpull/test_rate_limit_names.py @@ -0,0 +1,980 @@ +""" +Rate-limit names: the names a collation would fold into one, and one name +configured differently on two backends. + +A limit's name is the primary key of its window row. A primary key decides +identity with the column's collation, not with Python's ``==``, and MySQL's +usual collation folds case: `Reports` and `reports` are two limits to the +settings parser and one row to the database. They would share a window, +count each other's admissions, and throttle traffic neither was configured +for, with nothing raised anywhere. + +The same row is shared across backends. A name is keyed alone, with no +alias, so two backends that configure one name with different periods or +capacities count on one window, and `manage.py check` reports it. It +reports two spellings that differ only by case or accents on two backends +too, which MySQL's default collation counts on one window and PostgreSQL +and SQLite on two. +""" + +from contextlib import contextmanager +from datetime import timedelta +from io import StringIO + +import pytest +from django.conf import settings +from django.core import checks +from django.core.exceptions import ImproperlyConfigured +from django.core.management import call_command, load_command_class +from django.core.management.base import SystemCheckError +from django.db import connection +from django.db.models.functions import Now +from django.tasks import task_backends +from django.test import override_settings +from django.test.utils import CaptureQueriesContext + +from django_ox.models import OxTask +from oxpull.backend import OxpullBackend +from oxpull.models import OxpullRateWindow +from oxpull.rate_limits import rate_limits_from_options + +from . import tasks +from .test_rate_limits import ECHO, oxpull_worker, rate_limits + + +def options(*names): + return { + "RATE_LIMITS": { + name: {"limit": 10, "per": 60, "queues": ["default"]} for name in names + } + } + + +class TestNamesThatACollationWouldFold: + def test_two_names_differing_only_by_case_are_refused(self): + with pytest.raises(ImproperlyConfigured, match="differ only"): + rate_limits_from_options( + options("Reports", "reports"), "default", ["default"] + ) + + def test_the_message_names_both(self): + with pytest.raises(ImproperlyConfigured) as caught: + rate_limits_from_options( + options("Reports", "reports"), "default", ["default"] + ) + assert "'Reports'" in str(caught.value) + assert "'reports'" in str(caught.value) + + @pytest.mark.parametrize( + "pair", + [("emails", "EMAILS"), ("send-mail", "Send-Mail"), ("a", "A")], + ) + def test_it_holds_however_the_case_differs(self, pair): + with pytest.raises(ImproperlyConfigured, match="differ only"): + rate_limits_from_options(options(*pair), "default", ["default"]) + + def test_names_differing_by_more_than_case_are_accepted(self): + limits = rate_limits_from_options( + options("reports-daily", "reports-hourly"), "default", ["default"] + ) + assert sorted(limits) == ["reports-daily", "reports-hourly"] + + def test_one_name_is_fine(self): + limits = rate_limits_from_options(options("reports"), "default", ["default"]) + assert sorted(limits) == ["reports"] + + def test_no_limits_at_all_is_fine(self): + assert rate_limits_from_options({}, "default", ["default"]) == {} + + +# Stored spelling, configured spelling. The first pair differs by case, +# which every case-insensitive collation folds; the second by an accent, +# which MySQL's utf8mb4_unicode_ci and utf8mb4_0900_ai_ci fold as well and +# no Python string operation predicts. +RENAMES = [("Stripe", "stripe"), ("Café", "cafe")] + + +@pytest.mark.django_db +class TestALimitRenamedToASpellingTheCollationFolds: + """ + One configured name whose window row was written under another spelling. + + A limit deployed as `Stripe` and renamed to `stripe` leaves a row the + parser above cannot see: it only compares configured names with each + other. On MySQL the INSERT for `stripe` is refused as a duplicate of + `Stripe`, the SELECT keyed by the stored spelling has no `stripe` + entry, and claim_one() raised KeyError, which django-ox's run() does + not contain. The worker died on every claim, and five deaths in a + minute trip the supervisor's restart cap. + """ + + @pytest.mark.parametrize(("stored", "configured"), RENAMES) + def test_the_claim_survives_and_the_admission_lands_on_the_limits_row( + self, stored, configured + ): + OxpullRateWindow.objects.create(limit_name=stored, window_started_at=Now()) + tasks.plain_echo.enqueue("a") + with rate_limits({configured: {"limit": 5, "per": 60, "tasks": [ECHO]}}): + claimed = oxpull_worker().claim_one() + assert claimed is not None + # The database decides which row the configured name names: the + # stored row where the collation folds the two, a freshly seeded + # one where spelling is identity. Either way the admission lands + # on it. + assert OxpullRateWindow.objects.get(limit_name=configured).used == 1 + # The next refresh reads the same row under the configured name + # without another seed: the attribution is learned once. + tasks.plain_echo.enqueue("b") + with rate_limits({configured: {"limit": 5, "per": 60, "tasks": [ECHO]}}): + worker = oxpull_worker() + assert worker.claim_one() is not None + with CaptureQueriesContext(connection) as captured: + assert worker.claim_one() is None + assert not any("INSERT" in q["sql"].upper() for q in captured.captured_queries) + assert OxpullRateWindow.objects.get(limit_name=configured).used == 2 + rows = sorted(OxpullRateWindow.objects.values_list("limit_name", flat=True)) + if connection.vendor == "mysql": + assert rows == [stored] + else: + assert rows == sorted([stored, configured]) + + @pytest.mark.parametrize(("stored", "configured"), RENAMES) + def test_the_stored_rows_count_still_throttles(self, stored, configured): + """ + Where the collation folds the two, the stored row is the limit's + row and its count is the limit's count: a worker that adopted it + must honour a window that is already full, not read it as empty. + """ + if connection.vendor != "mysql": + pytest.skip("only a folding collation makes one row of the two") + OxpullRateWindow.objects.create( + limit_name=stored, window_started_at=Now(), used=5 + ) + tasks.plain_echo.enqueue("a") + with rate_limits({configured: {"limit": 5, "per": 60, "tasks": [ECHO]}}): + assert oxpull_worker().claim_one() is None + + +# Two configured names that no Python string operation folds together and +# that MySQL's utf8mb4_unicode_ci and utf8mb4_0900_ai_ci treat as one. +ACCENT_PAIRS = [("Café", "cafe"), ("naïve", "naive")] + + +@pytest.mark.django_db +class TestTwoConfiguredNamesTheDatabaseFoldsIntoOneRow: + """ + The parser refuses names that differ only by case, which is what every + case-insensitive collation folds. Accent folding is the collation's + alone: `casefold()` keeps `Café` and `cafe` apart, MySQL's primary key + does not. Both limits were seeded onto one row, both counted on it and + each throttled the other's traffic, with nothing raised anywhere. Only + the database can say which names it folds, so the worker asks it: the + seed writes one row, the second name's seed is refused, and the adopt + read for it comes back with a row another configured limit already + owns. That is refused at the first refresh, before any claim. + """ + + @pytest.mark.parametrize(("first", "second"), ACCENT_PAIRS) + def test_the_first_refresh_refuses_and_names_both(self, first, second): + if connection.vendor != "mysql": + pytest.skip("only a folding collation makes one row of the two") + tasks.plain_echo.enqueue("a") + config = { + first: {"limit": 5, "per": 60, "tasks": [ECHO]}, + second: {"limit": 5, "per": 60, "queues": ["default"]}, + } + with rate_limits(config), pytest.raises(ImproperlyConfigured) as caught: + oxpull_worker().claim_one() + assert repr(first) in str(caught.value) + assert repr(second) in str(caught.value) + assert OxTask.objects.get().status == OxTask.Status.READY + + @pytest.mark.parametrize(("first", "second"), ACCENT_PAIRS) + def test_a_row_stored_under_a_third_spelling_is_refused_too(self, first, second): + if connection.vendor != "mysql": + pytest.skip("only a folding collation makes one row of the two") + OxpullRateWindow.objects.create( + limit_name=first.upper(), window_started_at=Now() + ) + tasks.plain_echo.enqueue("a") + config = { + first: {"limit": 5, "per": 60, "tasks": [ECHO]}, + second: {"limit": 5, "per": 60, "queues": ["default"]}, + } + with rate_limits(config), pytest.raises(ImproperlyConfigured, match="one row"): + oxpull_worker().claim_one() + + @pytest.mark.parametrize(("first", "second"), ACCENT_PAIRS) + def test_where_spelling_is_identity_both_limits_get_their_own_row( + self, first, second + ): + if connection.vendor == "mysql": + pytest.skip("the folding collation is the other test's subject") + tasks.plain_echo.enqueue("a") + config = { + first: {"limit": 5, "per": 60, "tasks": [ECHO]}, + second: {"limit": 5, "per": 60, "queues": ["default"]}, + } + with rate_limits(config): + assert oxpull_worker().claim_one() is not None + rows = sorted(OxpullRateWindow.objects.values_list("limit_name", flat=True)) + assert rows == sorted([first, second]) + + +# Pairs as MySQL 8.4 matches them under utf8mb4_0900_ai_ci and +# utf8mb4_unicode_ci. Each pair in the second list is two names to both, and +# a comparison slightly wider than oxpull.W006's would call it one. +ONE_ROW_ON_MYSQL = [ + ("Stripe", "stripe"), + ("stripe", "stripé"), + ("café", "cafe\u0301"), + ("Straße", "strasse"), + ("\u1fb3", "\u03b1"), # alpha with ypogegrammeni, and alpha +] +TWO_ROWS_ON_MYSQL = [ + ("\u0439", "\u0438"), # Cyrillic short i is a letter, not i with a breve + ("a\u0363", "a"), # U+0363 is a small a written above, not an accent + ("\u1c90", "\u10d0"), # Georgian Mtavruli is newer than MySQL's weights + ("\u1fb3", "\u03b1\u03b9"), # casefold() spells the ypogegrammeni as iota + ("\u0b94", "\u0b92\u0bd7"), # Tamil au, precomposed and decomposed +] +FOLDING_COLLATIONS = ("utf8mb4_0900_ai_ci", "utf8mb4_unicode_ci") + + +def window_collation(): + """The window table's limit_name collation on MySQL, else None.""" + if connection.vendor != "mysql": + return None + with connection.cursor() as cursor: + cursor.execute( + "SELECT COLLATION_NAME FROM information_schema.columns " + "WHERE table_schema = DATABASE() AND table_name = %s " + "AND column_name = 'limit_name'", + [OxpullRateWindow._meta.db_table], + ) + return cursor.fetchone()[0] + + +def oxpull_backend(limits, queues=("default",)): + return { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": list(queues), + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": limits, + }, + } + + +# The installation each check test extends. Its one backend sorts first, +# names no limit and schedules the reconciler, so its own checks report +# nothing. It is written out here, not read from the suite's TASKS. Tests of +# another feature can add an alias there whose own check is an Error on +# purpose, and that Error would raise in every test here that expects none. +INSTALLATION = { + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": { + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, + }, + }, +} + + +@contextmanager +def backends(whole=None, /, **limits_by_alias): + """ + INSTALLATION, plus one OxpullBackend alias per keyword, and any alias + given whole in `whole`. + """ + added = {alias: oxpull_backend(limits) for alias, limits in limits_by_alias.items()} + with override_settings(TASKS={**INSTALLATION, **(whole or {}), **added}): + yield + + +def run_check(**options): + """ + `manage.py check`, as a deploy runs it. Warnings come back as text, and + an Error raises SystemCheckError, whose text is the whole report. + """ + out, err = StringIO(), StringIO() + call_command("check", stdout=out, stderr=err, **options) + return out.getvalue() + err.getvalue() + + +def check_report(**options): + """The whole `check` report, whether or not an Error raised.""" + try: + return run_check(**options) + except SystemCheckError as raised: + return str(raised) + + +def named(name, limit, per, **covers): + return {name: {"limit": limit, "per": per, **(covers or {"tasks": [ECHO]})}} + + +def stripe(limit, per, **covers): + return named("stripe", limit, per, **covers) + + +def emitted_by(check_id): + """The aliases whose own check() returns check_id, under current TASKS.""" + return [ + alias + for alias in sorted(settings.TASKS) + if isinstance(task_backends[alias], OxpullBackend) + and check_id in [message.id for message in task_backends[alias].check()] + ] + + +def messages(check_id): + """Every message with check_id, under current TASKS.""" + return [ + message + for alias in sorted(settings.TASKS) + if isinstance(task_backends[alias], OxpullBackend) + for message in task_backends[alias].check() + if message.id == check_id + ] + + +def levels(check_id): + """The level of every message with check_id, under current TASKS.""" + return [message.level for message in messages(check_id)] + + +class ProjectBackend(OxpullBackend): + """A project's own subclass of OxpullBackend.""" + + +PROJECT_BACKEND = f"{__name__}.ProjectBackend" + + +def ox_worker_start(alias, started): + """ + django-ox's `ox_worker` for one backend, run through BaseCommand.execute + as `manage.py` runs it, with handle() replaced so no worker loop starts. + Reaching handle() appends the backend to `started`. Returns what the + system checks wrote. An Error raises SystemCheckError before handle(). + """ + command = load_command_class("django_ox", "ox_worker") + command.handle = lambda *args, **options: started.append(options["backend"]) + out, err = StringIO(), StringIO() + call_command(command, "--backend", alias, skip_checks=False, stdout=out, stderr=err) + return out.getvalue() + err.getvalue() + + +@pytest.mark.django_db(databases="__all__") +class TestOneNameConfiguredDifferentlyOnTwoBackends: + """ + A limit's window row is keyed on its name alone, so every backend that + configures one name counts on one shared window, and nothing reported + two backends that disagree about it. Two periods cannot both hold on + one window. A shorter one rolls it and restarts its count, so the limit + is not enforced as configured. Two capacities over one period are a + shared budget, which can be deliberate. Each is a Warning with an id of + its own. Two spellings that differ only by case or accents are one window + on MySQL's default collation and two on PostgreSQL and SQLite. That is a + Warning too, under a third id. + + `check` runs here as a deploy runs it. From Django 6.1 that reads a + database feature on every alias for Django's own model checks, so these + tests allow every database. The class below proves this check reads + none. + """ + + def test_a_different_per_is_a_warning_naming_both_backends_and_periods(self): + with backends(billing=stripe(5, 1), reports=stripe(5, 60)): + report = run_check() + assert levels("oxpull.W005") == [checks.WARNING] + assert report.count("oxpull.W005") == 1 + assert "Rate limit 'stripe' has a different period" in report + assert "not enforced as configured" in report + assert "'billing' (limit=5, per=1)" in report + assert "'reports' (limit=5, per=60)" in report + assert "oxpull.W004" not in report + assert "oxpull.W006" not in report + + def test_a_different_limit_over_the_same_per_is_a_warning_of_its_own(self): + with backends( + billing=stripe(5, 60), reports=stripe(10, 60, queues=["default"]) + ): + report = run_check() + assert levels("oxpull.W004") == [checks.WARNING] + assert report.count("oxpull.W004") == 1 + assert "Rate limit 'stripe' has a different limit" in report + assert "'billing' (limit=5, per=60)" in report + assert "'reports' (limit=10, per=60)" in report + assert "oxpull.W005" not in report + + def test_a_different_per_is_the_one_reported_when_the_limit_differs_too(self): + with backends(billing=stripe(5, 1), reports=stripe(10, 60)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "oxpull.W004" not in report + + @pytest.mark.parametrize( + ("shorter", "longer"), + [(60, 61), (60, 90), (3600, timedelta(minutes=61))], + ) + def test_close_periods_are_reported(self, shorter, longer): + with backends(billing=stripe(5, shorter), reports=stripe(5, longer)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "oxpull.W004" not in report + + def test_it_is_reported_once_however_many_backends_name_it(self): + """ + From the first OxpullBackend alias in sort order, as oxpull.W002 is, + so one id silences it and a report does not repeat per alias. + """ + with backends(billing=stripe(5, 1), reports=stripe(5, 60), sms=stripe(5, 60)): + report = run_check() + assert emitted_by("oxpull.W005") == ["billing"] + assert report.count("oxpull.W005") == 1 + assert "'reports' (limit=5, per=60)" in report + assert "'sms' (limit=5, per=60)" in report + + def test_it_is_reported_by_the_first_backend_when_that_one_names_no_limit(self): + """ + INSTALLATION's `default` sorts first and configures no limit, the + usual shape of a project that limits one backend of several. + """ + with backends(reports=stripe(5, 1), zulu=stripe(5, 60)): + report = run_check() + assert emitted_by("oxpull.W005") == ["default"] + assert report.count("oxpull.W005") == 1 + + def test_a_disagreement_is_found_behind_backends_that_agree(self): + with backends(billing=stripe(5, 60), reports=stripe(5, 60), sms=stripe(5, 1)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "'sms' (limit=5, per=1)" in report + + def test_a_disagreement_is_found_between_backends_that_agree(self): + with backends(billing=stripe(5, 60), reports=stripe(5, 1), sms=stripe(5, 60)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "'reports' (limit=5, per=1)" in report + + def test_every_name_that_disagrees_is_reported(self): + with backends( + billing={**stripe(5, 1), **named("sms", 5, 60)}, + reports={**stripe(5, 60), **named("sms", 10, 60)}, + ): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "Rate limit 'stripe' has a different period" in report + assert report.count("oxpull.W004") == 1 + assert "Rate limit 'sms' has a different limit" in report + + @pytest.mark.parametrize( + ("billing", "reports"), + [ + # One configuration. Still one budget on MySQL and two elsewhere. + ((5, 60), (5, 60)), + # Two configurations. On MySQL that is oxpull.W005 unreported. + ((5, 1), (10, 60)), + ], + ) + def test_names_that_differ_only_by_case_are_a_warning(self, billing, reports): + with backends( + billing=named("Stripe", *billing), reports=named("stripe", *reports) + ): + report = run_check() + assert levels("oxpull.W006") == [checks.WARNING] + assert report.count("oxpull.W006") == 1 + assert "'billing' ('Stripe') and 'reports' ('stripe')" in report + assert "oxpull.W005" not in report + assert "oxpull.W004" not in report + + @pytest.mark.parametrize( + ("first", "second", "shown"), + [ + # Each pair is one name to casefold() and to MySQL's + # utf8mb4_0900_ai_ci. The first two are not one name to lower(). + ("Straße", "strasse", "'billing' ('Straße') and 'reports' ('strasse')"), + # The "fi" ligature can print like f and i, so it is escaped. + ("\ufb01le", "file", "'billing' ('\\ufb01le') and 'reports' ('file')"), + # The Kelvin sign. + ("\u212aey", "key", "'billing' ('\u212aey') and 'reports' ('key')"), + ], + ) + def test_names_equal_once_case_is_folded_are_a_warning(self, first, second, shown): + with backends(billing=named(first, 5, 60), reports=named(second, 5, 60)): + report = run_check() + assert report.count("oxpull.W006") == 1 + assert shown in report + + @pytest.mark.parametrize( + ("first", "second", "shown"), + [ + ("stripe", "stripé", "'billing' ('stripe') and 'reports' ('stripé')"), + ("Café", "cafe", "'billing' ('Café') and 'reports' ('cafe')"), + # One accent precomposed, the other a combining mark. They print + # alike, so both are shown as code points. + ( + "café", + "cafe\u0301", + "'billing' ('caf\\u00e9') and 'reports' ('cafe\\u0301')", + ), + ("Stripe", "STRIPÉ", "'billing' ('Stripe') and 'reports' ('STRIPÉ')"), + ], + ) + def test_names_that_differ_only_by_accents_are_a_warning( + self, first, second, shown + ): + with backends(billing=named(first, 5, 60), reports=named(second, 5, 60)): + report = run_check() + assert levels("oxpull.W006") == [checks.WARNING] + assert report.count("oxpull.W006") == 1 + assert shown in report + + @pytest.mark.parametrize( + ("billing", "reports", "sms"), + [("stripe", "Stripe", "STRIPE"), ("stripe", "Stripe", "stripé")], + ) + def test_three_spellings_on_three_backends_are_one_warning( + self, billing, reports, sms + ): + with backends( + billing=named(billing, 5, 60), + reports=named(reports, 5, 60), + sms=named(sms, 5, 60), + ): + report = run_check() + assert report.count("oxpull.W006") == 1 + assert ( + f"'billing' ({billing!r}), 'reports' ({reports!r}) and 'sms' ({sms!r})" + in report + ) + + @pytest.mark.parametrize(("first", "second"), ONE_ROW_ON_MYSQL + TWO_ROWS_ON_MYSQL) + def test_it_is_reported_exactly_when_mysql_keeps_one_row(self, first, second): + """ + On MySQL the suite's database uses utf8mb4_unicode_ci, which matches + these pairs as the default utf8mb4_0900_ai_ci does, and the row is + read back under the second spelling. + """ + with backends(billing=named(first, 5, 60), reports=named(second, 5, 60)): + report = run_check() + one_row = (first, second) in ONE_ROW_ON_MYSQL + assert report.count("oxpull.W006") == one_row + if window_collation() in FOLDING_COLLATIONS: + OxpullRateWindow.objects.create(limit_name=first, window_started_at=Now()) + found = OxpullRateWindow.objects.filter(limit_name=second).exists() + assert found == one_row + + @pytest.mark.parametrize( + ("first", "second", "one_row_on_unicode_ci"), + [ + # Georgian capital an folds to a letter newer than Unicode 3.2. + # utf8mb4_unicode_ci keeps the two apart. utf8mb4_0900_ai_ci + # makes one name of them, which the key misses. + ("\u10a0", "\u2d00", False), + # A mark on a letter that is neither Latin nor Greek stays in the + # key. Both collations make one name of these, which it misses. + ("\u0438\u0306", "\u0438", True), + ], + ) + def test_folds_the_key_leaves_to_the_database_are_not_reported( + self, first, second, one_row_on_unicode_ci + ): + with backends(billing=named(first, 5, 60), reports=named(second, 5, 60)): + report = run_check() + assert "oxpull.W006" not in report + if window_collation() == "utf8mb4_unicode_ci": + OxpullRateWindow.objects.create(limit_name=first, window_started_at=Now()) + found = OxpullRateWindow.objects.filter(limit_name=second).exists() + assert found == one_row_on_unicode_ci + + def test_a_case_variant_is_named_beside_the_names_own_disagreement(self): + with backends( + billing=stripe(5, 1), + reports=stripe(5, 60), + sms=named("STRIPE", 5, 60), + ): + report = run_check() + assert report.count("oxpull.W006") == 1 + assert ( + "'billing' ('stripe'), 'reports' ('stripe') and 'sms' ('STRIPE')" in report + ) + assert report.count("oxpull.W005") == 1 + assert "'sms' (limit" not in report + + @pytest.mark.parametrize( + ("first", "second"), + [ + # One period written two ways is one period to the worker. + (stripe(5, 60), stripe(5, timedelta(minutes=1))), + (stripe(5, 60), stripe(5, timedelta(seconds=60.4))), + # One budget shared on purpose, over different tasks. + (stripe(5, 60), stripe(5, 60, queues=["default"])), + # Names that differ by more than case are different windows. + (stripe(5, 1), named("stripe-reports", 5, 60)), + ], + ) + def test_configurations_that_agree_are_not_reported(self, first, second): + with backends(billing=first, reports=second): + report = run_check() + assert "oxpull.W005" not in report + assert "oxpull.W004" not in report + assert "oxpull.W006" not in report + + def test_a_backend_whose_limits_do_not_parse_is_left_to_its_own_error(self): + with ( + backends(billing=stripe(5, 1), broken=stripe(5, 0), reports=stripe(5, 60)), + pytest.raises(SystemCheckError) as caught, + ): + run_check() + report = str(caught.value) + assert "oxpull.E005" in report + assert report.count("oxpull.W005") == 1 + assert "'broken' (limit" not in report + + def test_each_backend_is_read_with_its_own_queues(self): + reports = oxpull_backend(stripe(5, 60, queues=["reports"]), queues=["reports"]) + with backends({"reports": reports}, billing=stripe(5, 1)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "oxpull.E005" not in report + + def test_a_backend_of_another_kind_sorting_first_does_not_hide_it(self): + immediate = {"BACKEND": "django.tasks.backends.immediate.ImmediateBackend"} + with backends({"aaa": immediate}, billing=stripe(5, 1), reports=stripe(5, 60)): + report = run_check() + assert report.count("oxpull.W005") == 1 + + def test_a_django_ox_backend_sorting_first_does_not_hide_it(self): + """ + A plain django-ox backend is not an OxpullBackend. Its check() never + runs this check, so it must not be the alias this is emitted from. + """ + plain = {"BACKEND": "django_ox.backend.OxBackend", "QUEUES": ["default"]} + with backends({"aaa": plain}, billing=stripe(5, 1), reports=stripe(5, 60)): + report = run_check() + assert emitted_by("oxpull.W005") == ["billing"] + assert report.count("oxpull.W005") == 1 + + def test_a_subclass_of_oxpull_backend_is_compared(self): + reports = {**oxpull_backend(stripe(5, 60)), "BACKEND": PROJECT_BACKEND} + with backends({"reports": reports}, billing=stripe(5, 1)): + report = run_check() + assert report.count("oxpull.W005") == 1 + assert "'reports' (limit=5, per=60)" in report + + def test_a_subclass_of_oxpull_backend_sorting_first_reports_it(self): + first = {"BACKEND": PROJECT_BACKEND, "QUEUES": ["default"]} + with backends({"aaa": first}, billing=stripe(5, 1), reports=stripe(5, 60)): + report = run_check() + assert emitted_by("oxpull.W005") == ["aaa"] + assert report.count("oxpull.W005") == 1 + + def test_it_is_reported_when_the_first_backend_fails_its_own_limit_check(self): + """ + Fixing the first backend's own oxpull.E006 should not be what it + takes to see this one. + """ + no_worker = { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": {"RATE_LIMITS": named("other", 5, 60, queues=["default"])}, + } + with ( + backends({"aaa": no_worker}, billing=stripe(5, 1), reports=stripe(5, 60)), + pytest.raises(SystemCheckError) as caught, + ): + run_check() + report = str(caught.value) + assert "oxpull.E006" in report + assert report.count("oxpull.W005") == 1 + + def test_the_suites_own_settings_report_none_of_them(self): + """ + The suite's TASKS as they are. Another feature's alias there can fail + its own check on purpose, so only these ids are asserted. + """ + report = check_report() + assert "oxpull.W005" not in report + assert "oxpull.W004" not in report + assert "oxpull.W006" not in report + + def test_a_different_per_says_why_the_limit_does_not_hold_and_the_fix(self): + with backends(billing=stripe(5, 1), reports=stripe(5, 60)): + report = run_check() + [message] = messages("oxpull.W005") + assert message.msg == ( + "Rate limit 'stripe' has a different period on backends 'billing' " + "(limit=5, per=1) and 'reports' (limit=5, per=60), and it is not " + "enforced as configured. A limit's window is keyed on its name alone, " + "so these backends roll one shared window, each at its own period. " + "Each roll restarts the count, so a backend on a longer period can " + "admit more than its limit in that period." + ) + assert message.hint == ( + "Use one period for this name on every backend, or give each " + "backend's limit a name of its own." + ) + assert message.msg in report + assert f"HINT: {message.hint}" in report + + def test_a_different_limit_says_what_is_shared_and_the_fix(self): + with backends(billing=stripe(5, 60), reports=stripe(10, 60)): + report = run_check() + [message] = messages("oxpull.W004") + assert message.msg == ( + "Rate limit 'stripe' has a different limit on backends 'billing' " + "(limit=5, per=60) and 'reports' (limit=10, per=60). A limit's window is " + "keyed on its name alone, so they share one window and count each " + "other's admissions. Each backend stops when the shared count reaches " + "its own limit, so together they admit up to the largest one." + ) + assert message.hint == ( + "If a shared budget is what you want, use the same limit for this name " + "on every backend. Otherwise give each backend's limit a name of its " + "own." + ) + assert message.msg in report + assert f"HINT: {message.hint}" in report + + def test_a_spelling_difference_says_what_each_database_does_and_the_fix(self): + with backends(billing=named("Stripe", 5, 60), reports=named("stripe", 5, 60)): + report = run_check() + [message] = messages("oxpull.W006") + assert message.msg == ( + "Rate limit names on backends 'billing' ('Stripe') and 'reports' " + "('stripe') differ only by case or accents. A limit's window is keyed on " + "its name, and the database's collation decides which spellings match. " + "On MySQL's default collation, which ignores case and accents, these " + "spellings are one name, so the backends share one window and count each " + "other's admissions. On PostgreSQL and SQLite, and on a binary MySQL " + "collation such as utf8mb4_bin, they are two separate limits." + ) + assert message.hint == ( + "Use one spelling of this name on every backend. If each backend " + "needs a limit of its own, put the backend's alias in each name." + ) + assert message.msg in report + assert f"HINT: {message.hint}" in report + + def test_two_spellings_on_one_backend_say_its_workers_refuse_them(self): + """ + The parser refuses a case difference within one backend, not an + accent. That backend's workers refuse the two on MySQL's default + collation, so it is not one of several backends counting together. + """ + with backends(billing={**named("stripe", 5, 60), **named("stripé", 5, 60)}): + report = run_check() + [message] = messages("oxpull.W006") + assert message.msg == ( + "Rate limit names 'stripe' and 'stripé' on backend 'billing' differ only " + "by case or accents. A limit's window is keyed on its name, and the " + "database's collation decides which spellings match. On MySQL's default " + "collation, which ignores case and accents, these spellings are one " + "name, so the workers for backend 'billing' refuse them before claiming " + "any task. On PostgreSQL and SQLite, and on a binary MySQL collation " + "such as utf8mb4_bin, they are two separate limits." + ) + assert message.hint == ( + "Keep one of these limits, or give each one a name that differs by " + "more than case or accents." + ) + assert message.msg in report + assert f"HINT: {message.hint}" in report + + def test_a_backend_with_two_spellings_beside_backends_with_one(self): + with backends( + billing={**named("stripe", 5, 60), **named("stripé", 5, 60)}, + reports=named("Stripe", 5, 60), + sms=named("STRIPE", 5, 60), + ): + report = run_check() + [message] = messages("oxpull.W006") + assert message.msg == ( + "Rate limit names on backends 'billing' ('stripe' and 'stripé'), " + "'reports' ('Stripe') and 'sms' ('STRIPE') differ only by case or " + "accents. A limit's window is keyed on its name, and the database's " + "collation decides which spellings match. On MySQL's default collation, " + "which ignores case and accents, these spellings are one name, so the " + "workers for backend 'billing' refuse them before claiming any task. " + "Backends 'reports' and 'sms' count each other's admissions. On " + "PostgreSQL and SQLite, and on a binary MySQL collation such as " + "utf8mb4_bin, they are two separate limits." + ) + assert report.count("oxpull.W006") == 1 + + def test_one_backend_with_two_spellings_beside_one_other_counts_with_none(self): + with backends( + billing={**named("stripe", 5, 60), **named("stripé", 5, 60)}, + reports=named("Stripe", 5, 60), + ): + run_check() + [message] = messages("oxpull.W006") + assert message.msg.startswith( + "Rate limit names on backends 'billing' ('stripe' and 'stripé') and " + "'reports' ('Stripe') differ only by case or accents." + ) + assert "the workers for backend 'billing' refuse them" in message.msg + assert "count each other's admissions" not in message.msg + + @pytest.mark.parametrize( + ("billing", "reports", "sms", "shown"), + [ + ( + "naïve", + "NAIVE", + "nai\u0308ve", + ( + "'billing' ('na\\u00efve'), 'reports' ('NAIVE') and " + "'sms' ('nai\\u0308ve')" + ), + ), + ( + "\u212aey", + "Key", + "KEY", + "'billing' ('\\u212aey'), 'reports' ('Key') and 'sms' ('KEY')", + ), + ( + # A character above U+FFFF is written with its eight-digit escape. + "caf\u00e9\U0001d42c", + "CAFE\U0001d42c", + "cafe\u0301\U0001d42c", + ( + "'billing' ('caf\\u00e9\\U0001d42c'), 'reports' ('CAFE\U0001d42c') " + "and 'sms' ('cafe\\u0301\\U0001d42c')" + ), + ), + ], + ) + def test_spellings_that_print_alike_are_shown_as_code_points( + self, billing, reports, sms, shown + ): + with backends( + billing=named(billing, 5, 60), + reports=named(reports, 5, 60), + sms=named(sms, 5, 60), + ): + report = run_check() + assert report.count("oxpull.W006") == 1 + assert shown in report + + def test_two_spellings_that_print_alike_on_one_backend_are_shown_as_code_points( + self, + ): + with backends(billing={**named("café", 5, 60), **named("cafe\u0301", 5, 60)}): + report = run_check() + assert report.count("oxpull.W006") == 1 + assert ( + "Rate limit names 'cafe\\u0301' and 'caf\\u00e9' on backend 'billing'" + in report + ) + + +class TestTheNameCheckReadsSettingsOnly: + """ + No database access is allowed here. `check` runs with `databases=[]`, + which leaves out the model checks that read a database feature from + Django 6.1 on. What is left includes every task backend's check. + """ + + def test_every_disagreement_is_reported_without_the_database(self): + with backends( + billing={**stripe(5, 1), **named("sms", 5, 60), **named("Mail", 5, 60)}, + reports={ + **stripe(5, 60), + **named("sms", 10, 60), + **named("mail", 5, 60), + }, + ): + report = run_check(databases=[]) + assert report.count("oxpull.W005") == 1 + assert report.count("oxpull.W004") == 1 + assert report.count("oxpull.W006") == 1 + + @pytest.mark.parametrize( + ("limits_by_alias", "reported"), + [ + # No limit named anywhere. + ({}, set()), + # One period written two ways, which is not reported. + ( + {"billing": stripe(5, 60), "reports": stripe(5, timedelta(minutes=1))}, + set(), + ), + # Each Warning alone, which does not raise. + ({"billing": stripe(5, 60), "reports": stripe(10, 60)}, {"oxpull.W004"}), + ({"billing": stripe(5, 1), "reports": stripe(5, 60)}, {"oxpull.W005"}), + ( + {"billing": named("Stripe", 5, 60), "reports": stripe(5, 60)}, + {"oxpull.W006"}, + ), + ], + ) + def test_a_configuration_with_no_error_is_checked_without_the_database( + self, limits_by_alias, reported + ): + with backends(**limits_by_alias): + report = run_check(databases=[]) + for check_id in ("oxpull.W004", "oxpull.W005", "oxpull.W006"): + assert report.count(check_id) == (check_id in reported) + + +@pytest.mark.django_db(databases="__all__") +class TestTheWorkerStartsBesideEveryWarning: + """ + django-ox's `ox_worker` runs every system check before it starts, as a + Django command does unless it opts out, whichever backend it serves. + An Error from any backend's check stops the workers of every backend. + One here would stop task processing at upgrade, and a supervisor that + restarts on exit would loop. So all three reports are Warnings, and + these tests run the real command's start to prove the worker is not + blocked by them. + """ + + @pytest.mark.parametrize("alias", ["billing", "reports", "default"]) + def test_ox_worker_starts_beside_a_different_per(self, alias): + started = [] + with backends(billing=stripe(5, 1), reports=stripe(5, 60)): + report = ox_worker_start(alias, started) + assert started == [alias] + assert report.count("oxpull.W005") == 1 + + def test_ox_worker_starts_beside_a_different_limit(self): + started = [] + with backends(billing=stripe(5, 60), reports=stripe(10, 60)): + report = ox_worker_start("billing", started) + assert started == ["billing"] + assert report.count("oxpull.W004") == 1 + + @pytest.mark.parametrize("alias", ["billing", "reports", "default"]) + def test_ox_worker_starts_beside_names_that_differ_only_by_case(self, alias): + """ + On PostgreSQL and SQLite these are two limits that work as meant, so + an Error would stop an installation that has nothing wrong with it. + """ + started = [] + with backends(billing=named("Stripe", 5, 60), reports=named("stripe", 5, 1)): + report = ox_worker_start(alias, started) + assert started == [alias] + assert report.count("oxpull.W006") == 1 + + def test_an_error_on_another_backend_still_stops_it(self): + """ + The control. It shows the start above runs the system checks, so a + worker that stopped running them could not pass those tests. + """ + started = [] + with ( + backends(billing=stripe(5, 1), broken=stripe(5, 0)), + pytest.raises(SystemCheckError, match=r"oxpull\.E005"), + ): + ox_worker_start("billing", started) + assert started == [] diff --git a/tests_oxpull/test_rate_limits.py b/tests_oxpull/test_rate_limits.py new file mode 100644 index 0000000..1591caa --- /dev/null +++ b/tests_oxpull/test_rate_limits.py @@ -0,0 +1,1149 @@ +""" +Rate-limit configuration, admission and shared-window behavior. + +The load-bearing tests are the ones a no-op limiter fails. Throughput is +asserted between a floor and a ceiling, so a limiter that blocks nothing +and a limiter that blocks everything both fail. A throttled task is +checked for the attempt and the lease it must not have spent, because the +claim UPDATE writes both and neither can be given back. And the exclusion +is proved on each claim path in turn: reverting either hook has to break +this file on the databases that take it. +""" + +import logging +import re +import time +from contextlib import contextmanager +from datetime import timedelta +from functools import partial + +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.core.management import call_command +from django.db import DatabaseError, IntegrityError, connections +from django.db.models import F, QuerySet +from django.db.models.functions import Now +from django.tasks import TaskResultStatus +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import batches +from oxpull.backend import OxpullBackend +from oxpull.models import OxpullBatch, OxpullRateWindow +from oxpull.rate_limits import ( + MAX_LIMIT_NAME_LENGTH, + RateLimit, + rate_limits_from_options, + refresh_interval_from_options, +) +from oxpull.worker import MAX_COUNT_FAILURES, MAX_RATE_ATTEMPTS, OxpullWorker + +from . import tasks +from .racing import THREADS, race, race_all +from .workers import admissions, run_workers + +pytestmark = pytest.mark.django_db + +ECHO = "tests_oxpull.tasks.plain_echo" +FAIL = "tests_oxpull.tasks.plain_flaky" +WINDOW_TABLE = OxpullRateWindow._meta.db_table + + +@contextmanager +def rate_limits(config, **options): + """The default backend, with these limits and the worker that applies them.""" + default = settings.TASKS["default"] + with override_settings( + TASKS={ + **settings.TASKS, + "default": { + **default, + "OPTIONS": { + **default["OPTIONS"], + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "RATE_LIMITS": config, + **options, + }, + }, + } + ): + yield + + +def oxpull_worker(**kwargs): + return OxpullWorker(backoff_initial=0, poll_interval=0.01, **kwargs) + + +def open_window(name, *, age=0): + """Put a limit's window at a known age on the database's own clock.""" + OxpullRateWindow.objects.update_or_create( + limit_name=name, + defaults={ + "window_id": 0, + "used": 0, + "window_started_at": Now() - timedelta(seconds=age), + }, + ) + return OxpullRateWindow.objects.get(limit_name=name) + + +def window(name): + return OxpullRateWindow.objects.get(limit_name=name) + + +@contextmanager +def captured_sql(alias): + statements = [] + + def wrapper(execute, sql, params, many, context): + statements.append(sql) + return execute(sql, params, many, context) + + with connections[alias].execute_wrapper(wrapper): + yield statements + + +# The savepoint counter increments once per atomic block in the process, +# so two runs of the same code path name their savepoints differently. It +# is the only part of a statement that is not a function of the path. +SAVEPOINT_NAME = re.compile(r"(SAVEPOINT) [`\"]?\w+[`\"]?") + + +def without_savepoint_names(statements): + return [SAVEPOINT_NAME.sub(r"\1 ", sql) for sql in statements] + + +def refresh_selects(statements): + return [ + sql + for sql in statements + if sql.lstrip().upper().startswith("SELECT") and WINDOW_TABLE in sql + ] + + +def failing_count(): + """ + A QuerySet.update that refuses the limiter's increment and its probe. + + Both are the one-column write on the window row, which is what the + fail-closed path counts; everything else, the seed and the roll + included, goes through. Set `.raising = False` to let the database + work again. + """ + real_update = QuerySet.update + + def failing(self, **kwargs): + if ( + failing.raising + and self.model is OxpullRateWindow + and set(kwargs) == {"used"} + ): + raise DatabaseError("no room on device") + return real_update(self, **kwargs) + + failing.raising = True + return failing + + +def alternating_count(): + """ + An increment that raises on one admission and matches nothing on the + next, so no admission is ever recorded and no statement ever fails + twice running. "Matches nothing" is what the increment returns when + the window rolled between the refresh and the write, and the worker + re-reads and tries again, MAX_RATE_ATTEMPTS times, so one admission + in that mode is that many zero rowcounts. The probe raises, so a + limit that closes stays closed. + """ + real_update = QuerySet.update + state = {"mode": "raise", "zeros": 0} + + def alternating(self, **kwargs): + if self.model is not OxpullRateWindow or set(kwargs) != {"used"}: + return real_update(self, **kwargs) + if isinstance(kwargs["used"], F): + # The reopen probe, used=F("used"): not an increment. + raise DatabaseError("no room on device") + if state["mode"] == "raise": + state["mode"], state["zeros"] = "zero", 0 + raise DatabaseError("no room on device") + state["zeros"] += 1 + if state["zeros"] == MAX_RATE_ATTEMPTS: + state["mode"] = "raise" + return 0 + + return alternating + + +# -- declaring limits -------------------------------------------------- + + +def test_a_limit_is_read_from_the_backend_that_configures_it(): + """ + OPTIONS is the only declaration site. A limit belongs to the alias + whose OPTIONS carry it, which is what lets one backend be limited + while another is not. + """ + backend = settings.TASKS["limited"] + + limits = rate_limits_from_options(backend["OPTIONS"], "limited", backend["QUEUES"]) + + assert limits == { + "hourly": RateLimit( + name="hourly", + limit=1, + per=3600, + task_paths=frozenset( + {"tests_oxpull.tasks.limited_echo", "tests_oxpull.tasks.limited_add"} + ), + queues=frozenset(), + ) + } + + +def test_default_backend_has_no_limits(): + backend = settings.TASKS["default"] + assert ( + rate_limits_from_options(backend["OPTIONS"], "default", backend["QUEUES"]) == {} + ) + + +def test_a_name_too_long_for_its_column_is_refused(): + """ + limit_name is the window table's primary key. An over-long name is a + DataError at the first admission on PostgreSQL and MySQL and silently + fine on SQLite, so it is refused where the configuration is read. + """ + name = "s" * (MAX_LIMIT_NAME_LENGTH + 1) + + with pytest.raises(ImproperlyConfigured, match="characters"): + rate_limits_from_options( + {"RATE_LIMITS": {name: {"limit": 5, "per": 60, "tasks": [ECHO]}}}, + "default", + ["default"], + ) + + +def test_a_name_the_length_of_its_column_is_accepted(): + name = "s" * MAX_LIMIT_NAME_LENGTH + limits = rate_limits_from_options( + {"RATE_LIMITS": {name: {"limit": 5, "per": 60, "tasks": [ECHO]}}}, + "default", + ["default"], + ) + assert set(limits) == {name} + + +def test_an_empty_limit_name_is_refused(): + with pytest.raises(ImproperlyConfigured, match="cannot be empty"): + rate_limits_from_options( + {"RATE_LIMITS": {"": {"limit": 5, "per": 60, "tasks": [ECHO]}}}, + "default", + ["default"], + ) + + +@pytest.mark.parametrize( + "value", + ["soon", None, -1, True, [0.25], float("inf"), float("nan"), 10**1000, -(10**1000)], +) +def test_a_refresh_interval_that_is_not_a_number_is_refused(value): + """ + The interval is validated where the configuration is read, so + `manage.py check` reports a bad value rather than the first worker of + a deploy raising at startup. inf and nan are floats that compare as + "not below zero", so a numeric check alone lets them through: inf + would stop the worker ever refreshing its view, and nan would fail + every comparison the refresh schedule makes. + """ + with pytest.raises(ImproperlyConfigured, match="RATE_REFRESH_INTERVAL"): + refresh_interval_from_options({"RATE_REFRESH_INTERVAL": value}) + + +def test_the_refresh_interval_defaults_and_accepts_a_number(): + assert refresh_interval_from_options({}) == 0.25 + assert refresh_interval_from_options({"RATE_REFRESH_INTERVAL": 1}) == 1.0 + + +def test_period_accepts_a_timedelta(): + limits = rate_limits_from_options( + { + "RATE_LIMITS": { + "a": {"limit": 5, "per": timedelta(minutes=2), "queues": ["default"]} + } + }, + "default", + ["default"], + ) + assert limits["a"].per == 120 + + +@pytest.mark.parametrize( + "config", + [ + {"a": {"limit": 5, "per": 0.5, "tasks": [ECHO]}}, + {"a": {"limit": 5, "per": 0, "tasks": [ECHO]}}, + {"a": {"limit": -1, "per": 60, "tasks": [ECHO]}}, + {"a": {"limit": 5, "tasks": [ECHO]}}, + {"a": {"per": 60, "tasks": [ECHO]}}, + {"a": {"limit": 5, "per": 60}}, + {"a": {"limit": 5, "per": 60, "tasks": ECHO}}, + {"a": {"limit": 5, "per": 60, "queues": ["nowhere"]}}, + {"a": {"limit": 5, "per": 60, "rate": 1}}, + {"a": "not a dict"}, + ], +) +def test_malformed_config_is_refused(config): + with pytest.raises(ImproperlyConfigured): + rate_limits_from_options({"RATE_LIMITS": config}, "default", ["default"]) + + +def test_rate_limits_must_be_a_dict(): + with pytest.raises(ImproperlyConfigured): + rate_limits_from_options({"RATE_LIMITS": [ECHO]}, "default", ["default"]) + + +# -- the system checks ------------------------------------------------- + + +def test_check_reports_malformed_limits(): + backend = OxpullBackend( + "default", {"OPTIONS": {"RATE_LIMITS": {"a": {"limit": 5, "per": 0}}}} + ) + assert "oxpull.E005" in [message.id for message in backend.check()] + + +def test_check_refuses_limits_without_a_worker_that_applies_them(): + with override_settings( + TASKS={ + **settings.TASKS, + "default": { + **settings.TASKS["default"], + "OPTIONS": { + "RATE_LIMITS": {"a": {"limit": 5, "per": 60, "tasks": [ECHO]}} + }, + }, + } + ): + backend = OxpullBackend("default", settings.TASKS["default"]) + errors = [message for message in backend.check() if message.id == "oxpull.E006"] + + assert len(errors) == 1 + assert "django_ox.worker.Worker" in errors[0].msg + + +def test_check_accepts_limits_with_the_oxpull_worker(): + with rate_limits({"a": {"limit": 5, "per": 60, "tasks": [ECHO]}}): + backend = OxpullBackend("default", settings.TASKS["default"]) + assert backend.check() == [] + + +def test_check_reports_a_refresh_interval_that_is_not_a_number(): + with rate_limits( + {"a": {"limit": 5, "per": 60, "tasks": [ECHO]}}, RATE_REFRESH_INTERVAL="soon" + ): + backend = OxpullBackend("default", settings.TASKS["default"]) + assert [message.id for message in backend.check()] == ["oxpull.E005"] + + +def test_check_reports_a_limited_task_path_that_is_not_a_task(): + """ + A limit matches on the stored task path, so a typo covers nothing and + throttles nothing while the queue looks healthy. + """ + with rate_limits( + {"a": {"limit": 5, "per": 60, "tasks": ["tests_oxpull.tasks.nope"]}} + ): + backend = OxpullBackend("default", settings.TASKS["default"]) + errors = backend.check() + + assert [message.id for message in errors] == ["oxpull.E005"] + assert "tests_oxpull.tasks.nope" in errors[0].msg + + +def test_check_reports_a_worker_class_that_is_not_a_worker(): + with rate_limits( + {"a": {"limit": 5, "per": 60, "tasks": [ECHO]}}, + WORKER_CLASS="oxpull.models.OxpullRateWindow", + ): + backend = OxpullBackend("default", settings.TASKS["default"]) + assert [m.id for m in backend.check()] == ["oxpull.E006"] + + +# -- zero cost when unused --------------------------------------------- + + +# The attribute a claim adds for a workflow node (oxpull.workflows.NODE_FLAG). +NODE_FLAG = "ox_wf_node" + + +def without_workflow_backends(): + """settings.TASKS without the backends that declare WORKFLOWS.""" + return override_settings( + TASKS={ + alias: config + for alias, config in settings.TASKS.items() + if not config.get("OPTIONS", {}).get("WORKFLOWS") + } + ) + + +def base_and_oxpull_statements(worker=None): + """ + The statements a `django_ox.worker.Worker` and then a + `oxpull.worker.OxpullWorker` run for one plain task. + """ + worker = worker or oxpull_worker() + alias = worker._db_alias + tasks.plain_echo.enqueue("a") + with captured_sql(alias) as base: + assert Worker(backoff_initial=0, poll_interval=0.01).run_once() is True + + OxTask.objects.all().delete() + tasks.plain_echo.enqueue("a") + with captured_sql(alias) as oxpull_sql: + assert worker.run_once() is True + return without_savepoint_names(base), without_savepoint_names(oxpull_sql) + + +def test_no_limits_configured_emits_exactly_the_base_worker_sql(): + """ + The whole cost argument rests on this. With nothing configured, and no + backend on the database declaring WORKFLOWS, the + `oxpull.worker.OxpullWorker`'s statements must be the + `django_ox.worker.Worker`'s statements, not statements that resemble them. + """ + with without_workflow_backends(): + assert oxpull_worker()._limits == {} + assert oxpull_worker()._records is False + base, oxpull_sql = base_and_oxpull_statements() + + assert oxpull_sql == base + assert refresh_selects(oxpull_sql) == [] + + +def test_beside_a_workflows_backend_the_claim_also_says_whether_a_task_is_a_node(): + """ + Where any backend on the database declares WORKFLOWS, every + `oxpull.worker.OxpullWorker` there records workflow outcomes, because a + claim is scoped to queues and not to backends. That costs one column on the + statement that returns the claimed row, and nothing else for a task that is + not a node. A worker on a backend without workflows first looks once for + the workflow node table, which is a cost per worker and not per claim, so + it is taken here first. + """ + worker = oxpull_worker() + assert worker._records is True + assert worker._recording() is True + base, oxpull_sql = base_and_oxpull_statements(worker) + + assert len(oxpull_sql) == len(base) + differing = [ + (base_line, oxpull_line) + for base_line, oxpull_line in zip(base, oxpull_sql, strict=True) + if base_line != oxpull_line + ] + assert len(differing) == 1, differing + assert NODE_FLAG not in differing[0][0] + assert NODE_FLAG in differing[0][1] + + +# -- the guarantee ----------------------------------------------------- + + +LIMIT = 10 +WORKERS = 4 + + +@pytest.mark.django_db(transaction=True) +def test_limit_caps_throughput_over_one_window(): + """ + The marquee test. A no-op limiter finishes all 100 tasks and fails the + ceiling by an order of magnitude; a limiter that blocks everything + fails the floor. + + The window outlasts any run, so every admission is counted against the + one budget, and the workers run until all of them have stopped taking + tasks, so the count is what the limiter let through and not what fitted + into a number of seconds. + """ + with rate_limits({"echo": {"limit": LIMIT, "per": 600, "tasks": [ECHO]}}): + for index in range(100): + tasks.plain_echo.enqueue(index) + open_window("echo") + + claims = run_workers(WORKERS, seconds=60, until_idle=True) + + admitted = len(admissions(claims)) + assert LIMIT <= admitted <= LIMIT + WORKERS - 1 + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == 100 - admitted + + +@pytest.mark.django_db(transaction=True) +class BoundedOxpullWorker(OxpullWorker): + """The production worker with a run loop that ends, so a test can run it.""" + + def run(self) -> None: + while self.run_once(): + pass + + +def test_the_real_command_applies_a_configured_limit(): + """ + End to end through ox_worker, not through OxpullWorker directly. + + Every other test here builds the worker itself, so all of them would + still pass if a configured limit never reached the command that runs + in production: a limit declared, a clean system check, and nothing + throttled. The command resolves WORKER_CLASS from the same settings + the limit is declared in, and only this test proves the two meet. + + The subclass replaces the run loop and nothing else, so the claim path + under test is the shipped one. + """ + limits = {"echo": {"limit": LIMIT, "per": 300, "tasks": [ECHO]}} + with rate_limits( + limits, WORKER_CLASS="tests_oxpull.test_rate_limits.BoundedOxpullWorker" + ): + for index in range(LIMIT + 5): + tasks.plain_echo.enqueue(index) + open_window("echo") + + with pytest.raises(SystemExit) as exit_code: + call_command("ox_worker", "--processes=1", verbosity=0) + + assert exit_code.value.code == 0 + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == 5 + + +@pytest.mark.django_db(transaction=True) +def test_the_next_window_is_what_admits_the_next_task(): + """ + The property in the time domain. One worker, because the guarantee is + exact only with one: with more, the overshoot could put the (N+1)-th + admission inside the first window and there would be nothing to + measure. + + Every instant compared is one the database wrote: open_window and the + roll both stamp window_started_at with Now(). The worker claims until it + has spent a window and been admitted once more, however long the machine + takes over that, so what bounds the run is what the limiter does rather + than a number of seconds. One worker is also the only writer of the + window row, so the row read after a claim is the row that admission + left behind. + """ + limit, period = 2, 1 + windows = [] + rolled = False + with rate_limits({"echo": {"limit": limit, "per": period, "tasks": [ECHO]}}): + for index in range(20): + tasks.plain_echo.enqueue(index) + open_window("echo") + worker = oxpull_worker() + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + db_task = worker.claim_one() + if db_task is None: + time.sleep(0.005) + continue + windows.append(window("echo")) + worker.execute(db_task) + if len(windows) > 1 and windows[-2].used >= limit: + rolled = True + break + + assert rolled, "the worker never spent a window and was admitted again" + spent, following = windows[-2:] + assert spent.used == limit + # The admission after a spent window is the first of the next one, and + # that window opened no sooner than a period after the spent one did. + assert (following.window_id, following.used) == (spent.window_id + 1, 1) + assert following.window_started_at - spent.window_started_at >= timedelta( + seconds=period + ) + + +@pytest.mark.django_db(transaction=True) +def test_racing_the_roll_opens_exactly_one_window(): + """ + Eight workers find the same window a second past its period and race to + open the next. The period is one no run outlasts, so the roll they race + for is the only one the database's clock allows, and a second window + can only be a second winner. The budget covers every task, so the run + ends on the empty queue rather than on a clock. + """ + with rate_limits({"echo": {"limit": 100, "per": 600, "tasks": [ECHO]}}): + for index in range(8): + tasks.plain_echo.enqueue(index) + open_window("echo", age=601) + + claims = run_workers(8, seconds=30, until_idle=True) + + row = window("echo") + + assert row.window_id == 1 + assert row.used == len(admissions(claims)) + + +@pytest.mark.django_db(transaction=True) +def test_racing_the_last_token_overshoots_by_at_most_one_per_worker(): + """ + Both workers that read the last token get it; the increment is + deliberately unconditional, because a compare-and-set that could + refuse would run after the attempt was already spent. What no worker + can do is take a second one. If this is ever tightened to == 1 the + published guarantee has been rewritten. + """ + workers = 8 + with rate_limits({"echo": {"limit": 1, "per": 600, "tasks": [ECHO]}}): + for index in range(20): + tasks.plain_echo.enqueue(index) + open_window("echo") + + first = len(admissions(run_workers(workers, seconds=0.5))) + second = len(admissions(run_workers(workers, seconds=0.5))) + + assert 1 <= first <= workers + assert second == 0 + + +@pytest.mark.django_db(transaction=True) +def test_exactly_one_racer_opens_the_window(): + """ + The roll's compare-and-set, on its own. The winner moves window_id, so + every loser's predicate stops matching and its rowcount is zero. + """ + with rate_limits({"echo": {"limit": 5, "per": 1, "tasks": [ECHO]}}): + open_window("echo", age=5) + workers = [oxpull_worker() for _ in range(THREADS)] + + results, errors = race_all( + [partial(worker._roll, "echo", 0, 1) for worker in workers] + ) + + row = window("echo") + + assert not errors + assert sum(results) == 1 + assert (row.window_id, row.used) == (1, 0) + + +@pytest.mark.django_db(transaction=True) +def test_racing_increments_lose_no_count(): + """ + The increment is what the bound's arithmetic counts, so it has to be + exact under contention even though it is never a gate. + """ + with rate_limits({"echo": {"limit": 100, "per": 600, "tasks": [ECHO]}}): + open_window("echo") + db_task = OxTask.objects.create( + task_path=ECHO, queue_name="default", enqueued_at=timezone.now() + ) + + def count(): + worker = oxpull_worker() + worker._refresh_view() + worker._count("echo", db_task) + + _, errors = race(count) + + row = window("echo") + + assert not errors + assert row.used == THREADS + + +@pytest.mark.django_db(transaction=True) +def test_an_unchanged_update_still_reports_a_rowcount(): + """ + Every decision here is a rowcount, and on MySQL that rests on Django + connecting with CLIENT_FOUND_ROWS: matched rows, not changed rows. + That is a connection flag, not a property of the SQL. + """ + OxpullRateWindow.objects.create( + limit_name="echo", window_started_at=timezone.now(), used=3 + ) + assert OxpullRateWindow.objects.filter(limit_name="echo").update(used=3) == 1 + + +# -- what a throttled task must not cost ------------------------------- + + +def test_a_throttled_task_burns_no_attempt_and_no_lease(): + with rate_limits({"echo": {"limit": 0, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + + for _ in range(5): + assert worker.run_once() is False + + assert worker.reap() == 0 + + row = OxTask.objects.get() + assert row.status == OxTask.Status.READY + assert row.attempts == 0 + assert row.lease_epoch == 0 + assert row.worker_ids == [] + assert row.locked_by is None + assert row.locked_at is None + assert row.started_at is None + + +def test_a_blocked_path_does_not_block_the_head_of_the_queue(): + with rate_limits({"echo": {"limit": 0, "per": 600, "tasks": [ECHO]}}): + for index in range(50): + tasks.plain_echo.using(priority=10).enqueue(index) + runnable = tasks.emails_echo.using(priority=0).enqueue("run me") + + assert oxpull_worker().run_once() is True + + assert OxTask.objects.get(id=runnable.id).status == OxTask.Status.SUCCESSFUL + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == 50 + + +def test_a_blocked_queue_does_not_block_another_queue(): + with rate_limits({"emails": {"limit": 0, "per": 600, "queues": ["emails"]}}): + for index in range(50): + tasks.emails_echo.using(priority=10).enqueue(index) + runnable = tasks.plain_echo.using(priority=0).enqueue("run me") + + assert oxpull_worker().run_once() is True + + assert OxTask.objects.get(id=runnable.id).status == OxTask.Status.SUCCESSFUL + + +# -- windows ----------------------------------------------------------- + + +def test_the_window_rolls_after_its_period(): + with rate_limits({"echo": {"limit": 1, "per": 1, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + tasks.plain_echo.enqueue("b") + worker = oxpull_worker() + + assert worker.run_once() is True + assert worker.run_once() is False + assert window("echo").used == 1 + + time.sleep(1.1) + assert worker.run_once() is True + + row = window("echo") + assert row.window_id == 1 + assert row.used == 1 + + +def test_a_retry_consumes_another_admission(): + with rate_limits({"flaky": {"limit": 5, "per": 600, "tasks": [FAIL]}}): + tasks.plain_flaky.enqueue(2) + worker = oxpull_worker() + + assert worker.run_once() is True # attempt 1 fails and requeues + assert OxTask.objects.get().status == OxTask.Status.READY + assert window("flaky").used == 1 + + assert worker.run_once() is True + + assert OxTask.objects.get().status == OxTask.Status.SUCCESSFUL + assert window("flaky").used == 2 + + +def test_a_task_under_two_limits_needs_both_and_counts_both(): + config = { + "by-path": {"limit": 5, "per": 600, "tasks": [ECHO]}, + "by-queue": {"limit": 0, "per": 600, "queues": ["default"]}, + } + with rate_limits(config): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + + assert worker.run_once() is False + assert window("by-path").used == 0 + + OxpullRateWindow.objects.filter(limit_name="by-queue").delete() + with rate_limits( + {**config, "by-queue": {"limit": 5, "per": 600, "queues": ["default"]}} + ): + assert oxpull_worker().run_once() is True + + assert window("by-path").used == 1 + assert window("by-queue").used == 1 + + +def test_a_limit_the_worker_has_never_seen_is_seeded_lazily(): + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + assert OxpullRateWindow.objects.count() == 0 + tasks.plain_echo.enqueue("a") + + assert oxpull_worker().run_once() is True + + row = window("echo") + assert (row.window_id, row.used) == (0, 1) + + +def test_a_blocked_worker_refreshes_no_faster_than_its_cooldown(): + """ + run() re-enters the claim on every task completion, not on the poll + interval, so without the cooldown a worker with fast in-flight work + would issue a refresh SELECT per completion. + """ + interval = 0.25 + config = { + "blocked": {"limit": 0, "per": 600, "tasks": [ECHO]}, + # A second, open limit, so the worker takes the cooldown rather + # than the every-limit-blocked path that sleeps to the window end. + "open": {"limit": 1000, "per": 600, "queues": ["emails"]}, + } + with rate_limits(config, RATE_REFRESH_INTERVAL=interval): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + alias = worker._db_alias + + with captured_sql(alias) as statements: + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + assert worker.claim_one() is None + + assert len(refresh_selects(statements)) <= 1 / interval + 1 + + +def test_the_postgresql_fragment_binds_one_array_per_column(): + """ + The fragment that reaches the raw claim, checked on every database + rather than only where it is used: it is the exclusion that a test + running elsewhere cannot see, and the array is one bound parameter + however many names it holds. + """ + config = { + "echo": {"limit": 0, "per": 600, "tasks": [ECHO]}, + "emails": {"limit": 0, "per": 600, "queues": ["emails"]}, + } + with rate_limits(config): + worker = oxpull_worker() + assert worker.claim_filter_sql() == ("", {}) + + worker._refresh_view() + fragment, params = worker.claim_filter_sql() + + assert fragment == ( + ' AND "task_path" <> ALL(%(ox_blocked_paths)s)' + ' AND "queue_name" <> ALL(%(ox_blocked_queues)s)' + ) + assert params == {"ox_blocked_paths": [ECHO], "ox_blocked_queues": ["emails"]} + + +def test_an_increment_that_raises_lets_the_task_run(monkeypatch, caplog): + """ + The claim is already committed by the time the count runs. Losing the + count is better than losing the task. + """ + monkeypatch.setattr(QuerySet, "update", failing_count()) + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert oxpull_worker().run_once() is True + + assert OxTask.objects.get().status == OxTask.Status.SUCCESSFUL + assert "runs uncounted" in caplog.text + assert window("echo").used == 0 + + +def test_a_counter_that_keeps_failing_stops_admitting(monkeypatch, caplog): + """ + A counter that cannot be written leaves `used` at zero, so every + refresh reads room that is not there and a worker would admit the + whole backlog at full speed with a warning per task. A run of + failures closes the limit instead, and it closes once. + """ + monkeypatch.setattr(QuerySet, "update", failing_count()) + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + for index in range(40): + tasks.plain_echo.enqueue(index) + worker = oxpull_worker() + + with caplog.at_level(logging.WARNING, logger="oxpull"): + admitted = sum(worker.run_once() for _ in range(40)) + + assert admitted == MAX_COUNT_FAILURES + assert ( + OxTask.objects.filter(status=OxTask.Status.READY).count() + == 40 - MAX_COUNT_FAILURES + ) + assert "admitting nothing under it" in caplog.text + # Once, on the transition, not once per refused claim. + assert caplog.text.count("admitting nothing under it") == 1 + + +def test_a_count_that_matched_nothing_does_not_end_the_run_of_failures( + monkeypatch, caplog +): + """ + The run that closes a limit is a run of admissions that were not + recorded. An increment that raised is one; an increment that ran and + matched no row is another, because the admission is no more on the + counter than if the statement had failed. Any statement that reached + the database used to end the run, so a counter that alternated + between raising and matching nothing never closed its limit and the + worker admitted the whole backlog at full speed, one warning per + task. Now the run ends only when an increment lands. + """ + monkeypatch.setattr(QuerySet, "update", alternating_count()) + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + for index in range(40): + tasks.plain_echo.enqueue(index) + worker = oxpull_worker() + + with caplog.at_level(logging.WARNING, logger="oxpull"): + admitted = sum(worker.run_once() for _ in range(40)) + + assert admitted == MAX_COUNT_FAILURES + assert ( + OxTask.objects.filter(status=OxTask.Status.READY).count() + == 40 - MAX_COUNT_FAILURES + ) + assert caplog.text.count("admitting nothing under it") == 1 + + +def test_a_closed_limit_admits_again_once_its_counter_takes_a_write(monkeypatch): + """ + Fail-closed has to be recoverable without a worker restart. A closed + limit admits nothing, so nothing else would ever run a count: the + refresh probes the row and reopens the limit when the write lands. + """ + failing = failing_count() + monkeypatch.setattr(QuerySet, "update", failing) + # No cooldown, so the probe rides the next claim rather than the next + # quarter second. What paces the probe is covered by its own test. + with rate_limits( + {"echo": {"limit": 20, "per": 600, "tasks": [ECHO]}}, RATE_REFRESH_INTERVAL=0 + ): + for index in range(20): + tasks.plain_echo.enqueue(index) + worker = oxpull_worker() + + while worker.run_once(): + pass + assert OxTask.objects.filter(status=OxTask.Status.READY).count() > 0 + + failing.raising = False + assert worker.run_once() is True + + assert window("echo").used == 1 + + +def failing_window_selects(armed): + """ + An execute wrapper that refuses every SELECT on the window table + while `armed["fail"]` is set. Everything else goes through. + """ + + def wrapper(execute, sql, params, many, context): + if ( + armed["fail"] + and sql.lstrip().upper().startswith("SELECT") + and WINDOW_TABLE in sql + ): + raise DatabaseError("connection reset by peer") + return execute(sql, params, many, context) + + return wrapper + + +def test_a_refresh_that_fails_after_the_claim_still_hands_the_task_over( + monkeypatch, caplog +): + """ + Everything after the claim is accounting, and accounting fails open: + the claim is committed and the attempt is spent, so the task must + run whether or not its count lands. The increment was already + covered. The re-read that follows an increment that matched no row + was not: it ran outside the handler, so a database fault there + escaped claim_one() after the claim had landed. The poll loop + contained the error but never received the task, which sat RUNNING + until the reaper took it back, one attempt poorer, and LOST if that + attempt was its last. + + The sequence: refresh, claim, another worker rolls the window, the + increment matches nothing, the re-read fails. The claimed task comes + back, the loss is logged, and the task runs. + """ + armed = {"fail": False} + real_claim = Worker.claim_one + + def claim_then_lose_the_window(self): + db_task = real_claim(self) + # Another worker rolls the window the moment the claim lands, + # and the database goes away before this worker can re-read it. + OxpullRateWindow.objects.filter(limit_name="echo").update( + window_id=F("window_id") + 1, used=0, window_started_at=Now() + ) + armed["fail"] = True + return db_task + + monkeypatch.setattr(Worker, "claim_one", claim_then_lose_the_window) + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + + with ( + connections["default"].execute_wrapper(failing_window_selects(armed)), + caplog.at_level(logging.WARNING, logger="oxpull"), + ): + db_task = worker.claim_one() + + assert db_task is not None + assert OxTask.objects.get(id=db_task.id).status == OxTask.Status.RUNNING + assert "runs uncounted" in caplog.text + worker.execute(db_task, inline=True) + + assert OxTask.objects.get().status == OxTask.Status.SUCCESSFUL + + +def refusing_window_inserts(armed): + """ + An execute wrapper that refuses every INSERT on the window table as a + duplicate while `armed["refuse"]` is set: the seed finding a row in + its way that the collation treats as the same name. + """ + + def wrapper(execute, sql, params, many, context): + if ( + armed["refuse"] + and sql.lstrip().upper().startswith("INSERT") + and WINDOW_TABLE in sql + ): + raise IntegrityError("duplicate key") + return execute(sql, params, many, context) + + return wrapper + + +def test_a_window_row_missing_after_the_claim_still_hands_the_task_over( + monkeypatch, caplog +): + """ + The re-read after an increment that matched no row can also find the + limit's row gone: the seed's INSERT is refused as a duplicate and the + read under the configured name returns nothing, which _adopt reports + as ImproperlyConfigured. Before the claim that is the right answer + and it propagates. After the claim it is accounting, and the handler + around the re-read caught DatabaseError alone, so the configuration + error escaped claim_one() with the task claimed and unreturned. The + claimed task now comes back, the loss is logged, and the task runs. + + The sequence: refresh, claim, the row is deleted under the worker, + the increment matches nothing, the re-read seeds and is refused, + and the adopt read finds no row. + """ + armed = {"refuse": False} + real_claim = Worker.claim_one + + def claim_then_lose_the_row(self): + db_task = real_claim(self) + OxpullRateWindow.objects.filter(limit_name="echo").delete() + armed["refuse"] = True + return db_task + + monkeypatch.setattr(Worker, "claim_one", claim_then_lose_the_row) + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + + with ( + connections["default"].execute_wrapper(refusing_window_inserts(armed)), + caplog.at_level(logging.WARNING, logger="oxpull"), + ): + db_task = worker.claim_one() + + assert db_task is not None + assert OxTask.objects.get(id=db_task.id).status == OxTask.Status.RUNNING + assert "runs uncounted" in caplog.text + assert "holds none that matches it" in caplog.text + worker.execute(db_task, inline=True) + + assert OxTask.objects.get().status == OxTask.Status.SUCCESSFUL + + +def test_a_refresh_that_fails_before_the_claim_claims_nothing(): + """ + The other side of the line. Before the claim nothing is spent, so a + database fault there fails closed: the error propagates, the poll + loop skips the pass, and the task stays READY for the next one. + """ + armed = {"fail": True} + with rate_limits({"echo": {"limit": 5, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + worker = oxpull_worker() + + with ( + connections["default"].execute_wrapper(failing_window_selects(armed)), + pytest.raises(DatabaseError), + ): + worker.claim_one() + + assert OxTask.objects.get().status == OxTask.Status.READY + + +# -- interaction with the other task-coordination features ------------- + + +def test_a_throttled_unique_task_still_deduplicates(): + """ + A throttled task is pending, and @unique promises unique while + pending, so throttling extends the dedup window. Releasing the lock + instead would stack duplicates up behind the throttle. + """ + path = "tests_oxpull.tasks.unique_echo" + with rate_limits({"unique": {"limit": 0, "per": 600, "tasks": [path]}}): + first = tasks.unique_echo.enqueue("a") + assert oxpull_worker().run_once() is False + + second = tasks.unique_echo.enqueue("a") + + assert second.id == first.id + assert OxTask.objects.count() == 1 + + +def test_a_batch_waits_for_its_throttled_member(): + with rate_limits({"echo": {"limit": 0, "per": 600, "tasks": [ECHO]}}): + batch = batches.create( + "throttled", on_complete=tasks.record_callback, on_complete_args=["done"] + ) + batch.add(tasks.plain_echo, 1) + batch.seal() + + assert oxpull_worker().run_once() is False + assert batches.reconcile.call() == 0 + + assert batch.status == OxpullBatch.Status.SEALED + assert "done" not in tasks.STATE + + +def test_a_throttled_task_reads_as_backlog(): + """ + The ready-age metric climbs while work is throttled, because the work + is queued and not running. A queue-age alert tuned before rate limits + were introduced will fire on it. + """ + from django_ox import stats + + with rate_limits({"echo": {"limit": 0, "per": 600, "tasks": [ECHO]}}): + tasks.plain_echo.enqueue("a") + assert oxpull_worker().run_once() is False + + assert stats.ready_count() == 1 + assert stats.oldest_ready_age() is not None + + +def test_a_throttled_task_still_reports_as_ready_to_its_caller(): + with rate_limits({"echo": {"limit": 0, "per": 600, "tasks": [ECHO]}}): + result = tasks.plain_echo.enqueue("a") + assert oxpull_worker().run_once() is False + + result.refresh() + assert result.status == TaskResultStatus.READY diff --git a/tests_oxpull/test_run_tasks_integration.py b/tests_oxpull/test_run_tasks_integration.py new file mode 100644 index 0000000..578f41c --- /dev/null +++ b/tests_oxpull/test_run_tasks_integration.py @@ -0,0 +1,94 @@ +""" +django_ox.testing.run_tasks() against oxpull's worker: a workflow chain whose +dependents are released by a commit callback of the outcome write, and a +rate limit that the claim enforces. + +Every test here carries commit_callbacks_wait: run_tasks() runs the outcome +write's callbacks itself, and the suite's own stand-in for a commit around +Worker.execute would otherwise run them first and leave them registered. +""" + +import re + +import pytest +from django.db import connection + +from django_ox import _run_tasks +from django_ox.compat import TaskResultStatus +from django_ox.models import OxTask +from django_ox.testing import run_tasks +from oxpull import workflows + +from . import tasks + +pytestmark = [pytest.mark.django_db, pytest.mark.commit_callbacks_wait] + + +def chain(name="chain"): + flow = workflows.create(name) + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.add("c", tasks.step, "c", depends_on="b") + flow.seal() + return flow + + +def test_a_workflow_chain_runs_in_order_inside_the_test_transaction(): + flow = chain() + results = run_tasks(backend="flows") + assert [r.return_value for r in results] == ["a", "b", "c"] + assert tasks.STATE["order"] == ["a", "b", "c"] + progress = flow.progress() + assert progress.succeeded == 3 + assert progress.pending == 0 + assert connection.in_atomic_block + + +def test_stepping_a_chain_one_attempt_at_a_time(): + chain() + assert [r.return_value for r in run_tasks(backend="flows", max_tasks=1)] == ["a"] + assert [r.return_value for r in run_tasks(backend="flows", max_tasks=1)] == ["b"] + assert [r.return_value for r in run_tasks(backend="flows")] == ["c"] + + +def test_a_failed_node_does_not_release_its_dependent(): + flow = workflows.create("fails") + flow.add("a", tasks.failing_step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + OxTask.objects.update(max_attempts=1) + results = run_tasks(backend="flows") + assert [r.status for r in results] == [TaskResultStatus.FAILED] + assert tasks.STATE["order"] == ["a"] + assert flow.progress().failed == 1 + + +@pytest.mark.django_db(transaction=True) +def test_a_workflow_chain_runs_in_order_in_autocommit(): + flow = chain("autocommit") + results = run_tasks(backend="flows") + assert [r.return_value for r in results] == ["a", "b", "c"] + assert flow.progress().succeeded == 3 + + +def test_a_rate_limit_holds_work_back_and_the_drain_returns(): + for value in range(3): + tasks.limited_echo.enqueue(value) + results = run_tasks(backend="limited") + assert [r.return_value for r in results] == [0] + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == 2 + + +def test_a_rate_limited_row_at_the_safety_limit_raises_conservatively(monkeypatch): + # The limit is applied when the claim runs, and the look at what is left + # does not claim, so it counts the held row as due work: documented as + # the conservative side of the safety limit. + monkeypatch.setattr(_run_tasks, "SAFETY_LIMIT", 1) + for value in range(2): + tasks.limited_echo.enqueue(value) + with pytest.raises( + RuntimeError, match=re.escape("'tests_oxpull.tasks.limited_echo'") + ): + run_tasks(backend="limited") + assert OxTask.objects.filter(status=OxTask.Status.SUCCESSFUL).count() == 1 + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == 1 diff --git a/tests_oxpull/test_schedule_isolation.py b/tests_oxpull/test_schedule_isolation.py new file mode 100644 index 0000000..e6abe35 --- /dev/null +++ b/tests_oxpull/test_schedule_isolation.py @@ -0,0 +1,685 @@ +""" +A schedule the database refuses, ahead of oxpull's own schedules, through +real `ox_worker --batch` processes. + +oxpull's batch and workflow reconcilers are ordinary django-ox schedule +entries, and nothing else completes a sealed batch or workflow. A schedule +ahead of them whose arguments the database refuses (a non-finite float: +PostgreSQL raises a DataError, MySQL an OperationalError and SQLite an +IntegrityError) is that schedule's failure. It must leave both reconcilers +to run, leave unique tasks deduplicated when two workers dispatch the same +ticks, and leave rate limits and read routing doing what they do without +it. + +Every test starts the worker a deployment runs, with oxpull's worker class, +on this session's test database, after proving which oxpull and which +django-ox that process imports. What is asserted is what the database holds +afterwards and how the process ended. Every wait is a deadlock guard with a +bound, never a measurement: a batch that exits does so on its own, and one +that does not has failed. SQLite is the control: it isolated this refusal +before, and PostgreSQL and MySQL did not. +""" + +import copy +import json +import os +import subprocess +import sys +import time +from datetime import timedelta +from pathlib import Path + +import pytest +from django.conf import settings +from django.db import DEFAULT_DB_ALIAS, connection, connections, transaction +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxScheduleTick, OxTask +from django_ox.stored import create_schedule +from django_ox.worker import Worker +from oxpull import batches, workflows +from oxpull.models import ( + OxpullBatch, + OxpullRateWindow, + OxpullUniqueLock, + OxpullWorkflow, +) +from oxpull.unique import unique_key + +from . import tasks +from .subprocesses import django_ox_path +from .test_workflows import drain, flow_worker + +REPO_ROOT = Path(__file__).resolve().parent.parent +SRC = REPO_ROOT / "src" + +pytestmark = [ + pytest.mark.django_db(transaction=True), + pytest.mark.skipif(os.name != "posix", reason="worker processes"), +] + +#: How long a test waits before calling a batch stuck. Every batch here +#: finishes in seconds; the failure these tests exist for was a batch that +#: never finished. +GUARD = 60 + +BATCH_RECONCILE = "oxpull.batches.reconcile" +WORKFLOW_RECONCILE = workflows.RECONCILE_TASK_PATH +BATCH_KEY = "oxpull-batch-reconcile" +WORKFLOW_KEY = "oxpull-workflow-reconcile" + +#: Serializable by Python's json module, which writes the bare token +#: Infinity, and refused by every supported database's JSON column. Nothing +#: django-ox or oxpull checks at start-up refuses it. +REFUSED = {"max_age": float("inf")} + +#: The same value as a stored row's form field holds it: the text typed. +REFUSED_TEXT = '{"max_age": Infinity}' + +VALUE_KIND = "iso.value" + +SUCCESSFUL = OxTask.Status.SUCCESSFUL + + +def twelve_hours_ago(): + """ + The project's wall clock twelve hours back, to the minute, naive. + + A schedule that runs daily at this time of day has a due tick twelve + hours old and its next one twelve hours off, so every pass, worker and + process in a test derives the same tick however long the test takes. A + minutely schedule would fire again across a minute boundary. + """ + now = timezone.now() + local = timezone.localtime(now).replace(tzinfo=None) if settings.USE_TZ else now + return (local - timedelta(hours=12)).replace(second=0, microsecond=0) + + +def as_stored(local): + """A naive project-local instant as the tick log stores it.""" + return timezone.make_aware(local) if settings.USE_TZ else local + + +def daily_cron_at(local): + return f"{local.minute} {local.hour} * * *" + + +def entry(task_path, *args, cron, refused=False): + """A SCHEDULES entry; a refused one carries REFUSED as its `value`.""" + config = {"task": task_path, "cron": cron} + if args: + config["args"] = list(args) + if refused: + config["kwargs"] = {"value": REFUSED} + return config + + +def with_history(*keys): + """ + One recorded tick per schedule a day ago, so each schedule's due tick + fires rather than anchoring a first sighting, which enqueues nothing + and so would never meet the refusal. + """ + now = timezone.now() + for key in keys: + OxScheduleTick.objects.create( + schedule_name=key, + scheduled_for=now - timedelta(days=1), + task_id=None, + created_at=now, + ) + + +def fired(key): + """(tick instant, task) for every tick of `key` that enqueued a task.""" + return [ + (tick.scheduled_for, OxTask.objects.get(id=tick.task_id)) + for tick in OxScheduleTick.objects.filter(schedule_name=key).exclude( + task_id=None + ) + ] + + +def deployment_tasks(**options_by_alias): + """ + settings.TASKS as a deployment runs it, with OPTIONS replaced per alias + (None removes one). + + "default" gets oxpull's worker class, which tests_oxpull/settings.py leaves + out. "flows_ungated" stops declaring WORKFLOWS: it is the suite's + deliberate oxpull.E007 fixture, and a system-check Error stops + ox_worker at start-up. Its alias stays because tests_oxpull.tasks declares a + task on it. + """ + config = copy.deepcopy(settings.TASKS) + config["flows_ungated"]["OPTIONS"]["WORKFLOWS"] = False + config["default"]["OPTIONS"]["WORKER_CLASS"] = "oxpull.worker.OxpullWorker" + for alias, options in options_by_alias.items(): + for name, value in options.items(): + if value is None: + config[alias]["OPTIONS"].pop(name, None) + else: + config[alias]["OPTIONS"][name] = value + return config + + +def deploy(tmp_path, config, *, router=False): + """ + The environment of a worker process on this session's test database, + with `config` as its TASKS. + + TASKS goes through json.loads, which reads the bare token Infinity back + as the float a settings file would hold. The replica alias is the + suite's never-written database; on SQLite it is moved into tmp_path so + that a read routed to it creates nothing in the checkout. `router` + installs tests_oxpull.routers.ReplicaReadRouter, which sends every django_ox + and oxpull read that names no database to that replica. + """ + lines = [ + "import copy, json", + f"from {settings.SETTINGS_MODULE} import * # noqa: F403", + f"TASKS = json.loads({json.dumps(config)!r})", + "DATABASES = copy.deepcopy(DATABASES) # noqa: F405", + ] + if connection.vendor == "sqlite": + replica = tmp_path / "replica.sqlite3" + lines.append(f"DATABASES['replica']['NAME'] = {str(replica)!r}") + if router: + lines.append("DATABASE_ROUTERS = ['tests_oxpull.routers.ReplicaReadRouter']") + lines += [ + "LOGGING = {", + " 'version': 1,", + " 'disable_existing_loggers': False,", + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},", + " 'loggers': {", + " 'django_ox': {'handlers': ['console'], 'level': 'INFO'},", + " 'oxpull': {'handlers': ['console'], 'level': 'INFO'},", + " },", + "}", + ] + (tmp_path / "isolation_settings.py").write_text("\n".join(lines) + "\n") + name = connection.settings_dict["NAME"] + if connection.vendor == "sqlite": + name = Path(name).resolve() + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "isolation_settings", + "PYTHONPATH": os.pathsep.join( + [str(SRC), django_ox_path(), str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(name), + } + + +def run_python(env, *args): + return subprocess.run( # noqa: S603 + [sys.executable, *args], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +PAIR_PROBE = """ +import django +django.setup() +import django_ox, oxpull +from django_ox.worker import worker_class +print(oxpull.__file__) +print(django_ox.__file__) +cls = worker_class(ALIAS) +print(cls.__module__ + "." + cls.__qualname__) +""" + + +def assert_the_pair(env, alias): + """ + The process imports this checkout's oxpull and the django-ox this + process runs against, and ox_worker would build oxpull's worker for + `alias`. A failure here is the set-up, not the behaviour under test. + """ + probe = run_python(env, "-c", PAIR_PROBE.replace("ALIAS", repr(alias))) + lines = probe.stdout.split() + if not ( + len(lines) == 3 + and Path(lines[0]).resolve().is_relative_to(SRC.resolve()) + and Path(lines[1]).resolve().is_relative_to(Path(django_ox_path())) + and lines[2] == "oxpull.worker.OxpullWorker" + ): + pytest.fail(f"unexpected worker set-up:\n{probe.stdout}{probe.stderr}") + + +ROUTER_PROBE = """ +import django +django.setup() +from django.db import router +from django_ox.models import OxScheduleTick +from oxpull.models import OxpullBatch +print(router.db_for_read(OxScheduleTick), router.db_for_read(OxpullBatch)) +try: + found = OxScheduleTick.objects.filter(schedule_name=KEY).exists() +except Exception as exc: + print(type(exc).__name__) +else: + print(found) +""" + + +def assert_reads_are_routed_away(env, key): + """ + The router reaches the worker process: a read that names no database + goes to the replica, and there it cannot see the tick row `key` this + test wrote. Every read that has to see it must name its database. + """ + probe = run_python(env, "-c", ROUTER_PROBE.replace("KEY", repr(key))) + lines = probe.stdout.splitlines() + if not (len(lines) == 2 and lines[0] == "replica replica" and lines[1] != "True"): + pytest.fail( + f"the router did not reach the process:\n{probe.stdout}{probe.stderr}" + ) + + +def start(env, log_path, backend="default"): + log = log_path.open("wb") + try: + return subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "django", + "ox_worker", + "--batch", + "--backend", + backend, + "--interval", + "0.05", + ], + cwd=REPO_ROOT, + env=env, + stdout=log, + stderr=log, + ) + finally: + log.close() + + +def finish(process, log_path): + """The exit code of a batch that ended on its own, or a failed test.""" + try: + return process.wait(timeout=GUARD) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + pytest.fail( + f"the batch did not finish within {GUARD}s:\n" + + log_path.read_text()[-4000:] + ) + + +def run_batch(env, log_path, backend="default"): + process = start(env, log_path, backend) + return finish(process, log_path), log_path.read_text() + + +def wait_for(predicate, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.05) + return False + + +def waiting_on_locks(): + """ + Sessions of this test database blocked on a lock right now, asked on a + connection of its own: PostgreSQL keeps one pg_stat_activity snapshot + per transaction, so asking inside the barrier's transaction would read + the same answer forever. + """ + probe = connections.create_connection(DEFAULT_DB_ALIAS) + try: + with probe.cursor() as cursor: + if probe.vendor == "postgresql": + cursor.execute( + "SELECT count(*) FROM pg_stat_activity " + "WHERE datname = current_database() AND wait_event_type = 'Lock'" + ) + else: + cursor.execute( + "SELECT COUNT(*) FROM performance_schema.data_lock_waits w " + "JOIN performance_schema.threads t " + "ON t.THREAD_ID = w.REQUESTING_THREAD_ID " + "WHERE t.PROCESSLIST_DB = DATABASE()" + ) + return cursor.fetchone()[0] + finally: + probe.close() + + +def sealed_and_run(): + """ + A sealed workflow and a sealed batch whose every task has run. Only a + reconciler completes either of them now, and the batch's on_complete + callback is enqueued by that completion. + """ + flow = workflows.create("isolation-flow") + flow.add("only", tasks.step, "only") + flow.seal() + assert drain(flow_worker()) == 1 + batch = batches.create( + "isolation-batch", + on_complete=tasks.record_callback, + on_complete_args=["isolation-batch-complete"], + ) + batch.add(tasks.plain_echo, 1) + batch.add(tasks.plain_echo, 2) + batch.seal() + members = Worker(backend_alias="default", backoff_initial=0, poll_interval=0.01) + assert drain(members) == 2 + assert set(OxTask.objects.values_list("status", flat=True)) == {SUCCESSFUL} + assert OxpullWorkflow.objects.get(pk=flow.id).status == OxpullWorkflow.Status.SEALED + assert OxpullBatch.objects.get(pk=batch.id).status == OxpullBatch.Status.SEALED + return flow, batch + + +def assert_reconciled(flow, batch, reconcilers, due): + """ + Each reconciler schedule fired its due tick once, into one task of its + own path that ran, and between them they completed the workflow and the + batch, whose callback ran too. + """ + for key, path in reconcilers.items(): + ((instant, task),) = fired(key) + assert instant == as_stored(due), key + assert (task.task_path, task.status) == (path, SUCCESSFUL), key + assert OxpullWorkflow.objects.get(pk=flow.id).status == ( + OxpullWorkflow.Status.COMPLETE + ) + assert OxpullBatch.objects.get(pk=batch.id).status == OxpullBatch.Status.COMPLETE + (callback,) = OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback") + assert callback.args == ["isolation-batch-complete"] + assert callback.status == SUCCESSFUL + + +def assert_refused_left_nothing(keys, names, logs): + """ + Each refused schedule fired nothing and was reported as its own failure + by some worker, and no pass was abandoned. A refused unique schedule's + lock row went with its rollback, so no lock names a refused key. + """ + for key in keys: + assert fired(key) == [], key + for name in names: + assert any( + f"Schedule {name} could not be dispatched this pass" in log for log in logs + ), name + for log in logs: + assert "Schedule dispatch failed" not in log, log + refused_lock = unique_key( + "default", tasks.unique_echo.module_path, "default", [], {"value": REFUSED} + ) + assert not OxpullUniqueLock.objects.filter(key=refused_lock).exists() + + +@pytest.mark.parametrize("router", [False, True], ids=["plain", "replica-read-router"]) +def test_refused_schedules_ahead_of_both_reconcilers_leave_them_to_complete( + router, tmp_path +): + """ + A refused plain schedule and a refused unique one, ahead of both + reconcilers in SCHEDULES. The second run of a deployment's batch job + meets the refusal. It has to complete the batch and the workflow, run + the batch's callback, and exit 0 by itself. + + With the replica read router the same run must come out the same: every + read the dispatch pass and the reconcilers make names its database. + """ + due = twelve_hours_ago() + cron = daily_cron_at(due) + schedules = { + "bad-plain": entry("tests_oxpull.tasks.plain_echo", cron=cron, refused=True), + "bad-unique": entry("tests_oxpull.tasks.unique_echo", cron=cron, refused=True), + BATCH_KEY: entry(BATCH_RECONCILE, cron=cron), + WORKFLOW_KEY: entry(WORKFLOW_RECONCILE, cron=cron), + } + with_history(*schedules) + flow, batch = sealed_and_run() + env = deploy( + tmp_path, deployment_tasks(default={"SCHEDULES": schedules}), router=router + ) + assert_the_pair(env, "default") + if router: + assert_reads_are_routed_away(env, BATCH_KEY) + + code, log = run_batch(env, tmp_path / "run.log") + + assert code == 0, log + assert_reconciled( + flow, batch, {BATCH_KEY: BATCH_RECONCILE, WORKFLOW_KEY: WORKFLOW_RECONCILE}, due + ) + assert_refused_left_nothing( + ["bad-plain", "bad-unique"], ["bad-plain", "bad-unique"], [log] + ) + # Nothing else was enqueued: the batch's two members, the workflow's + # node, one task per reconciler and the callback. + assert sorted(OxTask.objects.values_list("task_path", flat=True)) == sorted( + [ + BATCH_RECONCILE, + WORKFLOW_RECONCILE, + "tests_oxpull.tasks.plain_echo", + "tests_oxpull.tasks.plain_echo", + "tests_oxpull.tasks.record_callback", + "tests_oxpull.tasks.step", + ] + ) + assert not OxpullUniqueLock.objects.exists() + + +def test_refused_entries_ahead_of_stored_reconciler_rows(tmp_path, monkeypatch): + """ + The reconcilers as stored rows, which oxpull counts as scheduling them. + Settings schedules come before stored rows and rows come in primary-key + order, so a refused settings entry and a refused row written first are + both ahead of the two reconciler rows. + """ + # The registry is process-wide. Start it empty, as the stored-schedule + # test in test_workflows_hazard_reconcile_schedule.py does, so the + # settings channel fills it from SCHEDULABLE_TASKS below. + monkeypatch.setattr("django_ox.registry._registry", {}) + monkeypatch.setattr("django_ox.registry._discovered", True) + due = twelve_hours_ago() + cron = daily_cron_at(due) + config = deployment_tasks( + default={ + "SCHEDULES": { + "bad-plain": entry( + "tests_oxpull.tasks.plain_echo", cron=cron, refused=True + ) + }, + "SCHEDULE_SOURCE": "django_ox.stored.DatabaseScheduleSource", + "SCHEDULABLE_TASKS": { + VALUE_KIND: { + "task": "tests_oxpull.tasks.plain_echo", + "form": "tests_oxpull.forms.ValueArgs", + }, + BATCH_KEY: BATCH_RECONCILE, + WORKFLOW_KEY: WORKFLOW_RECONCILE, + }, + } + ) + with_history("bad-plain") + flow, batch = sealed_and_run() + boundary = timezone.now() - timedelta(days=1) + with override_settings(TASKS=config): + bad = create_schedule( + name="r-bad", + task_key=VALUE_KIND, + trigger="cron", + cron=cron, + arguments={"value": REFUSED_TEXT}, + start_time=boundary, + ) + batch_row = create_schedule( + name="batch-reconcile", + task_key=BATCH_KEY, + trigger="cron", + cron=cron, + start_time=boundary, + ) + flow_row = create_schedule( + name="workflow-reconcile", + task_key=WORKFLOW_KEY, + trigger="cron", + cron=cron, + start_time=boundary, + ) + assert bad.pk < batch_row.pk < flow_row.pk + env = deploy(tmp_path, config) + assert_the_pair(env, "default") + + code, log = run_batch(env, tmp_path / "run.log") + + assert code == 0, log + assert_reconciled( + flow, + batch, + { + f"db:{batch_row.pk}": BATCH_RECONCILE, + f"db:{flow_row.pk}": WORKFLOW_RECONCILE, + }, + due, + ) + assert_refused_left_nothing( + ["bad-plain", f"db:{bad.pk}"], ["bad-plain", "r-bad"], [log] + ) + + +def test_two_workers_dispatch_a_unique_task_once_behind_refused_schedules(tmp_path): + """ + Two batch workers over the same due ticks: two refused schedules, then + two schedules enqueueing the same unique call, then both reconcilers. + + On PostgreSQL and MySQL both workers are held inside the first unique + schedule's dispatch by a database barrier: this test holds an + uncommitted lock row for that call's key until both workers wait, one + on the key and the other on the first one's tick, then rolls it back. + Neither can run anything before its pass ends, so the second schedule's + enqueue finds the first one's task still pending and returns it: one + task, which both ticks name. SQLite has one writer, so its workers take + turns without a barrier, and a worker that meets the write lock there + can end its pass and run the task before the second tick fires. What + holds there is the unique contract itself: a second task only after the + first one finished. + """ + due = twelve_hours_ago() + cron = daily_cron_at(due) + schedules = { + "bad-plain": entry("tests_oxpull.tasks.plain_echo", cron=cron, refused=True), + "bad-unique": entry("tests_oxpull.tasks.unique_echo", cron=cron, refused=True), + "u-one": entry("tests_oxpull.tasks.unique_echo", "same", cron=cron), + "u-two": entry("tests_oxpull.tasks.unique_echo", "same", cron=cron), + BATCH_KEY: entry(BATCH_RECONCILE, cron=cron), + WORKFLOW_KEY: entry(WORKFLOW_RECONCILE, cron=cron), + } + with_history(*schedules) + env = deploy(tmp_path, deployment_tasks(default={"SCHEDULES": schedules})) + assert_the_pair(env, "default") + key = unique_key("default", tasks.unique_echo.module_path, "default", ["same"], {}) + + logs = [tmp_path / "w1.log", tmp_path / "w2.log"] + processes = [] + barrier = connection.vendor in ("postgresql", "mysql") + try: + if barrier: + with transaction.atomic(): + OxpullUniqueLock.objects.create( + key=key, + task_path=tasks.unique_echo.module_path, + queue_name="default", + created_at=timezone.now(), + ) + processes = [start(env, log) for log in logs] + held = wait_for(lambda: waiting_on_locks() >= 2, timeout=GUARD) + transaction.set_rollback(True) + assert held, "both workers never waited: " + "\n".join( + log.read_text()[-2000:] for log in logs + ) + else: + processes = [start(env, log) for log in logs] + codes = [ + finish(process, log) for process, log in zip(processes, logs, strict=True) + ] + finally: + for process in processes: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + text = [log.read_text() for log in logs] + assert codes == [0, 0], text + + ((first_at, first),) = fired("u-one") + ((second_at, second),) = fired("u-two") + assert first_at == second_at == as_stored(due) + uniques = list(OxTask.objects.filter(task_path=tasks.unique_echo.module_path)) + assert all(task.args == ["same"] for task in uniques) + assert all(task.status == SUCCESSFUL for task in uniques) + assert {task.id for task in uniques} == {first.id, second.id} + if barrier: + assert first.id == second.id, "the second tick enqueued a duplicate" + elif first.id != second.id: + assert first.finished_at <= second.enqueued_at, ( + "a second task was enqueued while the first was pending" + ) + # One lock for the call, naming the latest task; the refused schedule's + # lock rolled back with it. + (lock,) = OxpullUniqueLock.objects.all() + assert (lock.key, lock.task_id) == (key, second.id) + for reconciler in (BATCH_KEY, WORKFLOW_KEY): + ((_, task),) = fired(reconciler) + assert task.status == SUCCESSFUL + assert_refused_left_nothing( + ["bad-plain", "bad-unique"], ["bad-plain", "bad-unique"], text + ) + # Losing the race for a tick is not a schedule's failure. + for log in text: + for name in ("u-one", "u-two", BATCH_KEY, WORKFLOW_KEY): + assert f"Schedule {name} could not be dispatched" not in log, log + + +def test_a_refused_schedule_ahead_of_rate_limited_ones_leaves_the_limit_deciding( + tmp_path, +): + """ + Schedules on the "limited" backend, whose one-per-hour limit covers + limited_echo: a refused one, then two healthy ones. Both healthy ticks + enqueue; the limit admits one, holds the other READY without spending + an attempt, and the batch still ends, because a throttled task is not + one this worker can claim. + """ + due = twelve_hours_ago() + cron = daily_cron_at(due) + schedules = { + "l-bad": entry("tests_oxpull.tasks.limited_echo", cron=cron, refused=True), + "l-one": entry("tests_oxpull.tasks.limited_echo", "one", cron=cron), + "l-two": entry("tests_oxpull.tasks.limited_echo", "two", cron=cron), + } + with_history(*schedules) + env = deploy(tmp_path, deployment_tasks(limited={"SCHEDULES": schedules})) + assert_the_pair(env, "limited") + + code, log = run_batch(env, tmp_path / "run.log", backend="limited") + + assert code == 0, log + ((_, one),) = fired("l-one") + ((_, two),) = fired("l-two") + assert sorted([one.status, two.status]) == sorted([SUCCESSFUL, OxTask.Status.READY]) + held = one if one.status == OxTask.Status.READY else two + assert held.attempts == 0 + assert OxpullRateWindow.objects.get(limit_name="hourly").used == 1 + assert fired("l-bad") == [] + assert "Schedule l-bad could not be dispatched this pass" in log + assert "Schedule dispatch failed" not in log diff --git a/tests_oxpull/test_unique.py b/tests_oxpull/test_unique.py new file mode 100644 index 0000000..e265406 --- /dev/null +++ b/tests_oxpull/test_unique.py @@ -0,0 +1,1980 @@ +import contextlib +import threading +import uuid +import warnings +from datetime import timedelta + +import pytest +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.management import call_command +from django.db import OperationalError, connections, transaction +from django.db.models import F +from django.tasks import TaskResultStatus, default_task_backend +from django.tasks.exceptions import InvalidTask +from django.tasks.signals import task_enqueued +from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox import actions +from django_ox.backend import OxBackend +from django_ox.bulk import enqueue_many +from django_ox.models import OxTask +from django_ox.results import task_result_from_db +from django_ox.worker import Worker +from oxpull import backend as backend_module +from oxpull.backend import OxpullBackend +from oxpull.exceptions import UniqueEnqueueRace +from oxpull.models import OxpullUniqueLock +from oxpull.unique import is_registered, unique +from oxpull.worker import OxpullWorker + +from . import tasks +from .isolation import ( + callers_transaction, + isolation_for_new_connections, + opened_connections, + pg_sqlstate, + record_holder_checks, + result_fields, + session_isolation, + set_session_isolation, + set_transaction_isolation, +) + +pytestmark = pytest.mark.django_db + +LEVELS = ["READ COMMITTED", "REPEATABLE READ", "SERIALIZABLE"] + + +# -- marking ----------------------------------------------------------- + + +def test_decorator_registers_module_path(): + assert is_registered("tests_oxpull.tasks.unique_echo") + assert not is_registered("tests_oxpull.tasks.plain_echo") + + +def test_decorator_rejects_plain_functions(): + with pytest.raises(TypeError): + unique(lambda: None) + + +def test_bad_unique_tasks_shape_raises_at_init(): + with pytest.raises(ImproperlyConfigured): + OxpullBackend("default", {"OPTIONS": {"UNIQUE_TASKS": "not.a.list"}}) + + +def test_check_flags_unique_task_path_that_does_not_import(): + backend = OxpullBackend( + "default", {"OPTIONS": {"UNIQUE_TASKS": ["tests_oxpull.tasks.nope"]}} + ) + assert "oxpull.E002" in [message.id for message in backend.check()] + + +def test_check_flags_unique_task_path_that_is_not_a_task(): + backend = OxpullBackend( + "default", {"OPTIONS": {"UNIQUE_TASKS": ["tests_oxpull.tasks.STATE"]}} + ) + assert "oxpull.E002" in [message.id for message in backend.check()] + + +def test_check_reports_unconfigured_alias(): + backend = OxpullBackend("ghost", {}) + assert [message.id for message in backend.check()] == ["oxpull.E003"] + + +def test_split_router_raises_at_init(): + with ( + override_settings(DATABASE_ROUTERS=["tests_oxpull.routers.SplitTaskRouter"]), + pytest.raises(ImproperlyConfigured), + ): + OxpullBackend("default", {}) + + +def test_check_reports_split_router(): + backend = OxpullBackend("default", {}) + with override_settings(DATABASE_ROUTERS=["tests_oxpull.routers.SplitTaskRouter"]): + assert "oxpull.E004" in [message.id for message in backend.check()] + + +def test_split_rate_window_router_raises_at_init(): + with ( + override_settings( + DATABASE_ROUTERS=["tests_oxpull.routers.SplitRateWindowRouter"] + ), + pytest.raises(ImproperlyConfigured), + ): + OxpullBackend("default", {}) + + +def test_check_reports_a_router_that_splits_the_rate_window(): + # The window is the one oxpull table that is neither written in a + # task's transaction nor joined to the task table, so it needs its + # own case here: OxpullWorker reads and writes it on the alias the + # task tables use, and a router naming another database for it does + # not change which alias those statements name. + backend = OxpullBackend("default", {}) + with override_settings( + DATABASE_ROUTERS=["tests_oxpull.routers.SplitRateWindowRouter"] + ): + assert "oxpull.E004" in [message.id for message in backend.check()] + + +def test_check_warns_when_schedule_enqueues_unique_task(): + backend = OxpullBackend( + "default", + { + "OPTIONS": { + "SCHEDULES": { + "nightly": { + "task": "tests_oxpull.tasks.unique_echo", + "cron": "0 3 * * *", + } + } + } + }, + ) + assert "oxpull.W001" in [message.id for message in backend.check()] + + +def test_backend_check_passes_on_test_settings(): + assert default_task_backend.check() == [] + + +# -- dedup semantics --------------------------------------------------- + + +def test_duplicate_returns_existing_result(): + first = tasks.unique_echo.enqueue("x") + second = tasks.unique_echo.enqueue("x") + + assert second.id == first.id + assert second.status == TaskResultStatus.READY + assert OxTask.objects.count() == 1 + assert OxpullUniqueLock.objects.count() == 1 + assert OxpullUniqueLock.objects.get().task_id is not None + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + + +def test_duplicate_result_supports_refresh(): + first = tasks.unique_echo.enqueue("x") + second = tasks.unique_echo.enqueue("x") + second.refresh() + assert second.id == first.id + assert second.status == TaskResultStatus.READY + + +def test_different_args_are_different_tasks(): + tasks.unique_echo.enqueue("x") + tasks.unique_echo.enqueue("y") + assert OxTask.objects.count() == 2 + assert OxpullUniqueLock.objects.count() == 2 + + +def test_kwarg_order_is_deduplicated(): + tasks.unique_add.enqueue(a=1, b=2) + tasks.unique_add.enqueue(b=2, a=1) + assert OxTask.objects.count() == 1 + + +def test_positional_and_keyword_spellings_are_distinct_keys(): + # Deliberate: the key hashes args and kwargs as stored, with no + # signature binding. add(1, 2) and add(1, b=2) are different keys. + tasks.unique_add.enqueue(1, 2) + tasks.unique_add.enqueue(1, b=2) + assert OxTask.objects.count() == 2 + + +def test_plain_tasks_are_untouched(): + tasks.plain_echo.enqueue("x") + tasks.plain_echo.enqueue("x") + assert OxTask.objects.count() == 2 + assert OxpullUniqueLock.objects.count() == 0 + + +def test_options_configured_task_is_unique(): + first = tasks.configured_unique.enqueue("x") + second = tasks.configured_unique.enqueue("x") + assert second.id == first.id + assert OxTask.objects.count() == 1 + + +def test_queue_scopes_the_key(): + tasks.unique_echo.enqueue("x") + tasks.unique_echo.using(queue_name="emails").enqueue("x") + assert OxTask.objects.count() == 2 + + +def test_using_copies_stay_unique(): + # Task.using() returns a new frozen Task; uniqueness keys off + # module_path, which the copy shares. + first = tasks.unique_echo.using(priority=5).enqueue("x") + second = tasks.unique_echo.enqueue("x") + assert second.id == first.id + assert OxTask.objects.count() == 1 + + +def test_duplicate_result_reflects_the_stored_row(): + first = tasks.unique_echo.enqueue("x") + second = tasks.unique_echo.using(priority=7).enqueue("x") + assert second.id == first.id + # Row truth: the pending task was enqueued at priority 0; the + # duplicate's using() override does not rewrite history. + assert second.task.priority == 0 + + +def test_int_and_string_dict_keys_deduplicate_like_stored_json(): + # JSONField stringifies int keys at dump time, so these payloads + # store identically; the key must match them rather than raise over + # an unsortable mixed-key dict. + first = tasks.unique_echo.enqueue({1: "x", "y": 2}) + second = tasks.unique_echo.enqueue({"1": "x", "y": 2}) + assert second.id == first.id + assert OxTask.objects.count() == 1 + + +def test_no_signal_for_suppressed_duplicate(): + received = [] + + def receiver(sender, task_result, **kwargs): + received.append(task_result.id) + + task_enqueued.connect(receiver) + try: + first = tasks.unique_echo.enqueue("x") + tasks.unique_echo.enqueue("x") + finally: + task_enqueued.disconnect(receiver) + assert received == [first.id] + + +# -- lock release on every terminal path ------------------------------- + + +def test_lock_released_after_success(worker): + first = tasks.unique_echo.enqueue("x") + assert worker.run_once() is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.SUCCESSFUL + + second = tasks.unique_echo.enqueue("x") + + assert second.id != first.id + assert OxTask.objects.count() == 2 + lock = OxpullUniqueLock.objects.get() + assert str(lock.task_id) == second.id + + +def test_lock_released_after_terminal_failure(worker): + first = tasks.unique_fail_always.enqueue() + for _ in range(3): # MAX_ATTEMPTS in tests_oxpull/settings.py + assert worker.run_once() is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + + second = tasks.unique_fail_always.enqueue() + assert second.id != first.id + assert OxTask.objects.count() == 2 + + +def test_retry_is_not_a_duplicate(worker): + first = tasks.unique_flaky.enqueue(2) + assert worker.run_once() is True # attempt 1 fails -> back to READY + + row = OxTask.objects.get(id=first.id) + assert row.status == OxTask.Status.READY + assert row.attempts == 1 + + # The key is still held while the task waits to retry. + duplicate = tasks.unique_flaky.enqueue(2) + assert duplicate.id == first.id + assert OxTask.objects.count() == 1 + + assert worker.run_once() is True # attempt 2 succeeds + assert OxTask.objects.get(id=first.id).status == OxTask.Status.SUCCESSFUL + + +def test_reaper_lost_write_does_not_strand_the_lock(worker): + """ + The reaper settles the row via a queryset update with no signal at all + (django_ox worker.reap). Nothing here listens for signals, so the lock + must still release: staleness is derived from the row. + + It writes LOST rather than FAILED, and the lock releases on either. + Holding the key for an unobserved attempt would block the name for as + long as the vanished worker stays vanished, which is unbounded. + """ + first = tasks.unique_echo.enqueue("x") + OxTask.objects.filter(id=first.id).update( + status=OxTask.Status.RUNNING, + attempts=3, + max_attempts=3, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + assert worker.reap() == 1 + assert OxTask.objects.get(id=first.id).status == OxTask.Status.LOST + + second = tasks.unique_echo.enqueue("x") + assert second.id != first.id + assert OxTask.objects.count() == 2 + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + + +def test_reaper_requeue_keeps_the_lock(worker): + """A reclaimed task with attempts remaining goes back to READY; the + key stays held and a duplicate still resolves to it.""" + first = tasks.unique_echo.enqueue("x") + OxTask.objects.filter(id=first.id).update( + status=OxTask.Status.RUNNING, + attempts=1, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + assert worker.reap() == 1 + assert OxTask.objects.get(id=first.id).status == OxTask.Status.READY + + duplicate = tasks.unique_echo.enqueue("x") + assert duplicate.id == first.id + assert OxTask.objects.count() == 1 + + +def _claim_then_lose_the_worker(worker, task, *args, last_attempt): + """ + Enqueue a unique task, claim it, then age its lease past expiry, which + is how a worker that died mid-attempt leaves the row. + + With last_attempt the row is set one attempt short of its limit before + the claim, so the claim spends the final attempt and the in-memory row + it returns is what that execution holds: the database's attempts and + the lease epoch the claim handed out. + """ + result = task.enqueue(*args) + if last_attempt: + OxTask.objects.filter(id=result.id).update(attempts=F("max_attempts") - 1) + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == result.id + assert (db_task.attempts == db_task.max_attempts) is last_attempt + expired = timezone.now() - timedelta(seconds=9999) + OxTask.objects.filter(id=result.id).update( + locked_at=expired, lease_expires_at=expired + ) + return result, db_task + + +@pytest.mark.django_db(transaction=True) +def test_a_duplicate_coalesces_onto_an_expired_lease_until_the_reaper_writes_lost( + worker, +): + """ + A RUNNING holder whose worker has died still holds its key. + + Liveness is the row's status and nothing else. An expired lease says + only that renewals stopped, and a dead worker and a live one whose + renewal is late look the same from here, so the key is not released + on it. A duplicate enqueued in that gap returns the holder's result + and writes no row. The reaper settles it: on the last attempt it + writes LOST, which frees the key. The duplicate that coalesced is + left with a result that reads FAILED and no row of its own, and the + next enqueue creates one. `docs/features/unique-tasks.md` names the gap. + + Releasing on an expired lease instead would not close the gap, only + move it: a late renewal, or the reaper's requeue when attempts are + left, would then leave two pending rows for one key. + """ + first, _ = _claim_then_lose_the_worker( + worker, tasks.unique_echo, "x", last_attempt=True + ) + + duplicate = tasks.unique_echo.enqueue("x") + assert duplicate.id == first.id + assert duplicate.status == TaskResultStatus.RUNNING + assert OxTask.objects.count() == 1 + + assert worker.reap() == 1 + assert OxTask.objects.get(id=first.id).status == OxTask.Status.LOST + assert OxTask.objects.count() == 1 + duplicate.refresh() + assert duplicate.status == TaskResultStatus.FAILED + + third = tasks.unique_echo.enqueue("x") + assert third.id != first.id + assert OxTask.objects.count() == 2 + assert str(OxpullUniqueLock.objects.get().task_id) == third.id + + +@pytest.mark.django_db(transaction=True) +def test_a_duplicate_coalesces_onto_an_expired_lease_the_reaper_requeues(worker): + """ + The same gap with attempts left. The duplicate coalesces onto the + expired RUNNING holder, the reaper puts the row back to READY, which + is still pending, and the key stays with it, so the coalesced work + runs as the retry. + """ + first, _ = _claim_then_lose_the_worker( + worker, tasks.unique_echo, "x", last_attempt=False + ) + + assert tasks.unique_echo.enqueue("x").id == first.id + assert OxTask.objects.count() == 1 + + assert worker.reap() == 1 + assert OxTask.objects.get(id=first.id).status == OxTask.Status.READY + assert tasks.unique_echo.enqueue("x").id == first.id + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + ("task", "args", "outcome"), + [ + (tasks.unique_echo, ["x"], OxTask.Status.SUCCESSFUL), + (tasks.unique_fail_always, [], OxTask.Status.FAILED), + ], + ids=["successful", "failed"], +) +@pytest.mark.parametrize( + "worker_class", [Worker, OxpullWorker], ids=["django-ox", "oxpull"] +) +def test_a_lost_holders_late_outcome_lands_on_its_own_row_and_keeps_the_new_lock( + worker_class, task, args, outcome +): + """ + A LOST holder's execution can outlive its key, and its outcome write + touches its own row only. + + The reaper writes LOST without moving the lease epoch, so the + execution that lost the lease keeps the right to write its outcome + onto that row. The key is free by then: the next enqueue takes it + with a new row, and a worker claims that row while the first + execution is still running. Uniqueness deduplicates pending rows and + does not exclude executions. When the straggler finishes, its write + lands on the LOST row, the replacement stays RUNNING, and the lock + still names the replacement, so a further duplicate coalesces onto it. + + Run with both worker classes. A fleet can hold either, and an outcome + hook on OxpullWorker is where a change would most easily break this. + """ + worker = worker_class(backoff_initial=0, poll_interval=0.05) + first, straggler = _claim_then_lose_the_worker( + worker, task, *args, last_attempt=True + ) + assert worker.reap() == 1 + assert OxTask.objects.get(id=first.id).status == OxTask.Status.LOST + + second = task.enqueue(*args) + assert second.id != first.id + replacement = worker.claim_one() + assert replacement is not None and str(replacement.id) == second.id + + worker.execute(straggler) + + assert OxTask.objects.get(id=first.id).status == outcome + assert OxTask.objects.get(id=second.id).status == OxTask.Status.RUNNING + assert OxpullUniqueLock.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + assert task.enqueue(*args).id == second.id + assert OxTask.objects.count() == 2 + + +def test_pruned_task_row_releases_the_lock(): + first = tasks.unique_echo.enqueue("x") + # ox_prune deletes finished rows outright; the lock must read a + # missing row as "finished", not hold the key forever. + OxTask.objects.filter(id=first.id).delete() + + second = tasks.unique_echo.enqueue("x") + assert second.id != first.id + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + + +# -- transactions ------------------------------------------------------ + + +def test_duplicate_inside_caller_transaction_keeps_it_usable(): + first = tasks.unique_echo.enqueue("x") + with transaction.atomic(): + second = tasks.unique_echo.enqueue("x") + # The IntegrityError was confined to the inner savepoint; the + # caller's transaction is still usable. + assert OxTask.objects.count() == 1 + assert second.id == first.id + + +def test_two_enqueues_inside_one_transaction_deduplicate(): + # Same connection, same open transaction: the second INSERT conflicts + # with the caller's own uncommitted lock row, which surfaces + # immediately (no cross-transaction wait) and resolves to the first + # task, still visible on this connection. + with transaction.atomic(): + first = tasks.unique_echo.enqueue("x") + second = tasks.unique_echo.enqueue("x") + assert second.id == first.id + assert OxTask.objects.count() == 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("how", ["options", "session"]) +@pytest.mark.parametrize("level", LEVELS) +def test_two_enqueues_inside_one_transaction_deduplicate_at_every_level( + monkeypatch, level, how +): + """ + A transaction that enqueues the same unique task twice gets the first + task back at every level, whether the project sets the level in + OPTIONS["isolation_level"] or the session sets its own. + + The holder is the transaction's own row, not yet committed. On + PostgreSQL at REPEATABLE READ or SERIALIZABLE the enqueue asks for a + share lock on the holder's row, and a row this transaction wrote is + granted. No connection is opened on PostgreSQL. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite has one isolation level") + connections["default"].close() + if how == "options": + isolation_for_new_connections(monkeypatch, level) + else: + set_session_isolation(level) + checks = record_holder_checks(monkeypatch) + outcome = {} + try: + with opened_connections() as opened, transaction.atomic(): + outcome["level"] = session_isolation() + first = tasks.unique_echo.enqueue("x") + try: + outcome["second"] = tasks.unique_echo.enqueue("x") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + finally: + connections["default"].close() + + assert outcome["level"] == level + assert "race" not in outcome, outcome["race"] + assert outcome["second"].id == first.id + assert outcome["second"].status == TaskResultStatus.READY + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + if vendor == "postgresql": + assert opened == [] + assert checks == ([] if level == "READ COMMITTED" else ["granted"]) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("how", ["options", "session"]) +@pytest.mark.parametrize("change", ["discarded", "deleted", "finished"]) +@pytest.mark.parametrize("level", LEVELS) +def test_a_holder_this_transaction_settled_is_not_deduplicated_against( + monkeypatch, level, change, how +): + """ + A transaction that discards, deletes or finishes a key's holder and + then enqueues the same key queues a new task, at every level. A + snapshot shows the transaction's own changes, so the holder reads as + settled, or as gone, and the release path runs. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite has one isolation level") + first = tasks.unique_echo.enqueue("x") + connections["default"].close() + if how == "options": + isolation_for_new_connections(monkeypatch, level) + else: + set_session_isolation(level) + outcome = {} + try: + with transaction.atomic(): + outcome["level"] = session_isolation() + if change == "discarded": + assert actions.discard(first.id) + elif change == "deleted": + assert OxTask.objects.filter(id=first.id).delete()[0] == 1 + else: + assert ( + OxTask.objects.filter(id=first.id).update( + status=OxTask.Status.SUCCESSFUL + ) + == 1 + ) + with opened_connections() as opened: + try: + outcome["second"] = tasks.unique_echo.enqueue("x") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + finally: + connections["default"].close() + + assert outcome["level"] == level + assert "race" not in outcome, outcome["race"] + second = outcome["second"] + assert second.id != first.id, "deduplicated against a holder it had settled" + assert second.status == TaskResultStatus.READY + ready = OxTask.objects.filter(status=OxTask.Status.READY) + assert [str(task.id) for task in ready] == [second.id] + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + if vendor == "postgresql": + assert opened == [] + + +def test_rollback_discards_task_and_lock_together(): + with pytest.raises(ValidationError), transaction.atomic(): + tasks.unique_echo.enqueue("x") + raise ValidationError("abort") + + assert OxTask.objects.count() == 0 + assert OxpullUniqueLock.objects.count() == 0 + + # The key is free; the next enqueue wins cleanly. + result = tasks.unique_echo.enqueue("x") + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == result.id + + +def test_worker_still_runs_deduplicated_task_to_success(worker): + first = tasks.unique_add.enqueue(2, 3) + tasks.unique_add.enqueue(2, 3) + assert worker.run_once() is True + first.refresh() + assert first.status == TaskResultStatus.SUCCESSFUL + assert first.return_value == 5 + + +# -- diagnostics and routing ------------------------------------------- + + +@pytest.mark.django_db(databases=["default", "replica"]) +def test_replica_read_router_cannot_fake_staleness(): + # A router that sends reads to a lagged (here: empty) replica must + # not make a live lock look stale; every dedup-path read is pinned + # to the write database. + first = tasks.unique_echo.enqueue("pinned") + with override_settings(DATABASE_ROUTERS=["tests_oxpull.routers.ReplicaReadRouter"]): + second = tasks.unique_echo.enqueue("pinned") + assert second.id == first.id + assert OxTask.objects.using("default").count() == 1 + assert OxpullUniqueLock.objects.using("default").count() == 1 + + +# -- django-ox 0.3.x bulk enqueue and operator actions ------------------ + + +def test_enqueue_many_refuses_a_unique_task(): + """ + Repro against django-ox 0.3.1: enqueue_many() writes its rows with + one bulk INSERT and never reaches enqueue(), so no lock is taken. + Against 0.2.1 that call did not exist. Against 0.3.1 it wrote three + READY rows for one unique key beside a single lock that pointed at + the first, and every later enqueue() of the key kept returning that + first task, so the guarantee read as working while the duplicates + ran. + + Refusing is the fix: silently voiding the property the task is + marked with is worse than a call that says it cannot hold it. + """ + first = tasks.unique_echo.enqueue("a") + + with pytest.raises(InvalidTask) as excinfo: + enqueue_many(tasks.unique_echo, [(["a"], {}), (["b"], {})]) + assert "tests_oxpull.tasks.unique_echo" in str(excinfo.value) + assert "enqueue_many()" in str(excinfo.value) + + assert OxTask.objects.count() == 1 + assert OxpullUniqueLock.objects.count() == 1 + assert tasks.unique_echo.enqueue("a").id == first.id + + +def test_enqueue_many_refuses_an_options_configured_unique_task(): + """UNIQUE_TASKS is the same marking, so it gets the same refusal.""" + with pytest.raises(InvalidTask): + enqueue_many(tasks.configured_unique, [([1], {}), ([2], {})]) + assert OxTask.objects.count() == 0 + + +def test_enqueue_many_still_bulk_inserts_a_plain_task(): + """A non-unique task keeps the parent's bulk path, untouched.""" + results = enqueue_many(tasks.plain_echo, [(["a"], {}), (["b"], {}), (["c"], {})]) + assert len({result.id for result in results}) == 3 + assert OxTask.objects.count() == 3 + assert OxpullUniqueLock.objects.count() == 0 + # The promise is the returned list, which enqueue_many documents as being + # in input order. Every row written by one call shares an enqueued_at, so + # ordering the rows by it is a three-way tie and the sequence that comes + # back is the storage engine's: MySQL answers it differently from SQLite + # and PostgreSQL. Assert the order the product gives, and the contents of + # the table without an order it never promised. + assert [result.args for result in results] == [["a"], ["b"], ["c"]] + assert sorted(row.args for row in OxTask.objects.all()) == [["a"], ["b"], ["c"]] + + +def test_discarded_task_releases_the_lock(): + """ + django_ox.actions.discard closes a READY task without running it. + Staleness is derived from the row rather than from a signal, and + DISCARDED is neither READY nor RUNNING, so the key frees itself with + no change to the release path. + """ + first = tasks.unique_echo.enqueue("x") + assert actions.discard(first.id) is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.DISCARDED + + second = tasks.unique_echo.enqueue("x") + assert second.id != first.id + assert OxTask.objects.count() == 2 + lock = OxpullUniqueLock.objects.get() + assert str(lock.task_id) == second.id + + +def test_prune_clears_the_lock_of_a_discarded_task(): + first = tasks.unique_echo.enqueue("x") + assert actions.discard(first.id) is True + call_command("oxpull_prune") + assert OxpullUniqueLock.objects.count() == 0 + + +def test_retrying_a_settled_task_can_reopen_a_key_a_duplicate_took(worker): + """ + A documented consequence, pinned so it cannot change unnoticed. + + The lock is an enqueue-time gate, not a runtime one. Once a unique + task fails terminally its key is free, and the next enqueue takes it. + django_ox.actions.retry then puts the original row back on the queue + without passing the gate, so two READY rows share one key. Nothing in + oxpull can prevent that: retry writes the task table directly, and + oxpull does not own that table. + + An operator who retries a task that has already been superseded is + asking for the old attempt to run, which is what happens. + """ + first = tasks.unique_fail_always.enqueue() + while worker.run_once(): + pass + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + + second = tasks.unique_fail_always.enqueue() + assert second.id != first.id + + assert actions.retry(first.id) is True + ready = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_fail_always", status=OxTask.Status.READY + ) + assert ready.count() == 2 + assert OxpullUniqueLock.objects.count() == 1 + + +# -- a holder read behind a caller's snapshot --------------------------- + + +def assert_retry_outside_the_snapshot(race, level, found=None): + """ + PostgreSQL's refusal of a changed row, as the enqueue reports it. The + message names the database and the level, says nothing was created, and + says to retry in a new, short transaction at READ COMMITTED. `found` is + what the read on a connection of its own found, when one ran. + """ + assert isinstance(race, UniqueEnqueueRace), "no UniqueEnqueueRace was raised" + assert isinstance(race.__cause__, OperationalError) + assert pg_sqlstate(race.__cause__) == "40001" + message = str(race) + assert "on database 'default'" in message + assert f"inside a transaction at {level}," in message + assert "The enqueue created nothing." in message + assert "Retry it in a new, short transaction at READ COMMITTED" in message + if found is None: + assert "A fresh read found" not in message + else: + assert f"A fresh read found {found}." in message + + +def finish_elsewhere(task_id): + """Commit SUCCESSFUL onto the task row from a connection of the test's own.""" + other = connections.create_connection("default") + try: + table = other.ops.quote_name(OxTask._meta.db_table) + with other.cursor() as cursor: + cursor.execute( + f"UPDATE {table} SET status = %s WHERE id = %s", # noqa: S608 + [ + OxTask.Status.SUCCESSFUL, + OxTask._meta.pk.get_db_prep_value(uuid.UUID(task_id), other), + ], + ) + assert cursor.rowcount == 1 + finally: + other.close() + + +class BehindASnapshot: + """ + A caller's transaction on a thread of its own. It reads the holder, + which fixes the snapshot at REPEATABLE READ and SERIALIZABLE, waits + while the test changes the holder, and then enqueues the same key in + that transaction. After a UniqueEnqueueRace it checks the transaction + still answers a query, commits it, and retries the enqueue in a new + transaction at READ COMMITTED, as the message says to. + """ + + def __init__(self, level, holder_id, seen, enqueue): + self.outcome = {} + self._taken = threading.Event() + self._changed = threading.Event() + self._thread = threading.Thread( + target=self._run, args=(level, holder_id, seen, enqueue) + ) + self._thread.start() + assert self._taken.wait(timeout=10) + assert "error" not in self.outcome, self.outcome["error"] + + def _run(self, level, holder_id, seen, enqueue): + outcome = self.outcome + try: + if level != "READ COMMITTED": + set_session_isolation(level) + with transaction.atomic(): + # The first read fixes the snapshot. + assert OxTask.objects.get(id=holder_id).status == OxTask.Status[seen] + self._taken.set() + assert self._changed.wait(timeout=10) + with opened_connections() as opened: + try: + outcome["result"] = enqueue() + except UniqueEnqueueRace as exc: + outcome["race"] = exc + outcome["usable"] = OxTask.objects.filter(id=holder_id).count() + outcome["opened"] = [wrapper.alias for wrapper in opened] + outcome["closed"] = all( + wrapper.connection is None for wrapper in opened + ) + if "race" in outcome: + # The retry the message asks for, in a new transaction at + # READ COMMITTED. + with transaction.atomic(): + if connections["default"].vendor == "postgresql": + set_transaction_isolation("READ COMMITTED") + outcome["retried_level"] = session_isolation() + outcome["retried"] = enqueue() + except BaseException as exc: # pragma: no cover - reported by resume() + outcome["error"] = exc + finally: + self._taken.set() + connections.close_all() + + def resume(self): + """Let the enqueue run, and return what happened.""" + self._changed.set() + self._thread.join(timeout=30) + assert not self._thread.is_alive() + assert "error" not in self.outcome, self.outcome["error"] + return self.outcome + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_holder_retried_behind_a_snapshot_keeps_its_lock(worker, level): + """ + The duplicate path reads a settled holder with a locking read, so a + status the transaction's snapshot no longer agrees with cannot release + a live lock. + + A caller that read the holder as FAILED, then enqueued the same key + while an operator retry committed READY in between, used to read FAILED + again, delete the lock as stale, and insert a second READY row beside + the retried one. The engines part at the locking read. InnoDB's is a + current read, so the enqueue sees READY and returns the retried task. + PostgreSQL refuses to lock a row that changed after the snapshot began + (SQLSTATE 40001), which used to reach the caller raw. It is now + UniqueEnqueueRace, chained to that failure, saying to retry in a new, + short transaction at READ COMMITTED, and that retry returns the retried + task. A holder the snapshot calls settled is not read again elsewhere. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and there is no snapshot") + first = tasks.unique_fail_always.enqueue() + while worker.run_once(): + pass + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + + caller = BehindASnapshot( + level, first.id, "FAILED", tasks.unique_fail_always.enqueue + ) + assert actions.retry(first.id) is True + assert OxTask.objects.get(id=first.id).status == OxTask.Status.READY + outcome = caller.resume() + + if vendor == "postgresql": + assert_retry_outside_the_snapshot(outcome.get("race"), level) + assert "The row was the task that held the key" in str(outcome["race"]) + assert outcome["retried_level"] == "READ COMMITTED" + assert outcome["retried"].id == first.id + else: + assert "race" not in outcome, outcome["race"] + assert outcome["result"].id == first.id + assert outcome["opened"] == [] + ready = OxTask.objects.filter( + task_path="tests_oxpull.tasks.unique_fail_always", status=OxTask.Status.READY + ) + assert ready.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("change", ["finished", "lost", "pruned"]) +@pytest.mark.parametrize("seen", ["READY", "RUNNING"]) +@pytest.mark.parametrize("level", LEVELS) +def test_a_holder_settled_behind_a_snapshot_is_not_deduplicated_against( + monkeypatch, worker, level, seen, change +): + """ + A holder that settles after the caller's snapshot began does not keep + its key. Its worker finishes it, or the reaper records it LOST, or its + row is pruned after it finished. + + The holder read takes no lock, so inside a caller's transaction at + REPEATABLE READ, or at PostgreSQL's SERIALIZABLE, it answers from the + snapshot and still calls the holder READY or RUNNING. On PostgreSQL the + enqueue returned that settled task's result and queued nothing. The + enqueue now asks for a share lock on the holder's row without waiting, + in a savepoint it rolls back at once. PostgreSQL refuses a row that + changed after the snapshot began. The enqueue then reads the holder and + the key's lock once on a connection of its own, finds the task settled + or gone, and raises UniqueEnqueueRace saying to retry in a new, short + transaction at READ COMMITTED. That retry queues the new task. A key + share lock would have been granted after the outcome and LOST writes, + which change no key column. + + MySQL at REPEATABLE READ reads the committed status on a connection of + its own, as 1.2.0 does. READ COMMITTED reads the current row. InnoDB's + SERIALIZABLE makes this test's own read a locking read, which would + hold off the worker; test_concurrency pins that level. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and there is no snapshot") + first = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() if seen == "RUNNING" else None + checks = record_holder_checks(monkeypatch) + + caller = BehindASnapshot( + level, first.id, seen, lambda: tasks.unique_echo.enqueue("held") + ) + if db_task is None: + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == first.id + if change == "lost": + expired = timezone.now() - timedelta(hours=1) + OxTask.objects.filter(pk=db_task.pk).update( + attempts=db_task.max_attempts, locked_at=expired, lease_expires_at=expired + ) + assert worker.reap() == 1 + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.LOST + else: + worker.execute(db_task) + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.SUCCESSFUL + if change == "pruned": + assert OxTask.objects.filter(pk=db_task.pk).delete()[0] == 1 + outcome = caller.resume() + + snapshot = level != "READ COMMITTED" + if vendor == "postgresql" and snapshot: + found = { + "finished": "the task SUCCESSFUL", + "lost": "the task LOST", + "pruned": "the task's row gone", + }[change] + assert_retry_outside_the_snapshot(outcome.get("race"), level, found) + assert outcome["usable"] == 1 + assert outcome["retried_level"] == "READ COMMITTED" + second = outcome["retried"] + else: + assert "race" not in outcome, outcome["race"] + second = outcome["result"] + assert second.id != first.id, "deduplicated against a task that had settled" + ready = OxTask.objects.filter(status=OxTask.Status.READY) + assert [str(task.id) for task in ready] == [second.id] + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + assert checks == (["40001"] if vendor == "postgresql" and snapshot else []) + assert outcome["opened"] == (["default"] if snapshot else []) + assert outcome["closed"] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("change", ["renewed", "claimed", "requeued"]) +@pytest.mark.parametrize("level", LEVELS) +def test_a_holder_still_pending_but_changed_behind_a_snapshot( + monkeypatch, worker, level, change +): + """ + A holder that is still READY or RUNNING, but whose row changed after the + caller's snapshot began, is deduplicated. Its worker renewed the lease, + a worker claimed it, or the reaper put it back to READY. + + On PostgreSQL at REPEATABLE READ or SERIALIZABLE the share lock request + is refused (40001), and the enqueue reads the holder and the key's lock + once on a connection of its own. The task is still pending and the lock + still names it, so the enqueue returns the result built from that read, + not the snapshot's row, and closes the connection. No second task is + queued. MySQL at REPEATABLE READ reads the committed status elsewhere and + returns the snapshot's row, as 1.2.0 does. READ COMMITTED reads the + current row. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and there is no snapshot") + first = tasks.unique_echo.enqueue("held") + seen = "READY" if change == "claimed" else "RUNNING" + db_task = None + if change != "claimed": + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == first.id + if change == "requeued": + expired = timezone.now() - timedelta(hours=1) + OxTask.objects.filter(pk=db_task.pk).update( + locked_at=expired, lease_expires_at=expired + ) + + checks = record_holder_checks(monkeypatch) + caller = BehindASnapshot( + level, first.id, seen, lambda: tasks.unique_echo.enqueue("held") + ) + if change == "renewed": + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + elif change == "claimed": + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == first.id + else: + assert worker.reap() == 1 + assert OxTask.objects.get(pk=db_task.pk).status == OxTask.Status.READY + outcome = caller.resume() + + snapshot = level != "READ COMMITTED" + assert "race" not in outcome, outcome["race"] + result = outcome["result"] + assert result.id == first.id + if vendor == "mysql" and snapshot: + # 1.2.0's path: the status is read elsewhere, the result is the + # snapshot's row. + assert result.status == seen + else: + committed = task_result_from_db(OxTask.objects.get(id=first.id)) + assert result_fields(result) == result_fields(committed) + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == first.id + assert checks == (["40001"] if vendor == "postgresql" and snapshot else []) + assert outcome["opened"] == (["default"] if snapshot else []) + assert outcome["closed"] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_a_pending_holder_the_key_no_longer_names_is_not_deduplicated_against( + monkeypatch, worker, level +): + """ + The read on a connection of its own deduplicates only when the key's lock + still names the holder. + + The caller's snapshot sees the first task READY and holding the key. The + task then fails for good, another enqueue of the key queues a second task + and takes the lock, and an operator retry puts the first task back to + READY beside it, as django_ox.actions.retry may. The caller then enqueues + the key. Its snapshot still shows the first task holding the key, and + that row changed, so the enqueue reads again elsewhere. The first task is + pending there, but the lock names the second task. Returning the first + task would deduplicate against a task that no longer holds the key, so + the enqueue raises UniqueEnqueueRace. The retry at READ COMMITTED returns + the second task. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("MySQL reads the holder's status alone, as 1.2.0 does") + first = tasks.unique_fail_always.enqueue() + checks = record_holder_checks(monkeypatch) + caller = BehindASnapshot(level, first.id, "READY", tasks.unique_fail_always.enqueue) + while worker.run_once(): + pass + assert OxTask.objects.get(id=first.id).status == OxTask.Status.FAILED + second = tasks.unique_fail_always.enqueue() + assert second.id != first.id + assert actions.retry(first.id) is True + outcome = caller.resume() + + assert_retry_outside_the_snapshot( + outcome.get("race"), level, "the key held by another task" + ) + assert checks == ["40001"] + assert outcome["opened"] == ["default"] + assert outcome["closed"] + assert outcome["retried"].id == second.id + assert str(OxpullUniqueLock.objects.get().task_id) == second.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_fresh_read_asks_for_the_lock_of_the_key_being_enqueued( + monkeypatch, worker, level +): + """ + The read on a connection of its own reads the lock of the key being + enqueued, not any lock. + + Two keys are held by two RUNNING tasks, and the other key's lock is the + older row. The caller's snapshot sees the second task, whose lease is + then renewed, and the caller enqueues the second key. The fresh read + must find the second task named by its own key's lock, and the enqueue + returns it. A read that took another key's lock would find the key held + by another task and raise UniqueEnqueueRace. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("MySQL reads the holder's status alone, as 1.2.0 does") + other = tasks.unique_echo.enqueue("other") + older = timezone.now() - timedelta(hours=1) + assert ( + OxpullUniqueLock.objects.filter(task_id=other.id).update(created_at=older) == 1 + ) + held = tasks.unique_echo.enqueue("held") + claimed = {str(task.id): task for task in (worker.claim_one(), worker.claim_one())} + assert set(claimed) == {other.id, held.id} + db_task = claimed[held.id] + checks = record_holder_checks(monkeypatch) + caller = BehindASnapshot( + level, held.id, "RUNNING", lambda: tasks.unique_echo.enqueue("held") + ) + with worker._in_flight_lock: + worker._in_flight.add((db_task.pk, db_task.lease_epoch)) + assert worker.renew_leases() == 1 + outcome = caller.resume() + + assert "race" not in outcome, outcome["race"] + committed = task_result_from_db(OxTask.objects.get(id=held.id)) + assert result_fields(outcome["result"]) == result_fields(committed) + assert checks == ["40001"] + assert outcome["opened"] == ["default"] + assert outcome["closed"] + assert OxTask.objects.count() == 2 + named = OxpullUniqueLock.objects.values_list("task_id", flat=True) + assert sorted(str(task_id) for task_id in named) == sorted([other.id, held.id]) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("how", ["options", "session", "transaction"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_holder_check_follows_the_level_however_it_was_set(monkeypatch, level, how): + """ + A project sets PostgreSQL's level in OPTIONS["isolation_level"], which + the driver applies as each transaction begins. A session can set its + own, and a transaction can set its own as its first statement. The + driver knows only the first. The holder read asks the transaction with + current_setting('transaction_isolation'), in the same statement, so a + holder that finished after the snapshot began raises UniqueEnqueueRace + whichever way the level was set. Asked the driver, the other two would + read as READ COMMITTED and the enqueue would return the finished task. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + first = tasks.unique_echo.enqueue("x") + connections["default"].close() + if how == "options": + isolation_for_new_connections(monkeypatch, level) + outcome = {} + try: + if how == "session": + set_session_isolation(level) + with transaction.atomic(): + if how == "transaction": + set_transaction_isolation(level) + outcome["level"] = session_isolation() + assert OxTask.objects.get(id=first.id).status == OxTask.Status.READY + finish_elsewhere(first.id) + with opened_connections() as opened: + try: + outcome["second"] = tasks.unique_echo.enqueue("x") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + outcome["driver"] = connections["default"].connection.isolation_level + finally: + connections["default"].close() + + assert outcome["level"] == level + assert (outcome["driver"] is None) is (how != "options") + assert "second" not in outcome, "deduplicated against a task that had settled" + assert_retry_outside_the_snapshot(outcome.get("race"), level, "the task SUCCESSFUL") + assert [wrapper.alias for wrapper in opened] == ["default"] + assert all(wrapper.connection is None for wrapper in opened) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("how", ["atomic", "autocommit-off"]) +@pytest.mark.parametrize("level", LEVELS) +def test_a_winner_committed_behind_the_callers_snapshot_is_named_in_the_race( + level, how +): + """ + The known corner, and what the enqueue says about it. + + A caller's transaction reads, another enqueue of the same key commits, + and the caller then enqueues that key. At READ COMMITTED the enqueue + sees the winner and returns it. At MySQL's REPEATABLE READ, and at + PostgreSQL's REPEATABLE READ or SERIALIZABLE, the lock row is read from + the caller's snapshot, which cannot see the winner. The enqueue raises + UniqueEnqueueRace after five passes rather than queue a second task. + The message names the snapshot wherever the session reads one, and says + to retry the whole transaction. The retried transaction returns the + winner. At MySQL's SERIALIZABLE the caller's read locks, so the winner + could not commit in between. A connection with autocommit turned off is + the caller's transaction as surely as an atomic block, and the message + says so there too. + """ + vendor = connections["default"].vendor + if vendor == "sqlite": + pytest.skip("SQLite's writer waits on the open reader") + if vendor == "mysql" and level == "SERIALIZABLE": + pytest.skip("InnoDB SERIALIZABLE reads lock, and the winner would wait") + + snapshot_taken = threading.Event() + won = threading.Event() + outcome = {} + + def enqueue_behind_the_snapshot(): + try: + set_session_isolation(level) + with callers_transaction(how): + assert OxpullUniqueLock.objects.count() == 0 + snapshot_taken.set() + assert won.wait(timeout=10) + try: + outcome["second"] = tasks.unique_echo.enqueue("corner") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + if "race" in outcome: + with transaction.atomic(): + outcome["second"] = tasks.unique_echo.enqueue("corner") + except BaseException as exc: # pragma: no cover - reported below + outcome["error"] = exc + finally: + snapshot_taken.set() + connections.close_all() + + thread = threading.Thread(target=enqueue_behind_the_snapshot) + thread.start() + assert snapshot_taken.wait(timeout=10) + winner = tasks.unique_echo.enqueue("corner") + won.set() + thread.join(timeout=30) + assert not thread.is_alive() + + assert "error" not in outcome, outcome["error"] + assert outcome["second"].id == winner.id + if level == "READ COMMITTED": + assert "race" not in outcome + else: + message = str(outcome.get("race")) + assert "snapshot" in message + assert "retry the whole transaction" in message + assert OxTask.objects.count() == 1 + + +# Statements a duplicate of a live holder runs on the caller's connection in +# 1.2.0, as Django's query log records them: BEGIN, COMMIT and savepoints +# included. Keyed by engine, level and whether a caller's transaction is open. +STATEMENTS_IN_1_2_0 = { + ("postgresql", False): 10, + ("postgresql", True): 12, + ("mysql", False): 10, + ("mysql", True): 13, + ("sqlite", False): 8, + ("sqlite", True): 10, +} + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("ambient", [False, True], ids=["own", "ambient"]) +@pytest.mark.parametrize("level", LEVELS) +def test_what_a_duplicate_of_a_live_holder_runs(monkeypatch, worker, level, ambient): + """ + The statements a duplicate of a live, unchanged holder runs, against + 1.2.0's count. + + On PostgreSQL inside a caller's transaction the holder read also reads + the transaction's level, in the same statement. At REPEATABLE READ and + SERIALIZABLE three statements follow: a savepoint, the share lock + request without waiting, and the rollback to that savepoint, which is + never released instead. The lock is granted and the holder returned. + At READ COMMITTED, when the enqueue owns its transactions, on MySQL and + on SQLite the count is 1.2.0's, and the level is never asked with a + statement of its own. No connection is opened, except MySQL's at + REPEATABLE READ inside a caller's transaction, as in 1.2.0. + """ + vendor = connections["default"].vendor + if vendor == "sqlite" and level != "READ COMMITTED": + pytest.skip("SQLite has one isolation level") + holder = tasks.unique_echo.enqueue("held") + db_task = worker.claim_one() + assert db_task is not None and str(db_task.id) == holder.id + connection = connections["default"] + connection.close() + if vendor != "sqlite": + isolation_for_new_connections(monkeypatch, level) + connection.ensure_connection() + try: + with ( + CaptureQueriesContext(connection) as captured, + opened_connections() as opened, + ): + if ambient: + with transaction.atomic(): + result = tasks.unique_echo.enqueue("held") + else: + result = tasks.unique_echo.enqueue("held") + finally: + connection.close() + + assert result.id == holder.id + assert result.status == TaskResultStatus.RUNNING + sql = [query["sql"] for query in captured.captured_queries] + checked = vendor == "postgresql" and ambient and level != "READ COMMITTED" + mysql_level = vendor == "mysql" and ambient + expected = STATEMENTS_IN_1_2_0[(vendor, ambient)] + (3 if checked else 0) + assert len(sql) == expected, sql + share = [index for index, statement in enumerate(sql) if "FOR SHARE" in statement] + if checked: + (index,) = share + assert sql[index].endswith("FOR SHARE NOWAIT") + assert "KEY SHARE" not in sql[index] + assert sql[index - 1].startswith("SAVEPOINT ") + savepoint = sql[index - 1].split()[1] + assert sql[index + 1] == f"ROLLBACK TO SAVEPOINT {savepoint}" + assert not any( + statement == f"RELEASE SAVEPOINT {savepoint}" for statement in sql + ) + else: + assert share == [] + level_reads = [s for s in sql if "current_setting('transaction_isolation')" in s] + assert len(level_reads) == (1 if vendor == "postgresql" and ambient else 0) + assert not any("SHOW transaction_isolation" in statement for statement in sql) + asked = [s for s in sql if "@@transaction_isolation" in s] + assert len(asked) == (1 if mysql_level else 0) + assert [wrapper.alias for wrapper in opened] == ( + ["default"] if mysql_level and level == "REPEATABLE READ" else [] + ) + assert OxpullUniqueLock.objects.count() == 1 + + +# -- the enqueue's error paths, checked on every database ---------------- + + +class StubCursor: + def __init__(self, connection): + self.connection = connection + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + def execute(self, sql): + self.connection.executed.append(sql) + + def fetchone(self): + return (self.connection.reported,) + + +class StubConnection: + """A connection that answers the level query with a fixed string.""" + + def __init__(self, vendor, reported=None): + self.vendor = vendor + self.reported = reported + self.executed = [] + self.in_atomic_block = False + self.needs_rollback = False + + def cursor(self): + return StubCursor(self) + + +def refused(sqlstate, attribute="sqlstate"): + """ + An OperationalError as Django wraps a driver's, with that SQLSTATE. + psycopg 3 carries it as sqlstate, psycopg2 as pgcode. + """ + + class Cause(Exception): + pass + + cause = Cause() + setattr(cause, attribute, sqlstate) + exc = OperationalError(f"stubbed {sqlstate}") + exc.__cause__ = cause + return exc + + +def sqlstate_of(exc): + return pg_sqlstate(exc) + + +@pytest.mark.parametrize("attribute", ["sqlstate", "pgcode"]) +@pytest.mark.parametrize( + ("vendor", "sqlstate", "refusal"), + [ + ("postgresql", "40001", True), + ("postgresql", "55P03", False), + ("postgresql", None, False), + ("mysql", "40001", False), + ("sqlite", "40001", False), + ], +) +def test_only_postgresql_reports_a_refused_changed_row( + monkeypatch, vendor, sqlstate, refusal, attribute +): + """ + SQLSTATE 40001 is PostgreSQL's serialization_failure, and MySQL reports + its deadlocks under the same SQLSTATE. A deadlock keeps its own path, so + only PostgreSQL's code is read as a refused changed row. psycopg 3 and + psycopg2 carry the code under different names, and both are read, for + lock_not_available as well. + """ + monkeypatch.setattr( + backend_module, "connections", {"default": StubConnection(vendor)} + ) + exc = refused(sqlstate, attribute) + assert backend_module._is_serialization_failure(exc, "default") is refusal + waited = vendor == "postgresql" and sqlstate == "55P03" + assert backend_module._is_lock_wait_timeout(exc, "default") is waited + + +@pytest.mark.parametrize("attribute", ["sqlstate", "pgcode"]) +@pytest.mark.parametrize( + ("sqlstate", "answer"), + [ + (None, "current"), + ("55P03", "current"), + ("40001", "changed"), + ("57014", "raises"), + ], + ids=["granted", "lock-not-available", "serialization-failure", "query-canceled"], +) +def test_the_holder_check_rolls_back_its_savepoint_whatever_it_meets( + monkeypatch, sqlstate, answer, attribute +): + """ + The check asks for its lock in a savepoint and rolls that savepoint back + every time, granted or refused, and never releases it: released, the + share lock would stay with the caller's transaction until it commits. + Granted, or refused with lock_not_available because another transaction + is writing or locking the row, the check reports nothing. Refused with + serialization_failure, which says the row changed after the snapshot + began, it returns that failure for a fresh read to decide on. Anything + else propagates. Stubbed so it runs on every database, with the SQLSTATE + where psycopg 3 puts it and where psycopg2 does. On Django 6.1 the + savepoint is opened with savepoint_create(), never with the deprecated + savepoint(). + """ + calls = [] + refusal = None if sqlstate is None else refused(sqlstate, attribute) + + def lock(task_pk, db_alias): + calls.append(("lock", task_pk, db_alias)) + if refusal is not None: + raise refusal + + monkeypatch.setattr(OxpullBackend, "_lock_holder_nowait", staticmethod(lock)) + opens = ( + "savepoint_create" + if hasattr(backend_module.transaction, "savepoint_create") + else "savepoint" + ) + monkeypatch.setattr( + backend_module.transaction, + opens, + lambda using: calls.append(("savepoint", using)) or "sid", + ) + if opens != "savepoint": + + def deprecated(using=None): + raise AssertionError("opened with the deprecated transaction.savepoint()") + + monkeypatch.setattr(backend_module.transaction, "savepoint", deprecated) + monkeypatch.setattr( + backend_module.transaction, + "savepoint_rollback", + lambda sid, using: calls.append(("rollback", sid, using)), + ) + monkeypatch.setattr( + backend_module.transaction, + "savepoint_commit", + lambda sid, using: calls.append(("release", sid, using)), + ) + if answer == "raises": + with pytest.raises(OperationalError) as excinfo: + OxpullBackend._check_holder_row("pk", "default") + assert excinfo.value is refusal + elif answer == "changed": + assert OxpullBackend._check_holder_row("pk", "default") is refusal + else: + assert OxpullBackend._check_holder_row("pk", "default") is None + assert calls == [ + ("savepoint", "default"), + ("lock", "pk", "default"), + ("rollback", "sid", "default"), + ] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("path", ["unchanged", "finished"]) +@pytest.mark.parametrize("level", ["REPEATABLE READ", "SERIALIZABLE"]) +def test_the_holder_check_raises_no_warning(monkeypatch, level, path): + """ + A duplicate that runs the holder check raises no warning. Django 6.1 + deprecates transaction.savepoint() with RemovedInDjango70Warning, so a + project that turns warnings into errors would fail every such duplicate. + Warnings are errors while the enqueue runs. The holder's row is unchanged, + so the check is granted, or the task finished, so the check is refused and + the holder is read again on a new connection. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("the holder check is PostgreSQL's") + holder = tasks.unique_echo.enqueue("held") + checks = record_holder_checks(monkeypatch) + set_session_isolation(level) + outcome = {} + try: + with transaction.atomic(): + assert OxTask.objects.get(id=holder.id).status == OxTask.Status.READY + if path == "finished": + finish_elsewhere(holder.id) + with warnings.catch_warnings(): + warnings.simplefilter("error") + try: + outcome["result"] = tasks.unique_echo.enqueue("held") + except UniqueEnqueueRace as exc: + outcome["race"] = exc + finally: + connections["default"].close() + + if path == "unchanged": + assert "race" not in outcome, outcome["race"] + assert outcome["result"].id == holder.id + assert checks == ["granted"] + else: + assert_retry_outside_the_snapshot( + outcome.get("race"), level, "the task SUCCESSFUL" + ) + assert checks == ["40001"] + + +FRESH_READ_FINDS = { + "40001-settled": "the task SUCCESSFUL", + "40001-gone": "the task's row gone", + "40001-lock-gone": "the key's lock gone", + "40001-another-task": "the key held by another task", +} + +# A fresh read that fails, and the SQLSTATE it fails with. +FRESH_READ_FAILS = { + "40001-read-lock-timeout": "55P03", + "40001-read-statement-timeout": "57014", + "40001-read-refused": None, +} + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "answer", + [ + "read committed", + "granted", + "55P03", + "40001-pending", + *FRESH_READ_FINDS, + *FRESH_READ_FAILS, + "57014", + ], +) +def test_what_a_duplicate_does_with_each_answer_to_the_holder_check( + monkeypatch, answer +): + """ + Each answer the holder check can get, through a real enqueue inside a + caller's transaction on every database. The level the holder read + reports and the lock request are stubbed. So is the fresh read, where it + has to find something other than the holder as it stands, or fail. + + At READ COMMITTED the check does not run. Granted, or refused because + another transaction is changing the row (55P03), the enqueue returns the + holder and opens no connection. Refused because the row changed after + the snapshot began (40001), the enqueue reads the holder and the key's + lock once on a connection of its own. When the holder is pending and the + lock names it, the enqueue returns the result of that read. When that + read finds the task settled or gone, or the lock gone or naming another + task, the enqueue raises UniqueEnqueueRace, chained to the refusal, and + says what it found. When the read fails, by its lock timeout, its + statement timeout or a refused connection, the enqueue raises + UniqueEnqueueRace chained to that failure, with the failure's first line + in the message. Any other error from the check reaches the caller as it + was raised. Each time nothing is written, every connection opened is + closed, and the caller's transaction still answers a query. + """ + holder = tasks.unique_echo.enqueue("held") + level = "read committed" if answer == "read committed" else "repeatable read" + monkeypatch.setattr( + OxpullBackend, + "_isolation_level_sql", + staticmethod(lambda db_alias: f"'{level}'"), + ) + checked = [] + code = answer.split("-")[0] + + def lock(task_pk, db_alias): + checked.append(str(task_pk)) + if code in ("55P03", "40001", "57014"): + raise refused(code) + + monkeypatch.setattr(OxpullBackend, "_lock_holder_nowait", staticmethod(lock)) + + failure = None + if answer in FRESH_READ_FAILS: + failed_with = FRESH_READ_FAILS[answer] + failure = ( + refused(failed_with) + if failed_with + else OperationalError( + 'connection failed: connection to server at "127.0.0.1", port 1 ' + "failed: Connection refused\n\tIs the server running on that host?" + ) + ) + + def fresh(task_id, key, db_alias): + if failure is not None: + raise failure + row = OxTask.objects.get(id=task_id) + if answer == "40001-settled": + row.status = OxTask.Status.SUCCESSFUL + return row, row.pk + if answer == "40001-gone": + return None, None + if answer == "40001-lock-gone": + return row, None + return row, uuid.uuid4() + + if answer in FRESH_READ_FINDS or answer in FRESH_READ_FAILS: + monkeypatch.setattr(OxpullBackend, "_read_holder_fresh", staticmethod(fresh)) + with opened_connections() as opened, transaction.atomic(): + if answer in FRESH_READ_FINDS: + with pytest.raises(UniqueEnqueueRace) as excinfo: + tasks.unique_echo.enqueue("held") + assert sqlstate_of(excinfo.value.__cause__) == "40001" + message = str(excinfo.value) + assert f"A fresh read found {FRESH_READ_FINDS[answer]}." in message + assert "on database 'default'" in message + assert "inside a transaction at REPEATABLE READ," in message + elif answer in FRESH_READ_FAILS: + with pytest.raises(UniqueEnqueueRace) as excinfo: + tasks.unique_echo.enqueue("held") + assert excinfo.value.__cause__ is failure + message = str(excinfo.value) + first_line = str(failure).splitlines()[0] + assert f"A fresh read on a new connection failed ({first_line})." in message + assert "Is the server running" not in message + waited = FRESH_READ_FAILS[answer] == "55P03" + assert ("That read waits at most 100ms for a lock." in message) is waited + assert "on database 'default'" in message + assert "inside a transaction at REPEATABLE READ," in message + assert "The enqueue created nothing." in message + elif answer == "57014": + with pytest.raises(OperationalError) as excinfo: + tasks.unique_echo.enqueue("held") + assert sqlstate_of(excinfo.value) == "57014" + else: + result = tasks.unique_echo.enqueue("held") + assert result.id == holder.id + expected = task_result_from_db(OxTask.objects.get(id=holder.id)) + assert result_fields(result) == result_fields(expected) + assert OxTask.objects.count() == 1 + assert checked == ([] if answer == "read committed" else [holder.id]) + assert [wrapper.alias for wrapper in opened] == ( + ["default"] if answer == "40001-pending" else [] + ) + assert all(wrapper.connection is None for wrapper in opened) + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("owner", ["enqueue", "atomic", "autocommit-off"]) +def test_a_refused_changed_row_is_reissued_or_raised_by_who_owns_the_transaction( + monkeypatch, owner +): + """ + What the enqueue does with PostgreSQL's refusal of a changed row, checked + on every database by raising that failure in place of the first + resolution. When the enqueue owns its transactions the failure rolled + back only its own work, so the pass is reissued and the next one finds + the holder. Inside a caller's transaction the snapshot is the caller's, + so the enqueue raises UniqueEnqueueRace, chained to the failure, saying + to retry the whole transaction, and writes nothing. That transaction is + an atomic block, or a connection with autocommit turned off. + """ + holder = tasks.unique_echo.enqueue("held") + refusal = refused("40001") + existing_result = OxpullBackend._existing_result + calls = [] + + def refuse_the_first(self, *args, **kwargs): + calls.append(kwargs["ambient"]) + if len(calls) == 1: + raise refusal + return existing_result(self, *args, **kwargs) + + monkeypatch.setattr(OxpullBackend, "_existing_result", refuse_the_first) + monkeypatch.setattr( + backend_module, + "_is_serialization_failure", + lambda exc, db_alias: sqlstate_of(exc) == "40001", + ) + if owner == "enqueue": + assert tasks.unique_echo.enqueue("held").id == holder.id + assert calls == [False, False] + else: + with ( + callers_transaction(owner), + pytest.raises(UniqueEnqueueRace, match="new, short transaction") as excinfo, + ): + tasks.unique_echo.enqueue("held") + assert excinfo.value.__cause__ is refusal + assert calls == [True] + assert OxTask.objects.count() == 1 + assert str(OxpullUniqueLock.objects.get().task_id) == holder.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("owner", ["enqueue", "atomic", "autocommit-off"]) +def test_a_deadlock_is_reissued_only_when_the_enqueue_owns_its_transaction( + monkeypatch, owner +): + """ + An InnoDB deadlock rolls back the victim's whole transaction. When the + enqueue owns that transaction nothing else was lost, and the enqueue tries + again. Inside a caller's transaction the caller's work went with it, so the + error reaches the caller as raised and nothing is written. A connection + with autocommit turned off is a caller's transaction, as an atomic block + is. Checked on every database by raising MySQL's deadlock in place of the + first lock insert. + """ + from django.db.models.query import QuerySet + + deadlock = OperationalError( + 1213, "Deadlock found when trying to get lock; try restarting transaction" + ) + create = QuerySet.create + calls = [] + + def deadlock_the_first_lock_insert(self, **kwargs): + if self.model is OxpullUniqueLock: + calls.append(kwargs["key"]) + if len(calls) == 1: + raise deadlock + return create(self, **kwargs) + + monkeypatch.setattr(QuerySet, "create", deadlock_the_first_lock_insert) + monkeypatch.setattr( + backend_module, + "is_innodb_deadlock", + lambda exc, db_alias: bool(exc.args) and exc.args[0] == 1213, + ) + if owner == "enqueue": + result = tasks.unique_echo.enqueue("held") + assert len(calls) == 2 + assert [str(task.id) for task in OxTask.objects.all()] == [result.id] + assert str(OxpullUniqueLock.objects.get().task_id) == result.id + else: + with callers_transaction(owner), pytest.raises(OperationalError) as excinfo: + tasks.unique_echo.enqueue("held") + assert excinfo.value is deadlock + assert len(calls) == 1 + assert OxTask.objects.count() == 0 + assert OxpullUniqueLock.objects.count() == 0 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("ambient", [False, True], ids=["own", "ambient"]) +def test_a_refused_lock_update_is_reissued_or_raised_by_who_owns_the_transaction( + monkeypatch, ambient +): + """ + PostgreSQL's SERIALIZABLE can refuse the UPDATE that names the new task + on its lock. The refusal reached the caller raw. When the enqueue owns + its transaction it now tries again. Inside a caller's transaction it + raises UniqueEnqueueRace, chained to the refusal, and writes nothing. + Checked on every database by raising PostgreSQL's failure in place of + the first UPDATE. + """ + from django.db.models.query import QuerySet + + refusal = refused("40001") + update = QuerySet.update + calls = [] + + def refuse_the_first_lock_update(self, **kwargs): + if self.model is OxpullUniqueLock: + calls.append(kwargs) + if len(calls) == 1: + raise refusal + return update(self, **kwargs) + + monkeypatch.setattr(QuerySet, "update", refuse_the_first_lock_update) + monkeypatch.setattr( + backend_module, + "_is_serialization_failure", + lambda exc, db_alias: sqlstate_of(exc) == "40001", + ) + if ambient: + with ( + transaction.atomic(), + pytest.raises(UniqueEnqueueRace, match="new, short transaction") as excinfo, + ): + tasks.unique_echo.enqueue("held") + assert excinfo.value.__cause__ is refusal + assert len(calls) == 1 + assert OxTask.objects.count() == 0 + assert OxpullUniqueLock.objects.count() == 0 + else: + result = tasks.unique_echo.enqueue("held") + assert len(calls) == 2 + assert [str(task.id) for task in OxTask.objects.all()] == [result.id] + assert str(OxpullUniqueLock.objects.get().task_id) == result.id + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "released", [False, True], ids=["first-pass", "after-a-release"] +) +@pytest.mark.parametrize("ambient", [False, True], ids=["own", "ambient"]) +def test_a_serialization_failure_outside_the_resolution_propagates_raw( + monkeypatch, ambient, released +): + """ + Only a refusal met after the task is written, or while resolving a + duplicate, is reissued or reported as UniqueEnqueueRace. PostgreSQL's + SERIALIZABLE can also fail the task write itself with SQLSTATE 40001. + That failure says nothing about the holder or the lock, so it reaches + the caller once, as any other enqueue at that level raises it, and + nothing is written. The same holds on the pass after one that released + a settled holder's lock. Checked on every database. + """ + if released: + settled = tasks.unique_echo.enqueue("held") + OxTask.objects.filter(id=settled.id).update(status=OxTask.Status.FAILED) + + refusal = refused("40001") + calls = [] + + def refuse(self, task, args, kwargs): + calls.append(task.module_path) + raise refusal + + monkeypatch.setattr(OxBackend, "enqueue", refuse) + monkeypatch.setattr( + backend_module, + "_is_serialization_failure", + lambda exc, db_alias: sqlstate_of(exc) == "40001", + ) + with ( + transaction.atomic() if ambient else contextlib.nullcontext(), + pytest.raises(OperationalError) as excinfo, + ): + tasks.unique_echo.enqueue("held") + assert excinfo.value is refusal + assert calls == ["tests_oxpull.tasks.unique_echo"] + assert OxTask.objects.count() == (1 if released else 0) + assert OxpullUniqueLock.objects.count() == 0 + + +SNAPSHOT_NAMES = [ + ("mysql", "REPEATABLE-READ", "MySQL's REPEATABLE READ"), + ("postgresql", "repeatable read", "PostgreSQL's REPEATABLE READ"), + ("postgresql", "serializable", "PostgreSQL's SERIALIZABLE"), + ("mysql", "READ-COMMITTED", None), + ("mysql", "SERIALIZABLE", None), + ("postgresql", "read committed", None), + ("sqlite", None, None), +] + + +@pytest.mark.parametrize(("vendor", "reported", "names"), SNAPSHOT_NAMES) +def test_the_race_message_names_a_snapshot_only_where_the_session_reads_one( + monkeypatch, vendor, reported, names +): + """ + After five passes the message names the likely cause when the enqueue + ran inside a transaction whose session reads from a snapshot. That was + said for MySQL alone, and for a MySQL session at any level. At READ + COMMITTED and at MySQL's SERIALIZABLE the lock read is current, so it is + not the cause there. Outside a transaction the level is not asked. + """ + connection = StubConnection(vendor, reported) + monkeypatch.setattr(backend_module, "connections", {"default": connection}) + inside = default_task_backend._race_message( + tasks.unique_echo, "default", ambient=True + ) + asked = list(connection.executed) + outside = default_task_backend._race_message( + tasks.unique_echo, "default", ambient=False + ) + + assert inside.startswith(outside) + assert "after 5 attempts." in outside + assert "snapshot" not in outside + assert connection.executed == asked + assert len(asked) == (0 if vendor == "sqlite" else 1) + if names is None: + assert inside == outside + else: + assert names in inside + assert "snapshot" in inside + assert "retry the whole transaction" in inside + + +@pytest.mark.parametrize(("vendor", "reported", "names"), SNAPSHOT_NAMES) +def test_the_lock_wait_message_names_a_snapshot_only_where_the_session_reads_one( + monkeypatch, vendor, reported, names +): + """ + Inside a caller's transaction the lock-wait message says to retry the + whole transaction. It gave MySQL's REPEATABLE READ as the reason on every + database and at every level. It now names a snapshot only where the + session reads one, as the race message does. Outside a transaction the + level is not asked. + """ + connection = StubConnection(vendor, reported) + monkeypatch.setattr(backend_module, "connections", {"default": connection}) + exc = OperationalError("stopped waiting") + exc.__cause__ = Exception("stopped waiting") + outside = default_task_backend._lock_wait_message( + tasks.unique_echo, exc, "default", ambient_atomic=False + ) + assert connection.executed == [] + inside = default_task_backend._lock_wait_message( + tasks.unique_echo, exc, "default", ambient_atomic=True + ) + + assert inside.startswith(outside) + assert "whole transaction" not in outside + assert "whole transaction" in inside + if names is None: + assert "snapshot" not in inside + assert "REPEATABLE READ" not in inside + else: + assert names in inside + assert "snapshot" in inside diff --git a/tests_oxpull/test_upgrade.py b/tests_oxpull/test_upgrade.py new file mode 100644 index 0000000..5d026f2 --- /dev/null +++ b/tests_oxpull/test_upgrade.py @@ -0,0 +1,146 @@ +""" +The upgrade path, from the schema a released version left behind. + +Every other test in this suite runs against a database Django built by +applying all migrations to an empty file. That is not the path anyone +takes. A user arrives with the schema their last release created and +runs `migrate`, and nothing here exercised that until this file. + +0.3.1 is the last published version and it shipped 0001 and 0002 only. +Migration 0003 creates the table rate limiting counts in; it went out in +0.4.0, which was never published. An installation with the earlier oxpull +0.3.1 schema has an unapplied 0003, and the CHANGELOG tells them to run `migrate oxpull` +before enabling RATE_LIMITS. This asserts what happens if they do, and +what happens if they do not. +""" + +import pytest +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.db.utils import DatabaseError, OperationalError, ProgrammingError +from django.utils import timezone + +from oxpull.models import OxpullRateWindow + +# The state a 0.3.1 install is in: batches exist, rate limiting does not. +PUBLISHED_0_3_1 = "0002_oxpullbatch_oxpullbatchmember" + +MISSING_TABLE_ERRORS = (OperationalError, ProgrammingError, DatabaseError) + + +def _latest_migration() -> str: + """ + The newest oxpull migration, read from the graph rather than named. + + Restoring to a hard-coded migration silently rolls back every one + added after it was written, which breaks the tests that run next + rather than this file. Asking the graph cannot go stale. + """ + executor = MigrationExecutor(connection) + executor.loader.build_graph() + leaves = [ + name for app, name in executor.loader.graph.leaf_nodes() if app == "oxpull" + ] + assert len(leaves) == 1, f"expected one oxpull leaf migration, got {leaves}" + return leaves[0] + + +def _migrate(target: str) -> None: + """Move the oxpull app to `target`.""" + executor = MigrationExecutor(connection) + executor.loader.build_graph() + executor.migrate([("oxpull", target)]) + + +@pytest.fixture +def at_published_schema(): + """ + Rewinds oxpull to the 0.3.1 schema for one test, then restores it. + + The restore runs even when the test fails, because a database left at + 0002 breaks every test that follows it in the same session. + """ + latest = _latest_migration() + _migrate(PUBLISHED_0_3_1) + try: + yield + finally: + _migrate(latest) + + +@pytest.mark.django_db(transaction=True) +def test_the_published_schema_has_no_rate_limit_table(at_published_schema): + """ + Pins the premise. If 0003 ever moves into an earlier migration this + fails, and the rest of this file stops meaning anything. + """ + assert OxpullRateWindow._meta.db_table not in connection.introspection.table_names() + + +@pytest.mark.django_db(transaction=True) +def test_using_rate_limits_before_migrating_fails_on_the_missing_table( + at_published_schema, +): + """ + The footgun the CHANGELOG warns about, made concrete: on the 0.3.1 + schema the limiter's own table is not there, and the failure is a + database error rather than anything oxpull can explain. + """ + with pytest.raises(MISSING_TABLE_ERRORS): + OxpullRateWindow.objects.filter(limit_name="anything").exists() + + +@pytest.mark.django_db(transaction=True) +def test_migrating_from_the_published_schema_creates_the_rate_limit_table(): + """ + The upgrade from the earlier oxpull 0.3.1 schema, then + `migrate`. The table has to exist afterwards and be usable, not merely + be recorded as applied. + """ + _migrate(PUBLISHED_0_3_1) + try: + assert ( + OxpullRateWindow._meta.db_table + not in connection.introspection.table_names() + ) + _migrate(_latest_migration()) + assert OxpullRateWindow._meta.db_table in connection.introspection.table_names() + + # Applied is not the same as working: write and read a window. + OxpullRateWindow.objects.create( + limit_name="upgrade-check", + window_started_at=timezone.now(), + used=1, + ) + assert OxpullRateWindow.objects.get(limit_name="upgrade-check").used == 1 + OxpullRateWindow.objects.filter(limit_name="upgrade-check").delete() + finally: + _migrate(_latest_migration()) + + +@pytest.mark.django_db(transaction=True) +def test_the_upgrade_leaves_batch_rows_alone(): + """ + 0003 adds a table and must not touch the batch table containing the existing + `oxpull` 0.3.1 data. A row written before the upgrade survives it. + """ + from oxpull.models import OxpullBatch + + table = OxpullRateWindow._meta.db_table + _migrate(PUBLISHED_0_3_1) + try: + # The row is written on the old schema, which is the whole point: + # asserting it survives is only meaningful if a schema change + # actually happened around it. + assert table not in connection.introspection.table_names() + batch = OxpullBatch.objects.create( + name="pre-upgrade", created_at=timezone.now() + ) + + _migrate(_latest_migration()) + + assert table in connection.introspection.table_names() + assert OxpullBatch.objects.filter(pk=batch.pk).exists() + assert OxpullBatch.objects.get(pk=batch.pk).name == "pre-upgrade" + finally: + _migrate(_latest_migration()) diff --git a/tests_oxpull/test_worker_subclass.py b/tests_oxpull/test_worker_subclass.py new file mode 100644 index 0000000..22b7f6d --- /dev/null +++ b/tests_oxpull/test_worker_subclass.py @@ -0,0 +1,514 @@ +""" +Worker subclasses that narrow what they claim, on top of rate limits. + +django-ox promises that a Worker subclass overriding claim_filter_q() alone +keeps its condition on every database. On PostgreSQL the worker gives up the +single-statement claim, which reads only claim_filter_sql(), for the +SELECT ... FOR UPDATE SKIP LOCKED path, which applies claim_filter_q(). + +A subclass of OxpullWorker narrows what it claims through those same hooks, +and the worker adds its rate limits to whatever they return. These tests hold +both halves. The rows a subclass leaves alone stay READY, and a limit admits +what it should whether or not the subclass calls super(). A subclass that +writes both hooks keeps the single statement, and so does one that writes +neither. The rows claimed are asserted on every database. The claim path is +asserted on PostgreSQL, the one database where it is a choice. +""" + +import logging +from contextlib import contextmanager + +import pytest +from django.conf import settings +from django.core.management import call_command +from django.db import connections +from django.db.models import Q +from django.db.models.functions import Now +from django.test import override_settings + +from django_ox.models import OxTask +from django_ox.worker import Worker, worker_class +from oxpull.models import OxpullRateWindow +from oxpull.worker import OxpullWorker + +from . import tasks +from .workers import admissions, run_workers + +pytestmark = pytest.mark.django_db + +ECHO = "tests_oxpull.tasks.plain_echo" +LIMITED = "tests_oxpull.tasks.record_callback" +RESERVED_QUEUE = "emails" +RESERVED_PATH = "tests_oxpull.tasks.plain_flaky" +TASK_TABLE = OxTask._meta.db_table +NOTICE = "claim_filter_sql_missing" + +SINGLE_STATEMENT = "single statement" +SELECT_FOR_UPDATE = "select for update" +COMPARE_AND_SET = "compare and set" + + +def _narrowed(inherited, mine): + return mine if inherited is None else inherited & mine + + +class LeavesAQueue(OxpullWorker): + """Leaves one queue to another pool, through claim_filter_q() alone.""" + + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), ~Q(queue_name=RESERVED_QUEUE)) + + +class LeavesATaskPath(OxpullWorker): + """Leaves one task to another pool, through claim_filter_q() alone.""" + + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), ~Q(task_path=RESERVED_PATH)) + + +class LeavesAQueueMixin: + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), ~Q(queue_name=RESERVED_QUEUE)) + + +class LeavesAQueueThroughAMixin(LeavesAQueueMixin, OxpullWorker): + """The same condition, supplied by a mixin listed ahead of OxpullWorker.""" + + +class LeavesAQueueInBothHooks(OxpullWorker): + """The same condition, written in both hooks, each through super().""" + + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), ~Q(queue_name=RESERVED_QUEUE)) + + def claim_filter_sql(self): + fragment, params = super().claim_filter_sql() + return ( + fragment + ' AND "queue_name" <> %(reserved_queue)s', + {**params, "reserved_queue": RESERVED_QUEUE}, + ) + + +class LeavesAQueueWithoutSuper(OxpullWorker): + """The queue condition, returned alone. It never calls super().""" + + def claim_filter_q(self): + return ~Q(queue_name=RESERVED_QUEUE) + + +class LeavesAQueueFromAStaticMethod(OxpullWorker): + """The queue condition, from a static method, as django-ox allows.""" + + @staticmethod + def claim_filter_q(): + return ~Q(queue_name=RESERVED_QUEUE) + + +class LeavesAQueueInBothHooksWithoutSuper(OxpullWorker): + """The queue condition in both hooks, neither calling super().""" + + def claim_filter_q(self): + return ~Q(queue_name=RESERVED_QUEUE) + + def claim_filter_sql(self): + return ' AND "queue_name" <> %(reserved_queue)s', { + "reserved_queue": RESERVED_QUEUE + } + + +class ClaimsOnlyTheDefaultQueue(OxpullWorker): + """An allow-list rather than an exclusion, through claim_filter_q() alone.""" + + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), Q(queue_name="default")) + + +class OverridesNeitherHook(OxpullWorker): + """Changes how it runs and leaves both claim filter hooks alone.""" + + def run(self): + return super().run() + + +class PlainLeavesAQueue(Worker): + """django-ox's own Worker with the same override, to compare claim costs.""" + + def claim_filter_q(self): + return _narrowed(super().claim_filter_q(), ~Q(queue_name=RESERVED_QUEUE)) + + +class _RunsUntilIdle: + """A run loop that ends, so the real command returns. The claim is untouched.""" + + def run(self): + while self.run_once(): + pass + + +class LeavesAQueueUntilIdle(_RunsUntilIdle, LeavesAQueue): + pass + + +class LeavesATaskPathUntilIdle(_RunsUntilIdle, LeavesATaskPath): + pass + + +class LeavesAQueueInBothHooksUntilIdle(_RunsUntilIdle, LeavesAQueueInBothHooks): + pass + + +class LeavesAQueueWithoutSuperUntilIdle(_RunsUntilIdle, LeavesAQueueWithoutSuper): + pass + + +class LeavesAQueueInBothHooksWithoutSuperUntilIdle( + _RunsUntilIdle, LeavesAQueueInBothHooksWithoutSuper +): + pass + + +# class, what it excludes, whether it writes claim_filter_sql() too +SHAPES = { + "queue-q-only": (LeavesAQueue, "queue", False), + "path-q-only": (LeavesATaskPath, "path", False), + "queue-q-mixin": (LeavesAQueueThroughAMixin, "queue", False), + "queue-q-no-super": (LeavesAQueueWithoutSuper, "queue", False), + "queue-q-staticmethod": (LeavesAQueueFromAStaticMethod, "queue", False), + "allow-list-q-only": (ClaimsOnlyTheDefaultQueue, "queue", False), + "queue-both-hooks": (LeavesAQueueInBothHooks, "queue", True), + "queue-both-hooks-no-super": (LeavesAQueueInBothHooksWithoutSuper, "queue", True), +} + +COMMAND_SHAPES = { + "queue-q-only": (LeavesAQueueUntilIdle, "queue", False), + "path-q-only": (LeavesATaskPathUntilIdle, "path", False), + "queue-q-no-super": (LeavesAQueueWithoutSuperUntilIdle, "queue", False), + "queue-both-hooks": (LeavesAQueueInBothHooksUntilIdle, "queue", True), + "queue-both-hooks-no-super": ( + LeavesAQueueInBothHooksWithoutSuperUntilIdle, + "queue", + True, + ), +} + + +def one_per_hour(limit): + return {"spend": {"limit": limit, "per": 3600, "tasks": [LIMITED]}} + + +@contextmanager +def configured(worker, limits=None): + """The default backend, running this worker class, with these limits.""" + default = settings.TASKS["default"] + options = { + **default["OPTIONS"], + "WORKER_CLASS": f"{worker.__module__}.{worker.__qualname__}", + } + if limits is not None: + options["RATE_LIMITS"] = limits + with override_settings( + TASKS={**settings.TASKS, "default": {**default, "OPTIONS": options}} + ): + yield + + +def built(): + """The worker ox_worker would build from WORKER_CLASS.""" + return worker_class("default")(backoff_initial=0, poll_interval=0.01) + + +@contextmanager +def captured(alias): + statements = [] + + def wrapper(execute, sql, params, many, context): + statements.append(sql) + return execute(sql, params, many, context) + + with connections[alias].execute_wrapper(wrapper): + yield statements + + +def on_the_task_table(statements): + return [" ".join(sql.split()) for sql in statements if TASK_TABLE in sql] + + +def claim_path(statements): + upper = [sql.upper() for sql in on_the_task_table(statements)] + if any(sql.startswith("UPDATE") and "RETURNING" in sql for sql in upper): + return SINGLE_STATEMENT + if any( + sql.startswith("SELECT") and "FOR UPDATE SKIP LOCKED" in sql for sql in upper + ): + return SELECT_FOR_UPDATE + return COMPARE_AND_SET + + +def enqueue_excluded(kind): + if kind == "queue": + return tasks.emails_echo.using(priority=100).enqueue("excluded") + return tasks.plain_flaky.using(priority=100).enqueue(1) + + +def drain(worker): + """Claim until the worker finds nothing, recording each row and its path.""" + claims = [] + for _ in range(10): + with captured(worker._db_alias) as statements: + row = worker.claim_one() + if row is None: + return claims + claims.append((str(row.id), claim_path(statements))) + raise AssertionError("still claiming after more rows than the test enqueued") + + +def notices(caplog): + return [r for r in caplog.records if getattr(r, "event", None) == NOTICE] + + +def used(): + return dict(OxpullRateWindow.objects.values_list("limit_name", "used")) + + +def on_postgresql(): + return connections["default"].vendor == "postgresql" + + +def assert_untouched(result): + row = OxTask.objects.get(id=result.id) + assert (row.status, row.attempts, row.lease_epoch) == (OxTask.Status.READY, 0, 0) + + +# limits, labels claimed in order, window counts afterwards +STATES = { + "no-limits": (None, ["limited", "limited", "allowed"], {}), + "limit-not-spent": ( + one_per_hour(100), + ["limited", "limited", "allowed"], + {"spend": 2}, + ), + "limit-spent-by-the-claims": ( + one_per_hour(1), + ["limited", "allowed"], + {"spend": 1}, + ), + "limit-spent-before-start": (one_per_hour(1), ["allowed"], {"spend": 1}), +} + + +@pytest.mark.parametrize("state", list(STATES)) +@pytest.mark.parametrize("shape", list(SHAPES)) +def test_a_narrowing_subclass_claims_only_what_it_allows(shape, state, caplog): + cls, kind, writes_sql = SHAPES[shape] + limits, expected, counts = STATES[state] + with configured(cls, limits), caplog.at_level(logging.WARNING, logger="django_ox"): + if state == "limit-spent-before-start": + # A real admission by another worker, then a fresh worker, as + # after a restart or a deploy inside the window. + spend = tasks.record_callback.enqueue("spend") + assert built().run_once() is True + assert OxTask.objects.get(id=spend.id).status == OxTask.Status.SUCCESSFUL + caplog.clear() + worker = built() + assert type(worker) is cls + excluded = enqueue_excluded(kind) + labels = {excluded.id: "excluded"} + for index in range(2): + limited = tasks.record_callback.using(priority=50).enqueue( + f"limited-{index}" + ) + labels[limited.id] = "limited" + labels[tasks.plain_echo.using(priority=0).enqueue("allowed").id] = "allowed" + + claims = drain(worker) + + assert [labels.get(task_id, task_id) for task_id, _ in claims] == expected + assert_untouched(excluded) + assert used() == counts + if on_postgresql(): + if writes_sql: + assert {path for _, path in claims} == {SINGLE_STATEMENT} + assert notices(caplog) == [] + else: + assert {path for _, path in claims} == {SELECT_FOR_UPDATE} + assert len(notices(caplog)) == 1 + + +def test_a_spent_queue_limit_keeps_a_task_path_condition(): + """The queue half of the rate-limit fragment, under a path condition.""" + limits = {"mail": {"limit": 1, "per": 3600, "queues": [RESERVED_QUEUE]}} + with configured(LeavesATaskPath, limits): + tasks.emails_echo.enqueue("spend") + assert built().run_once() is True + worker = built() + excluded = tasks.plain_flaky.using(priority=100).enqueue(1) + limited = tasks.emails_echo.using(priority=50).enqueue("limited") + allowed = tasks.plain_echo.using(priority=0).enqueue("allowed") + labels = {excluded.id: "excluded", limited.id: "limited", allowed.id: "allowed"} + + claims = drain(worker) + + assert [labels.get(task_id, task_id) for task_id, _ in claims] == ["allowed"] + assert_untouched(excluded) + assert_untouched(limited) + assert used() == {"mail": 1} + + +def run_the_command(): + with pytest.raises(SystemExit) as exit_code: + call_command("ox_worker", "--processes=1", verbosity=0) + assert exit_code.value.code == 0 + + +def statuses(results): + return sorted(OxTask.objects.get(id=result.id).status for result in results) + + +@pytest.mark.parametrize("shape", list(COMMAND_SHAPES)) +def test_the_command_admits_the_limit_and_leaves_the_excluded_rows(shape): + """ + End to end through ox_worker. The worker spends the limit itself, part + way through, and from then on every claim is made with the limit spent. + """ + cls, kind, _ = COMMAND_SHAPES[shape] + with configured(cls, one_per_hour(3)): + excluded = [enqueue_excluded(kind) for _ in range(5)] + limited = [ + tasks.record_callback.using(priority=50).enqueue(f"limited-{index}") + for index in range(8) + ] + allowed = [tasks.plain_echo.using(priority=0).enqueue(i) for i in range(4)] + + run_the_command() + + for result in excluded: + assert_untouched(result) + assert ( + statuses(limited) == [OxTask.Status.READY] * 5 + [OxTask.Status.SUCCESSFUL] * 3 + ) + assert statuses(allowed) == [OxTask.Status.SUCCESSFUL] * 4 + assert used() == {"spend": 3} + + +@pytest.mark.parametrize("shape", list(COMMAND_SHAPES)) +def test_the_command_restarted_inside_a_spent_window_keeps_the_condition(shape, caplog): + cls, kind, writes_sql = COMMAND_SHAPES[shape] + with configured(cls, one_per_hour(1)): + limited = [ + tasks.record_callback.using(priority=50).enqueue(f"limited-{index}") + for index in range(2) + ] + run_the_command() + assert statuses(limited) == [OxTask.Status.READY, OxTask.Status.SUCCESSFUL] + + excluded = enqueue_excluded(kind) + allowed = tasks.plain_echo.using(priority=0).enqueue("allowed") + caplog.clear() + with caplog.at_level(logging.WARNING, logger="django_ox"): + run_the_command() + + assert_untouched(excluded) + assert statuses([allowed]) == [OxTask.Status.SUCCESSFUL] + assert statuses(limited) == [OxTask.Status.READY, OxTask.Status.SUCCESSFUL] + assert used() == {"spend": 1} + if on_postgresql(): + assert len(notices(caplog)) == (0 if writes_sql else 1) + + +def test_a_subclass_overriding_neither_hook_keeps_the_single_statement(caplog): + """ + Nothing to apply beyond the rate limits, so nothing to give up. The first + claim spends the limit and the second is made with it spent. + """ + with ( + configured(OverridesNeitherHook, one_per_hour(1)), + caplog.at_level(logging.WARNING, logger="django_ox"), + ): + worker = built() + labels = {} + for index in range(2): + limited = tasks.record_callback.using(priority=50).enqueue( + f"limited-{index}" + ) + labels[limited.id] = "limited" + labels[tasks.emails_echo.using(priority=0).enqueue("other").id] = "other" + + claims = drain(worker) + + assert [labels.get(task_id, task_id) for task_id, _ in claims] == [ + "limited", + "other", + ] + assert used() == {"spend": 1} + if on_postgresql(): + assert [path for _, path in claims] == [SINGLE_STATEMENT] * 2 + assert notices(caplog) == [] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("cls", [LeavesAQueue, LeavesAQueueWithoutSuper]) +def test_the_limit_bounds_concurrent_subclass_workers(cls): + """ + The published bound, N + W - 1, on the path that applies claim_filter_q(). + A limiter that stops excluding once the limit is spent claims the reserved + queue and blows the ceiling; one that excludes everything fails the floor. + A subclass that drops the limits from its hook blows the ceiling too. + + The window outlasts any run and the workers run until all of them have + stopped taking tasks, so neither bound depends on how fast they are. + """ + limit, workers = 10, 4 + with configured(cls, {"echo": {"limit": limit, "per": 600, "tasks": [ECHO]}}): + reserved = [tasks.emails_echo.using(priority=10).enqueue(i) for i in range(30)] + for index in range(100): + tasks.plain_echo.enqueue(index) + OxpullRateWindow.objects.create( + limit_name="echo", window_id=0, used=0, window_started_at=Now() + ) + + claims = run_workers(workers, seconds=60, worker_class=cls, until_idle=True) + + admitted = len(admissions(claims)) + assert statuses(reserved) == [OxTask.Status.READY] * 30 + assert limit <= admitted <= limit + workers - 1 + ready = OxTask.objects.filter(task_path=ECHO, status=OxTask.Status.READY).count() + assert ready == 100 - admitted + + +def one_claim(worker): + with captured(worker._db_alias) as statements: + row = worker.claim_one() + assert row is not None + return row, on_the_task_table(statements) + + +def verbs(statements): + return [sql.split(" ", 1)[0].upper() for sql in statements] + + +def test_on_postgresql_the_fallback_costs_what_it_costs_django_ox(): + """ + A subclass that writes claim_filter_q() alone claims exactly as django-ox's + own Worker does with the same override. OxpullWorker itself, with a limit + spent, keeps the single statement. + """ + if not on_postgresql(): + pytest.skip("only PostgreSQL has a single-statement claim to give up") + with configured(LeavesAQueue, one_per_hour(1)): + tasks.record_callback.enqueue("spend") + assert built().run_once() is True + for index in range(3): + tasks.plain_echo.enqueue(index) + tasks.record_callback.using(priority=50).enqueue("limited") + + _, subclass = one_claim(built()) + oxpull_row, oxpull_sql = one_claim( + OxpullWorker(backoff_initial=0, poll_interval=0.01) + ) + _, plain = one_claim(PlainLeavesAQueue(backoff_initial=0, poll_interval=0.01)) + + assert oxpull_row.task_path == ECHO + assert verbs(oxpull_sql) == ["UPDATE"] + assert verbs(subclass) == verbs(plain) + assert claim_path(subclass) == claim_path(plain) == SELECT_FOR_UPDATE + assert used() == {"spend": 1} diff --git a/tests_oxpull/test_workflows.py b/tests_oxpull/test_workflows.py new file mode 100644 index 0000000..f01cdc2 --- /dev/null +++ b/tests_oxpull/test_workflows.py @@ -0,0 +1,1276 @@ +""" +Workflows: the test plan from WORKFLOWS-DESIGN.md. + +The load-bearing tests are the ones that prove ordering is enforced by +the claim statement and not by anything a process has to remember. Two of +them are written as a matched pair -- the same diamond drained by the +gating worker and by the `django_ox.worker.Worker` that does not carry the gate -- +so the guard is proven rather than asserted. If the gate ever stops +reaching the emitted SQL, test_a_worker_without_the_gate_runs_the_diamond +_out_of_order keeps passing and its partner fails, which is the shape of +a rule that holds on two databases and not the third, and the reason the pair +exists. +""" + +import copy +import logging +import subprocess +import sys +import threading +import time +import uuid +from datetime import timedelta + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.db import connections, transaction +from django.db.models import QuerySet +from django.test import override_settings +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import unique as unique_module +from oxpull import workflows +from oxpull.exceptions import ( + UnknownWorkflowNode, + WorkflowNodeExists, + WorkflowSealed, +) +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.worker import OxpullWorker + +from . import tasks +from .racing import THREADS, race, race_all + +pytestmark = pytest.mark.django_db + + +def flow_worker(**kwargs): + """A worker on the backend that declares WORKFLOWS.""" + return OxpullWorker( + backend_alias="flows", backoff_initial=0, poll_interval=0.01, **kwargs + ) + + +def free_worker(**kwargs): + """ + The `django_ox.worker.Worker`: no dependency gate in its claim. + + Its claim does not filter on backend_name, so it will happily take + the "flows" rows the gating worker is holding back. That is not a + contrivance for the test; it is exactly what an install that forgets + WORKER_CLASS gets, and it is what oxpull.E007 exists to refuse. + """ + return Worker(backoff_initial=0, poll_interval=0.01, **kwargs) + + +def drain(worker, limit=50): + """ + Run tasks until the worker can claim nothing more. + + An empty claim is not proof that the workflow is finished. A worker's + release of what it finished is best-effort: a release chunk that loses + two lock conflicts is logged and left to the reconciler, and workers + racing on one SQLite database do lose them. Django opens a DEFERRED + transaction, so the release has to upgrade a read transaction to a write + one, and SQLite refuses that upgrade with SQLITE_BUSY the instant another + connection holds the write lock, skipping the busy handler rather than + risking a deadlock. The timeout in tests_oxpull/settings.py never applies, both + attempts go at once, and the node the release was holding stays WAITING, + which no claim admits. So every worker comes back empty with work still + in the workflow. A caller whose count has to be complete finishes on + settle(), which is the reconciler that every workflow deployment runs. + """ + ran = 0 + while ran < limit and worker.run_once(): + ran += 1 + return ran + + +def settle(flow, limit, worker=None): + """ + One reconciler pass and the drain after it, if the workflow has a node + left. Returns how many tasks that drain ran. + + `create()` and `seal()` refuse a workflow whose backend has no schedule + running the reconciler, so this is the deployment the product describes + and not a second chance the test gives itself. It is called with every + worker thread joined, so no lock is held and nothing here can lose a + conflict: one pass releases what an inline release left, and one drain + runs it. A run whose inline releases all won reaches this with nothing + pending and it does nothing. + + One pass is enough. A deferred chunk's ids go to their workflow's skip + queue, which the backstop retries before anything else, and the drain + after it is the only writer left, so nothing it releases can lose a + conflict either. + + It lives here rather than beside any one race because three of them end + on it now, on two backends, which is why the worker is an argument. + """ + if not flow.progress().pending: + return 0 + reconcile() + return drain(worker if worker is not None else flow_worker(), limit) + + +def order(): + return list(tasks.STATE.get("order", [])) + + +def deferrals(caplog): + """ + What the product logged about a release it did not complete: a chunk + that lost both its attempts, or one that raised. + + A race that leaves work for the reconciler must have said so. These are + the two events that say it, and asserting one of them is present is what + stops a silently lost wakeup from passing as a documented deferral. + """ + return [ + record + for record in caplog.records + if getattr(record, "event", "") + in ("workflow_release_deferred", "workflow_release_failed") + ] + + +def reconcile(): + """Run one reconciler pass inline, as a scheduled worker would.""" + return workflows._reconcile() + + +def stored(flow): + return OxpullWorkflow.objects.get(pk=flow.id) + + +def diamond(name="diamond", **create_kwargs): + """ + extract -> {transform-a, transform-b} -> load, with the priorities + set so that dependency order is the exact reverse of claim order. + + That inversion is deliberate and load-bearing. A DAG declared in + topological order runs correctly under a worker with no gate at all, + simply because the claim orders by priority then enqueued_at -- so a + test built on one proves nothing about the gate. Here the ungated + claim reaches `load` first, and only the dependency conjunct can + stop it. Its partner test drains this same DAG with the + `django_ox.worker.Worker` and asserts it comes out backwards. + """ + flow = workflows.create(name, **create_kwargs) + flow.add("extract", tasks.step.using(priority=0), "extract") + flow.add( + "transform-a", + tasks.step.using(priority=5), + "transform-a", + depends_on="extract", + ) + flow.add( + "transform-b", + tasks.step.using(priority=5), + "transform-b", + depends_on="extract", + ) + flow.add( + "load", + tasks.step.using(priority=10), + "load", + depends_on=["transform-a", "transform-b"], + ) + flow.seal() + return flow + + +# -- shape and configuration ------------------------------------------- + + +def test_create_returns_an_open_workflow(): + flow = workflows.create("nightly") + assert flow.name == "nightly" + assert flow.status == OxpullWorkflow.Status.OPEN + assert stored(flow).total_count is None + assert stored(flow).failed_count is None + + +def test_get_returns_the_same_workflow(): + flow = workflows.create("fetch") + assert workflows.get(flow.id).id == flow.id + assert workflows.get(str(flow.id)).name == "fetch" + + +def test_get_raises_for_an_unknown_workflow(): + with pytest.raises(OxpullWorkflow.DoesNotExist): + workflows.get("6c1b3f52-0000-4000-8000-000000000000") + + +def test_over_long_names_are_rejected_identically_on_every_database(): + with pytest.raises(ValueError, match="128 characters"): + workflows.create("x" * 129) + flow = workflows.create("names") + with pytest.raises(ValueError, match="128 characters"): + flow.add("y" * 129, tasks.step, "x") + + +def test_a_callback_made_unique_after_create_is_refused_at_completion( + monkeypatch, caplog +): + """ + As batches refuse it, and for the reason batches._reject_unique gives. + A callback that was an ordinary task at create() and is unique by the + time the workflow completes must not be enqueued: the workflow stays + SEALED and the reconciler says which task and why, every tick, until + the deploy is fixed. + """ + flow = workflows.create( + "later", on_complete=tasks.record_callback, on_complete_args=["c"] + ) + flow.seal() + monkeypatch.setattr( + unique_module, + "_registry", + unique_module._registry | {"tests_oxpull.tasks.record_callback"}, + ) + with caplog.at_level(logging.ERROR, logger="oxpull.workflows"): + assert reconcile() == 0 + row = stored(flow) + assert row.status == OxpullWorkflow.Status.SEALED + assert row.callback_task_id is None + assert not OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback" + ).exists() + assert "'tests_oxpull.tasks.record_callback'" in caplog.text + assert "unique" in caplog.text + + +def test_callback_that_does_not_import_fails_at_create(): + with pytest.raises(ImproperlyConfigured, match="Workflow callback"): + workflows.create("bad", on_complete="tests_oxpull.tasks.nope") + + +def test_a_backend_without_the_gate_is_refused_at_add(): + """ + The whole product is ordering, so a backend that would not enforce it + fails on the line that creates the work, not three hours later with a + healthy-looking queue. + """ + flow = workflows.create("ungated") + with pytest.raises(ImproperlyConfigured, match="WORKFLOWS"): + flow.add("first", tasks.default_step, "first") + assert OxTask.objects.count() == 0 + assert OxpullWorkflowNode.objects.count() == 0 + + +def test_a_backend_that_declares_workflows_without_a_gating_worker_is_an_error(): + """oxpull.E007: declared but not enforced is the silent failure.""" + from django.tasks import task_backends + + errors = task_backends["flows_ungated"].check() + assert [error.id for error in errors] == ["oxpull.E007"] + assert "would run as soon as it was enqueued" in errors[0].msg + + +def test_a_non_bool_workflows_option_is_an_error(): + from django.tasks import task_backends + from django.test import override_settings as _override + + with _override( + TASKS={ + **__import__("django.conf", fromlist=["settings"]).settings.TASKS, + "flows_ungated": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": {"WORKFLOWS": "yes"}, + }, + } + ): + errors = task_backends["flows_ungated"].check() + assert [error.id for error in errors] == ["oxpull.E007"] + assert "must be True or False" in errors[0].msg + + +def test_depends_on_must_name_a_node_that_exists(): + flow = workflows.create("unknown-dep") + with pytest.raises(UnknownWorkflowNode): + flow.add("second", tasks.step, "second", depends_on="first") + # The refusal rolls the whole add() back, enqueue included. + assert OxTask.objects.count() == 0 + + +def test_a_duplicate_node_name_is_refused(): + flow = workflows.create("dupe") + flow.add("first", tasks.step, "a") + with pytest.raises(WorkflowNodeExists): + flow.add("first", tasks.step, "b") + assert OxpullWorkflowNode.objects.count() == 1 + assert OxTask.objects.count() == 1 + + +def test_a_repeated_dependency_is_one_edge(): + flow = workflows.create("repeat") + flow.add("first", tasks.step, "a") + flow.add("second", tasks.step, "b", depends_on=["first", "first"]) + assert OxpullWorkflowEdge.objects.count() == 1 + + +def test_add_after_seal_raises_and_enqueues_nothing(): + flow = workflows.create("sealed") + flow.add("first", tasks.step, "a") + flow.seal() + with pytest.raises(WorkflowSealed): + flow.add("second", tasks.step, "b") + assert OxTask.objects.count() == 1 + assert stored(flow).total_count == 1 + + +def test_seal_twice_raises(): + flow = workflows.create("twice") + flow.seal() + with pytest.raises(WorkflowSealed): + flow.seal() + + +# -- the gate ---------------------------------------------------------- + + +def test_the_diamond_runs_in_dependency_order(): + """ + Fan-out then fan-in, one worker, nothing but the claim statement + deciding what may run. + """ + flow = diamond() + assert drain(flow_worker()) == 4 + + ran = order() + assert ran[0] == "extract" + assert ran[-1] == "load" + assert sorted(ran[1:3]) == ["transform-a", "transform-b"] + assert reconcile() == 1 + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +def put_back_to_ready(flow, names=("transform-a", "transform-b", "load")): + """ + Move waiting nodes to READY by hand, their edges still unsatisfied. + + The shape django-ox's retry or an application's own UPDATE leaves, and + the case the claim's dependency check exists for: nothing released + these rows, so only the claim can hold them. + """ + ids = flow.node_ids() + moved = OxTask.objects.filter( + pk__in=[ids[name] for name in names], status=OxTask.Status.WAITING + ).update(status=OxTask.Status.READY) + assert moved == len(names) + + +def test_no_worker_claims_a_waiting_node(): + """ + The diamond as add() leaves it: extract READY, everything below it + WAITING. The `django_ox.worker.Worker`'s claim carries no dependency check, and + it still takes only extract, because no claim reads a WAITING row. + """ + flow = diamond("waiting-diamond") + statuses = OxTask.objects.filter(pk__in=flow.node_ids().values()).values_list( + "status", flat=True + ) + assert sorted(statuses) == [ + OxTask.Status.READY, + OxTask.Status.WAITING, + OxTask.Status.WAITING, + OxTask.Status.WAITING, + ] + assert drain(free_worker()) == 1 + assert order() == ["extract"] + + +def test_a_worker_without_the_gate_runs_the_diamond_out_of_order(): + """ + The proof that the claim's check is what holds a READY row back, and + not the order in which the rows happen to be enqueued. + + The partner of test_the_gate_holds_ready_rows_whose_edges_are_unsatisfied. + The same DAG with its blocked rows put back to READY, drained by a + worker whose claim carries no dependency conjunct. It runs `load` + before either of its parents, which is what the gated worker must + never do. If this test ever starts passing *and* its partner starts + failing, the gate has stopped reaching the emitted SQL. + """ + put_back_to_ready(diamond("ungated-diamond")) + assert drain(free_worker()) == 4 + + # Straight down the priority order, with every dependency violated: + # the fan-in first, then the transforms, then the root they all + # needed. Nothing consulted an edge. + ran = order() + assert ran == ["load", "transform-a", "transform-b", "extract"] + # The real assertion: this worker emits no dependency conjunct at all. + assert free_worker().claim_filter_q() is None + + +def test_the_gate_holds_ready_rows_whose_edges_are_unsatisfied(): + """ + The same rows, drained by the worker whose claim carries the check. It + runs extract first and load last. Each outcome write marks the edges it + satisfies, which is what lets the next level through. + """ + put_back_to_ready(diamond("gated-ready-diamond")) + assert drain(flow_worker()) == 4 + ran = order() + assert ran[0] == "extract" + assert ran[-1] == "load" + + +def test_the_gate_holds_a_child_while_a_parent_is_still_pending(): + flow = workflows.create("held") + flow.add("parent", tasks.step, "parent") + flow.add("child", tasks.step, "child", depends_on="parent") + flow.seal() + + worker = flow_worker() + # The child waits, so no claim can reach it once the parent is taken + # out of the running. + OxTask.objects.filter(task_path="tests_oxpull.tasks.step", args=["parent"]).update( + status=OxTask.Status.RUNNING, locked_by="someone-else" + ) + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.WAITING + assert worker.claim_one() is None + assert flow.progress().blocked == 1 + + # A SUCCESSFUL written by hand releases nothing on its own. The + # reconciler's backstop marks the edge and releases the child. + OxTask.objects.filter(args=["parent"]).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + assert worker.run_once() is False + reconcile() + assert worker.run_once() is True + assert order() == ["child"] + + +def test_a_task_outside_a_workflow_is_untouched_by_the_gate(): + tasks.step.enqueue("loose") + assert drain(flow_worker()) == 1 + assert order() == ["loose"] + + +def _as_uuid(value): + """ + One spelling for an id, whatever produced it. + + TaskResult.id is a str, the ORM hands back UUID objects, and a raw + cursor hands back a UUID on PostgreSQL and 32 undashed hex characters + on SQLite. Comparing sets across those without normalising would make + this test pass or fail on the storage format rather than on the + predicate. + """ + return uuid.UUID(str(value)) + + +def _rows_the_orm_path_allows(): + return { + _as_uuid(pk) + for pk in OxTask.objects.filter(status=OxTask.Status.READY) + .filter(workflows.runnable_filter_q()) + .values_list("id", flat=True) + } + + +def _rows_the_sql_path_allows(): + """ + The raw fragment, executed, against the same rows. + + The fragment is written for the single-statement PostgreSQL claim and + quotes its identifiers the way PostgreSQL does. SQLite accepts the + same quoting; MySQL does not without ANSI_QUOTES, and MySQL never + reads this path in production, so there it is unexecutable by + construction rather than untested by omission. The caller skips it. + """ + fragment, params = workflows.runnable_filter_sql() + table = OxTask._meta.db_table + sql = ( + f'SELECT "id" FROM "{table}" WHERE "status" = %(ox_ready)s' # noqa: S608 + f"{fragment}" + ) + with connections["default"].cursor() as cursor: + cursor.execute(sql, {"ox_ready": OxTask.Status.READY, **params}) + return {_as_uuid(row[0]) for row in cursor.fetchall()} + + +def test_the_two_claim_paths_agree_on_the_same_rows(): + """ + claim_filter_q and claim_filter_sql are one predicate with two + implementations, and PostgreSQL is the only database that reads the + second. A divergence would be a rule that applies on two databases + and not the third, which no test run short of the third would catch. + + Asserting the fragment's shape would not establish this. A fragment + can start with " AND NOT EXISTS (", mention both table names, and + still select different rows. So the two predicates are run against the + same table, in every state the gate distinguishes, and their result + sets are compared. + + Every downstream row is put back to READY first, because the gate + judges READY rows and add() leaves these WAITING. The states are the + ones where a plausible mis-write diverges: an upstream that succeeded, + one still pending, one that failed, one whose row is gone with nothing + recorded (blocks), one whose row is gone after its edge was marked + satisfied (runs), an edge marked satisfied over an upstream still + pending (the mark decides), a node with two upstreams where only one + is done, and a task in no workflow at all. + """ + fragment, params = workflows.runnable_filter_sql() + assert fragment.startswith(" AND NOT EXISTS (") + assert params == { + "ox_wf_success": OxTask.Status.SUCCESSFUL, + "ox_wf_unsatisfied": False, + } + assert OxpullWorkflowEdge._meta.db_table in fragment + assert OxTask._meta.db_table in fragment + # Both features conjoined, in the order the fragment builds them. + worker = flow_worker() + assert worker.claim_filter_sql()[0] == fragment + assert worker.claim_filter_q() is not None + + loose = tasks.step.enqueue("loose") + + def chain(name, label): + flow = workflows.create(name) + flow.add("up", tasks.step, f"u{label}") + down = flow.add("down", tasks.step, f"d{label}", depends_on="up") + return flow, flow.node_ids()["up"], down + + _, done_up, after_done = chain("succeeded-upstream", 1) + OxTask.objects.filter(id=done_up).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + _, _, after_pending = chain("pending-upstream", 2) + _, failed_up, after_failed = chain("failed-upstream", 3) + OxTask.objects.filter(id=failed_up).update( + status=OxTask.Status.FAILED, finished_at=timezone.now() + ) + _, gone_up, after_gone = chain("pruned-upstream", 4) + OxTask.objects.filter(id=gone_up).delete() + _, marked_gone_up, after_marked_gone = chain("pruned-after-success", 6) + OxpullWorkflowEdge.objects.filter(upstream_task_id=marked_gone_up).update( + satisfied=True + ) + OxTask.objects.filter(id=marked_gone_up).delete() + _, marked_pending_up, after_marked_pending = chain("marked-pending", 7) + OxpullWorkflowEdge.objects.filter(upstream_task_id=marked_pending_up).update( + satisfied=True + ) + + half = workflows.create("one-of-two") + half.add("a", tasks.step, "u5a") + half.add("b", tasks.step, "u5b") + after_half = half.add("down", tasks.step, "d5", depends_on=["a", "b"]) + OxTask.objects.filter(id=half.node_ids()["a"]).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + + OxTask.objects.filter(status=OxTask.Status.WAITING).update( + status=OxTask.Status.READY + ) + + allowed = _rows_the_orm_path_allows() + # Pin what the predicate is, not only that the two agree: two + # identically-wrong implementations would agree with each other. + assert _as_uuid(loose.id) in allowed + assert _as_uuid(after_done.id) in allowed + assert _as_uuid(after_marked_gone.id) in allowed + assert _as_uuid(after_marked_pending.id) in allowed + assert _as_uuid(after_gone.id) not in allowed + assert _as_uuid(after_pending.id) not in allowed + assert _as_uuid(after_failed.id) not in allowed + assert _as_uuid(after_half.id) not in allowed + + if connections["default"].vendor == "mysql": + pytest.skip( + "the fragment quotes identifiers the PostgreSQL way; MySQL " + "never executes this path" + ) + assert _rows_the_sql_path_allows() == allowed + + +# -- atomicity --------------------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_a_failing_edge_insert_leaves_no_orphan_task(monkeypatch): + """ + The invariant the gate rests on: a worker cannot see a node's task + row without also seeing every edge that gates it. They commit + together, so a failure anywhere in add() leaves neither. + """ + flow = workflows.create("atomic") + flow.add("parent", tasks.step, "parent") + before = OxTask.objects.count() + + real = QuerySet.bulk_create + + def boom(self, objs, *args, **kwargs): + if self.model is OxpullWorkflowEdge: + raise RuntimeError("edge insert failed") + return real(self, objs, *args, **kwargs) + + monkeypatch.setattr(QuerySet, "bulk_create", boom) + with pytest.raises(RuntimeError): + flow.add("child", tasks.step, "child", depends_on="parent") + + assert OxTask.objects.count() == before + assert OxpullWorkflowNode.objects.filter(name="child").count() == 0 + + +# -- failure and cancellation ------------------------------------------ + + +def test_a_permanently_failing_upstream_cancels_its_whole_cone(): + """ + Three levels, cancelled in one pass rather than one level per tick. + """ + flow = workflows.create( + "doomed", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + flow.add("root", tasks.failing_step, "root") + flow.add("mid", tasks.step, "mid", depends_on="root") + flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.seal() + + worker = flow_worker() + # Three attempts, then FAILED; nothing downstream may run meanwhile. + for _ in range(3): + assert worker.run_once() is True + assert OxTask.objects.get(args=["root"]).status == OxTask.Status.FAILED + assert worker.run_once() is False + assert order() == ["root", "root", "root"] + + assert reconcile() == 1 + assert ( + OxTask.objects.filter(status=OxTask.Status.DISCARDED) + .values_list("args", flat=True) + .count() + == 2 + ) + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert row.failed_count == 1 + assert row.cancelled_count == 2 + assert row.callback_task_id is not None # on_complete fires + assert row.success_task_id is None # on_success does not + assert flow.progress().unattributed == 2 + + +def test_a_sibling_branch_still_runs_when_one_branch_fails(): + """ + Cancellation is a cone, not a workflow-wide abort. Celery's chord + leaves the rest of the header running on a failure; so does this, and + it also refuses to run the fan-in that depended on the failure. + """ + flow = workflows.create("half-doomed") + flow.add("root", tasks.step, "root") + flow.add("good", tasks.step, "good", depends_on="root") + flow.add("bad", tasks.failing_step, "bad", depends_on="root") + flow.add("join", tasks.step, "join", depends_on=["good", "bad"]) + flow.seal() + + assert drain(flow_worker()) == 5 # root, good, bad x3 + assert "good" in order() + assert "join" not in order() + + assert reconcile() == 1 + assert OxTask.objects.get(args=["join"]).status == OxTask.Status.DISCARDED + assert stored(flow).cancelled_count == 1 + + +def test_a_running_upstream_does_not_cancel_anything(): + """ + Only a node that has settled without succeeding cancels its cone. A + node that is merely still running is not a verdict. + """ + flow = workflows.create("in-flight") + flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + OxTask.objects.filter(args=["root"]).update(status=OxTask.Status.RUNNING) + + assert reconcile() == 0 + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.WAITING + + +def test_an_operator_discarding_a_node_cancels_its_cone(): + """ + A node a person discarded is an upstream that settled without + succeeding, so the workflow cancels its cone. The cancelled count is + the workflow's own: the rows its cancellation changed, here one + (child). root is the operator's and shows only as unattributed. + """ + flow = workflows.create("discarded") + root = flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + discarded = OxTask.objects.filter( + id__in=flow.node_ids().values(), status=OxTask.Status.DISCARDED + ) + + assert actions.discard(root.id) is True + assert discarded.count() == 1 # the operator's + assert reconcile() == 1 + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.DISCARDED + assert discarded.count() == 2 # the workflow cancelled one row, child + row = stored(flow) + assert row.failed_count == 0 + # The stored cancelled_count must be that same 1, not 2. Completion + # stores every DISCARDED node today, so the stored value is asserted in + # test_operator_discards_are_not_counted_as_nodes_the_workflow_cancelled, + # an expected failure until completion stores the workflow's own count. + assert flow.progress().unattributed == 2 + + +# -- crash and resume -------------------------------------------------- + + +def test_a_node_reaped_to_lost_cancels_its_cone_and_completes(worker): + """ + Crash mid-workflow. The reaper settles a dead worker's task through a + queryset update that emits no signal at all, so anything counting + task_finished would hang here. LOST says the attempt was never + observed, and "we do not know it worked" is not "it worked", so the + cone is cancelled rather than released. + """ + flow = workflows.create("crashed") + root = flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + OxTask.objects.filter(id=root.id).update( + status=OxTask.Status.RUNNING, + attempts=3, + max_attempts=3, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + assert worker.reap() == 1 + assert OxTask.objects.get(id=root.id).status == OxTask.Status.LOST + + assert reconcile() == 1 + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.DISCARDED + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert row.failed_count == 0 + assert flow.progress().unattributed == 2 + + +def test_a_node_reaped_with_attempts_remaining_resumes_the_workflow(worker): + """ + The other half of crash-and-resume: a requeued node keeps the + workflow open, runs, and releases its child with no reconciler help. + """ + flow = workflows.create("resumed") + root = flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + OxTask.objects.filter(id=root.id).update( + status=OxTask.Status.RUNNING, + attempts=1, + locked_by="dead-worker", + locked_at=timezone.now() - timedelta(seconds=9999), + ) + + assert worker.reap() == 1 + assert OxTask.objects.get(id=root.id).status == OxTask.Status.READY + assert reconcile() == 0 + assert flow.progress().blocked == 1 + + assert drain(flow_worker()) == 2 + assert order() == ["root", "child"] + assert reconcile() == 1 + + +def test_a_retried_node_holds_the_workflow_open(): + flow = workflows.create("retried") + root = flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert OxTask.objects.get(id=root.id).status == OxTask.Status.FAILED + + assert actions.retry(root.id) is True + assert reconcile() == 0 + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.WAITING + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + +def test_reconciling_a_complete_workflow_is_a_no_op(): + """ + A duplicate completion arriving at a workflow that is already + complete changes nothing and enqueues no second callback. + """ + flow = diamond( + "idempotent", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + drain(flow_worker()) + assert reconcile() == 1 + first = stored(flow).callback_task_id + + assert reconcile() == 0 + assert reconcile() == 0 + assert stored(flow).callback_task_id == first + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 1 + ) + + +def test_an_unsealed_workflow_never_completes(): + flow = workflows.create("never-sealed") + flow.add("only", tasks.step, "only") + assert drain(flow_worker()) == 1 + assert reconcile() == 0 + assert flow.status == OxpullWorkflow.Status.OPEN + + +def test_progress_reports_blocked_separately_from_pending(): + flow = diamond("progress") + progress = flow.progress() + assert (progress.total, progress.pending, progress.blocked) == (4, 4, 3) + + assert flow_worker().run_once() is True # extract + progress = flow.progress() + assert (progress.total, progress.succeeded, progress.blocked) == (4, 1, 1) + + +# -- concurrency ------------------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_workers_racing_to_finish_a_fan_in_run_it_exactly_once(caplog): + """ + The classic double-fire, and the reason this design does not have one. + Eight workers finish the leaves of a fan-in at the same instant, and + each of them releases the join it was holding back. Exactly one worker + gets the join, because a claim is a compare-and-set on one row and + carries the gate besides. + + The release is best-effort (see drain), and this is the shape that meets + the contention: eight completions against one child. Whichever of them + frees it, the join runs once and last. So the run ends where a + deployment's would, on settle(), and the two drains are added up. + + Asserted five ways: the fan-in ran once, it ran last, no worker raised, + nothing was left waiting, and a run that left the reconciler work said + so in the log rather than losing the wakeup in silence. + """ + flow = workflows.create("fan-in") + flow.add("root", tasks.step, "root") + for index in range(THREADS): + flow.add( + f"leaf-{index}", tasks.slow_step, f"leaf-{index}", 0.05, depends_on="root" + ) + flow.add( + "join", + tasks.step, + "join", + depends_on=[f"leaf-{index}" for index in range(THREADS)], + ) + flow.seal() + assert flow_worker().run_once() is True # root + + def work(): + worker = flow_worker() + # A ceiling under race()'s 30 second join. The loop leaves as soon + # as no row is READY or RUNNING, so a passing run never waits it out. + deadline = time.monotonic() + 25 + ran = 0 + try: + while time.monotonic() < deadline: + if worker.run_once(): + ran += 1 + continue + if not OxTask.objects.filter( + status__in=(OxTask.Status.READY, OxTask.Status.RUNNING) + ).exists(): + break + time.sleep(0.005) + finally: + connections.close_all() + return ran + + with caplog.at_level(logging.WARNING, logger="oxpull"): + results, errors = race(work) + settled = settle(flow, THREADS + 1) + + assert errors == [] + assert sum(results) + settled == THREADS + 1 # every leaf, and the join + assert settled == 0 or deferrals(caplog), ( + f"the reconciler had {settled} task(s) to run and the product logged " + "no deferred or failed release, so a wakeup went missing in silence" + ) + ran = order() + assert ran.count("join") == 1 + assert ran[-1] == "join" + assert OxTask.objects.filter(status=OxTask.Status.SUCCESSFUL).count() == THREADS + 2 + # The backstop is the backstop, not the mechanism: once the deployment's + # reconciler has run, it has nothing of its own left to do. + assert workflows._release_backstop("default") == 0 + assert flow.progress().pending == 0 + + +@pytest.mark.django_db(transaction=True) +def test_racing_reconcilers_fire_the_callbacks_once(): + """ + The compare-and-set rowcount enqueues each callback once, the same + mechanism batches use and the same reason: never check-then-act. The + enqueued task then runs like any other, which is at least once. + """ + flow = diamond( + "raced", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + drain(flow_worker()) + + results, errors = race(reconcile) + + assert errors == [] + assert sorted(results) == [0] * (THREADS - 1) + [1] + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 2 + ) + + +@pytest.mark.django_db(transaction=True) +def test_racing_add_and_seal_never_strands_a_node_outside_the_count(): + flow = workflows.create("sealing") + + def add_or_refuse(index): + def call(): + try: + flow.add(f"node-{index}", tasks.step, f"node-{index}") + except WorkflowSealed: + return "refused" + return "added" + + return call + + def seal(): + flow.seal() + return "sealed" + + results, errors = race_all( + [seal, *[add_or_refuse(index) for index in range(THREADS - 1)]] + ) + + assert errors == [] + added = results.count("added") + row = stored(flow) + assert row.status == OxpullWorkflow.Status.SEALED + assert OxpullWorkflowNode.objects.count() == added + assert OxTask.objects.count() == added + assert row.total_count == added + + +@pytest.mark.django_db(transaction=True) +def test_a_retry_cannot_interleave_with_the_cancellation_that_would_discard_it(): + """ + The one race cancellation could have had, and the lock that closes it. + + An operator retries a FAILED upstream at the moment the reconciler is + deciding to discard its downstream. Without the SELECT ... FOR UPDATE + on the doomed upstream rows, the reconciler can read FAILED, the + retry can succeed, the child can become legitimately runnable, and + the reconciler's UPDATE can then discard it -- and no compare-and-set + sees the difference, because the child is READY before and after and + retry() bumps the parent's lease_epoch, not the child's. + + With the lock, retry()'s plain UPDATE must wait for our transaction, + so the two orders are the only outcomes: the child is discarded and + the retry lands afterwards, or the retry lands first and the parent + is not FAILED when we look. Both are asserted; a torn state -- the + parent successfully retried *and* the child discarded in the same + pass -- is not reachable and is what would fail here. + """ + flow = workflows.create("retry-race") + root = flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + for _ in range(3): + worker.run_once() + assert OxTask.objects.get(id=root.id).status == OxTask.Status.FAILED + + started = threading.Event() + + def cancel(): + started.set() + try: + return workflows._cancel_unreachable(flow.id, "default") + finally: + connections.close_all() + + def retry(): + assert started.wait(timeout=10) + try: + return actions.retry(root.id) + finally: + connections.close_all() + + _, errors = race_all([cancel, retry]) + assert errors == [] + + root_row = OxTask.objects.get(id=root.id) + child_row = OxTask.objects.get(args=["child"]) + if child_row.status == OxTask.Status.DISCARDED: + # We cancelled; the retry either lost the row or landed after. + assert root_row.status in (OxTask.Status.FAILED, OxTask.Status.READY) + else: + # The retry went first, so nothing was doomed to cancel. + assert root_row.status == OxTask.Status.READY + assert child_row.status == OxTask.Status.WAITING + + +# -- routing ----------------------------------------------------------- + + +def test_split_router_refuses_workflow_operations(): + class SplitRouter: + def db_for_write(self, model, **hints): + if model._meta.app_label == "oxpull": + return "replica" + return "default" + + with ( + override_settings(DATABASE_ROUTERS=[SplitRouter()]), + pytest.raises(ImproperlyConfigured, match="one database"), + ): + workflows.create("split") + + +@pytest.mark.django_db(transaction=True) +def test_a_node_added_inside_a_caller_transaction_is_never_visible_alone(): + """ + The gate's invariant, checked from outside: while add() is in flight, + another connection sees neither the task row nor its edge; once it + commits it sees both. There is no instant at which the task row is + claimable without its gate. + """ + flow = workflows.create("visibility") + flow.add("root", tasks.step, "root") + inside = threading.Event() + looked = threading.Event() + observed = [] + + def adder(): + try: + with transaction.atomic(): + flow.add("child", tasks.step, "child", depends_on="root") + inside.set() + # The transaction stays open until the observer has read + # both tables, so its reads land mid-transaction however + # slowly its thread and connection start. + assert looked.wait(timeout=10) + finally: + connections.close_all() + return "added" + + def observer(): + try: + assert inside.wait(timeout=10) + observed.append( + ( + OxTask.objects.filter(args=["child"]).exists(), + OxpullWorkflowEdge.objects.count(), + ) + ) + finally: + looked.set() + connections.close_all() + return "observed" + + results, errors = race_all([adder, observer]) + assert errors == [] + assert sorted(results) == ["added", "observed"] + # Mid-transaction the other connection saw neither half. + assert observed == [(False, 0)] + assert OxTask.objects.filter(args=["child"]).exists() + assert OxpullWorkflowEdge.objects.count() == 1 + + +# -- the reconciler must be scheduled (oxpull.W003) -------------------- + + +def without_the_workflow_reconciler(): + """ + settings.TASKS with every SCHEDULES entry naming the workflow + reconciler removed. The suite's settings schedule it, as a project + must before workflows.create() and seal() accept a workflow. + """ + from django.conf import settings as django_settings + + rebuilt = copy.deepcopy(django_settings.TASKS) + for params in rebuilt.values(): + schedules = params.get("OPTIONS", {}).get("SCHEDULES") + if not isinstance(schedules, dict): + continue + for name, config in list(schedules.items()): + if ( + isinstance(config, dict) + and config.get("task") == workflows.RECONCILE_TASK_PATH + ): + del schedules[name] + return rebuilt + + +CHECK_SCRIPT = """ +import sys +import django +from django.conf import settings + +SCHEDULES = { + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, +} +options = { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "WORKFLOWS": True, + # The batch reconciler is scheduled in both runs, so oxpull.W002 stays + # silent and the workflow reconciler is the only thing that can be missing. + "SCHEDULES": { + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, +} +if "--scheduled" in sys.argv: + options["SCHEDULES"].update(SCHEDULES) + +settings.configure( + SECRET_KEY="check-only", + USE_TZ=True, + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + INSTALLED_APPS=["django_ox", "oxpull"], + DATABASES={ + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } + }, + TASKS={ + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": options, + } + }, +) +django.setup() +from django.core.management import execute_from_command_line + +execute_from_command_line(["manage.py", "check"]) +""" + + +def _run_check(*args): + """`manage.py check` in a fresh interpreter, exactly as an operator runs it.""" + return subprocess.run( # noqa: S603 + [sys.executable, "-c", CHECK_SCRIPT, *args], + capture_output=True, + text=True, + check=False, + ) + + +def test_manage_py_check_reports_an_unscheduled_reconciler(): + """ + oxpull.W003, through the real management command. + + Not through backend.check(): this project has a recorded failure + where a test proved a command shape the package does not ship, and + the whole point of this check is that it reaches an operator who + types `manage.py check` and reads the output. The command is what is + asserted, in a fresh interpreter, with settings that are correct in + every other respect -- a gating worker, WORKFLOWS declared, nothing + else wrong -- so the only thing the warning can be about is the + missing schedule. + """ + result = _run_check() + combined = result.stdout + result.stderr + assert "oxpull.W003" in combined, combined + assert "oxpull.workflows.reconcile" in combined, combined + # A warning, not an error: `manage.py check` and every command that + # runs the checks -- ox_worker included -- must still start. + assert result.returncode == 0, combined + + +def test_manage_py_check_is_silent_once_the_reconciler_is_scheduled(): + result = _run_check("--scheduled") + combined = result.stdout + result.stderr + assert "oxpull.W003" not in combined, combined + assert result.returncode == 0, combined + assert "no issues" in combined, combined + + +def test_the_reconciler_schedule_may_live_on_another_backend(): + """ + The reconciler is global -- it sweeps every SEALED workflow whatever + backend enqueued it -- so one schedule anywhere is a correct install, + and a warning per workflow backend would be noise an operator learns + to ignore. + """ + from django.tasks import task_backends + + with override_settings(TASKS=without_the_workflow_reconciler()): + ids = [m.id for m in task_backends["flows"].check()] + assert "oxpull.W003" in ids, ids + + scheduled = { + **without_the_workflow_reconciler(), + "cron": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": { + "SCHEDULES": { + "wf": { + "task": workflows.RECONCILE_TASK_PATH, + "cron": "* * * * *", + }, + }, + }, + }, + } + with override_settings(TASKS=scheduled): + ids = [m.id for m in task_backends["flows"].check()] + assert "oxpull.W003" not in ids, ids + + +def test_the_warning_names_the_path_the_schedule_must_carry(): + """ + The hint is only useful if the string it tells an operator to paste + is the string the schedule dispatcher resolves. Pin them together. + """ + from django.tasks import task_backends + + with override_settings(TASKS=without_the_workflow_reconciler()): + warning = next( + m for m in task_backends["flows"].check() if m.id == "oxpull.W003" + ) + assert workflows.RECONCILE_TASK_PATH in warning.hint + assert workflows.reconcile.module_path == workflows.RECONCILE_TASK_PATH diff --git a/tests_oxpull/test_workflows_at_scale.py b/tests_oxpull/test_workflows_at_scale.py new file mode 100644 index 0000000..4053aaf --- /dev/null +++ b/tests_oxpull/test_workflows_at_scale.py @@ -0,0 +1,150 @@ +""" +The two bounds in the reconciler, exercised. + +`MAX_CANCEL_PASSES` and `BACKSTOP_PAGE` exist so one reconciler tick +cannot run unboundedly, and both carry the same promise in their comments: +what a pass does not finish, the next tick continues. Nothing tested either. +Every other workflow in the suite is one to five nodes, so neither constant +had ever been reached by a test, and the "next tick continues" claim was +prose. + +These build workflows large enough to cross each bound. They are slower than +the rest of the suite by design: a test that stays inside the limit is not +testing the limit. +""" + +from __future__ import annotations + +import pytest + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow +from oxpull.workflows import BACKSTOP_PAGE, MAX_CANCEL_PASSES + +from . import tasks +from .test_workflows import reconcile, stored + +pytestmark = pytest.mark.django_db + + +def chain(name: str, length: int, task=None): + """A -> B -> C -> ... of `length` nodes, each blocked on the last.""" + task = task or tasks.step + flow = workflows.create(name) + previous = None + for i in range(length): + node = f"n{i:04d}" + if previous is None: + flow.add(node, task, node) + else: + flow.add(node, task, node, depends_on=previous) + previous = node + flow.seal() + return flow + + +def fan_out(name: str, width: int, task=None): + """ + One root with `width` children, all blocked on it. + + The backend comes from the task, not from create(): tasks.deferred_step + is bound to the second workflow backend. + """ + task = task or tasks.step + flow = workflows.create(name) + flow.add("root", task, "root") + for i in range(width): + flow.add(f"c{i:04d}", task, f"c{i:04d}", depends_on="root") + flow.seal() + return flow + + +def _cancelled(flow) -> int: + ids = flow.node_ids().values() + return OxTask.objects.filter(id__in=ids, status=OxTask.Status.DISCARDED).count() + + +def test_a_chain_deeper_than_the_cancel_bound_needs_more_than_one_tick(): + """ + _cancel_unreachable walks one level of the DAG per iteration and stops + after MAX_CANCEL_PASSES. A doomed chain longer than that cannot be + finished by a single tick, and the next one has to pick up where it + stopped. Both halves are asserted: the bound holds, and it is not + permanent. + """ + depth = MAX_CANCEL_PASSES + 6 + flow = chain("deep", depth) + ids = flow.node_ids() + + # Kill the root. Everything downstream is now unreachable. + OxTask.objects.filter(id=ids["n0000"]).update(status=OxTask.Status.FAILED) + + first = reconcile() + after_first = _cancelled(flow) + assert after_first > 0, "the first pass cancelled nothing" + assert after_first < depth - 1, ( + f"the whole chain of {depth} was cancelled in one pass, so this test " + f"never reached MAX_CANCEL_PASSES={MAX_CANCEL_PASSES}" + ) + + # The promise the bound's comment makes. + for _ in range(10): + reconcile() + if _cancelled(flow) >= depth - 1: + break + assert _cancelled(flow) == depth - 1, ( + "later ticks did not continue from where the bounded pass stopped" + ) + # _reconcile() returns completed workflows, not cancelled nodes; a doomed + # chain completes none, so the count above is the assertion that matters. + assert first == 0 + + +def test_a_fan_out_wider_than_the_release_bound_releases_across_ticks(monkeypatch): + """ + The reconciler's backstop marks and releases at most BACKSTOP_PAGE edges a + page. With one page a visit, a fan-out wider than that cannot be released + by one tick, and the next tick goes on from the page it stopped at. + """ + monkeypatch.setattr(workflows, "VISIT_PAGES", 1) + width = BACKSTOP_PAGE + 25 + flow = fan_out("wide", width, task=tasks.deferred_step) + ids = flow.node_ids() + + waiting = OxTask.objects.filter(id__in=ids.values(), status=OxTask.Status.WAITING) + assert waiting.count() == width, "the children were not all waiting" + + # The root succeeds by a route that does not release eagerly, which is + # what leaves the backstop to do the work. + OxTask.objects.filter(id=ids["root"]).update(status=OxTask.Status.SUCCESSFUL) + + released = workflows._release_backstop("default") + assert released == BACKSTOP_PAGE, ( + f"one pass released {released}, expected the bound {BACKSTOP_PAGE}; " + "this test did not reach it" + ) + assert waiting.count() == width - BACKSTOP_PAGE + + assert workflows._release_backstop("default") == width - BACKSTOP_PAGE + assert waiting.count() == 0, "the second pass did not finish the release" + + +def test_a_deep_chain_completes_and_fires_its_callback_once(): + """ + The happy path at a depth no other test reaches: a chain longer than the + cancel bound runs to completion in order, and the workflow completes once. + """ + depth = MAX_CANCEL_PASSES + 6 + flow = chain("deep-ok", depth) + + from .test_workflows import drain, flow_worker + + ran = drain(flow_worker(), limit=depth + 10) + assert ran == depth, f"drained {ran} of {depth}" + + order = list(tasks.STATE.get("order", [])) + assert order == [f"n{i:04d}" for i in range(depth)], "ran out of order" + + reconcile() + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE diff --git a/tests_oxpull/test_workflows_callers.py b/tests_oxpull/test_workflows_callers.py new file mode 100644 index 0000000..e46d614 --- /dev/null +++ b/tests_oxpull/test_workflows_callers.py @@ -0,0 +1,205 @@ +""" +Workflow calls inside a caller's transaction, and the transactions a call owns. + +A call owns its transaction when the connection is in autocommit and no atomic +block is open. Then add(), seal(), the cancellation and completion run their +transaction again after a deadlock (InnoDB's 1213, PostgreSQL's 40P01) or a +serialization failure where each says so. Inside a caller's transaction, +whether the caller opened it with atomic or by turning autocommit off, the +server rolled back the caller's work too, so the error propagates and the +caller runs its own transaction again. + +A caller that touches several workflows in one transaction takes them in +ascending id order. Two callers that add to the same two workflows in opposite +orders each hold one workflow row and wait for the other's, and the database +ends one of them with a deadlock. + +Shortcut: the simulated conflicts raise from Workflow._take_for_add, the first +statement of add()'s transaction, with the error code the database would send. +""" + +from __future__ import annotations + +import contextlib +import threading +import time +from typing import Any + +import pytest +from django.db import OperationalError, connection, connections, transaction + +from oxpull import workflows +from oxpull.models import OxpullWorkflowNode + +from . import tasks + +pytestmark = pytest.mark.django_db(transaction=True) + +needs_row_locks = pytest.mark.skipif( + connection.vendor == "sqlite", + reason="SQLite has no row locks, and its writers serialise", +) + + +class SimulatedDriverError(Exception): + """The driver error a conflict carries as its cause.""" + + +def conflict(code: Any) -> OperationalError: + if connection.vendor == "mysql": + return OperationalError(code, "simulated") + cause = SimulatedDriverError("simulated") + cause.sqlstate = code # type: ignore[attr-defined] + error = OperationalError("simulated") + error.__cause__ = cause + return error + + +DEADLOCK_CODE = {"postgresql": "40P01", "mysql": 1213} + + +def code_of(error: OperationalError) -> Any: + if connection.vendor == "mysql": + return error.args[0] if error.args else None + return getattr(error.__cause__, "sqlstate", None) + + +SERIALIZATION_CODE = {"postgresql": "40001", "mysql": 1213} + + +def losing_the_first_take(monkeypatch, code: Any) -> list[int]: + """Make add()'s first attempt lose `code` at its first statement.""" + real = workflows.Workflow._take_for_add + calls: list[int] = [] + + def take_for_add(self, db_alias): + calls.append(1) + if len(calls) == 1: + raise conflict(code) + return real(self, db_alias) + + monkeypatch.setattr(workflows.Workflow, "_take_for_add", take_for_add) + return calls + + +@needs_row_locks +def test_an_add_that_owns_its_transaction_runs_it_again_after_a_deadlock(monkeypatch): + flow = workflows.create("owned-deadlock") + calls = losing_the_first_take(monkeypatch, DEADLOCK_CODE[connection.vendor]) + + flow.add("a", tasks.step, "a") + + assert len(calls) == 2 + assert list(flow.node_ids()) == ["a"] + + +@needs_row_locks +@pytest.mark.parametrize("kind", ["deadlock", "serialization failure"]) +def test_an_add_in_a_transaction_opened_by_turning_autocommit_off_does_not_run_it_again( + monkeypatch, kind +): + flow = workflows.create("autocommit-off") + codes = DEADLOCK_CODE if kind == "deadlock" else SERIALIZATION_CODE + calls = losing_the_first_take(monkeypatch, codes[connection.vendor]) + + connection.set_autocommit(False) + try: + with pytest.raises(OperationalError): + flow.add("a", tasks.step, "a") + finally: + connection.rollback() + connection.set_autocommit(True) + + assert len(calls) == 1 + assert flow.node_ids() == {} + + +@needs_row_locks +def test_an_add_in_an_atomic_block_does_not_run_it_again(monkeypatch): + flow = workflows.create("atomic") + calls = losing_the_first_take(monkeypatch, DEADLOCK_CODE[connection.vendor]) + + with pytest.raises(OperationalError), transaction.atomic(): + flow.add("a", tasks.step, "a") + + assert len(calls) == 1 + assert flow.node_ids() == {} + + +def _run_callers(flows, orders, *, meet): + """ + Two caller transactions, "one" and "two", each adding one node to two of + `flows` in the order given. With `meet`, each waits after its first add() + until the other has made its own. Without it, "two" starts once "one" has + made its first add(), and "one" waits half a second before its second, so + "two" is already waiting on the lock. Returns what each raised, or None. + """ + both_started = threading.Barrier(2, timeout=30) + one_added = threading.Event() + outcome: dict[str, BaseException | None] = {} + + def caller(label, names): + try: + with transaction.atomic(): + first, second = (flows[name] for name in names) + first.add(f"{label}-first", tasks.step, label) + if meet: + with contextlib.suppress(threading.BrokenBarrierError): + both_started.wait() + elif label == "one": + one_added.set() + time.sleep(0.5) + second.add(f"{label}-second", tasks.step, label) + outcome[label] = None + except BaseException as exc: + outcome[label] = exc + both_started.abort() + one_added.set() + finally: + connections.close_all() + + one = threading.Thread(target=caller, args=("one", orders[0])) + two = threading.Thread(target=caller, args=("two", orders[1])) + one.start() + if not meet: + assert one_added.wait(30) + two.start() + for thread in (one, two): + thread.join(120) + assert not thread.is_alive() + return outcome + + +def two_workflows(): + flows = {name: workflows.create(name) for name in ("w1", "w2")} + ascending = tuple(flow.name for flow in sorted(flows.values(), key=lambda f: f.id)) + return flows, ascending + + +@needs_row_locks +def test_callers_taking_two_workflows_in_opposite_orders_deadlock_and_nothing_retries(): + """ + The contract's reason. One caller is ended by the deadlock detector, the + error reaches it, and nothing of its transaction is left. The other + commits both its nodes. + """ + flows, ascending = two_workflows() + outcome = _run_callers(flows, (ascending, ascending[::-1]), meet=True) + + raised = {label: exc for label, exc in outcome.items() if exc is not None} + assert len(raised) == 1, outcome + ((loser, error),) = raised.items() + assert isinstance(error, OperationalError), error + assert code_of(error) == DEADLOCK_CODE[connection.vendor], error + winner = "two" if loser == "one" else "one" + names = set(OxpullWorkflowNode.objects.values_list("name", flat=True)) + assert names == {f"{winner}-first", f"{winner}-second"}, names + + +@needs_row_locks +def test_callers_that_take_two_workflows_in_ascending_id_order_both_commit(): + flows, ascending = two_workflows() + outcome = _run_callers(flows, (ascending, ascending), meet=False) + + assert outcome == {"one": None, "two": None}, outcome + assert OxpullWorkflowNode.objects.count() == 4 diff --git a/tests_oxpull/test_workflows_concurrency.py b/tests_oxpull/test_workflows_concurrency.py new file mode 100644 index 0000000..2fecdb7 --- /dev/null +++ b/tests_oxpull/test_workflows_concurrency.py @@ -0,0 +1,224 @@ +""" +The gate and the reconciler under real concurrency, at sizes the rest of the +suite does not reach. + +The existing race tests are thorough but small: two-parent fan-ins, two +workers, five-node diamonds. These use the same harness on wider and deeper +graphs, and put the reconciler and the workers on the same workflow at the +same time, which nothing else does. + +What is being defended, in one line each: + + * the DAG order survives many workers, not just one; + * a node never runs twice however many workers race for it; + * cancelling a doomed cone while workers are draining it does not let a + discarded node run, and does not lose a node that was still runnable. +""" + +from __future__ import annotations + +import logging + +import pytest + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .racing import THREADS, race, race_all +from .test_workflows import ( + deferrals, + drain, + flow_worker, + reconcile, + settle, + stored, +) + +pytestmark = pytest.mark.django_db + + +def chain(name: str, length: int): + """ + A chain whose priorities run backwards, so claim order is the reverse of + dependency order and only the gate can produce the right sequence. A + chain declared in topological order drains correctly with no gate at + all, which is the trap diamond() documents. + """ + flow = workflows.create(name) + previous = None + for i in range(length): + node = f"n{i:03d}" + task = tasks.step.using(priority=length - i) + if previous is None: + flow.add(node, task, node) + else: + flow.add(node, task, node, depends_on=previous) + previous = node + flow.seal() + return flow + + +def wide_fan_in(name: str, parents: int): + """ + `parents` roots, all of which must succeed before one child runs. + + The child carries the highest priority, so an ungated claim reaches it + first and only the dependency conjunct holds it back. Without this the + test passes with no gate at all, because the child was enqueued last. + """ + flow = workflows.create(name) + names = [f"p{i:03d}" for i in range(parents)] + for node in names: + flow.add(node, tasks.step.using(priority=0), node) + flow.add("child", tasks.step.using(priority=9), "child", depends_on=names) + flow.seal() + return flow + + +@pytest.mark.django_db(transaction=True) +def test_a_deep_chain_keeps_its_order_under_many_workers(caplog): + """ + Order is a property of the gate, not of there being one worker. Eight + workers race down a chain far longer than any other test's graph; the + only sequence any of them can produce is the declared one. + + Forty links is forty releases, each of them taken with seven other + workers claiming against the same SQLite file, so this is the most + releases the suite puts under contention. A link whose release loses + both its tries leaves the next node WAITING and every worker comes back + empty short of the end, which is the reconciler's to finish: the run + ends on settle(), and both drains are counted. Order is what is under + test, and it is asserted over the whole run whichever drain ran which + node. + """ + depth = 40 + flow = chain("deep-race", depth) + + with caplog.at_level(logging.WARNING, logger="oxpull"): + results, errors = race(lambda: drain(flow_worker(), depth)) + settled = settle(flow, depth) + + assert not errors, f"workers raised: {errors[:2]}" + ran = list(tasks.STATE.get("order", [])) + assert ran == [f"n{i:03d}" for i in range(depth)], ( + f"the chain ran out of order under {len(results)} workers" + ) + assert len(ran) == len(set(ran)), f"a node ran twice: {ran}" + assert sum(results) + settled == depth + assert settled == 0 or deferrals(caplog), ( + f"the reconciler had {settled} task(s) to run and the product logged " + "no deferred or failed release, so a wakeup went missing in silence" + ) + + +@pytest.mark.django_db(transaction=True) +def test_a_wide_fan_in_releases_its_child_exactly_once(caplog): + """ + Every parent finishing at once is the moment a counter-based design + double-fires or hangs. There is no counter, but the claim still has to + admit the child exactly once with many workers asking together. + + Twenty-four parents finishing on eight workers is also the most lock + contention this suite puts on one release, and the release that frees + the child is the one that meets it: about once in a hundred runs it + loses both its tries and leaves the child to the reconciler. That is the + product's documented answer to a lost lock conflict, so the run ends on + `settle` rather than at the last empty claim, and both counts are added + up: the child is run exactly once, by a worker, whichever of the two + released it. A run that left the reconciler work must also have said so + in the log, which is what separates a documented deferral from a lost + wakeup. + """ + parents = 24 + flow = wide_fan_in("wide-in", parents) + + total = parents + 1 + with caplog.at_level(logging.WARNING, logger="oxpull"): + results, errors = race(lambda: drain(flow_worker(), total)) + settled = settle(flow, total) + + assert not errors, f"workers raised: {errors[:2]}" + ran = list(tasks.STATE.get("order", [])) + assert ran.count("child") == 1, f"the child ran {ran.count('child')} times" + assert ran[-1] == "child", "the child ran before its last parent" + assert len(ran) == total + assert sum(results) + settled == total + assert settled == 0 or deferrals(caplog), ( + f"the reconciler had {settled} task(s) to run and the product logged " + "no deferred or failed release, so a wakeup went missing in silence" + ) + + +@pytest.mark.django_db(transaction=True) +def test_cancelling_a_doomed_cone_while_workers_drain_it_runs_nothing_discarded(): + """ + The reconciler and the workers on one workflow at the same instant. + + Half the graph is doomed by a failed root and half is independent and + still runnable. Whatever interleaving happens, a discarded node must not + have run, and the runnable half must not be lost. + """ + flow = workflows.create("half-doomed") + flow.add("bad-root", tasks.step, "bad-root") + for i in range(8): + flow.add(f"doomed{i}", tasks.step, f"doomed{i}", depends_on="bad-root") + for i in range(8): + flow.add(f"free{i}", tasks.step, f"free{i}") + flow.seal() + ids = flow.node_ids() + + OxTask.objects.filter(id=ids["bad-root"]).update(status=OxTask.Status.FAILED) + + # Four reconcilers and four workers, released together. + targets = [reconcile] * 4 + [lambda: drain(flow_worker(), 20)] * 4 + _, errors = race_all(targets) + assert not errors, f"raised: {errors[:2]}" + + for _ in range(5): + reconcile() + + ran = set(tasks.STATE.get("order", [])) + discarded_names = { + args[0] + for args in OxTask.objects.filter( + id__in=ids.values(), status=OxTask.Status.DISCARDED + ).values_list("args", flat=True) + if args + } + + # Without these two the assertion below is vacuous: nothing cancelled, or + # nothing run, and "no overlap" is true for uninteresting reasons. + assert discarded_names, "the reconcilers cancelled nothing, so this proves nothing" + assert ran, "no worker ran anything, so this proves nothing" + assert not (ran & discarded_names), ( + f"these ran and were also discarded: {sorted(ran & discarded_names)}" + ) + assert {f"free{i}" for i in range(8)} <= ran, ( + f"independent nodes were lost: {sorted({f'free{i}' for i in range(8)} - ran)}" + ) + assert len(list(tasks.STATE.get("order", []))) == len(ran), "a node ran twice" + + +@pytest.mark.django_db(transaction=True) +def test_racing_reconcilers_complete_a_wide_workflow_once(): + """ + Completion is a compare-and-set. Eight reconcilers on a finished + workflow must produce one completion, not eight. + """ + parents = 12 + flow = wide_fan_in("wide-complete", parents) + drain(flow_worker(), parents + 1) + + results, errors = race(reconcile) + assert not errors, f"reconcilers raised: {errors[:2]}" + assert len(results) == THREADS, ( + f"only {len(results)} of {THREADS} reconcilers ran; a race between one " + "thread and itself proves nothing" + ) + + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + assert sum(results) == 1, ( + f"{sum(results)} reconcilers each claimed the completion; expected 1" + ) diff --git a/tests_oxpull/test_workflows_cost.py b/tests_oxpull/test_workflows_cost.py new file mode 100644 index 0000000..fe3573c --- /dev/null +++ b/tests_oxpull/test_workflows_cost.py @@ -0,0 +1,236 @@ +""" +What the dependency gate costs a claim, measured rather than argued. + +The gate is on the claim path, so a regression in what it costs is a +regression in throughput. These tests pin that cost inside the repository, +where the suite reaches it. + +Two decisions about method. + +**Rows examined, not wall-clock.** These run on whatever machine happens to +be free, and a loaded machine changes a wall-clock reading without changing +the query. `EXPLAIN (ANALYZE)` reports rows touched server-side, which a +loaded laptop does not change. + +**A ratio against a control on the same table, never an absolute number.** +An absolute figure against an empty baseline measures "the table grew by +50,000 rows" and "the gate is expensive" together, and attributes both to +the gate. Every assertion here compares two queries over identical data, +differing only in the conjunct under test. + +What is asserted, and what is only recorded: + + * **Asserted:** a claim's cost does not grow with the number of blocked + nodes in front of it. They wait in django-ox's WAITING status, which no + claim reads, and it is the property that would regress silently. + * **Asserted, generously:** what the gate costs per READY row it judges. + The gate still guards READY rows nothing released, such as a django-ox + retry, so the blocked rows are put back to READY for that measurement. +""" + +from __future__ import annotations + +import pytest +from django.db import connection + +from django_ox.models import OxTask +from oxpull import workflows + +# transaction=True, not the default wrapper: VACUUM refuses to run inside a +# transaction, and it reaches only rows no open transaction can still see. +pytestmark = pytest.mark.django_db(transaction=True) + +postgres_only = pytest.mark.skipif( + connection.vendor != "postgresql", + reason="EXPLAIN ANALYZE row counts are read from the PostgreSQL plan", +) + + +def vacuum_and_analyze() -> None: + """ + Settle the two tables the claim reads, before measuring. + + Without this the measurement is order-dependent, and the order it + depends on is the whole suite's. Most of the suite runs each test inside + a transaction that is rolled back, and on PostgreSQL the rows of a + rolled-back test stay in the table and in every index as dead entries + until a vacuum reaches them. A bitmap index scan counts each dead entry + as a row it examined, so the same claim reports 3 rows on its own and + 1,549 after those tests, and whether autovacuum has been by in between + is timing. + + ANALYZE alone corrects the column statistics and leaves the dead entries + where they are. VACUUM removes them and sets the visibility map, and the + ANALYZE with it re-sizes the tables, so every input to the plan is a row + this test wrote. That is what makes a ratio comparable between runs at + all. + """ + with connection.cursor() as cursor: + cursor.execute("VACUUM (ANALYZE) django_ox_oxtask") + cursor.execute("VACUUM (ANALYZE) oxpull_oxpullworkflowedge") + + +def rows_examined(queryset) -> int: + """ + Total rows touched by the plan, server-side. + + Summed across every node of the plan rather than read off the top, + because the gate's cost is in the anti-join underneath, not in the + number of rows the query finally returns. + """ + plan = queryset.explain(analyze=True, format="JSON") + if isinstance(plan, str): + import json + + plan = json.loads(plan) + root = plan[0]["Plan"] if isinstance(plan, list) else plan["Plan"] + + total = 0 + + def walk(node: dict) -> None: + nonlocal total + total += int(node.get("Actual Rows", 0)) * int(node.get("Actual Loops", 1)) + for child in node.get("Plans", []): + walk(child) + + walk(root) + return total + + +def build(blocked: int, task) -> None: + """One never-succeeding root with `blocked` children waiting on it.""" + flow = workflows.create(f"cost-{blocked}-{task.name}") + flow.add("root", task, "root") + for i in range(blocked): + flow.add(f"b{i:05d}", task, f"b{i:05d}", depends_on="root") + flow.seal() + + +def claim_candidates(*, gated: bool): + """ + The shape of a worker's claim: the next row it would actually take. + + The status and run_after conjuncts are part of the claim in django-ox, + and status is what keeps a waiting node out of the candidate set. + """ + from django.db.models import Q + from django.utils import timezone + + qs = OxTask.objects.filter(status=OxTask.Status.READY).filter( + Q(run_after__isnull=True) | Q(run_after__lte=timezone.now()) + ) + if gated: + qs = qs.filter(workflows.runnable_filter_q()) + return qs.order_by("-priority", "enqueued_at")[:1] + + +def claim_candidate_count() -> int: + """The READY rows a claim chooses among, counted rather than planned.""" + from django.db.models import Q + from django.utils import timezone + + return ( + OxTask.objects.filter(status=OxTask.Status.READY) + .filter(Q(run_after__isnull=True) | Q(run_after__lte=timezone.now())) + .count() + ) + + +@postgres_only +@pytest.mark.parametrize("blocked", [50, 400]) +def test_the_gate_costs_a_bounded_multiple_per_candidate_row(blocked): + """ + The gate's cost per row a claim chooses among, over the same table. + + The control is the candidate count and not the ungated claim's plan. + django-ox 1.1.0 rebuilt ox_dequeue_idx as (status, -priority, + enqueued_at), so the ungated claim reads rows in claim order and stops + at the first one: it examines 2 rows whatever the table holds, and a + ratio against it measures how many nodes are blocked rather than what + the gate costs. Against django-ox 1.0.0's index the ungated claim + sorted every candidate, which is what the ratio used to be taken + against. The ungated figure is still printed, because the gap between + it and the gated one is what a gated backend gives up on PostgreSQL. + """ + from . import tasks + + build(blocked, tasks.step) + # The blocked nodes wait, and a waiting node is no candidate at all. Put + # back to READY, as a django-ox retry leaves a row, they are what the gate + # has to judge. + OxTask.objects.filter(status=OxTask.Status.WAITING).update( + status=OxTask.Status.READY + ) + + vacuum_and_analyze() + candidates = claim_candidate_count() + ungated = rows_examined(claim_candidates(gated=False)) + gated = rows_examined(claim_candidates(gated=True)) + + assert candidates > 0, "no candidate rows, so the ratio is meaningless" + per_candidate = gated / candidates + + # The measurement is the point of this test: a run that passes silently + # tells you nothing about whether the ratio moved. + print( # noqa: T201 + f"\n [cost] blocked={blocked} candidates={candidates} " + f"ungated={ungated} gated={gated} per_candidate={per_candidate:.1f} " + f"vs_ungated={gated / max(ungated, 1):.1f}x" + ) + + # Generous on purpose. This is a regression tripwire for the shape of the + # cost, not a performance budget: it should catch the gate becoming an + # order of magnitude worse, and should not fail on planner variation. + assert per_candidate < 60, ( + f"the gate examined {per_candidate:.1f} rows per candidate row" + ) + + +@postgres_only +def test_waiting_nodes_keep_the_claim_flat_as_they_pile_up(): + """ + The property the waiting status exists for, and the one that would + regress without anyone noticing. + + Blocked nodes wait, so they are outside every worker's candidate set, + and a claim costs the same with four hundred of them in front of it as + with fifty. Measured at both sizes. + + Two assertions, because the two halves of that sentence fail apart. The + candidate count says whether the blocked nodes stayed out of the set, + counted rather than planned: each build() leaves its root READY and + everything under it WAITING, so the set holds one row after the first + build and two after the second. The cost is then taken per candidate + row, as the gate's own measurement above takes it, because the second + root is a candidate the first reading did not have and a claim that + chooses among two rows reads both. + """ + from . import tasks + + build(50, tasks.deferred_step) + vacuum_and_analyze() + small_candidates = claim_candidate_count() + small = rows_examined(claim_candidates(gated=True)) + + build(400, tasks.deferred_step) + vacuum_and_analyze() + large_candidates = claim_candidate_count() + large = rows_examined(claim_candidates(gated=True)) + + print( # noqa: T201 + f"\n [cost] waiting: 50 blocked={small} rows over {small_candidates} " + f"candidates, 450 blocked={large} rows over {large_candidates}" + ) + + assert (small_candidates, large_candidates) == (1, 2), ( + f"the two roots are the only candidates, but a claim chooses among " + f"{small_candidates} rows of 51 and {large_candidates} of 452: the " + "blocked nodes are in the candidate set" + ) + assert small > 0 + growth = (large / large_candidates) / (small / small_candidates) + assert growth < 3, ( + f"claim cost grew {growth:.1f}x per candidate row when blocked nodes " + "went from 50 to 450; waiting nodes are supposed to be outside the " + "candidate set" + ) diff --git a/tests_oxpull/test_workflows_deferred.py b/tests_oxpull/test_workflows_deferred.py new file mode 100644 index 0000000..4545ab5 --- /dev/null +++ b/tests_oxpull/test_workflows_deferred.py @@ -0,0 +1,403 @@ +""" +Releasing waiting workflow nodes, on "flows_deferred". + +A node with dependencies is inserted WAITING and moves to READY only once +every edge into it is marked satisfied. That is a blocked -> ready +transition, so most of what is below is about it: who makes it, what happens +when nobody does, and what two workers finishing a fan-in at the same +instant do to each other. + +"flows_deferred" still carries the removed OPTIONS["DEFER_BLOCKED_NODES"], +which changes nothing, so every test here also shows the leftover option is +ignored. The helpers are shared with the hazard tests that name this +backend. + +The claim still checks the edges. That is the safety story: the release +decides from marks only a writer that saw the upstream succeed can make, and +an extra conjunct in the claim can only remove rows from the candidate set, +so no interleaving here can make a node run early or twice. Every test below +is therefore a liveness test, deliberately. +""" + +import contextlib +import logging + +import pytest +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .racing import race_all +from .test_workflows import deferrals, settle + +pytestmark = pytest.mark.django_db + +READY = OxTask.Status.READY +WAITING = OxTask.Status.WAITING + + +def deferring_worker(**kwargs): + return OxpullWorker( + backend_alias="flows_deferred", backoff_initial=0, poll_interval=0.01, **kwargs + ) + + +def order(): + return list(tasks.STATE.get("order", [])) + + +def run_after(label): + return OxTask.objects.get(args=[label]).run_after + + +def status(label): + return OxTask.objects.get(args=[label]).status + + +def chain(name="chain"): + """parent -> child, on the second workflow backend.""" + flow = workflows.create(name) + flow.add("parent", tasks.deferred_step, "parent") + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.seal() + return flow + + +def fan_in(name="fan-in"): + """{a, b} -> child, the case two workers can finish at the same instant.""" + flow = workflows.create(name) + flow.add("a", tasks.deferred_step, "a") + flow.add("b", tasks.deferred_step, "b") + flow.add("child", tasks.deferred_step, "child", depends_on=["a", "b"]) + flow.seal() + return flow + + +# -- the status -------------------------------------------------------- + + +def test_a_blocked_node_waits_and_a_free_node_does_not(): + chain() + assert (status("parent"), run_after("parent")) == (READY, None) + assert (status("child"), run_after("child")) == (WAITING, None) + + +def test_a_waiting_row_is_outside_the_candidate_set_of_every_worker(): + """ + Not merely refused by the gate: absent from the candidate set. The + ungated `django_ox.worker.Worker` is the sharpest way to say it, because the + gate is the only thing that ever held it back before. + """ + chain() + OxTask.objects.filter(args=["parent"]).update( + status=OxTask.Status.RUNNING, locked_by="someone-else" + ) + assert deferring_worker().claim_one() is None + assert Worker(backoff_initial=0, poll_interval=0.01).claim_one() is None + + +def test_a_node_whose_parent_already_succeeded_is_released_as_it_is_added(): + """ + add() reads the upstream as SUCCESSFUL, marks the edge and releases the + node in its own transaction, so nothing is owed a later release. + """ + flow = workflows.create("late-add") + flow.add("parent", tasks.deferred_step, "parent") + assert deferring_worker().run_once() is True + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.seal() + assert status("child") == READY + assert deferring_worker().run_once() is True + assert order() == ["parent", "child"] + + +def test_a_callers_own_run_after_is_never_overwritten(): + """ + run_after is a public argument and a stronger statement than ours: the + caller has said when this may run. The node waits with that time on its + row, and the release keeps it. + """ + later = timezone.now() + timezone.timedelta(days=1) + flow = workflows.create("scheduled") + flow.add("parent", tasks.deferred_step, "parent") + flow.add( + "child", + tasks.deferred_step.using(run_after=later), + "child", + depends_on="parent", + ) + flow.seal() + assert (status("child"), run_after("child")) == (WAITING, later) + assert deferring_worker().run_once() is True # the parent + assert (status("child"), run_after("child")) == (READY, later) + assert deferring_worker().run_once() is False + + +# -- the release ------------------------------------------------------- + + +def test_the_worker_releases_its_dependents_in_the_same_pass(): + """ + No reconciler, no second tick, no waiting: the worker that writes + SUCCESSFUL releases what that task was holding back, and the very next + claim takes it. + + This is the inline release on its own, and it is the reason the race + tests may end on the reconciler without that becoming the only thing + they prove. One worker on one connection, and no other writer anywhere, + so no chunk here can lose a lock conflict and the release either happens + in the pass or the product is broken. The reconciler is never run. + """ + chain() + worker = deferring_worker() + assert worker.run_once() is True + assert status("child") == READY + assert worker.run_once() is True + assert order() == ["parent", "child"] + + +def test_a_fan_in_child_is_released_only_by_the_last_parent(): + fan_in() + worker = deferring_worker() + assert worker.run_once() is True + assert status("child") == WAITING, "one parent of two must not release the child" + assert worker.run_once() is True + assert status("child") == READY + assert worker.run_once() is True + assert order()[-1] == "child" + + +def test_a_failed_parent_releases_nothing(): + flow = workflows.create("doomed") + flow.add("parent", tasks.failing_deferred_step, "parent") + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.seal() + worker = deferring_worker() + for _ in range(10): + if not worker.run_once(): + break + # Drop the retry backoff so the attempts are exhausted here rather + # than in real time. + OxTask.objects.filter(args=["parent"], status=READY).update(run_after=None) + assert status("parent") == OxTask.Status.FAILED + assert status("child") == WAITING + # And the reconciler cancels it rather than releasing it: a doomed cone + # is cancellation's business, not the release's. + workflows._reconcile() + assert status("child") == OxTask.Status.DISCARDED + + +def test_a_retrys_backoff_is_not_mistaken_for_a_release(): + """ + The release moves only WAITING rows. A node that failed and is going + round again is READY with a real backoff time, and a second successful + write on its upstream must not touch it. + """ + flow = workflows.create("retrying") + parent = flow.add("parent", tasks.deferred_step, "parent") + flow.add("child", tasks.failing_deferred_step, "child", depends_on="parent") + flow.seal() + worker = deferring_worker() + assert worker.run_once() is True # parent succeeds, child released + assert status("child") == READY + assert worker.run_once() is True # child fails, retries with a backoff + backoff = run_after("child") + assert backoff is not None + OxTask.objects.filter(id=parent.id).update(status=READY) + assert worker.run_once() is True + assert run_after("child") == backoff + + +# -- the backstop ------------------------------------------------------ + + +def test_the_reconciler_releases_what_the_worker_never_saw(): + """ + Worker._write_outcome is the only place django-ox writes SUCCESSFUL, so + on the normal path the worker's release covers it. This is everything + else: a status written out of band, a worker that died between the + outcome write and the release. All of them are bounded by one reconciler + tick, and that bound is why oxpull.W003 exists. + """ + chain() + OxTask.objects.filter(args=["parent"]).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + assert status("child") == WAITING + assert deferring_worker().run_once() is False + workflows._reconcile() + assert status("child") == READY + assert deferring_worker().run_once() is True + assert order() == ["child"] + + +def test_the_backstop_reaches_an_unsealed_workflow(): + """ + The release backstop covers OPEN workflows, unlike the sealed sweep. An + OPEN workflow's nodes run before it is sealed, so a waiting node in one + needs the backstop just as much. + """ + flow = workflows.create("open") + parent = flow.add("parent", tasks.deferred_step, "parent") + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + OxTask.objects.filter(id=parent.id).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + workflows._reconcile() + assert status("child") == READY + + +def test_a_pruned_upstream_that_left_no_record_keeps_its_dependents_waiting(): + """ + A node whose upstream row is gone, with nothing recorded about how that + upstream finished, is held: it stays WAITING through the reconciler's + pass. A missing row does not say the upstream succeeded, and this one + never ran at all, so releasing the node would run work whose input was + never produced. Holding is stuck work an operator can see; releasing is + wrongly run work. + """ + flow = workflows.create("pruned") + parent = flow.add("parent", tasks.deferred_step, "parent") + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.seal() + OxTask.objects.filter(id=parent.id).delete() + assert status("child") == WAITING + workflows._reconcile() + assert status("child") == WAITING, ( + f"the reconciler released the child to {status('child')} although its " + "upstream is gone and nothing recorded that it succeeded" + ) + + +# -- the transition under contention ----------------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_a_fan_in_released_by_racing_workers_runs_its_child_exactly_once(caplog): + """ + The transition, hammered. + + Eight workers drain a fan-in whose two parents can be claimed at the + same instant, so the release runs concurrently on both, against the + same child. What is asserted is an invariant and not a count: every + node ran exactly once, in dependency order, and nothing was left + waiting. + + Correctness never depended on the release: the child's claim is a + compare-and-set on one row and still carries the gate. Liveness is what + is under test, and the release is only half of what provides it. It is + best-effort by design: a chunk that loses two lock conflicts is logged + and left to the reconciler, and on SQLite it loses them readily, because + Django opens a DEFERRED transaction and the engine refuses the read-to- + write upgrade at once rather than waiting out the busy timeout. The + reconciler is the other half, and create() and seal() refuse a backend + that runs no schedule for it, so the run ends where a deployment's + would: the workers drain, then one reconciler pass and the drain after + it, with every thread joined. + + Two things are asserted about that ending, and together they are + stronger than asking the backstop to have had nothing to do. The + workflow is finished, with nothing waiting and nothing for the backstop + left. And whatever the reconciler had to run, the product said so first: + a wakeup that went missing without a word fails here. What it must never + take is a node run twice or out of order, which is asserted over both + drains. test_the_worker_releases_its_dependents_in_the_same_pass is + where the inline release is asserted on its own, with no contention. + """ + tasks.STATE["order"] = [] + flow = fan_in("raced") + total = 3 + + def work(): + worker = deferring_worker() + ran = 0 + while ran < 10 and worker.run_once(): + ran += 1 + return ran + + with caplog.at_level(logging.WARNING, logger="oxpull"): + results, errors = race_all([work] * 8) + settled = settle(flow, total, worker=deferring_worker()) + + assert not errors, errors[:3] + assert settled == 0 or deferrals(caplog), ( + f"the reconciler had {settled} task(s) to run and the product logged " + "no deferred or failed release, so a wakeup went missing in silence" + ) + ran = order() + assert sorted(ran) == ["a", "b", "child"], ran + assert ran.index("child") == 2, ran + assert sum(results) + settled == total + assert OxTask.objects.filter(status=OxTask.Status.SUCCESSFUL).count() == 3 + assert flow.progress().pending == 0 + # Now, and only now, the backstop is owed nothing. + assert workflows._release_backstop("default") == 0 + + +@pytest.mark.django_db(transaction=True) +def test_a_release_racing_an_operator_retry_of_the_same_upstream(): + """ + django_ox.actions.retry moves a settled upstream back to READY while a + release may be marking and releasing at that instant. + + The property under test is not that the release is correctly ordered + against this out-of-band traffic, but that the write paths do not + deadlock, do not raise, and cannot make a child claimable before its + parent has ever succeeded. + + That last one is asserted as a necessary condition rather than a + timestamp comparison: `succeeded` is set before any SUCCESSFUL write is + issued, so a child claimed while it is clear is a child that was + claimable with no successful parent behind it. + """ + import threading + + tasks.STATE["order"] = [] + flow = chain("contended") + parent = flow.node_ids()["parent"] + child = flow.node_ids()["child"] + succeeded = threading.Event() + violations = [] + + def releaser(): + for _ in range(20): + succeeded.set() + OxTask.objects.filter(id=parent).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + workflows._satisfy_edges_into([child], "default") + workflows.release_downstream([child], "default") + return True + + def retrier(): + for _ in range(20): + row = OxTask.objects.filter(id=parent).first() + if row is not None and row.status == OxTask.Status.SUCCESSFUL: + OxTask.objects.filter(id=parent).update(status=OxTask.Status.FAILED) + # Whether a retry is legal depends on the status another + # thread just wrote; the outcome is not the subject. + with contextlib.suppress(Exception): + actions.retry(parent) + return True + + def claimer(): + worker = deferring_worker() + for _ in range(20): + claimed = worker.claim_one() + if claimed is None: + continue + if claimed.id == child and not succeeded.is_set(): + violations.append(claimed.id) + OxTask.objects.filter(id=claimed.id).update( + status=OxTask.Status.READY, locked_by=None, locked_at=None + ) + return True + + _, errors = race_all([releaser, retrier, claimer] * 2) + assert not errors, errors[:3] + assert not violations, violations diff --git a/tests_oxpull/test_workflows_guarantees.py b/tests_oxpull/test_workflows_guarantees.py new file mode 100644 index 0000000..299d535 --- /dev/null +++ b/tests_oxpull/test_workflows_guarantees.py @@ -0,0 +1,258 @@ +""" +The guarantees a workflow makes that no single query can enforce. + +Each test here is written against a property an operator depends on: that +an unscheduled reconciler is reported rather than silently tolerated, that +blocked rows do not become permanent, that an idle sealed workflow costs no +write on a tick, that a worker without the gate never runs a node twice, +and that cancellation survives a concurrent operator retry. + +One property is accepted rather than enforced and is marked +`xfail(strict)`, so it stays visible without holding the suite red. Strict +turns an unexpected pass into a failure, so changing that behaviour forces +this file to be revisited. +""" + +import pytest +from django.db import connections +from django.tasks import task_backends +from django.test import override_settings +from django.test.utils import CaptureQueriesContext + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import ( + diamond, + flow_worker, + put_back_to_ready, + reconcile, + stored, + without_the_workflow_reconciler, +) + +pytestmark = pytest.mark.django_db + + +# The reconciler schedule, and what happens without one + + +def test_a_workflow_backend_with_no_reconciler_schedule_is_flagged(): + """ + A backend with workflows enabled and no `oxpull.workflows.reconcile` + schedule warns, as oxpull.W003. + + Unflagged, nothing cancels an unreachable + cone and nothing ever completes a workflow, so callbacks never fire + and blocked READY rows accumulate in the claim's candidate set + permanently. The claim cost is linear in that count, which makes it a + silent, unbounded throughput regression on the hot path. + + _check_workflow_reconciler now covers it, following the idiom + _check_unique_schedules already set for reading OPTIONS['SCHEDULES']. + """ + with override_settings(TASKS=without_the_workflow_reconciler()): + backend = task_backends["flows"] + # the 'flows' backend schedules nothing + assert "SCHEDULES" not in backend.options + errors = backend.check() + ids = {e.id for e in errors} + assert "oxpull.W003" in ids or any( + "reconcile" in str(e.msg).lower() for e in errors + ), ( + "no check warns that oxpull.workflows.reconcile is unscheduled; " + f"checks returned {[(e.id, e.msg) for e in errors]}" + ) + + +@pytest.mark.xfail( + strict=True, + reason=( + "Accepted, not fixed: with no reconciler scheduled there is no " + "bounded-time recovery. _check_workflow_reconciler warns at " + "manage.py check, and detection is not recovery. Fixing this means " + "recovering without a schedule; if that ever lands, strict xfail " + "turns the unexpected pass into a failure and this test is rewritten." + ), +) +def test_without_the_reconciler_blocked_rows_are_permanent(): + """ + The consequence, measured. A failed root leaves its whole cone WAITING + forever if no reconciler ever runs, and the workflow never completes. + + Everything up to the final assertion is true of the shipped code and + is worth pinning. The final assertion is the state we would want and + do not have. + """ + flow = diamond("no-reconciler") + ids = flow.node_ids() + OxTask.objects.filter(id=ids["extract"]).update(status=OxTask.Status.FAILED) + + worker = flow_worker() + assert worker.claim_one() is None, "nothing should be claimable" + blocked = flow.progress().blocked + assert blocked == 3, blocked + # No reconciler runs. The rows still wait, indefinitely. + assert ( + OxTask.objects.filter(id__in=ids.values(), status=OxTask.Status.WAITING).count() + == 3 + ) + assert stored(flow).status == OxpullWorkflow.Status.SEALED + # The expected failure. See the xfail reason above. + assert blocked == 0, ( + "3 workflow nodes wait forever, and the workflow never completes" + ) + + +# What a reconciler tick costs when nothing is cancellable + + +def test_an_idle_sealed_workflow_costs_no_write_on_a_reconciler_tick(): + """ + Eight sealed workflows with nothing cancellable produce no UPDATE on + the workflow table. + + Opening a transaction and issuing a real UPDATE before reading + anything would cost a write transaction per sealed workflow per tick, + whether or not anything was cancellable. At a once-a-minute cron that + is a write per workflow per minute forever, and on SQLite those + serialise against the workers. Both the cancel path and + _complete_if_finished read a cheap filter first. + """ + flows = [] + for i in range(8): + f = workflows.create(f"idle-{i}") + f.add("only", tasks.step, f"n{i}") + f.seal() + flows.append(f) + # Nothing is doomed: every node is simply still READY. + conn = connections["default"] + with CaptureQueriesContext(conn) as ctx: + reconcile() + writes = [ + q + for q in ctx.captured_queries + if q["sql"].lstrip().upper().startswith("UPDATE") + ] + updates_on_workflow = [ + q for q in writes if OxpullWorkflow._meta.db_table in q["sql"] + ] + assert len(updates_on_workflow) == 0, ( + f"{len(updates_on_workflow)} no-op UPDATE(s) on the workflow table for " + f"8 idle sealed workflows, one write transaction each, every tick:\n" + + "\n".join(q["sql"][:120] for q in updates_on_workflow[:3]) + ) + + +# What an ungated worker on the same queue does + + +@pytest.mark.django_db(transaction=True) +def test_an_ungated_worker_runs_nodes_out_of_order_but_never_twice(): + """ + A worker on an ungated backend draining the same queue violates the + DAG order. It never runs a node twice. + + Both halves matter. The claim is a compare-and-set on one row on + every path, so no arrangement of workers can double-execute; the gate + is only ever an extra conjunct on which rows are candidates, so a + worker without it simply ignores the ordering. The failure mode of a + mixed fleet is correctness of the DAG, not duplication. + """ + from django_ox.worker import Worker + + tasks.STATE["order"] = [] + # Put back to READY, the shape a django-ox retry leaves, so the ungated + # worker can reach the nodes the WAITING status would hold from it. + put_back_to_ready(diamond("mixed-fleet")) + gated, ungated = flow_worker(), Worker(backoff_initial=0, poll_interval=0.01) + + ran = 0 + while ran < 50: + # Alternate: the ungated worker wins some claims, the gated one others. + if ungated.run_once(): + ran += 1 + continue + if gated.run_once(): + ran += 1 + continue + break + + order = list(tasks.STATE.get("order", [])) + assert ran == 4, ran + # Never twice: exactly one execution per node. + assert sorted(order) == ["extract", "load", "transform-a", "transform-b"] + assert len(order) == len(set(order)), f"a node ran twice: {order}" + # Out of order: the ungated worker took `load` first, before its parents. + assert order[0] == "load", ( + f"expected the ungated worker to violate the DAG, got {order}" + ) + + +# The cancel path's lock ordering under contention + + +@pytest.mark.django_db(transaction=True) +def test_cancel_survives_a_concurrent_operator_retry_under_contention(): + """ + Six threads alternating reconciler passes and operator retries against + one workflow, on file-backed SQLite, raise no OperationalError. + + The cancel path opens with a write to take the lock before reading. + This hammers that ordering rather than trusting it: papering over + shows up as + OperationalError('database is locked'), correct lock ordering shows up + as waits. + """ + import contextlib + import threading + + from django.db import OperationalError + from django.db import connections as conns + + from django_ox import actions + + flow = diamond("contended") + ids = flow.node_ids() + OxTask.objects.filter(id=ids["extract"]).update(status=OxTask.Status.FAILED) + + errors, barrier = [], threading.Barrier(6) + + def reconciler(): + barrier.wait() + try: + for _ in range(15): + workflows._reconcile() + except OperationalError as exc: # the failure this guards against + errors.append(("reconcile", str(exc))) + finally: + conns.close_all() + + def retrier(): + barrier.wait() + try: + for _ in range(15): + row = OxTask.objects.filter(id=ids["extract"]).first() + if row and row.status == OxTask.Status.FAILED: + # A losing racer is the point; only a lock error matters. + with contextlib.suppress(Exception): + actions.retry(row.id) + OxTask.objects.filter(id=ids["extract"]).update( + status=OxTask.Status.FAILED + ) + except OperationalError as exc: + errors.append(("retry", str(exc))) + finally: + conns.close_all() + + threads = [threading.Thread(target=reconciler) for _ in range(3)] + [ + threading.Thread(target=retrier) for _ in range(3) + ] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + + assert not errors, f"lock contention surfaced as errors: {errors[:3]}" diff --git a/tests_oxpull/test_workflows_hazard_a_correctly_configured_worker_on_another.py b/tests_oxpull/test_workflows_hazard_a_correctly_configured_worker_on_another.py new file mode 100644 index 0000000..1ebfb28 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_correctly_configured_worker_on_another.py @@ -0,0 +1,612 @@ +""" +A correctly configured worker on another backend alias, sharing a queue +name, must not claim a workflow node before its upstreams have succeeded. + +A django-ox claim filters on status, run_after and queue_name, never on +backend_name, and a worker started without --queues takes every queue its +own backend lists (every queue when QUEUES is empty). So a second +OxpullBackend alias on the same database, with WORKER_CLASS set correctly and +no WORKFLOWS, reaches a workflow's nodes. tests_oxpull/settings.py is itself this +layout: "limited" (WORKER_CLASS = OxpullWorker, RATE_LIMITS, no WORKFLOWS) +serves queue "default" beside "flows". Its "default" alias serves the same +names but sets no WORKER_CLASS, so where a test needs "default" to be a +correctly configured OxpullBackend alias it constructs OxpullWorker directly (in +process) or adds WORKER_CLASS (the real process). + +What a fix has to do to pass: + + * Ordering holds whatever the other alias runs, for two reasons that are + tested apart. Every OxpullWorker gates its claim whenever any backend on + its database declares WORKFLOWS, so the gate holds a blocked node even + after it has been put back to READY. And every blocked node waits in + django-ox's WAITING status, including one the caller gave its own + run_after, so a worker that does not gate at all cannot claim it either. + * The other alias still runs the nodes, in order. A fix that stops it + claiming, or has it skip workflow rows, fails here: each test requires + the runnable node to be claimed, or every node to run. + * Nothing is refused. add() and seal() accept the nodes and `manage.py + check` exits 0, because an Error would stop every ox_worker on upgrade. + Instead the check reports a Warning, naming both aliases, whenever a + backend without WORKFLOWS serves a queue that a WORKFLOWS backend uses. + An empty QUEUES serves every queue, on either side. The Warning is + still needed once this version gates everywhere: a check in one process + cannot see which worker another deploy runs on that queue. + +Tests that still fail are marked xfail(strict=True, raises=AssertionError), +so a broken setup (the wrong oxpull imported, say) reports as a failure +rather than as the expected one. The WAITING status closed the tests that a +held row answers. The gate everywhere and the Warning are still open. + +Not covered: that the Warning is limited to one database. django-ox routes +every backend's task rows to a single database, so one settings module +cannot hold two aliases on different databases. +""" + +from __future__ import annotations + +import json +import os +import re +import signal +import subprocess +import sys +import time +from datetime import timedelta +from pathlib import Path + +import pytest +from django.conf import settings +from django.db import connection +from django.tasks import task_backends +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .subprocesses import django_ox_path +from .test_workflows import diamond, drain, order, reconcile + +REPO_ROOT = Path(__file__).resolve().parent.parent +SRC = REPO_ROOT / "src" + +NODES = ["extract", "transform-a", "transform-b", "load"] +EDGES = [ + ("extract", "transform-a"), + ("extract", "transform-b"), + ("transform-a", "load"), + ("transform-b", "load"), +] + + +def out_of_order(sequence): + """Edges whose downstream was claimed before, or without, its upstream.""" + first: dict[str, int] = {} + for index, label in enumerate(sequence): + first.setdefault(label, index) + return [ + (upstream, downstream) + for upstream, downstream in EDGES + if downstream in first + and (upstream not in first or first[upstream] > first[downstream]) + ] + + +def check_ids(*aliases): + return {alias: [m.id for m in task_backends[alias].check()] for alias in aliases} + + +def queues(*aliases): + return {alias: sorted(task_backends[alias].queues) for alias in aliases} + + +def oxpull_worker(alias): + """This branch's OxpullWorker, correctly configured, on `alias`.""" + return OxpullWorker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + + +def free_worker(alias): + """django-ox's own Worker, which has no dependency gate, on `alias`.""" + return Worker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + + +def root_and_child(name, *, child_run_after=None, **using): + """ + root (priority 0) -> child (priority 10): a claim that ignores the + dependency takes child first. + """ + flow = workflows.create(name) + root = flow.add("root", tasks.step.using(priority=0, **using), "root") + child_task = tasks.step.using(priority=10, **using) + if child_run_after is not None: + child_task = child_task.using(run_after=child_run_after) + child = flow.add("child", child_task, "child", depends_on="root") + flow.seal() + return root, child + + +def claimed_root(claimed, root, child): + """ + The assertion for a single claim, and what to say when it fails. + + The runnable root has to be the one claimed: None would pass a fix that + stops the other alias claiming at all, and child is the ordering bug. + """ + if claimed is None: + return False, "nothing" + label = {str(root.id): "root", str(child.id): "child"}.get( + str(claimed.pk), f"an unrelated task {claimed.args}" + ) + return label == "root", f"{label} from queue {claimed.queue_name!r}" + + +def rows(root, child): + root_row = OxTask.objects.get(pk=root.id) + child_row = OxTask.objects.get(pk=child.id) + return ( + f"root {root_row.status}, child {child_row.status} with run_after " + f"{child_row.run_after}" + ) + + +# -- in process, the suite's own settings --------------------------------- + + +@pytest.mark.django_db +def test_a_worker_on_a_plain_alias_sharing_the_queue_claims_the_root_not_the_child(): + """ + "default" is an OxpullBackend with no WORKFLOWS, serving ["default", + "emails"]; "flows" serves the same names. The worker is OxpullWorker on + that alias, which is what OPTIONS['WORKER_CLASS'] = OxpullWorker would + start (tests_oxpull/settings.py leaves the key out; the claim is the same). + """ + root, child = root_and_child("cross-alias-pair") + claimed = oxpull_worker("default").claim_one() + + ok, what = claimed_root(claimed, root, child) + assert ok, ( + f"OxpullWorker(backend_alias='default') claimed {what}; {rows(root, child)}. " + f"check ids: {check_ids('default', 'flows')}; queues: " + f"{queues('default', 'flows')}" + ) + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "A `oxpull.worker.OxpullWorker` on a backend without workflows puts no " + "dependency condition in its claim, so a blocked node put back to READY is " + "claimed ahead of its upstream." + ), +) +@pytest.mark.django_db +def test_a_worker_on_a_plain_alias_gates_a_blocked_node_that_is_not_deferred(): + """ + The gate on its own. The child is put back to READY after the build, the + shape a django-ox retry or an application's own UPDATE leaves, so + nothing but the dependency condition in this worker's claim can hold it. + A fix that only holds blocked nodes in WAITING fails here. + """ + root, child = root_and_child("cross-alias-gate") + moved = OxTask.objects.filter(pk=child.id, status=OxTask.Status.WAITING).update( + status=OxTask.Status.READY, run_after=None + ) + if moved != 1: + pytest.fail(f"setup: the child was not WAITING; {rows(root, child)}") + claimed = oxpull_worker("default").claim_one() + + ok, what = claimed_root(claimed, root, child) + assert ok, ( + f"OxpullWorker(backend_alias='default') claimed {what}; {rows(root, child)}. " + f"check ids: {check_ids('default', 'flows')}" + ) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "own_run_after", + [False, True], + ids=["without-a-run-after", "with-its-own-run-after-already-due"], +) +def test_a_worker_without_the_gate_cannot_claim_a_blocked_node(own_run_after): + """ + The held row on its own. django-ox's Worker on "default" stands in for + a process using the base worker or an earlier version on that queue, which no check + in this process can see. It has no gate, so only the node's own row can + keep it out. The second case gives the child a run_after that is already + due: a fix that holds only nodes without one fails there. + """ + child_run_after = timezone.now() - timedelta(hours=1) if own_run_after else None + root, child = root_and_child("cross-alias-fence", child_run_after=child_run_after) + claimed = free_worker("default").claim_one() + + ok, what = claimed_root(claimed, root, child) + assert ok, ( + f"django_ox Worker(backend_alias='default') claimed {what}; " + f"{rows(root, child)}; the caller's run_after was {child_run_after}" + ) + + +@pytest.mark.django_db +def test_a_rate_limited_backends_worker_runs_every_node_after_its_parents(): + """ + "limited" carries RATE_LIMITS and WORKER_CLASS = OxpullWorker, serves + ["default"], and declares no WORKFLOWS. The DAG is test_workflows.diamond, + whose priorities make claim order the reverse of dependency order. + """ + diamond("cross-alias-limited") + worker = oxpull_worker("limited") + # Drain, then one reconciler pass as the schedule would run it, so a + # waiting node is released the way production releases it when the + # worker that ran its upstream does not. + for _ in range(10): + drain(worker) + reconcile() + if len(order()) >= len(NODES): + break + ran = order() + + assert sorted(ran) == sorted(NODES) and out_of_order(ran) == [], ( + f"OxpullWorker(backend_alias='limited') ran {ran}; edges violated: " + f"{out_of_order(ran)}. check ids: " + f"{check_ids('limited', 'default', 'flows')}; queues: " + f"{queues('limited', 'flows')}" + ) + + +# -- in process, an alias with empty QUEUES -------------------------------- + + +def _catch_all_tasks(): + """ + The suite's TASKS, with "flows" moved to its own queue "pipelines" and a + plain OxpullBackend "catchall" whose QUEUES is empty. No other alias names + "pipelines", so only the empty list can put the nodes in its claim. + """ + configured = {alias: dict(config) for alias, config in settings.TASKS.items()} + configured["flows"]["QUEUES"] = ["pipelines"] + configured["catchall"] = { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": [], + "OPTIONS": {"WORKER_CLASS": "oxpull.worker.OxpullWorker"}, + } + return configured + + +@pytest.mark.django_db +def test_a_plain_alias_with_empty_queues_claims_the_root_not_the_child(): + with override_settings(TASKS=_catch_all_tasks()): + root, child = root_and_child("catch-all", queue_name="pipelines") + claimed = oxpull_worker("catchall").claim_one() + + ok, what = claimed_root(claimed, root, child) + assert ok, ( + f"OxpullWorker(backend_alias='catchall') claimed {what}; " + f"{rows(root, child)}. check ids: {check_ids('catchall', 'flows')}; " + f"queues: {queues('catchall', 'flows')}" + ) + + +# -- `manage.py check`, in a fresh interpreter ----------------------------- + +CHECK_SCRIPT = """ +import json +import sys + +import django +from django.conf import settings + +settings.configure( + SECRET_KEY="check-only", + USE_TZ=True, + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + INSTALLED_APPS=["django_ox", "oxpull"], + DATABASES={"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}}, + TASKS=json.loads(sys.argv[1]), +) +django.setup() +import oxpull + +print("oxpull from", oxpull.__file__) +from django.core.management import execute_from_command_line + +execute_from_command_line(["manage.py", "check"]) +""" + +# Both reconcilers scheduled, so oxpull.W002 and oxpull.W003 have nothing to +# say and the only thing left to report is the overlap. +RECONCILERS = { + "oxpull-batch-reconcile": {"task": "oxpull.batches.reconcile", "cron": "* * * * *"}, + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "cron": "* * * * *", + }, +} + +# Alias names that cannot turn up in a message by accident, the way +# "default" (also a queue name) or "flows" (inside "workflows") would, so +# finding one in a Warning means the Warning named that alias. +PLAIN = "shop" +FLOWS = "etl" + +# The reconcilers resolve their Task on the "default" task backend, so that +# alias has to exist. It carries the schedules on a queue of its own that no +# workflow backend below serves, unless that backend's QUEUES is empty. +HOUSEKEEPING = { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["housekeeping"], + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "SCHEDULES": RECONCILERS, + }, +} + + +def two_aliases(plain_queues, flow_queues, *, plain_workflows=False): + """ + A user's TASKS after adding Workflows on a second backend: the + existing backend keeps its correctly configured OxpullWorker, and the + new one declares WORKFLOWS. None means the QUEUES key is left out, which + Django reads as ["default"]. + """ + plain = { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": {"WORKER_CLASS": "oxpull.worker.OxpullWorker"}, + } + if plain_workflows: + plain["OPTIONS"]["WORKFLOWS"] = True + flows = { + "BACKEND": "oxpull.backend.OxpullBackend", + "OPTIONS": {"WORKER_CLASS": "oxpull.worker.OxpullWorker", "WORKFLOWS": True}, + } + if plain_queues is not None: + plain["QUEUES"] = plain_queues + if flow_queues is not None: + flows["QUEUES"] = flow_queues + return {"default": HOUSEKEEPING, PLAIN: plain, FLOWS: flows} + + +OVERLAPPING = { + "a-shared-queue-name": two_aliases(["default", "emails"], ["default"]), + "both-on-djangos-default-queue": two_aliases(None, None), + "a-catch-all-plain-alias": two_aliases([], ["pipelines"]), + "a-catch-all-workflow-alias": two_aliases(["emails"], []), +} + +APART = { + "queues-kept-apart": two_aliases(["default"], ["pipelines"]), + # A missing QUEUES key is ["default"], not every queue. + "plain-alias-left-on-djangos-default-queue": two_aliases(None, ["pipelines"]), + # Both backends gate, so sharing a queue is a correct install. + "both-aliases-declare-workflows": two_aliases( + ["default", "emails"], ["default"], plain_workflows=True + ), +} + + +def run_check(configured): + result = subprocess.run( # noqa: S603 + [sys.executable, "-c", CHECK_SCRIPT, json.dumps(configured)], + env={ + **os.environ, + "PYTHONPATH": os.pathsep.join([str(SRC), django_ox_path()]), + }, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + combined = result.stdout + result.stderr + # Prove which oxpull answered before believing what it said. A failure + # here is the setup, not the defect, so it must not count as the + # expected failure. + if f"oxpull from {SRC}" not in combined: + pytest.fail(f"the check did not import oxpull from {SRC}:\n{combined}") + return result, combined + + +def reported(combined, section): + """ + The messages `manage.py check` printed under WARNINGS: or ERRORS:, each + with its hint. Django prints every message at the start of a line and + indents its hint. + """ + match = re.search(rf"^{section}:\n(.*?)(?:\n\n|\Z)", combined, re.S | re.M) + if match is None: + return [] + return re.split(r"\n(?=\S)", match.group(1)) + + +def names(alias, message): + return re.search(rf"(? leaf, every node SUCCESSFUL. The root's row is pruned by the + default ox_prune statuses while the leaf still waits; a healthy + reconciler ticks in between. The leaf runs, because the gate reads the + pruned root as SUCCESSFUL. The workflow must then complete clean. + """ + caplog.set_level(logging.INFO, logger="oxpull") + step = tasks.step if backend == "gate" else tasks.deferred_step + worker = flow_worker() if backend == "gate" else deferring_worker() + + flow = _with_callbacks(f"long-clean-{backend}-{horizon}") + root = flow.add("root", step, "root") + leaf = flow.add("leaf", step, "leaf", depends_on="root") + flow.seal() + + assert worker.run_once() is True + assert order() == ["root"] + assert _status(root.id) == OxTask.Status.SUCCESSFUL + # A reconciler on its once-a-minute schedule: nothing to complete yet. + assert reconcile() == 0 + + said = _prune_past(horizon, [root.id]) + assert _status(leaf.id) == OxTask.Status.READY + + # The gate reads the pruned root as satisfied and runs the leaf. + assert worker.run_once() is True + assert order() == ["root", "leaf"] + assert _status(leaf.id) == OxTask.Status.SUCCESSFUL + + assert reconcile() == 1 + _run_callbacks() + row = stored(flow) + assert row.failed_count == 0 + assert tasks.STATE.get("complete") == 1 + # Completion reads the same absence as "outcome unknown". + assert row.success_task_id is not None and tasks.STATE.get("success") == 1, ( + "every node SUCCESSFUL and the leaf ran on the gate's reading of the " + "pruned root as SUCCESSFUL, yet on_success was withheld: " + + _record(flow, caplog, said) + ) + + +def test_a_delayed_node_past_the_prune_horizon_still_fires_on_success( + caplog, held_clock +): + """ + signup -> reminder, with the reminder scheduled after its upstream via + run_after: the shape of a drip campaign, scaled from days to seconds. + No row is written by hand. At the default --older-than 7d the same + sequence is a reminder scheduled more than seven days after signup. + + The clock stands still while signup runs, so the reminder is not due + however long that takes, and is then moved past the reminder's time, + which puts signup's finished_at more than the prune horizon behind it. + """ + caplog.set_level(logging.INFO, logger="oxpull") + worker = flow_worker() + due = held_clock.at + timedelta(seconds=2) + + flow = _with_callbacks("drip") + signup = flow.add("signup", tasks.step, "signup") + reminder = flow.add( + "reminder", + tasks.step.using(run_after=due), + "reminder", + depends_on="signup", + ) + flow.seal() + + assert worker.run_once() is True + assert order() == ["signup"] + assert worker.run_once() is False # the reminder is not due yet + assert reconcile() == 0 + + held_clock.at = due + timedelta(milliseconds=200) + out = StringIO() + call_command("ox_prune", "--older-than", "1s", stdout=out) + assert not OxTask.objects.filter(pk=signup.id).exists(), out.getvalue() + + assert worker.run_once() is True + assert order() == ["signup", "reminder"] + assert _status(reminder.id) == OxTask.Status.SUCCESSFUL + + assert reconcile() == 1 + _run_callbacks() + row = stored(flow) + assert row.success_task_id is not None and tasks.STATE.get("success") == 1, ( + "both nodes SUCCESSFUL, on_success withheld: " + + _record(flow, caplog, out.getvalue()) + ) + + +@pytest.mark.parametrize("horizon", HORIZONS) +def test_a_failed_branch_pruned_while_a_sibling_waits_is_counted_as_failed( + horizon, caplog +): + """ + root -> {bad, good}. bad fails every attempt while good is still + READY; a healthy reconciler ticks and cannot complete the workflow. + ox_prune --include-failed then deletes bad's FAILED row, good runs, and + the workflow completes. The workflow's record must still say a node + failed. + """ + caplog.set_level(logging.INFO, logger="oxpull") + worker = flow_worker() + + flow = _with_callbacks(f"long-failed-{horizon}") + flow.add("root", tasks.step, "root") + bad = flow.add("bad", tasks.failing_step, "bad", depends_on="root") + # Lower priority, so the claim takes every attempt of bad first and + # good is still READY when bad's row is pruned. + good = flow.add("good", tasks.step.using(priority=-5), "good", depends_on="root") + flow.seal() + + for _ in range(4): + assert worker.run_once() is True + assert order() == ["root", "bad", "bad", "bad"] + assert _status(bad.id) == OxTask.Status.FAILED + assert _status(good.id) == OxTask.Status.READY + assert reconcile() == 0 + + said = _prune_past(horizon, [bad.id], "--include-failed") + assert _status(good.id) == OxTask.Status.READY + + assert worker.run_once() is True + assert order() == ["root", "bad", "bad", "bad", "good"] + assert reconcile() == 1 + _run_callbacks() + row = stored(flow) + assert row.success_task_id is None # withheld, correctly + assert row.failed_count == 1, ( + "a node FAILED all 3 attempts, and the completed workflow records " + "failed_count=0: " + _record(flow, caplog, said) + ) + + +def test_an_operator_discard_of_a_failed_node_is_still_counted_as_failed(caplog): + """ + root -> {bad, good}. bad fails every attempt while good is still + READY. An operator triaging the failure discards bad (the django-ox + admin's "Discard selected tasks"), so default ox_prune can reclaim it. + good runs and the workflow completes. + """ + caplog.set_level(logging.INFO, logger="oxpull") + worker = flow_worker() + + flow = _with_callbacks("acknowledged") + flow.add("root", tasks.step, "root") + bad = flow.add("bad", tasks.failing_step, "bad", depends_on="root") + good = flow.add("good", tasks.step.using(priority=-5), "good", depends_on="root") + flow.seal() + + for _ in range(4): + assert worker.run_once() is True + assert _status(bad.id) == OxTask.Status.FAILED + assert _status(good.id) == OxTask.Status.READY + assert reconcile() == 0 + + assert actions.discard(bad.id) is True + assert worker.run_once() is True + assert order() == ["root", "bad", "bad", "bad", "good"] + assert reconcile() == 1 + _run_callbacks() + row = stored(flow) + assert row.success_task_id is None # withheld, correctly + assert row.failed_count == 1, ( + "a node FAILED all 3 attempts and was discarded afterwards; the " + "completed workflow records failed_count=0: " + _record(flow, caplog) + ) diff --git a/tests_oxpull/test_workflows_hazard_a_downstream_node_can_start_while_an_earlier.py b/tests_oxpull/test_workflows_hazard_a_downstream_node_can_start_while_an_earlier.py new file mode 100644 index 0000000..ec819ab --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_downstream_node_can_start_while_an_earlier.py @@ -0,0 +1,427 @@ +""" +A downstream node starts while an earlier, fenced-off +execution of its upstream is still running. + +The claim gate reads the upstream's row. django-ox 1.2.0 fences rows, not +executions: when a lease is handed over with attempts left, the row goes +back to READY with the epoch moved on while the first execution's thread +keeps going. A second execution then succeeds, the row reads SUCCESSFUL, +and the child is claimable, while the first execution is still inside the +upstream's body producing that node's side effects. + +Two production routes, both through the real worker: + +- test_..._while_a_requeued_upstream_execution_is_live: the reaper requeues + a live execution. Either an operator shortens the lease with + actions.expire_lease (documented "It does not stop the task") and another + worker reaps, or the worker's own renewal UPDATE cannot reach the + database for longer than LOCK_TIMEOUT and its own run() loop's reaper + takes the row. The upstream is a truncate-and-reload; the first execution + is slow to start its reload and the child reads the table mid-reload. + +- test_..._while_a_stuck_upstream_execution_is_live: the TASK_TIMEOUT grace + backstop. The node blocks in a C call past the deadline and the grace; the + watchdog writes READY with the epoch moved and recycles the worker, which + then drains a healthy sibling task while the stuck thread is known to be + alive. Nothing is injected: the only inputs are OPTIONS and a body that + blocks on a lock. + +Each test asserts what the Workflows copy promises ("a node becomes +claimable only once every node it depends on has succeeded"): no execution +of a downstream node begins while any execution of its upstream is still +inside the upstream's body, and the child eventually runs. A fix that makes +the child wait passes; a fix that never runs the child does not. +""" + +import threading +import time + +import pytest +from django.conf import settings +from django.db import OperationalError, connections +from django.tasks import task +from django.test import override_settings + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +WAIT = 30.0 +ROWS = 5 +FACTORIES = {"flows": flow_worker, "flows_deferred": deferring_worker} + +# Shared between worker threads and the test. Module state rather than +# tasks.STATE because a worker thread resolves the node by its dotted path. +_GUARD = threading.Lock() +TABLE: list[int] = [] # what the upstream loads and the child reads +LIVE: dict[str, set[int]] = {} # label -> executions inside the body now +CALLS: dict[str, int] = {} # label -> executions started +SEEN: list[dict] = [] # one entry per downstream execution, taken at entry +CTL: dict = {} + + +@pytest.fixture(autouse=True) +def _shared_state(): + _reset() + yield + for key in ("before_truncate", "finish_reload", "sibling_release"): + if key in CTL: + CTL[key].set() + if "c_gate" in CTL and CTL["c_gate"].locked(): + CTL["c_gate"].release() + _reset() + + +def _reset(): + TABLE.clear() + LIVE.clear() + CALLS.clear() + SEEN.clear() + CTL.clear() + + +@task(backend="flows") +def reload_table(label): + """ + Truncate-and-reload. Execution 1 is the slow one: in "python" mode it + stalls before its truncate and again after the first row (a slow source + read); in "c" mode it blocks in a C call, where TaskTimeout cannot land. + Every later execution reloads straight through. + """ + with _GUARD: + n = CALLS[label] = CALLS.get(label, 0) + 1 + LIVE.setdefault(label, set()).add(n) + try: + if n == 1: + CTL["exec1_started"].set() + if CTL["mode"] == "c": + CTL["c_gate"].acquire(timeout=WAIT) + else: + CTL["before_truncate"].wait(WAIT) + TABLE.clear() + TABLE.append(0) + CTL["mid_reload"].set() + CTL["finish_reload"].wait(WAIT) + TABLE.extend(range(1, ROWS)) + else: + TABLE.clear() + TABLE.extend(range(ROWS)) + return tasks._record(label) + finally: + with _GUARD: + LIVE[label].discard(n) + + +@task(backend="flows") +def read_table(label): + """The child: reads what its upstream loaded, and notes what was live.""" + with _GUARD: + live = {name: sorted(ns) for name, ns in LIVE.items() if ns} + probe = CTL.get("probe") + SEEN.append( + { + "label": label, + "live": live, + "table": list(TABLE), + "context": probe() if probe else None, + } + ) + return tasks._record(label) + + +@task(backend="flows", queue_name="emails") +def sibling_step(label): + """Another task in flight on the same worker; its queue has no timeout.""" + CTL["sibling_started"].set() + CTL["sibling_release"].wait(WAIT) + return label + + +class Runner: + """A target on its own thread, the way a pool thread would run it.""" + + def __init__(self, target): + self.errors: list[BaseException] = [] + self.thread = threading.Thread(target=self._run, args=(target,), daemon=True) + + def _run(self, target): + try: + target() + except BaseException as exc: + self.errors.append(exc) + finally: + connections.close_all() + + def start(self): + self.thread.start() + return self + + +def wait_until(predicate, what, timeout=WAIT): + deadline = time.monotonic() + timeout + while not predicate(): + assert time.monotonic() < deadline, f"timed out waiting for: {what}" + time.sleep(0.01) + + +def chain(alias, name): + flow = workflows.create(name) + root = flow.add("root", reload_table.using(backend=alias), "root") + child = flow.add( + "child", read_table.using(backend=alias), "child", depends_on="root" + ) + flow.seal() + return flow, root, child + + +def row(result): + return OxTask.objects.get(id=result.id) + + +def settle(factory, rounds=3): + """Give the child every chance a fix might use to run it later.""" + for _ in range(rounds): + reconcile() + drain(factory()) + + +def assert_no_child_overlapped(child): + child_runs = [seen for seen in SEEN if seen["label"] == "child"] + assert child_runs, "the child never ran" + assert row(child).status == OxTask.Status.SUCCESSFUL + for seen in child_runs: + assert not seen["live"], ( + "the child started while execution(s) " + f"{seen['live'].get('root')} of its upstream 'root' were still " + f"inside root's body; the child read {seen['table']} " + f"(a full load is {list(range(ROWS))}); context {seen['context']}" + ) + assert seen["table"] == list(range(ROWS)), seen + + +@pytest.mark.xfail( + strict=True, + reason=( + "A downstream node starts while an earlier execution of its upstream, " + "whose lease was handed to another worker, is still running the " + "upstream's body." + ), +) +@pytest.mark.parametrize("alias", sorted(FACTORIES)) +@pytest.mark.parametrize("how", ["operator_expires_lease", "renewals_fail"]) +def test_a_downstream_node_never_starts_while_a_requeued_upstream_execution_is_live( + alias, how +): + factory = FACTORIES[alias] + CTL.update( + mode="python", + exec1_started=threading.Event(), + before_truncate=threading.Event(), + mid_reload=threading.Event(), + finish_reload=threading.Event(), + ) + _flow, root, child = chain(alias, f"overlap-{how}") + + # Execution 1 of root, on a real worker with a real lease. + refused = [] + if how == "renewals_fail": + holder = factory(lock_timeout=1.0) + + def cannot_reach_the_database(): + refused.append(time.monotonic()) + raise OperationalError("injected: renewal cannot reach the database") + + holder.renew_leases = cannot_reach_the_database + runner = Runner(holder.run).start() + else: + holder = factory() + runner = Runner(holder.run_once).start() + try: + assert CTL["exec1_started"].wait(WAIT), "execution 1 never started" + claimed = row(root) + assert claimed.status == OxTask.Status.RUNNING + assert claimed.attempts == 1 < claimed.max_attempts + assert claimed.lease_expires_at is not None + epoch1 = claimed.lease_epoch + assert epoch1 >= 1 + + # The lease is handed over while execution 1 is alive. + if how == "renewals_fail": + wait_until( + lambda: row(root).status != OxTask.Status.RUNNING, + "the holder's own reaper to requeue root", + ) + assert refused, "no renewal was attempted" + else: + assert actions.expire_lease(root.id) is True + assert factory().reap() == 1 + requeued = row(root) + assert requeued.status == OxTask.Status.READY + assert requeued.lease_epoch == epoch1 + 1 + assert runner.thread.is_alive() and LIVE["root"] == {1} + + # Execution 2 of root, on another worker, succeeds. The child is + # still gated, so root is the only row this claim can take. + other = factory() + assert other.run_once() is True + assert CALLS["root"] == 2 + done = row(root) + assert done.status == OxTask.Status.SUCCESSFUL + assert done.lease_epoch == epoch1 + 2 + assert list(range(ROWS)) == TABLE + # The precondition for the hazard, independent of the child: + # root's row is SUCCESSFUL while execution 1 is inside the body. + assert runner.thread.is_alive() and LIVE["root"] == {1} + + # Execution 1 resumes: it truncates and has loaded one row. + CTL["before_truncate"].set() + assert CTL["mid_reload"].wait(WAIT) + CTL["probe"] = lambda: {"execution_1_thread_alive": runner.thread.is_alive()} + + # Whatever a worker can claim now. Not asserted: a fix may refuse it. + other.run_once() + finally: + CTL["before_truncate"].set() + CTL["finish_reload"].set() + if how == "renewals_fail": + holder.request_stop() + runner.thread.join(WAIT) + assert not runner.thread.is_alive() + assert runner.errors == [] + assert CALLS["root"] == 2 and LIVE["root"] == set() + # Execution 1's outcome write was refused by the lease: the row is + # execution 2's. + assert row(root).status == OxTask.Status.SUCCESSFUL + assert row(root).lease_epoch == epoch1 + 2 + + settle(factory) + assert_no_child_overlapped(child) + + +@pytest.mark.xfail( + strict=True, + reason=( + "A downstream node starts while an upstream execution that overran its " + "timeout and grace period is still stuck inside the upstream's body." + ), +) +@pytest.mark.parametrize("alias", sorted(FACTORIES)) +def test_a_downstream_node_never_starts_while_a_stuck_upstream_execution_is_live( + alias, +): + """ + The backstop path. The overlap here is not an interleaving the test + picks: from the moment the backstop writes READY until the recycling + process exits, the stuck thread is alive by the worker's own account, + and the drain keeps the process up for as long as a sibling needs, up to + recycle_drain_budget (LOCK_TIMEOUT, 300 s by default). + """ + factory = FACTORIES[alias] + backend = settings.TASKS[alias] + timeouts = { + **settings.TASKS, + alias: { + **backend, + "OPTIONS": { + **backend["OPTIONS"], + "TASK_TIMEOUTS": {"default": 0.5}, + "TASK_TIMEOUT_GRACE": 0.5, + }, + }, + } + c_gate = threading.Lock() + c_gate.acquire() + CTL.update( + mode="c", + c_gate=c_gate, + exec1_started=threading.Event(), + sibling_started=threading.Event(), + sibling_release=threading.Event(), + ) + with override_settings(TASKS=timeouts): + _flow, root, child = chain(alias, "overlap-backstop") + sibling = sibling_step.using(backend=alias).enqueue("sibling") + + # The production worker loop, concurrency 2, a real (non-zero) + # backoff, the default drain budget rule (= LOCK_TIMEOUT). + holder = OxpullWorker( + backend_alias=alias, + concurrency=2, + poll_interval=0.01, + backoff_initial=0.2, + lock_timeout=60, + ) + assert holder.timeouts.for_queue("default") == 0.5 + assert holder.timeouts.for_queue("emails") is None + runner = Runner(holder.run).start() + try: + assert CTL["exec1_started"].wait(WAIT), "execution 1 never started" + assert CTL["sibling_started"].wait(WAIT), "the sibling never started" + epoch1 = row(root).lease_epoch + assert epoch1 >= 1 + + # The backstop: TaskTimeout could not land in the C call, the + # grace passed, the attempt is recorded failed with attempts + # left, the epoch moved, and the worker is recycling. + wait_until( + lambda: row(root).status == OxTask.Status.READY, + "the grace backstop to hand root back", + ) + handed = row(root) + assert handed.lease_epoch == epoch1 + 1 + assert handed.attempts == 1 < handed.max_attempts + assert "did not stop" in handed.errors[-1]["traceback"] + wait_until(lambda: holder.recycling, "the holder to start recycling") + # The drain is holding the process open on the sibling, and + # execution 1 is still inside root's body. + assert runner.thread.is_alive() + assert row(sibling).status == OxTask.Status.RUNNING + assert LIVE["root"] == {1} + + CTL["probe"] = lambda: { + "holder_recycling": holder.recycling, + "holder_run_returned": not runner.thread.is_alive(), + "execution_1_live": sorted(LIVE.get("root", ())), + } + + # Another worker takes root once its backoff passes. + other = factory() + wait_until( + lambda: ( + other.run_once() or row(root).status == OxTask.Status.SUCCESSFUL + ), + "another worker to run execution 2 of root", + ) + assert row(root).status == OxTask.Status.SUCCESSFUL + assert CALLS["root"] == 2 + assert row(root).lease_epoch == epoch1 + 2 + # Root's row is SUCCESSFUL while the recycling process still runs + # execution 1. + assert LIVE["root"] == {1} + assert runner.thread.is_alive() + + # Whatever a worker can claim now. Not asserted: a fix may refuse it. + other.run_once() + finally: + if c_gate.locked(): + c_gate.release() + CTL["sibling_release"].set() + runner.thread.join(WAIT) + assert not runner.thread.is_alive() + assert runner.errors == [] + assert holder.recycling + # The stuck thread returns from the C call; its pending TaskTimeout + # lands, its write is refused, and it leaves the body. + wait_until(lambda: LIVE["root"] == set(), "execution 1 to leave the body") + assert row(root).status == OxTask.Status.SUCCESSFUL + assert row(sibling).status == OxTask.Status.SUCCESSFUL + + settle(factory) + assert_no_child_overlapped(child) diff --git a/tests_oxpull/test_workflows_hazard_a_node_that_builds_workflow_structure_is.py b/tests_oxpull/test_workflows_hazard_a_node_that_builds_workflow_structure_is.py new file mode 100644 index 0000000..9a9d85b --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_node_that_builds_workflow_structure_is.py @@ -0,0 +1,397 @@ +""" +A node that builds workflow structure is not re-executable. + +Nothing refuses building a workflow from inside a node, and the nodes of +an OPEN workflow already run, so the fan-out-after-discovery shape is +invited: extract runs, finds N items, and adds N transforms that depend +on itself. Workflow.add(), Workflow.seal() and workflows.create() each +commit in their own transaction, whatever lease the calling node holds. + +django-ox runs a task at least once. A node body runs again when: + +- lease_lost: its lease is taken while the body is still running (a + stall past LOCK_TIMEOUT, or django_ox.actions.expire_lease, which is + documented as "It does not stop the task"). The reaper requeues the + row with the epoch bumped, and the fence refuses execution 1's own + SUCCESSFUL write, although its body returned normally. +- raised: the body raises after its structure committed, and the attempt + is retried like any other. + +Execution 2 then meets what execution 1 committed. add() raises +WorkflowNodeExists, or WorkflowSealed if execution 1 also sealed, so +every remaining attempt fails and the node ends FAILED. The reconciler +then discards the child execution 1 added, because its only upstream is +FAILED. A node that calls create() starts a second sub-workflow of the +same name instead; both run in full and both fire their callbacks. + +Everything runs through the production path. A real OxpullWorker claims +and executes the node, on a thread when its lease is to be lost. The +lease is expired with the public action and taken by a second worker's +real reaper. Every later attempt is claimed and executed by a worker, and +the reconciler runs as the task a SCHEDULES entry enqueues, claimed by a +gating worker. No task row is written by hand. + +Each test asserts the re-executable outcome: the node whose body +succeeded is SUCCESSFUL, the structure it built exists once and runs +once, and the workflow completes clean. A fix that instead refuses +building structure from inside a node would fail these at their +preconditions and needs its own tests. +""" + +import threading +import time + +import pytest +from django.db import connections +from django.tasks import task + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, order, stored + +pytestmark = pytest.mark.django_db(transaction=True) + +WAIT = 30.0 + +REEXECUTIONS = ["lease_lost", "raised"] + +# label -> the node's controls. Module state, because a worker thread +# resolves the task by its dotted path and must reach the Events the test +# holds. +_CONTROL: dict[str, dict] = {} + + +@pytest.fixture(autouse=True) +def _control(): + _CONTROL.clear() + yield + for ctl in _CONTROL.values(): + ctl["release"].set() + _CONTROL.clear() + + +def _arm(label, mode): + _CONTROL[label] = { + "mode": mode, + "calls": 0, + "bodies_completed": 0, + "raised": [], + "started": threading.Event(), + "release": threading.Event(), + "released": None, + } + + +def _enter(label): + _CONTROL[label]["calls"] += 1 + + +def _after_structure(label): + """ + Execution 1 only: outlive the lease, or fail transiently. The + structure it built is already committed either way. + """ + ctl = _CONTROL[label] + if ctl["calls"] == 1: + if ctl["mode"] == "lease_lost": + ctl["started"].set() + ctl["released"] = ctl["release"].wait(timeout=60) + else: + raise RuntimeError( + f"{label}: transient failure after its structure committed" + ) + ctl["bodies_completed"] += 1 + tasks._record(label) + return label + + +def _raising(label, exc): + _CONTROL[label]["raised"].append(type(exc).__name__) + + +@task(backend="flows") +def grafting_step(workflow_id, label, seal_in_body): + """Discovers its work, then adds a child that depends on itself.""" + _enter(label) + try: + flow = workflows.get(workflow_id) + flow.add(f"{label}-child", tasks.step, f"{label}-child", depends_on=label) + if seal_in_body: + flow.seal() + except Exception as exc: + _raising(label, exc) + raise + return _after_structure(label) + + +@task(backend="flows") +def spawning_step(label): + """Starts a sub-workflow of its own.""" + _enter(label) + sub = workflows.create( + f"sub-of-{label}", + on_success=tasks.record_callback, + on_success_args=[f"sub-of-{label}-success"], + ) + sub.add("leaf", tasks.step, f"{label}-leaf") + sub.seal() + return _after_structure(label) + + +def first_execution(root_id, label, mode): + """ + Run execution 1 of the node through a real worker, and leave the row + READY for a second execution the way production does. + """ + ctl = _CONTROL[label] + if mode == "raised": + assert flow_worker().run_once() is True + row = OxTask.objects.get(id=root_id) + assert row.status == OxTask.Status.READY, row.status + assert row.attempts == 1 + assert [e["exception_class_path"] for e in row.errors] == [ + "builtins.RuntimeError" + ] + return {"epoch_at_claim": row.lease_epoch, "epoch_after": row.lease_epoch} + + executor = flow_worker() + errors: list[BaseException] = [] + + def run(): + try: + executor.run_once() + except BaseException as exc: + errors.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + try: + assert ctl["started"].wait(WAIT), "execution 1 never reached its structure" + row = OxTask.objects.get(id=root_id) + assert row.status == OxTask.Status.RUNNING + assert row.attempts == 1 < row.max_attempts + # A 1.2.0 lease: a stored expiry and a claimed epoch. + assert row.lease_expires_at is not None + assert row.lease_epoch >= 1 + claimed = row.lease_epoch + assert actions.expire_lease(root_id) is True + assert flow_worker().reap() == 1 + row = OxTask.objects.get(id=root_id) + assert row.status == OxTask.Status.READY + assert row.lease_epoch == claimed + 1 + assert thread.is_alive(), "execution 1 must outlive its lease" + finally: + ctl["release"].set() + thread.join(WAIT) + assert not thread.is_alive() + assert errors == [] + assert ctl["released"] is True, "execution 1 timed out instead of finishing" + assert ctl["bodies_completed"] == 1 + row = OxTask.objects.get(id=root_id) + # Execution 1's body returned normally and its SUCCESSFUL write was + # refused by the fence: the row is still the requeued one. + assert row.status == OxTask.Status.READY + assert row.lease_epoch == claimed + 1 + return {"epoch_at_claim": claimed, "epoch_after": row.lease_epoch} + + +def reconciler_tick(): + """ + One reconciler tick, run the way a SCHEDULES entry runs it: enqueued + as the task, then claimed and executed by a gating worker. + """ + result = workflows.reconcile.enqueue() + runner = flow_worker() + deadline = time.monotonic() + WAIT + while OxTask.objects.get(id=result.id).status in ( + OxTask.Status.READY, + OxTask.Status.RUNNING, + ): + assert time.monotonic() < deadline, "no worker ran the reconciler" + if not runner.run_once(): + time.sleep(0.01) + row = OxTask.objects.get(id=result.id) + assert row.status == OxTask.Status.SUCCESSFUL, row.errors + return row.return_value + + +def node_rows(flow): + ids = flow.node_ids() + rows = {row.id: row for row in OxTask.objects.filter(id__in=list(ids.values()))} + return {name: rows[tid] for name, tid in ids.items()} + + +def describe_node(row): + return ( + f"{row.status} attempts={row.attempts}/{row.max_attempts} " + f"epoch={row.lease_epoch} " + f"errors={[e['exception_class_path'] for e in row.errors]}" + ) + + +def callbacks_enqueued(label): + return OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback", args=[label] + ).count() + + +@pytest.mark.xfail( + strict=True, + reason=( + "A node that adds to its own workflow cannot run again: every later " + "execution raises on the structure the first one committed, so the " + "node ends FAILED and the child it added is discarded." + ), +) +@pytest.mark.parametrize("seal", ["seal_afterwards", "seal_in_body"]) +@pytest.mark.parametrize("mode", REEXECUTIONS) +def test_a_node_that_grafts_onto_its_own_workflow_survives_running_again(mode, seal): + flow = workflows.create( + f"graft-{mode}-{seal}", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + root = flow.add("root", grafting_step, str(flow.id), "root", seal == "seal_in_body") + _arm("root", mode) + + epochs = first_execution(root.id, "root", mode) + # Execution 1's structure is committed before anything runs it again. + assert "root-child" in flow.node_ids() + if seal == "seal_in_body": + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + drain(flow_worker()) + ctl = _CONTROL["root"] + assert ctl["calls"] >= 2, "the node was never run a second time" + if seal == "seal_afterwards": + # Whoever seals once discovery is over; here, after the node ran. + flow.seal() + reconciler_tick() + drain(flow_worker()) + reconciler_tick() + + nodes = node_rows(flow) + row = stored(flow) + problems = [] + if nodes["root"].status != OxTask.Status.SUCCESSFUL: + problems.append( + f"root is {nodes['root'].status}: execution 1 committed its " + f"structure, and every later execution raised {ctl['raised']}" + ) + if nodes["root-child"].status != OxTask.Status.SUCCESSFUL: + problems.append( + f"root-child, which execution 1 of root added, is " + f"{nodes['root-child'].status}" + ) + if order().count("root-child") != 1: + problems.append(f"root-child ran {order().count('root-child')} time(s)") + if ( + row.status != OxpullWorkflow.Status.COMPLETE + or row.failed_count != 0 + or row.cancelled_count != 0 + or row.success_task_id is None + ): + problems.append( + f"workflow {row.status}, failed_count={row.failed_count}, " + f"cancelled_count={row.cancelled_count}, on_success " + f"{'withheld' if row.success_task_id is None else 'enqueued'}" + ) + assert not problems, "\n".join( + [ + f"re-executed via {mode}, {seal}", + *problems, + f"epochs {epochs}", + f"nodes: { ({n: describe_node(r) for n, r in nodes.items()}) }", + ( + f"body entered {ctl['calls']} time(s), returned normally " + f"{ctl['bodies_completed']}; order={order()}" + ), + ( + f"callbacks enqueued: complete={callbacks_enqueued('complete')}, " + f"success={callbacks_enqueued('success')}" + ), + ] + ) + + +@pytest.mark.xfail( + strict=True, + reason=( + "A node that starts a sub-workflow starts a second one of the same name " + "when it runs again, and both run in full and fire their callbacks." + ), +) +@pytest.mark.parametrize("mode", REEXECUTIONS) +def test_a_node_that_starts_a_sub_workflow_starts_exactly_one(mode): + parent = workflows.create(f"parent-{mode}") + root = parent.add("root", spawning_step, "root") + parent.seal() + _arm("root", mode) + + epochs = first_execution(root.id, "root", mode) + # Execution 1 created and sealed its sub-workflow before anything runs + # the node again. + subs = OxpullWorkflow.objects.filter(name="sub-of-root") + assert subs.count() == 1 + assert subs.get().status == OxpullWorkflow.Status.SEALED + + drain(flow_worker()) + ctl = _CONTROL["root"] + assert ctl["calls"] >= 2, "the node was never run a second time" + reconciler_tick() + drain(flow_worker()) + + root_row = OxTask.objects.get(id=root.id) + sub_rows = list(subs.order_by("created_at")) + leaf_runs = order().count("root-leaf") + success_callbacks = callbacks_enqueued("sub-of-root-success") + problems = [] + if len(sub_rows) != 1: + problems.append( + f"{len(sub_rows)} workflows named 'sub-of-root', started by one " + "node that succeeded once" + ) + if leaf_runs != 1: + problems.append(f"the sub-workflow's leaf ran {leaf_runs} time(s)") + if success_callbacks != 1: + problems.append( + f"the sub-workflow's on_success was enqueued {success_callbacks} time(s)" + ) + if root_row.status != OxTask.Status.SUCCESSFUL: + problems.append(f"root is {root_row.status}") + assert not problems, "\n".join( + [ + f"re-executed via {mode}", + *problems, + f"epochs {epochs}", + f"root: {describe_node(root_row)}", + "sub-workflows: " + + str( + [ + ( + str(s.id)[:8], + s.status, + s.total_count, + s.success_task_id is not None, + ) + for s in sub_rows + ] + ), + ( + f"body entered {ctl['calls']} time(s), returned normally " + f"{ctl['bodies_completed']}; order={order()}" + ), + ( + "record_callback ran for sub success: " + f"{tasks.STATE.get('sub-of-root-success', 0)}" + ), + ] + ) diff --git a/tests_oxpull/test_workflows_hazard_a_stored_admin_edited_reconciler_schedule.py b/tests_oxpull/test_workflows_hazard_a_stored_admin_edited_reconciler_schedule.py new file mode 100644 index 0000000..ba30675 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_stored_admin_edited_reconciler_schedule.py @@ -0,0 +1,566 @@ +""" +A stored (admin-edited) reconciler schedule on a +backend whose QUEUES lack the default backend's queue. + +oxpull.workflows.reconcile and oxpull.batches.reconcile are built lazily by +batches._build_task_on_a_queue, which takes the queue from the DEFAULT +backend: "default" if that backend accepts it, else its first queue. A +django-ox 1.2.0 stored schedule is dispatched by the backend whose OPTIONS +name SCHEDULE_SOURCE, and DatabaseScheduleSource._to_schedule rebinds the +registry task with Task.using(backend=), which re-runs +validate_task. When that backend's QUEUES does not contain the queue the +default backend handed the reconciler, the rebind raises InvalidTask, and +_build logs "Skipping stored schedule ..." and drops the row. The row has +no queue_name column, so nothing on the row can fix it. The settings +channel raises ImproperlyConfigured for the same layout instead. + +The layout: "default" keeps its own QUEUES (it contains "default"), and a +"cron" backend with QUEUES ["pipelines"] reads schedule rows and exposes +the reconciler through SCHEDULABLE_TASKS. No SCHEDULES block anywhere. + +Every non-control test asserts what a user needs: either the row the +admin and create_schedule accept is one the dispatching worker builds +and runs (so workflows complete, dead cones are cancelled, missed +releases are released, and batches complete), or the configuration is +refused where an operator cannot miss it. Valid refusal points are +create_schedule, the admin form, workflows.create(), batches.create(), +or a backend check reporting an Error that also stops `ox_worker`. + +Every test named test_control_* runs the same row, cron backend, QUEUES +and worker as its regression case, with one difference: +SCHEDULABLE_TASKS exposes the reconciler already bound to the cron +backend's own queue (reconcile.using(backend="cron", +queue_name="pipelines"), built below). These controls isolate failures +caused by the queue the reconciler is built on from harness failures. +One more control shows the settings channel refuses the same layout loudly. + +Batch tests import nothing from oxpull.workflows, so `-k batch` also runs +against oxpull 1.2.0's src (a control run on the released package). +""" + +import copy +import logging +from datetime import timedelta + +import pytest +from django.conf import settings +from django.core import checks +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.db import OperationalError +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.stored import create_schedule +from django_ox.worker import Worker +from oxpull import batches +from oxpull.models import OxpullBatch + +from . import tasks + +pytestmark = pytest.mark.django_db + +WORKFLOW_RECONCILE = "oxpull.workflows.reconcile" +BATCH_RECONCILE = "oxpull.batches.reconcile" +# The controls' reconcilers, bound to the cron backend's own queue. +WORKFLOW_RECONCILE_ON_CRON = f"{__name__}.workflow_reconcile_on_cron" +BATCH_RECONCILE_ON_CRON = f"{__name__}.batch_reconcile_on_cron" +ON_CRON = { + WORKFLOW_RECONCILE: WORKFLOW_RECONCILE_ON_CRON, + BATCH_RECONCILE: BATCH_RECONCILE_ON_CRON, +} +ROW_KEY = "reconcile" +ROW_NAME = "reconcile-every-minute" +SKIP_LOG = "Skipping stored schedule" + + +def __getattr__(name): + """ + The controls' reconcilers, built on access as oxpull builds its own: + the "cron" alias exists only inside each test's override_settings. The + function is the same one, so the task row still stores the reconciler's + public path and the worker resolves it by that path. + """ + if name == "workflow_reconcile_on_cron": + from oxpull import workflows + + return workflows.reconcile.using(backend="cron", queue_name="pipelines") + if name == "batch_reconcile_on_cron": + return batches.reconcile.using(backend="cron", queue_name="pipelines") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +class Refused(Exception): + """The layout was refused where an operator cannot miss it.""" + + +@pytest.fixture(autouse=True) +def empty_registry(monkeypatch): + """ + django-ox's schedulable registry is process-wide and refuses a second, + different kind under one key. Start it empty, as django-ox's own + test_dst_schedules does; the settings channel refills it from + SCHEDULABLE_TASKS on every lookup, under whatever TASKS is active. + """ + monkeypatch.setattr("django_ox.registry._registry", {}) + monkeypatch.setattr("django_ox.registry._discovered", True) + + +def layout(exposed, *, channel="rows"): + """ + settings.TASKS as the hazard describes it. + + `exposed` is the SCHEDULABLE_TASKS mapping the cron backend carries; the + row names ROW_KEY. Every SCHEDULES block is removed, so the cron backend + is the only thing that can dispatch a reconciler. flows_ungated is + dropped: it is oxpull.E007 by design, and an Error there would be + mistaken for a refusal of this layout. + """ + rebuilt = {} + for alias, params in copy.deepcopy(settings.TASKS).items(): + if alias == "flows_ungated": + continue + params.get("OPTIONS", {}).pop("SCHEDULES", None) + rebuilt[alias] = params + if channel == "rows": + options = { + "SCHEDULE_SOURCE": "django_ox.stored.DatabaseScheduleSource", + "SCHEDULABLE_TASKS": dict(exposed), + } + else: + options = { + "SCHEDULES": { + ROW_KEY: {"task": exposed[ROW_KEY], "cron": "* * * * *"}, + }, + } + rebuilt["cron"] = { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["pipelines"], + "OPTIONS": options, + } + return rebuilt + + +def hazard(path): + """The row's key exposes the reconciler exactly as a project would.""" + return {ROW_KEY: path} + + +def control(path): + """ + The row's key exposes the reconciler bound to the cron queue. The public + path is exposed as well, under a key no row names, so oxpull 1.2.0's + batch gate (which looks for that path in SCHEDULABLE_TASKS) counts this + layout exactly as it counts the hazard's. + """ + return {ROW_KEY: ON_CRON[path], "public-path": path} + + +def refuse_on_check_errors(): + """ + `manage.py check` Errors stop every management command, ox_worker too. + + Django's check_tasks, which is how `manage.py check` reaches every + backend's check() (oxpull's and django-ox's checks all live there; neither + registers a check of its own). Called directly rather than through + run_checks(): the admin test installs contrib apps for its duration, and + their checks stay registered for the rest of the process. + """ + from django.tasks.checks import check_tasks + + errors = [m for m in check_tasks() if m.level >= checks.ERROR] + if errors: + raise Refused(errors) + + +def write_row(**fields): + """ + The row, through django_ox.stored.create_schedule: the write path the + admin's save_model calls, and the documented programmatic one. + """ + fields = { + "name": ROW_NAME, + "task_key": ROW_KEY, + "trigger": "cron", + "cron": "* * * * *", + # A row fires from its boundary; five minutes back puts the latest + # minute tick after it whatever second this runs at. + "start_time": timezone.now() - timedelta(minutes=5), + **fields, + } + try: + return create_schedule(**fields) + except ValidationError as exc: + raise Refused(exc) from exc + + +def cron_worker(): + """What `ox_worker --backend cron` builds: django-ox's Worker on that alias.""" + return Worker(backend_alias="cron", backoff_initial=0, poll_interval=0.01) + + +def run_until_idle(worker, limit=50): + ran = 0 + while ran < limit and worker.run_once(): + ran += 1 + return ran + + +def skipped_lines(caplog): + return "\n".join( + r.getMessage() for r in caplog.records if SKIP_LOG in r.getMessage() + ) + + +def dispatch_pass(caplog): + """ + One pass of the cron worker's loop: build (worker start), dispatch due + ticks, run what was enqueued. Returns (tasks enqueued, skip log lines). + """ + with caplog.at_level(logging.WARNING, logger="django_ox"): + dispatcher = cron_worker() + enqueued = dispatcher.dispatch_schedules() + run_until_idle(dispatcher) + return enqueued, skipped_lines(caplog) + + +# -- the row is accepted, then never built -------------------------------- + + +@pytest.mark.parametrize("path", [WORKFLOW_RECONCILE, BATCH_RECONCILE]) +def test_a_reconciler_row_create_schedule_accepts_is_built_by_its_worker(path, caplog): + """ + create_schedule validates the row (registry membership, trigger, cron) + and saves it enabled. The worker `ox_worker --backend cron` builds must + then hold it among its schedules, or the write must have been refused. + """ + with override_settings(TASKS=layout(hazard(path))): + try: + refuse_on_check_errors() + row = write_row() + except Refused: + return + assert row.enabled + with caplog.at_level(logging.WARNING, logger="django_ox"): + built = [schedule.name for schedule in cron_worker().schedules] + assert ROW_NAME in built, ( + f"create_schedule accepted an enabled row naming {path}, and the " + f"worker that dispatches it built {built}.\nWorker log:\n" + + skipped_lines(caplog) + ) + + +class _Operator: + """A staff user holding every permission, as the admin's request carries.""" + + is_active = is_staff = is_superuser = is_authenticated = True + pk = id = 1 + + def has_perm(self, perm, obj=None): + return True + + def has_perms(self, perms, obj=None): + return True + + def has_module_perms(self, app_label): + return True + + +def admin_save_then_build(exposed, caplog, rf): + """ + The row through the admin: OxScheduleAdmin.get_form() builds the form + the add view renders, the POST data is validated by it, and + OxScheduleAdmin.save_model() writes it. Only the HTTP view layer is + skipped (tests_oxpull/settings.py installs no admin, auth or sessions app, so + they are installed for the duration). Then the worker for the cron + backend is built. Raises Refused if anything on the way refuses. + """ + installed = [ + *settings.INSTALLED_APPS, + "django.contrib.contenttypes", + "django.contrib.auth", + "django.contrib.messages", + "django.contrib.admin", + ] + with override_settings(TASKS=layout(exposed), INSTALLED_APPS=installed): + from django.contrib import admin + + from django_ox.admin import OxScheduleAdmin + from django_ox.models import OxSchedule + + refuse_on_check_errors() + model_admin = OxScheduleAdmin(OxSchedule, admin.site) + request = rf.post("/admin/django_ox/oxschedule/add/") + request.user = _Operator() + form_class = model_admin.get_form(request, None, change=False) + form = form_class( + data={ + "name": ROW_NAME, + "task_key": ROW_KEY, + "trigger": "cron", + "cron": "* * * * *", + "every_seconds": "", + "phase_seconds": "0", + "arguments": "{}", + "enabled": "on", + "end_time": "", + "starting_deadline_seconds": "", + } + ) + if not form.is_valid(): + raise Refused(form.errors) + try: + model_admin.save_model(request, form.save(commit=False), form, change=False) + except ValidationError as exc: + raise Refused(exc) from exc + row = OxSchedule.objects.get(name=ROW_NAME) + assert row.enabled + with caplog.at_level(logging.WARNING, logger="django_ox"): + built = [schedule.name for schedule in cron_worker().schedules] + return built, skipped_lines(caplog) + + +@pytest.mark.parametrize("path", [WORKFLOW_RECONCILE, BATCH_RECONCILE]) +def test_a_reconciler_row_the_admin_saves_is_built_by_its_worker(path, caplog, rf): + """The hazard's layout, through the admin form and save_model.""" + try: + built, log = admin_save_then_build(hazard(path), caplog, rf) + except Refused: + return # refused in front of the person, which is acceptable + assert ROW_NAME in built, ( + f"The admin form validated and saved an enabled row naming {path}, " + f"and the worker that dispatches it built {built}.\nWorker log:\n{log}" + ) + + +@pytest.mark.parametrize("path", [WORKFLOW_RECONCILE, BATCH_RECONCILE]) +def test_control_the_admin_saves_and_the_worker_builds_a_cron_queue_reconciler( + path, caplog, rf +): + """ + Control: the same form data and save, with the reconciler exposed on + the cron queue. Nothing refuses and the row builds, so the admin test + above cannot pass through an unrelated refusal. + """ + built, log = admin_save_then_build(control(path), caplog, rf) + assert log == "" + assert ROW_NAME in built + + +# -- what it costs: workflows ---------------------------------------------- + + +def run_workflows_reconciled_by_the_row(caplog): + """ + Two sealed workflows on "flows", drained by a real gating worker: + "doomed" (root fails three times; leaf depends on it) and "fine" (one + node, succeeds). Then one pass of the cron worker that dispatches the + row. Raises Refused if the layout is refused anywhere on the way. + """ + from oxpull import workflows + + from .test_workflows import drain, flow_worker, stored + + refuse_on_check_errors() + try: + doomed = workflows.create( + "doomed", on_complete=tasks.record_callback, on_complete_args=["doomed"] + ) + doomed.add("root", tasks.failing_step, "root") + doomed.add("leaf", tasks.step, "leaf", depends_on="root") + doomed.seal() + fine = workflows.create( + "fine", on_success=tasks.record_callback, on_success_args=["fine"] + ) + fine.add("only", tasks.step, "only") + fine.seal() + except ImproperlyConfigured as exc: + raise Refused(exc) from exc + write_row() + + drain(flow_worker()) + assert OxTask.objects.get(args=["root"]).status == OxTask.Status.FAILED + assert OxTask.objects.get(args=["only"]).status == OxTask.Status.SUCCESSFUL + assert OxTask.objects.get(args=["leaf"]).status == OxTask.Status.WAITING + assert stored(doomed).status == stored(fine).status == "SEALED" + + enqueued, log = dispatch_pass(caplog) + return { + "enqueued": enqueued, + "reconciler_rows": OxTask.objects.filter(task_path=WORKFLOW_RECONCILE).count(), + "log": log, + "doomed": stored(doomed), + "fine": stored(fine), + "leaf": OxTask.objects.get(args=["leaf"]).status, + } + + +def assert_reconciled(seen): + report = ( + f"cron worker enqueued {seen['enqueued']} task(s), " + f"{seen['reconciler_rows']} reconciler row(s) exist; doomed is " + f"{seen['doomed'].status}, fine is {seen['fine'].status}, leaf is " + f"{seen['leaf']}.\nWorker log:\n{seen['log']}" + ) + assert seen["doomed"].status == "COMPLETE", report + assert seen["leaf"] == OxTask.Status.DISCARDED, report + assert seen["doomed"].callback_task_id is not None, report + assert seen["fine"].status == "COMPLETE", report + assert seen["fine"].success_task_id is not None, report + + +def test_workflows_reconciled_only_by_a_stored_row_complete_and_cancel_dead_cones( + caplog, +): + """ + The hazard's layout. The accepted row must run the reconciler: the dead + cone cancelled, both workflows COMPLETE, both callbacks enqueued. + """ + with override_settings(TASKS=layout(hazard(WORKFLOW_RECONCILE))): + try: + seen = run_workflows_reconciled_by_the_row(caplog) + except Refused: + return + assert_reconciled(seen) + + +def test_control_workflows_complete_when_the_row_runs_a_cron_queue_reconciler( + caplog, +): + """Control: the same run, with the reconciler on the cron queue.""" + with override_settings(TASKS=layout(control(WORKFLOW_RECONCILE))): + seen = run_workflows_reconciled_by_the_row(caplog) + assert seen["log"] == "" + assert seen["enqueued"] == 1 + assert_reconciled(seen) + + +def test_control_the_settings_channel_refuses_the_same_layout_loudly(): + """ + Control: the same reconciler in the cron backend's OPTIONS["SCHEDULES"] + prevents the worker from starting. + """ + with ( + override_settings(TASKS=layout(hazard(WORKFLOW_RECONCILE), channel="settings")), + pytest.raises( + ImproperlyConfigured, match="Queue 'default' is not valid for backend" + ), + ): + cron_worker() + + +def run_a_missed_release(caplog, monkeypatch): + """ + parent -> child on "flows_deferred". The parent runs on a real worker + whose release fails (only release_downstream is made to raise, as a lock + wait timeout or dropped connection would; the worker logs it and moves + on). Then one pass of the cron worker. Returns (child's status, whether a + worker then claims it, tasks enqueued, skip log). + """ + from .test_workflows_deferred import chain, deferring_worker + + def child_status(): + return OxTask.objects.get(args=["child"]).status + + refuse_on_check_errors() + try: + chain() + except ImproperlyConfigured as exc: + raise Refused(exc) from exc + write_row() + + def lock_wait_timeout(*args, **kwargs): + raise OperationalError("simulated: lock wait timeout exceeded") + + with monkeypatch.context() as patched: + patched.setattr("oxpull.worker.release_downstream", lock_wait_timeout) + assert deferring_worker().run_once() is True + assert OxTask.objects.get(args=["parent"]).status == OxTask.Status.SUCCESSFUL + assert child_status() == OxTask.Status.WAITING + assert deferring_worker().run_once() is False + + enqueued, log = dispatch_pass(caplog) + return child_status(), deferring_worker().run_once(), enqueued, log + + +def test_with_deferred_nodes_a_missed_release_is_released_by_the_stored_row( + caplog, monkeypatch +): + """ + The reconciler is the only backstop for a release the worker missed. The + accepted row must run it, and the child must become claimable. + """ + with override_settings(TASKS=layout(hazard(WORKFLOW_RECONCILE))): + try: + released, ran, enqueued, log = run_a_missed_release(caplog, monkeypatch) + except Refused: + return + assert released == OxTask.Status.READY and ran is True, ( + f"child still {released} after the cron worker's pass " + f"(enqueued {enqueued}); a deferring worker claimed it: {ran}.\n" + f"Worker log:\n{log}" + ) + + +def test_control_a_missed_release_is_released_by_a_cron_queue_reconciler( + caplog, monkeypatch +): + """Control: the same run, with the reconciler on the cron queue.""" + with override_settings(TASKS=layout(control(WORKFLOW_RECONCILE))): + released, ran, enqueued, log = run_a_missed_release(caplog, monkeypatch) + assert (released, ran, enqueued, log) == (OxTask.Status.READY, True, 1, "") + + +# -- what it costs: batches (the same helper, and 1.2.0's gate passes it) --- + + +def run_a_batch_reconciled_by_the_row(caplog): + """ + A sealed one-member batch on "default", its member run by a real worker, + then one pass of the cron worker. Returns (the 1.2.0 gate's answer, tasks + enqueued, batch status, skip log). Raises Refused if anything refuses. + """ + gate = batches.reconcile_schedule_aliases() + refuse_on_check_errors() + try: + batch = batches.create("nightly") + batch.add(tasks.plain_echo, "x") + batch.seal() + except ImproperlyConfigured as exc: + raise Refused(exc) from exc + write_row() + run_until_idle(Worker(backoff_initial=0, poll_interval=0.01)) + assert OxTask.objects.get(task_path="tests_oxpull.tasks.plain_echo").status == ( + OxTask.Status.SUCCESSFUL + ) + assert OxpullBatch.objects.get(pk=batch.id).status == "SEALED" + + enqueued, log = dispatch_pass(caplog) + return gate, enqueued, OxpullBatch.objects.get(pk=batch.id).status, log + + +def test_a_batch_reconciled_only_by_a_stored_row_completes(caplog): + """ + oxpull 1.2.0 refuses batches.create() unless something schedules the + batch reconciler, and counts a row-reading backend that exposes it. + This layout passes that gate. The accepted row must then complete the + batch, or something must refuse. + """ + with override_settings(TASKS=layout(hazard(BATCH_RECONCILE))): + try: + gate, enqueued, status, log = run_a_batch_reconciled_by_the_row(caplog) + except Refused: + return + assert status == "COMPLETE", ( + f"reconcile_schedule_aliases() said {gate}, create() and seal() " + f"accepted the batch, create_schedule accepted the row; the cron " + f"worker enqueued {enqueued} task(s) and the batch is {status}.\n" + f"Worker log:\n{log}" + ) + + +def test_control_a_batch_completes_when_the_row_runs_a_cron_queue_reconciler(caplog): + """Control: reconciler on the cron queue; also passes on `oxpull` 1.2.0.""" + with override_settings(TASKS=layout(control(BATCH_RECONCILE))): + gate, enqueued, status, log = run_a_batch_reconciled_by_the_row(caplog) + assert (gate, enqueued, status, log) == (["cron"], 1, "COMPLETE", "") diff --git a/tests_oxpull/test_workflows_hazard_a_support_engineer_sets_an_abandoned_open.py b/tests_oxpull/test_workflows_hazard_a_support_engineer_sets_an_abandoned_open.py new file mode 100644 index 0000000..b3c48cb --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_support_engineer_sets_an_abandoned_open.py @@ -0,0 +1,337 @@ +""" +An abandoned OPEN workflow (or batch) is sealed by hand in SQL. + +A builder that dies between add() and seal() leaves its workflow OPEN for +good: the reconciler sweeps SEALED rows only. The obvious support fix is +one statement, UPDATE ... SET status = 'SEALED'. That leaves total_count +NULL, because seal() is its only writer, and the completing transaction +reads `(row.total_count or 0)`, turning an unknown denominator into zero. +Every node that is DISCARDED, LOST or pruned then stops counting as +unattributed, and on_success fires for a run that was not clean. + +Each scenario runs twice: + +- hand_sealed_sql: the literal UPDATE a support engineer would issue. + These assert the correct behaviour and fail today. +- control_public_seal: the same state sealed through workflows.get(id).seal() + (or batches.get(id).seal()) from a fresh handle, the public way to finish + an abandoned build from another process. These pass today. They show the + assertions can pass, and that the only difference is the NULL + total_count. + +A sealed row with no total now completes on its live count, which is the +total seal() would have stored, and the completion writes it. The +assertions ask for that much and no more: the row completes and enqueues +on_complete, and nothing enqueues on_success or records a completion +whose unattributed count disagrees with progress(). Refusing to complete +would strand the row SEALED, so it fails them too. +""" + +import logging +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command +from django.db import connections +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.models import OxpullBatch, OxpullWorkflow +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_batches import reconcile as reconcile_batches +from .test_workflows import drain, flow_worker +from .test_workflows import reconcile as reconcile_workflows + +pytestmark = pytest.mark.django_db + +HAND_SEALED = "hand_sealed_sql" +CONTROL = "control_public_seal" +SEAL_HOW = [HAND_SEALED, CONTROL] + +CALLBACKS = { + "on_complete": tasks.record_callback, + "on_complete_args": ["complete"], + "on_success": tasks.record_callback, + "on_success_args": ["success"], +} + +# Enough ticks for any fix that needs more than one pass to settle. +TICKS = 3 + + +def hand_seal(model, name): + """ + The support fix, as literal SQL: + + UPDATE SET status = 'SEALED', sealed_at = + WHERE name = AND status = 'OPEN' + + Keyed on the name, which is what an engineer reading the table has. + """ + connection = connections[oxpull_db_alias()] + qn = connection.ops.quote_name + with connection.cursor() as cursor: + cursor.execute( + f"UPDATE {qn(model._meta.db_table)} " # noqa: S608 + f"SET {qn('status')} = %s, {qn('sealed_at')} = %s " + f"WHERE {qn('name')} = %s AND {qn('status')} = %s", + [ + "SEALED", + connection.ops.adapt_datetimefield_value(timezone.now()), + name, + "OPEN", + ], + ) + assert cursor.rowcount == 1 + + +def seal(how, module, model, handle, members): + """Seal an abandoned build one of two ways, and prove which state it left.""" + if how == HAND_SEALED: + hand_seal(model, handle.name) + else: + # A fresh handle, as a shell in another process would build it. + module.get(handle.id).seal() + row = model.objects.get(pk=handle.id) + assert row.status == "SEALED" + if how == HAND_SEALED: + assert row.total_count is None # the state the hazard needs + else: + assert row.total_count == members + + +def lose_to_the_reaper(worker, task_id): + """ + Drive one task to LOST through django-ox 1.2.0's own lease path. + + Each attempt is a real claim (a stored lease expiry and a bumped + epoch), then actions.expire_lease, then the real reaper. Attempts 1 + and 2 are requeued; attempt 3 is exhausted and written LOST. No task + row is written by hand. + """ + max_attempts = OxTask.objects.get(id=task_id).max_attempts + for attempt in range(1, max_attempts + 1): + claimed = worker.claim_one() + assert claimed is not None and str(claimed.id) == str(task_id) + held = OxTask.objects.get(id=task_id) + assert held.status == OxTask.Status.RUNNING + assert held.lease_expires_at is not None + assert held.lease_epoch >= attempt + assert actions.expire_lease(task_id) is True + assert worker.reap() == 1 + lost = OxTask.objects.get(id=task_id) + assert lost.status == OxTask.Status.LOST + assert lost.attempts == lost.max_attempts + + +def judge(*, how, model, handle, reconcile, logger_name, event, caplog): + """ + Run the reconciler and return what a correct design must not do. + + The run is not clean by construction, and progress() says so: that + is asserted first, so a zero below cannot come from a scenario that + provoked nothing. A workflow's blocked child is still pending here; + only the reconciler's cancellation settles it. + """ + before = handle.progress() + assert before.failed + before.unattributed > 0, before + + with caplog.at_level(logging.INFO, logger=logger_name): + passes = [reconcile() for _ in range(TICKS)] + row = model.objects.get(pk=handle.id) + after = handle.progress() + logged = [ + record.unattributed_count + for record in caplog.records + if getattr(record, "event", None) == event + ] + success_rows = OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback", args=["success"] + ).count() + observed = { + "sealed_by": how, + "reconcile_passes": passes, + "status": row.status, + "total_count": row.total_count, + "failed_count": row.failed_count, + "cancelled_count": getattr(row, "cancelled_count", "n/a"), + "callback_task_id": row.callback_task_id, + "success_task_id": row.success_task_id, + "on_success_task_rows": success_rows, + "logged_unattributed": logged, + "progress": { + "total": after.total, + "succeeded": after.succeeded, + "failed": after.failed, + "unattributed": after.unattributed, + }, + } + + problems = [] + if row.status != "COMPLETE" or row.callback_task_id is None: + problems.append( + f"the sealed row did not complete: it is {row.status} with " + f"callback_task_id {row.callback_task_id}" + ) + if row.success_task_id is not None or success_rows: + problems.append( + f"on_success was enqueued for a run that was not clean: " + f"progress() reports {after.failed} failed and " + f"{after.unattributed} unattributed of {after.total}" + ) + if logged and logged != [after.unattributed]: + problems.append( + f"the completion recorded unattributed={logged} while " + f"progress() reports {after.unattributed}" + ) + return problems, observed + + +def report(problems, observed): + return "\n".join([*problems, f"observed: {observed}"]) + + +# -- workflows --------------------------------------------------------- + + +@pytest.mark.parametrize("how", SEAL_HOW) +def test_hand_sealed_workflow_with_a_discarded_root_withholds_on_success(how, caplog): + """ + root -> child. The builder dies before seal(); an operator discards + root, the abandoned work. Hand-sealed, the reconciler cancels child + and completes, and on_success fires over two DISCARDED nodes. + """ + flow = workflows.create(f"abandoned-discarded-{how}", **CALLBACKS) + root = flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + assert actions.discard(root.id) is True + + seal(how, workflows, OxpullWorkflow, flow, members=2) + problems, observed = judge( + how=how, + model=OxpullWorkflow, + handle=flow, + reconcile=reconcile_workflows, + logger_name="oxpull.workflows", + event="workflow_completed", + caplog=caplog, + ) + assert not problems, report(problems, observed) + + +@pytest.mark.parametrize("how", SEAL_HOW) +def test_hand_sealed_workflow_with_a_lost_root_withholds_on_success(how, caplog): + """ + root -> child. The builder dies before seal(); root's worker dies on + every attempt and the reaper writes it LOST. Hand-sealed, the + reconciler cancels child and completes, and on_success fires for a + run whose root was never observed to finish and whose child never ran. + """ + flow = workflows.create(f"abandoned-lost-{how}", **CALLBACKS) + root = flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + lose_to_the_reaper(flow_worker(), root.id) + + seal(how, workflows, OxpullWorkflow, flow, members=2) + problems, observed = judge( + how=how, + model=OxpullWorkflow, + handle=flow, + reconcile=reconcile_workflows, + logger_name="oxpull.workflows", + event="workflow_completed", + caplog=caplog, + ) + assert not problems, report(problems, observed) + + +@pytest.mark.parametrize("how", SEAL_HOW) +def test_hand_sealed_workflow_with_a_pruned_failed_node_withholds_on_success( + how, caplog +): + """ + Two independent nodes. The builder dies before seal(); one node fails + all three attempts, the other succeeds. The workflow sits OPEN until + ox_prune --include-failed deletes the FAILED row. Hand-sealed, it + completes with failed_count 0 and on_success fires over a failure. + """ + flow = workflows.create(f"abandoned-pruned-{how}", **CALLBACKS) + doomed = flow.add("fails", tasks.failing_step, "fails") + kept = flow.add("succeeds", tasks.step, "succeeds") + assert drain(flow_worker()) == 4 # fails x3, succeeds + assert OxTask.objects.get(id=doomed.id).status == OxTask.Status.FAILED + assert OxTask.objects.get(id=kept.id).status == OxTask.Status.SUCCESSFUL + + # Stands in for the week the abandoned workflow sat OPEN. + OxTask.objects.filter(id=doomed.id).update( + finished_at=timezone.now() - timedelta(days=8) + ) + call_command("ox_prune", "--include-failed", stdout=StringIO()) + assert not OxTask.objects.filter(id=doomed.id).exists() + assert OxTask.objects.filter(id=kept.id).exists() + + seal(how, workflows, OxpullWorkflow, flow, members=2) + problems, observed = judge( + how=how, + model=OxpullWorkflow, + handle=flow, + reconcile=reconcile_workflows, + logger_name="oxpull.workflows", + event="workflow_completed", + caplog=caplog, + ) + assert not problems, report(problems, observed) + + +# -- the same expression in batches.py ----------------------------------- + + +@pytest.mark.parametrize("how", SEAL_HOW) +def test_hand_sealed_batch_with_a_discarded_member_withholds_on_success( + how, worker, caplog +): + batch = batches.create(f"abandoned-batch-discarded-{how}", **CALLBACKS) + doomed = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + assert actions.discard(doomed.id) is True + assert worker.run_once() is True + assert worker.run_once() is False + + seal(how, batches, OxpullBatch, batch, members=2) + problems, observed = judge( + how=how, + model=OxpullBatch, + handle=batch, + reconcile=reconcile_batches, + logger_name="oxpull.batches", + event="batch_completed", + caplog=caplog, + ) + assert not problems, report(problems, observed) + + +@pytest.mark.parametrize("how", SEAL_HOW) +def test_hand_sealed_batch_with_a_lost_member_withholds_on_success(how, worker, caplog): + batch = batches.create(f"abandoned-batch-lost-{how}", **CALLBACKS) + doomed = batch.add(tasks.plain_echo, "a") + batch.add(tasks.plain_echo, "b") + lose_to_the_reaper(worker, doomed.id) + assert worker.run_once() is True + assert worker.run_once() is False + + seal(how, batches, OxpullBatch, batch, members=2) + problems, observed = judge( + how=how, + model=OxpullBatch, + handle=batch, + reconcile=reconcile_batches, + logger_name="oxpull.batches", + event="batch_completed", + caplog=caplog, + ) + assert not problems, report(problems, observed) diff --git a/tests_oxpull/test_workflows_hazard_a_unique_enqueue_inside_a_transaction_can.py b/tests_oxpull/test_workflows_hazard_a_unique_enqueue_inside_a_transaction_can.py new file mode 100644 index 0000000..7325fb0 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_unique_enqueue_inside_a_transaction_can.py @@ -0,0 +1,501 @@ +""" +A unique enqueue inside an open transaction can freeze the +workflow reconciler on PostgreSQL and MySQL. + +The state. A unique task ran as a node of a SEALED workflow (W1) and ended +FAILED or LOST, and no reconciler tick has cancelled W1's cone yet. The +key's OxpullUniqueLock row still names that dead task row. + +The trigger. Application code enqueues the same key inside a transaction +it keeps open: a view under ATOMIC_REQUESTS, Workflow.add() or Batch.add() +inside atomic(), or a bare enqueue inside atomic(). The enqueue meets the +stale lock, and conflict resolution (OxpullBackend._resolve_conflict) +re-reads the dead holder with SELECT ... FOR UPDATE before it releases the +key. That row lock belongs to the caller's transaction and lives until the +caller commits. + +The stall. The next reconciler tick sweeps SEALED workflows serially, +oldest first. It reaches W1, takes W1's workflow row lock in +_cancel_one_level, and then blocks locking the same dead row. Every SEALED +workflow behind W1 waits with it: nothing is cancelled or completed, and no +callback is enqueued. + +Each test asserts what a correct reconciler does. An unrelated sealed +workflow (W2), whose only node has succeeded, is completed within a few +seconds of the tick starting, whatever some caller's transaction holds. +The dead row and the new enqueue come from real paths: a real worker runs +the attempts, and the operator's expire_lease plus a real reaper produce +LOST. The two controls run the identical harness and pass today: a +non-unique enqueue inside atomic(), and the unique enqueue with no +transaction open. + +add() now refuses a unique task as a node, so no unique lock row can name +a node's row and the state above cannot be built. The tests still try the +hazard's own build first. When add() refuses the unique root, saying it is +unique, the dead unique row is the application's own enqueue of that call, +taken to FAILED or LOST by the same real paths before W1 exists, and W1's +root is an ordinary task that dies the same way. Workflow.add() inside +atomic() is refused before it enqueues, so that holder's transaction is +open and holds nothing of the key. Every other holder still meets the dead +lock inside its transaction, and the tick must still complete W2 in time. +Both tests catch the PostgreSQL and MySQL failure caused by accepting the +unique root without preventing this lock interaction. + +SQLite is skipped. It has no row locks, so _resolve_conflict takes none +there, and the lock this hazard is about cannot exist. +""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from typing import Any + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.db import DatabaseError, connections, transaction +from django.http import HttpRequest, HttpResponse +from django.tasks.exceptions import TaskException +from django.test import Client, override_settings +from django.urls import path + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.models import OxpullUniqueLock, OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile, stored + +pytestmark = pytest.mark.django_db(transaction=True) + +# How long one reconciler tick is given to reach and complete W2. A tick +# over two small workflows takes milliseconds; three seconds is generous +# under load and far below either database's lock wait. +TICK_BOUND = 3.0 + +DEAD = ["failed", "lost"] + +# What add() may raise when it refuses a unique node. The message must also +# say the task is unique, so an unrelated error does not count. +REFUSALS = (ImproperlyConfigured, TaskException, ValueError) + + +@pytest.fixture(autouse=True) +def _row_locking_database(): + if not connections["default"].features.has_select_for_update: + pytest.skip( + "SQLite has no row locks: _resolve_conflict takes no FOR UPDATE " + "there, so the lock this hazard is about is never taken." + ) + + +def _unique_node(): + """tests_oxpull.tasks.unique_fail_always (@unique), on the gated backend.""" + return tasks.unique_fail_always.using(backend="flows") + + +# -- the state ---------------------------------------------------------- + + +def _kill(task_id: Any, dead: str) -> None: + """Take the one claimable row, `task_id`, to FAILED or LOST with real workers.""" + if dead == "failed": + # Three real attempts, each raising. + drain(flow_worker()) + return + # Two real failing attempts, then a claim whose worker goes quiet. + # The operator expires its lease, and another worker's reaper takes + # the row, attempts exhausted, to LOST. + worker = flow_worker() + assert worker.run_once() and worker.run_once() + claimed = worker.claim_one() + assert claimed is not None and str(claimed.id) == str(task_id) + assert claimed.attempts == claimed.max_attempts + assert actions.expire_lease(claimed.id) + assert flow_worker().reap() == 1 + + +def _doomed_workflow(dead: str) -> tuple[Any, str, str, str]: + """ + W1: root -> child, sealed, with the root driven to FAILED or LOST by + real workers, and a dead unique row whose lock row still names it. No + reconciler tick has run, so the child is still WAITING. + + Returns (W1, root id, child id, dead unique row id). The hazard's own + build makes the unique task W1's root, and the last two ids are one + row. When add() refuses that root, the unique row is the application's + own enqueue of the call, killed before W1 is built, and W1's root is an + ordinary failing task. + """ + expected = OxTask.Status.FAILED if dead == "failed" else OxTask.Status.LOST + flow = workflows.create("w1-doomed") + try: + root = flow.add("root", _unique_node()) + except REFUSALS as exc: + assert "unique" in str(exc).lower(), f"add() refused the root: {exc!r}" + held = _unique_node().enqueue() + _kill(held.id, dead) + assert OxTask.objects.get(id=held.id).status == expected + root = flow.add("root", tasks.failing_step, "root") + else: + held = root + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + _kill(root.id, dead) + assert OxTask.objects.get(id=root.id).status == expected + assert OxTask.objects.get(id=child.id).status == OxTask.Status.WAITING + assert stored(flow).status == OxpullWorkflow.Status.SEALED + # The key still names the dead row, so the next enqueue of that key + # goes through conflict resolution rather than a clean insert. + assert OxpullUniqueLock.objects.filter(task_id=held.id).exists() + return flow, str(root.id), str(child.id), str(held.id) + + +def _healthy_workflow() -> Any: + """W2: one node, succeeded, sealed. Created after W1, so swept after it.""" + flow = workflows.create("w2-healthy") + only = flow.add("only", tasks.step, "only") + flow.seal() + assert flow_worker().run_once() + assert OxTask.objects.get(id=only.id).status == OxTask.Status.SUCCESSFUL + assert stored(flow).status == OxpullWorkflow.Status.SEALED + return flow + + +# -- the application transaction --------------------------------------- + + +class Hold: + """One application transaction, parked on an Event until released.""" + + def __init__(self) -> None: + self.holding = threading.Event() + self.release = threading.Event() + self.out: dict[str, Any] = {} + + def enqueued(self, result: Any) -> None: + self.out["new_id"] = str(result.id) + + def refused(self, exc: BaseException) -> None: + self.out["refused"] = exc + + def park(self) -> None: + self.out["transaction_open"] = connections["default"].in_atomic_block + self.holding.set() + self.out["released"] = self.release.wait(180) + + +_VIEW_HOLD: dict[str, Hold] = {} + + +def enqueue_and_park(request: HttpRequest) -> HttpResponse: + hold = _VIEW_HOLD["hold"] + hold.enqueued(_unique_node().enqueue()) + hold.park() + return HttpResponse("queued") + + +urlpatterns = [path("enqueue-and-park/", enqueue_and_park)] + + +def _atomic_request(hold: Hold) -> None: + """A real request through Django's handler, with ATOMIC_REQUESTS on.""" + _VIEW_HOLD["hold"] = hold + database = connections.settings["default"] + previous = database.get("ATOMIC_REQUESTS", False) + database["ATOMIC_REQUESTS"] = True + try: + response = Client().get("/enqueue-and-park/") + finally: + database["ATOMIC_REQUESTS"] = previous + hold.out["status_code"] = response.status_code + + +def _enqueue_in_atomic(hold: Hold) -> None: + with transaction.atomic(): + hold.enqueued(_unique_node().enqueue()) + hold.park() + + +def _workflow_add_in_atomic(hold: Hold) -> None: + flow = workflows.create("w3-open") + with transaction.atomic(): + try: + hold.enqueued(flow.add("again", _unique_node())) + except REFUSALS as exc: + # Refused before it enqueued: the transaction stays open and + # holds nothing of the key. + hold.refused(exc) + hold.park() + + +def _batch_add_in_atomic(hold: Hold) -> None: + batch = batches.create("b-open") + with transaction.atomic(): + hold.enqueued(batch.add(_unique_node())) + hold.park() + + +def _plain_enqueue_in_atomic(hold: Hold) -> None: + with transaction.atomic(): + hold.enqueued(tasks.failing_step.enqueue("again")) + hold.park() + + +def _unique_enqueue_without_a_transaction(hold: Hold) -> None: + hold.enqueued(_unique_node().enqueue()) + hold.park() + + +# id -> (holder, enqueues the dead node's unique key, caller's transaction open) +HOLDERS: dict[str, tuple[Callable[[Hold], None], bool, bool]] = { + "atomic-requests-view": (_atomic_request, True, True), + "enqueue-in-atomic": (_enqueue_in_atomic, True, True), + "workflow-add-in-atomic": (_workflow_add_in_atomic, True, True), + "batch-add-in-atomic": (_batch_add_in_atomic, True, True), + "control-plain-enqueue-in-atomic": (_plain_enqueue_in_atomic, False, True), + "control-unique-enqueue-no-transaction": ( + _unique_enqueue_without_a_transaction, + True, + False, + ), +} + + +def _start_holder(target: Callable[[Hold], None], hold: Hold) -> threading.Thread: + def run() -> None: + try: + target(hold) + except BaseException as exc: + hold.out["error"] = exc + finally: + hold.holding.set() + connections.close_all() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + return thread + + +def _start_reconciler(out: dict[str, Any]) -> threading.Thread: + def run() -> None: + started = time.monotonic() + try: + out["completed"] = reconcile() + except BaseException as exc: + out["error"] = exc + finally: + out["elapsed"] = time.monotonic() - started + connections.close_all() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + return thread + + +# -- observations (reported in failure messages, never asserted) -------- + + +def _row_lock_held(task_id: str) -> bool: + """Does another transaction hold this task row's lock right now?""" + try: + with transaction.atomic(): + list( + OxTask.objects.select_for_update(nowait=True) + .filter(id=task_id) + .values_list("id", flat=True) + ) + except DatabaseError: + return True + return False + + +def _lock_waits() -> list[str]: + """The statements in this test database that are waiting on a lock.""" + connection = connections["default"] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute( + "SELECT wait_event || ': ' || query FROM pg_stat_activity " + "WHERE datname = current_database() AND wait_event_type = 'Lock' " + "AND pid <> pg_backend_pid()" + ) + elif connection.vendor == "mysql": + cursor.execute( + "SELECT t.trx_query FROM information_schema.innodb_trx t " + "JOIN information_schema.processlist p " + "ON p.id = t.trx_mysql_thread_id " + "WHERE t.trx_state = 'LOCK WAIT' AND p.db = DATABASE()" + ) + else: + return [] + return [" ".join(str(row[0]).split())[:220] for row in cursor.fetchall()] + + +# -- the tests ----------------------------------------------------------- + + +@pytest.mark.parametrize("dead", DEAD) +@pytest.mark.parametrize("holder", list(HOLDERS)) +def test_a_transaction_that_reenqueued_a_dead_unique_node_does_not_stall_the_reconciler( + holder, dead +): + target, is_unique, transaction_open = HOLDERS[holder] + w1, _root_id, child_id, held_id = _doomed_workflow(dead) + w2 = _healthy_workflow() + assert stored(w1).created_at < stored(w2).created_at + + hold = Hold() + tick: dict[str, Any] = {} + tick_thread = None + started = None + released_after = None + with override_settings(ROOT_URLCONF=__name__): + holder_thread = _start_holder(target, hold) + try: + assert hold.holding.wait(30), "the holder never reached its park" + assert "error" not in hold.out, f"the holder raised {hold.out['error']!r}" + # Preconditions. The variant's transaction is open or not as it + # says, and a unique enqueue released the dead key and queued new + # work, rather than handing back the dead row. + assert hold.out["transaction_open"] is transaction_open + if "refused" in hold.out: + assert "unique" in str(hold.out["refused"]).lower() + elif is_unique: + assert hold.out["new_id"] != held_id + held_locked = _row_lock_held(held_id) + + started = time.monotonic() + tick_thread = _start_reconciler(tick) + tick_thread.join(TICK_BOUND) + stalled = tick_thread.is_alive() + waits = _lock_waits() if stalled else [] + w2_during = stored(w2).status + child_during = OxTask.objects.get(id=child_id).status + finally: + hold.release.set() + if started is not None: + released_after = time.monotonic() - started + holder_thread.join(60) + if tick_thread is not None: + tick_thread.join(120) + + assert not holder_thread.is_alive(), "the holder did not finish after release" + assert "error" not in hold.out, f"the holder raised {hold.out['error']!r}" + assert tick_thread is not None and not tick_thread.is_alive() + assert "error" not in tick, f"the reconciler raised {tick['error']!r}" + if holder == "atomic-requests-view": + assert hold.out["status_code"] == 200 + + print( # noqa: T201 - the observation is the evidence, read with -s + f"\n[observe] {connections['default'].vendor} {holder}/{dead}: " + f"dead unique row lock held by the holder={held_locked}; tick stalled past " + f"{TICK_BOUND}s={stalled}; tick took {tick['elapsed']:.2f}s, holder " + f"released {released_after:.2f}s after the tick started; W2 during=" + f"{w2_during}; W1 child during={child_during}; waiting={waits}" + ) + + # After the holder commits, the next tick finishes W1 as any correct + # design must: its child is cancelled and it completes. + reconcile() + problems = [] + if stalled: + problems.append( + f"the reconciler tick was still running {TICK_BOUND:.0f}s after it " + f"started, behind an application transaction ({holder}, dead root " + f"{dead}); it returned only {tick['elapsed']:.2f}s in, after the " + f"holder was released at {released_after:.2f}s. Dead unique row " + f"locked by the holder: {held_locked}. Waiting on a lock: {waits}" + ) + if w2_during != OxpullWorkflow.Status.COMPLETE: + problems.append( + f"unrelated workflow W2, whose only node had SUCCEEDED, was still " + f"{w2_during} {TICK_BOUND:.0f}s into the tick; it sits behind W1 " + f"in the sweep" + ) + if stored(w2).status != OxpullWorkflow.Status.COMPLETE: + problems.append(f"W2 is {stored(w2).status} after release and a second tick") + child_after = OxTask.objects.get(id=child_id).status + if (stored(w1).status, child_after) != ( + OxpullWorkflow.Status.COMPLETE, + OxTask.Status.DISCARDED, + ): + problems.append( + f"W1 is {stored(w1).status} with child {child_after} after release " + f"and a second tick" + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("dead", DEAD) +def test_a_second_reconciler_does_not_queue_behind_a_stalled_one(dead): + """ + Running more reconcilers must not be the thing that fails too. A second + tick started while the first is blocked has to be able to finish W2. + """ + w1, _root_id, _child_id, held_id = _doomed_workflow(dead) + w2 = _healthy_workflow() + assert stored(w1).created_at < stored(w2).created_at + + hold = Hold() + first: dict[str, Any] = {} + second: dict[str, Any] = {} + threads: list[threading.Thread] = [] + holder_thread = _start_holder(_enqueue_in_atomic, hold) + try: + assert hold.holding.wait(30), "the holder never reached its park" + assert "error" not in hold.out, f"the holder raised {hold.out['error']!r}" + assert hold.out["transaction_open"] is True + assert hold.out["new_id"] != held_id + + threads.append(_start_reconciler(first)) + # Let the first tick reach W1. If nothing blocks it, it simply + # finishes and the loop ends. + deadline = time.monotonic() + TICK_BOUND + while ( + time.monotonic() < deadline and threads[0].is_alive() and not _lock_waits() + ): + # Not faster: InnoDB refreshes information_schema.innodb_trx only + # after 100 ms without a read, so a tighter poll keeps reading the + # snapshot taken before the first tick blocked. + time.sleep(0.25) + first_waits = _lock_waits() + + threads.append(_start_reconciler(second)) + threads[1].join(TICK_BOUND) + first_alive, second_alive = (thread.is_alive() for thread in threads) + waits = _lock_waits() if (first_alive or second_alive) else [] + w2_during = stored(w2).status + finally: + hold.release.set() + holder_thread.join(60) + for thread in threads: + thread.join(120) + + assert not holder_thread.is_alive() + assert "error" not in hold.out, f"the holder raised {hold.out['error']!r}" + assert all(not thread.is_alive() for thread in threads) + assert "error" not in first and "error" not in second + + print( # noqa: T201 - the observation is the evidence, read with -s + f"\n[observe] {connections['default'].vendor} two reconcilers/{dead}: " + f"first waiting when the second started={first_waits}; after " + f"{TICK_BOUND}s first alive={first_alive}, second alive={second_alive}; " + f"first took {first['elapsed']:.2f}s, second {second['elapsed']:.2f}s; " + f"W2 during={w2_during}; waiting={waits}" + ) + + problems = [] + if second_alive: + problems.append( + f"a second reconciler started beside the stalled one was still " + f"running {TICK_BOUND:.0f}s later (took {second['elapsed']:.2f}s); " + f"waiting on a lock: {waits}" + ) + if w2_during != OxpullWorkflow.Status.COMPLETE: + problems.append( + f"W2 was still {w2_during} with two reconcilers running; neither reached it" + ) + assert not problems, "\n".join(problems) diff --git a/tests_oxpull/test_workflows_hazard_a_workflow_build_that_raises_part_way_leaves.py b/tests_oxpull/test_workflows_hazard_a_workflow_build_that_raises_part_way_leaves.py new file mode 100644 index 0000000..6d48fb9 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_workflow_build_that_raises_part_way_leaves.py @@ -0,0 +1,374 @@ +""" +A workflow build that raises part-way. + +Each add() is its own atomic block (workflows.py:419), so outside a +caller's transaction every node commits the moment it is added. The +module docstring's own example (workflows.py:5-12) builds with no +transaction, and nothing tells a caller to wrap one. When a later add() +raises, or code between two adds raises, the committed prefix is READY +with its gates satisfied, and workers claim it at once. + +A caller then has three things to do, and each one goes wrong: + +1. leave the workflow OPEN: the prefix runs, and a dead root's cone + stays READY for good, because the reconciler sweeps SEALED only; +2. run the build again: create() makes a second workflow, so the + prefix runs twice; +3. seal() it as cleanup: total_count freezes at the prefix and, once + the prefix succeeds, on_success fires for nodes that were never added. + +Every test asserts the behaviour a user needs, so it fails today. +All of them run with real commits (transaction=True) and a real +OxpullWorker. The controls at the bottom pass today, and show the +instrument can see a node run and a callback fire. +""" + +import contextlib +import time +from pprint import pformat + +import pytest +from django.db import transaction +from django.tasks import task + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.exceptions import UnknownWorkflowNode, WorkflowSealed +from oxpull.models import OxpullWorkflow +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .racing import race_all +from .test_workflows import drain, flow_worker, order, reconcile + +pytestmark = pytest.mark.django_db(transaction=True) + +RECONCILER_TICKS = 3 + + +def build_checkout(flow, first_node=tasks.step): + """ + The documented build, with a typo in the third node's depends_on. + + UnknownWorkflowNode is one of several things that can raise here; + WorkflowNodeExists, InvalidTask, a database error or a killed + request leave the same committed prefix. + """ + flow.add("charge", first_node, "charge") + flow.add("ship", tasks.step, "ship", depends_on="charge") + flow.add("email", tasks.step, "email", depends_on=["ship", "chrage"]) + flow.seal() + + +def customer_cleanup_after_a_failed_build(flow): + """ + What a caller does once the build has raised. + + Today there is nothing to call: workflows.__all__ (workflows.py:105-118) + has no abort or cancel for an OPEN workflow. A fix that adds one calls + it here. A fix that rolls a failed build back needs nothing here. + """ + + +def on_its_own_connection(target): + """Run target on another thread, which sees only committed rows.""" + results, errors = race_all([target]) + assert errors == [], errors + return results[0] + + +def settle(worker, rounds=3): + """ + Drain, pausing so a zero-backoff retry is eligible, until idle. + + django-ox notes that on SQLite a retry written with no backoff can be + ineligible for up to a millisecond, which would end one drain early. + """ + ran = 0 + for _ in range(rounds): + ran += drain(worker) + time.sleep(0.02) + return ran + + +def workflow_row(flow): + return OxpullWorkflow.objects.filter(pk=flow.id).values().first() + + +# -- exit 1: leave it OPEN -------------------------------------------- + + +@pytest.mark.xfail( + strict=True, + reason=( + "Each add() commits on its own, so a build that raises part-way leaves " + "READY nodes that a worker claims and runs" + ), +) +def test_a_build_that_raises_part_way_leaves_nothing_a_worker_can_claim(): + """ + Brief's fix criterion: claim_one() is None after a failed build. + + The claim runs on another connection, so it sees only what add() + committed, which is what a worker process sees. + """ + flow = workflows.create( + "checkout", + on_success=tasks.record_callback, + on_success_args=["success"], + ) + with pytest.raises(UnknownWorkflowNode): + build_checkout(flow) + customer_cleanup_after_a_failed_build(flow) + + def claim_and_run(): + worker = flow_worker() + first = worker.claim_one() + claimed = None if first is None else list(first.args) + if first is not None: + worker.execute(first, inline=True) + return claimed, drain(worker) + + claimed, drained = on_its_own_connection(claim_and_run) + observed = {"first_claim": claimed, "ran": order()} + context = { + "drained_after_first": drained, + "reconcile": reconcile(), + "workflow": workflow_row(flow), + } + assert observed == {"first_claim": None, "ran": []}, pformat(context) + + +@pytest.mark.xfail( + strict=True, + reason=( + "In a build left OPEN after an error, a node whose upstream failed stays " + "pending forever, because the reconciler only cancels in sealed workflows" + ), +) +def test_an_abandoned_build_whose_root_failed_does_not_hold_a_ready_node_forever(): + """ + The committed root fails for good. Its child can never run, and + nothing ends it: the reconciler cancels cones in SEALED workflows + only, so the child stays pending (WAITING) and ox_prune, which deletes + settled rows, never reaches it. + """ + flow = workflows.create("checkout-failing-root") + with pytest.raises(UnknownWorkflowNode): + build_checkout(flow, first_node=tasks.failing_step) + customer_cleanup_after_a_failed_build(flow) + committed = flow.node_ids() + + ran = on_its_own_connection(lambda: settle(flow_worker())) + ticks = [reconcile() for _ in range(RECONCILER_TICKS)] + + statuses = dict( + OxTask.objects.filter(id__in=committed.values()).values_list("id", "status") + ) + by_name = {name: statuses.get(task_id) for name, task_id in committed.items()} + left_ready = sorted( + name + for name, status in by_name.items() + if status in (OxTask.Status.READY, OxTask.Status.WAITING) + ) + context = { + "tasks_run": ran, + "order": order(), + "reconcile_ticks": ticks, + "node_status": by_name, + "workflow_status": (workflow_row(flow) or {}).get("status"), + "progress_blocked": flow.progress().blocked if workflow_row(flow) else None, + } + assert left_ready == [], pformat(context) + + +# -- exit 2: run the build again --------------------------------------- + + +@task +def build_checkout_in_a_task(order_id): + """ + A build run as a task, which is how a build usually runs. + + The first attempt dies between two adds, as a request or a task does + when a lookup times out. django-ox's retry runs the body again, and a + retry is exactly what a transient failure asks for. + """ + attempts = int(tasks.STATE.get("build_attempts", 0)) + 1 + tasks.STATE["build_attempts"] = attempts + flow = workflows.create( + f"checkout-{order_id}", + on_success=tasks.record_callback, + on_success_args=["success"], + ) + flow.add("charge", tasks.step, "charge") + flow.add("ship", tasks.step, "ship", depends_on="charge") + if attempts <= int(tasks.STATE.get("fail_attempts", 0)): + raise TimeoutError("address lookup timed out") + flow.add("email", tasks.step, "email", depends_on="ship") + flow.seal() + return str(flow.id) + + +def run_build_task_and_everything_it_enqueues(): + build_checkout_in_a_task.enqueue(42) + worker = flow_worker() + ran = on_its_own_connection(lambda: settle(worker)) + ticks = [reconcile() for _ in range(RECONCILER_TICKS)] + # Callbacks are ordinary tasks; run them the way a worker would. + ran += on_its_own_connection(lambda: settle(flow_worker())) + return ran, ticks + + +@pytest.mark.xfail( + strict=True, + reason=( + "Retrying a build that raised creates a second workflow, so the nodes " + "added before the error run twice" + ), +) +def test_retrying_a_build_that_raised_runs_each_node_once(): + """ + The build task raises on attempt 1 and succeeds on attempt 2. The + checkout must charge once. + """ + tasks.STATE["fail_attempts"] = 1 + ran, ticks = run_build_task_and_everything_it_enqueues() + + ran_order = order() + observed = {name: ran_order.count(name) for name in ("charge", "ship", "email")} + context = { + "build_attempts": tasks.STATE.get("build_attempts"), + "tasks_run": ran, + "order": ran_order, + "reconcile_ticks": ticks, + "workflows": sorted( + OxpullWorkflow.objects.values_list("status", "total_count") + ), + "on_success_runs": tasks.STATE.get("success"), + } + assert observed == {"charge": 1, "ship": 1, "email": 1}, pformat(context) + + +# -- exit 3: seal() as cleanup ----------------------------------------- + + +@pytest.mark.xfail( + strict=True, + reason=( + "Sealing a build that raised part-way completes it with only the nodes " + "that were added, and on_success fires" + ), +) +def test_sealing_a_failed_build_does_not_fire_on_success(): + """ + Brief's fix criterion: success_task_id is None after sealing a failed + build. The DAG the caller declared had three nodes; one was never + added. on_success says the run was clean, and it was not. + + seal() raising for a failed build, or the workflow row no longer + existing, are both acceptable ways to refuse; what is not acceptable + is on_success firing. + """ + flow = workflows.create( + "checkout-sealed-as-cleanup", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + with pytest.raises(UnknownWorkflowNode): + build_checkout(flow) + seal_refused = True + with contextlib.suppress(WorkflowSealed, OxpullWorkflow.DoesNotExist): + flow.seal() + seal_refused = False + + ran = on_its_own_connection(lambda: settle(flow_worker())) + ticks = [reconcile() for _ in range(RECONCILER_TICKS)] + ran += on_its_own_connection(lambda: settle(flow_worker())) + + row = workflow_row(flow) or {} + observed = { + "success_task_id": row.get("success_task_id"), + "on_success_runs": tasks.STATE.get("success"), + } + context = { + "seal_refused": seal_refused, + "tasks_run": ran, + "order": order(), + "reconcile_ticks": ticks, + "status": row.get("status"), + "total_count": row.get("total_count"), + "callback_task_id": row.get("callback_task_id"), + "on_complete_runs": tasks.STATE.get("complete"), + } + assert observed == {"success_task_id": None, "on_success_runs": None}, pformat( + context + ) + + +# -- controls: pass today ---------------------------------------------- + + +def test_control_the_same_build_inside_one_transaction_leaves_nothing(): + """ + The mechanism, shown by its absence: the identical build inside + transaction.atomic(using=oxpull_db_alias()) commits nothing, so a + worker on another connection finds nothing and nothing is left over. + """ + with ( + pytest.raises(UnknownWorkflowNode), + transaction.atomic(using=oxpull_db_alias()), + ): + flow = workflows.create( + "checkout-atomic", + on_success=tasks.record_callback, + on_success_args=["success"], + ) + build_checkout(flow) + + claimed = on_its_own_connection(lambda: flow_worker().claim_one()) + assert claimed is None + assert order() == [] + assert OxTask.objects.count() == 0 + assert OxpullWorkflow.objects.count() == 0 + + +def test_control_a_build_task_that_does_not_raise_runs_each_node_once(): + """ + The same task path with no failure: every node runs once, the + workflow completes and on_success runs. So the instrument sees node + runs and callbacks, and test_retrying... and test_sealing... are not + reading zeros they could never have observed. + """ + tasks.STATE["fail_attempts"] = 0 + _, ticks = run_build_task_and_everything_it_enqueues() + + ran_order = order() + assert tasks.STATE.get("build_attempts") == 1 + assert {name: ran_order.count(name) for name in ("charge", "ship", "email")} == { + "charge": 1, + "ship": 1, + "email": 1, + } + assert ticks[0] == 1 + (row,) = OxpullWorkflow.objects.values("status", "total_count", "success_task_id") + assert row["status"] == OxpullWorkflow.Status.COMPLETE + assert row["total_count"] == 3 + assert row["success_task_id"] is not None + assert tasks.STATE.get("success") == 1 + + +def test_control_the_build_runs_on_a_thread_that_sees_committed_rows_only(): + """ + on_its_own_connection really is another connection: a row inside an + uncommitted transaction on this thread is invisible to it. + """ + with transaction.atomic(using=oxpull_db_alias()): + tasks.step.enqueue("uncommitted") + seen = on_its_own_connection( + lambda: OxTask.objects.filter(args=["uncommitted"]).exists() + ) + assert seen is False + assert OxTask.objects.filter(args=["uncommitted"]).exists() diff --git a/tests_oxpull/test_workflows_hazard_a_workflow_callback_that_becomes_unique.py b/tests_oxpull/test_workflows_hazard_a_workflow_callback_that_becomes_unique.py new file mode 100644 index 0000000..e63e37c --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_a_workflow_callback_that_becomes_unique.py @@ -0,0 +1,324 @@ +""" +A workflow whose stored callback stops passing the completion checks after +create() stays SEALED, says so at every tick, and keeps its rows. + +create() validates on_complete/on_success once and stores only the dotted +path. Completion imports that path again and re-runs _require_ox_backend +and _reject_unique inside the transaction that wins SEALED -> COMPLETE. +When a later deploy or config change makes the path unique, removes it, or +binds it to a backend that is not django-ox, that completion is refused. + +The rule is the one 1.2.0 documents for batches, applied to workflows: +- the workflow stays SEALED and neither callback is enqueued. Completing + anyway would drop the callback, or coalesce a unique one onto another + workflow's task; +- the refusal is visible: every tick logs an ERROR about this workflow that + names the callback and why it was refused; +- oxpull_prune keeps its edge and node rows while it is stranded; +- nothing is lost: once the deploy is fixed, the next tick completes it, + both callbacks run, and prune then reaps it. + +Each change is made the way production makes it, after create(): +- unique_tasks_option: the operator adds the path to UNIQUE_TASKS. The + override resets task_backends, as a process restart would. +- unique_decorator: the real oxpull.unique.unique decorator is applied to + the module-level task, as a deploy that adds @unique would. +- renamed: the module no longer has the name, as after a rename or move. +- moved_off_django_ox: the module-level name is now a Task on a backend + that is not django-ox, as after @task(backend="immediate"). + +The reconciler runs as the task a SCHEDULES entry enqueues, claimed and +executed by a worker, and prune runs through the real management command. +A healthy workflow sealed beside the stranded one completes on the first +tick and is reaped by the same prune call, so a reconciler that never ran +and a prune that deletes nothing cannot pass. +""" + +import io +import logging +from contextlib import ExitStack + +import pytest +from django.conf import settings +from django.core.management import call_command +from django.tasks import task_backends +from django.test import override_settings +from django.utils.module_loading import import_string + +from django_ox.backend import OxBackend +from django_ox.models import OxTask +from oxpull import unique as unique_module +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import drain, flow_worker, stored + +pytestmark = pytest.mark.django_db + +CALLBACK = "tests_oxpull.tasks.record_callback" +TICKS = 5 +CHANGES = [ + "unique_tasks_option", + "unique_decorator", + "renamed", + "moved_off_django_ox", +] + +# A word the refusal must carry, besides the callback's name, to say why +# the callback was refused. Matched case-insensitively against the log +# message and every exception in the record's chain, type names included. +REFUSAL_REASON = { + "unique_tasks_option": "unique", + "unique_decorator": "unique", + "renamed": "import", + "moved_off_django_ox": "django-ox", +} + + +def premise(condition, message): + """ + A fact the test stands on, not the behaviour under test. pytest.fail is + not an AssertionError, so a strict marker never absorbs it. + """ + if not condition: + pytest.fail(f"premise: {message}") + + +def change_the_callback(how, monkeypatch, stack): + """Make the stored callback path fail the completion checks, for real.""" + if how == "unique_tasks_option": + default = settings.TASKS["default"] + options = default["OPTIONS"] + stack.enter_context( + override_settings( + TASKS={ + **settings.TASKS, + "default": { + **default, + "OPTIONS": { + **options, + "UNIQUE_TASKS": [*options["UNIQUE_TASKS"], CALLBACK], + }, + }, + } + ) + ) + premise( + CALLBACK in task_backends["default"].unique_task_paths, + "UNIQUE_TASKS did not take", + ) + elif how == "unique_decorator": + monkeypatch.setattr(unique_module, "_registry", set(unique_module._registry)) + unique_module.unique(tasks.record_callback) + premise(unique_module.is_registered(CALLBACK), "@unique did not register") + elif how == "renamed": + monkeypatch.delattr(tasks, "record_callback") + try: + import_string(CALLBACK) + except ImportError: + pass + else: + pytest.fail(f"premise: {CALLBACK} still imports") + elif how == "moved_off_django_ox": + monkeypatch.setattr( + tasks, "record_callback", tasks.record_callback.using(backend="immediate") + ) + premise( + not isinstance(import_string(CALLBACK).get_backend(), OxBackend), + "the callback is still on a django-ox backend", + ) + else: # pragma: no cover + raise AssertionError(how) + + +def reconciler_tick(): + """ + One reconciler tick, run the way a SCHEDULES entry runs it: enqueued as + the task, then claimed and executed by a worker. Returns the reconcile + task's final status and return value. + """ + result = workflows.reconcile.enqueue() + runner = flow_worker() + for _ in range(20): + if OxTask.objects.get(id=result.id).status not in ( + OxTask.Status.READY, + OxTask.Status.RUNNING, + ): + break + runner.run_once() + row = OxTask.objects.get(id=result.id) + premise( + row.status not in (OxTask.Status.READY, OxTask.Status.RUNNING), + f"the reconcile task never finished: {row.status}", + ) + return row.status, row.return_value + + +def two_node_workflow(name, callback, label): + flow = workflows.create( + name, + on_complete=callback, + on_complete_args=[f"{label}-complete"], + on_success=callback, + on_success_args=[f"{label}-success"], + ) + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + return flow + + +def rows_of(flow): + """(edge rows, node rows) oxpull still holds for this workflow.""" + return ( + OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count(), + OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count(), + ) + + +def prune(): + out = io.StringIO() + call_command("oxpull_prune", "--older-than", "0", stdout=out) + return out.getvalue().strip().replace("\n", "; ") + + +def record_text(record): + """A record's message and every exception in its chain, as one string.""" + parts = [record.getMessage()] + exc = record.exc_info[1] if record.exc_info else None + seen = set() + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + parts.append(f"{type(exc).__name__}: {exc}") + exc = exc.__cause__ or exc.__context__ + return " | ".join(parts) + + +def refusals(records, flow, how): + """ + ERROR records about this workflow that name its callback and say why the + callback was refused. + """ + reason = REFUSAL_REASON[how] + found = [] + for record in records: + if record.levelno < logging.ERROR: + continue + text = record_text(record) + about = ( + getattr(record, "workflow_id", None) == str(flow.id) or str(flow.id) in text + ) + if about and "record_callback" in text and reason in text.lower(): + found.append(record) + return found + + +def describe(records): + return [ + (record.levelname, getattr(record, "event", None), record_text(record)[:240]) + for record in records + ] or "no records at ERROR" + + +@pytest.mark.parametrize("how", CHANGES) +def test_a_workflow_whose_callback_changed_after_create_stays_sealed_and_says_so( + how, monkeypatch, caplog +): + flow = two_node_workflow(f"callback-{how}", tasks.record_callback, "stranded") + healthy = two_node_workflow(f"healthy-{how}", tasks.plain_echo, "healthy") + row = stored(flow) + premise( + (row.on_complete_path, row.on_success_path) == (CALLBACK, CALLBACK), + f"stored callbacks {row.on_complete_path}, {row.on_success_path}", + ) + premise(drain(flow_worker()) == 4, "the worker did not run all four nodes") + node_ids = [*flow.node_ids().values(), *healthy.node_ids().values()] + premise( + set(OxTask.objects.filter(id__in=node_ids).values_list("status", flat=True)) + == {OxTask.Status.SUCCESSFUL}, + "not every node succeeded", + ) + premise(rows_of(flow) == (1, 2), f"rows before anything: {rows_of(flow)}") + + problems = [] + with ExitStack() as stack: + change_the_callback(how, monkeypatch, stack) + + ticks = [] + for tick in range(1, TICKS + 1): + caplog.clear() + with caplog.at_level(logging.ERROR, logger="oxpull"): + ticks.append(reconciler_tick()) + # Anything a fix enqueued gets its chance to run. + drain(flow_worker()) + if not refusals(caplog.records, flow, how): + problems.append( + f"tick {tick} (reconcile task {ticks[-1]}) logged no ERROR " + f"about workflow {flow.id} naming {CALLBACK!r} and " + f"{REFUSAL_REASON[how]!r}: {describe(caplog.records)}" + ) + + pruned = prune() + row = stored(flow) + progress = flow.progress() + callback_rows = OxTask.objects.filter(task_path=CALLBACK).count() + stranded_rows = rows_of(flow) + + healthy_row = stored(healthy) + premise( + healthy_row.status == OxpullWorkflow.Status.COMPLETE + and healthy_row.callback_task_id is not None + and healthy_row.success_task_id is not None, + f"the healthy workflow sealed beside it did not complete: " + f"{healthy_row.status}, ticks {ticks}", + ) + premise( + rows_of(healthy) == (0, 0), + f"oxpull_prune did not reap the completed healthy workflow " + f"({rows_of(healthy)} edge and node rows left): {pruned}", + ) + + if row.status != OxpullWorkflow.Status.SEALED: + problems.append( + f"left SEALED for {row.status} while its callback could not be " + f"enqueued (progress total={progress.total} pending={progress.pending} " + f"succeeded={progress.succeeded}); ticks {ticks}" + ) + if row.callback_task_id is not None or row.success_task_id is not None: + problems.append( + f"recorded callback ids callback_task_id={row.callback_task_id} " + f"success_task_id={row.success_task_id} for a refused callback" + ) + if callback_rows: + problems.append( + f"{callback_rows} task row(s) for the refused callback {CALLBACK} exist" + ) + if stranded_rows != (1, 2): + problems.append( + f"oxpull_prune --older-than 0 left {stranded_rows[0]} edge and " + f"{stranded_rows[1]} node row(s) of the stranded workflow, not 1 and " + f"2: {pruned}" + ) + assert not problems, f"[{how}] " + " | ".join(problems) + + # The deploy is fixed: the path imports again, is not unique, and is on + # django-ox. The held completion goes through on the next tick. + monkeypatch.undo() + restored = reconciler_tick() + drain(flow_worker()) + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE, ( + f"[{how}] still {row.status} after the callback was put back; tick {restored}" + ) + ran = (tasks.STATE.get("stranded-complete"), tasks.STATE.get("stranded-success")) + assert ran == (1, 1), ( + f"[{how}] on_complete and on_success ran {ran} times after the fix; " + f"callback_task_id={row.callback_task_id} " + f"success_task_id={row.success_task_id}" + ) + pruned = prune() + assert rows_of(flow) == (0, 0), ( + f"[{how}] oxpull_prune left {rows_of(flow)} edge and node rows once the " + f"workflow completed: {pruned}" + ) diff --git a/tests_oxpull/test_workflows_hazard_add_fires_task_enqueued_before_it_validates.py b/tests_oxpull/test_workflows_hazard_add_fires_task_enqueued_before_it_validates.py new file mode 100644 index 0000000..7c2d03d --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_add_fires_task_enqueued_before_it_validates.py @@ -0,0 +1,242 @@ +""" +Workflow.add() sends task_enqueued before it +validates the caller's input. + +add() takes the workflow row first, so a workflow that is no longer OPEN +refuses before anything is enqueued and no signal is sent. Then it +enqueues, and django-ox's enqueue sends task_enqueued right after its +INSERT (add() sends it itself for a node born WAITING, at the same point). +Only then does add() resolve depends_on and insert the node row. When one +of those refuses, add() rolls its block back, the task row with it, and the +receiver has already been handed an id that never commits. + +The regression tests assert what a caller would expect: an add() that +raises UnknownWorkflowNode, WorkflowNodeExists or WorkflowSealed sends +no task_enqueued, and every id a receiver was handed can be read back +once the caller's transaction commits. + +Two controls pass today and are meant to. They bound the consequence: +* A receiver that defers its effect with transaction.on_commit, as + `docs/features/unique-tasks.md` and `docs/features/batches.md` advise, + sees none of the refused adds, both when add() owns the transaction + and inside a caller's atomic block that catches the refusal and + commits. +* add() refuses a unique task as a node before it enqueues anything, so + that refusal sends no signal and writes no row. +""" + +from contextlib import contextmanager, suppress + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.db import transaction +from django.tasks import task, task_backends +from django.tasks.exceptions import TaskResultDoesNotExist +from django.tasks.signals import task_enqueued + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.exceptions import UnknownWorkflowNode, WorkflowNodeExists, WorkflowSealed +from oxpull.models import OxpullWorkflow +from oxpull.unique import unique + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile, stored + + +@unique +@task(backend="flows") +def unique_step(label): + return label + + +@contextmanager +def enqueued_ids(*, deferred=False): + """ + The ids task_enqueued hands receivers while the block runs. + + deferred=False records at send time, as a receiver with an immediate + external effect would. deferred=True records from transaction.on_commit, + the pattern documented in `docs/features/unique-tasks.md` and + `docs/features/batches.md`. + """ + seen = [] + + def receiver(sender, task_result, **kwargs): + if deferred: + transaction.on_commit(lambda: seen.append(str(task_result.id))) + else: + seen.append(str(task_result.id)) + + task_enqueued.connect(receiver, weak=False, dispatch_uid="add-enqueued-ids") + try: + yield seen + finally: + task_enqueued.disconnect(dispatch_uid="add-enqueued-ids") + + +# -- the refused adds --------------------------------------------------- +# +# Each builds its workflow before any receiver is connected and returns +# the refused call and the exception it raises. + + +def _unknown_dependency(): + flow = workflows.create("unknown-dependency") + flow.add("extract", tasks.step, "extract") + # A typo in depends_on. + return ( + lambda: flow.add("load", tasks.step, "load", depends_on="extarct"), + UnknownWorkflowNode, + ) + + +def _duplicate_node_name(): + flow = workflows.create("duplicate-name") + flow.add("extract", tasks.step, "extract") + return ( + lambda: flow.add("extract", tasks.step, "extract-again"), + WorkflowNodeExists, + ) + + +def _add_after_seal(): + flow = workflows.create("sealed") + flow.add("extract", tasks.step, "extract") + flow.seal() + return (lambda: flow.add("late", tasks.step, "late"), WorkflowSealed) + + +def _add_after_complete(): + """COMPLETE through the real path: a worker runs it, the reconciler completes.""" + flow = workflows.create("complete") + flow.add("extract", tasks.step, "extract") + flow.seal() + assert drain(flow_worker()) == 1 + assert reconcile() == 1 + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + return (lambda: flow.add("late", tasks.step, "late"), WorkflowSealed) + + +REFUSALS = [ + pytest.param(_unknown_dependency, id="unknown-dependency"), + pytest.param(_duplicate_node_name, id="duplicate-node-name"), + pytest.param(_add_after_seal, id="add-after-seal"), + pytest.param(_add_after_complete, id="add-after-complete"), +] + + +def refusals_marked(reason): + """ + REFUSALS, with the refusals that still send a signal marked xfail. + + add() checks the workflow's status before it enqueues, so the sealed and + complete cases send nothing. A bad depends_on or a duplicate name is + found after the enqueue. + """ + still_sends = pytest.mark.xfail(strict=True, reason=reason) + return [ + pytest.param(_unknown_dependency, id="unknown-dependency", marks=still_sends), + pytest.param(_duplicate_node_name, id="duplicate-node-name", marks=still_sends), + pytest.param(_add_after_seal, id="add-after-seal"), + pytest.param(_add_after_complete, id="add-after-complete"), + ] + + +# transaction=True so add() owns a real transaction, as a bare call does. +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "refusal", + refusals_marked( + "add() sends task_enqueued before it resolves depends_on and inserts the " + "node, so a refused add hands receivers the id of a task that is rolled " + "back" + ), +) +def test_a_refused_add_sends_no_task_enqueued(refusal): + attempt, refused_with = refusal() + rows_before = OxTask.objects.count() + + with enqueued_ids() as seen, pytest.raises(refused_with): + attempt() + + # (signals sent, task rows added). The rollback already keeps the + # second at 0; the first is the phantom. + assert (len(seen), OxTask.objects.count() - rows_before) == (0, 0) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "refusal", + refusals_marked( + "A refused add() still sends task_enqueued, and the id it hands " + "receivers does not exist after the caller commits" + ), +) +def test_every_id_a_receiver_was_handed_exists_after_the_caller_commits(refusal): + """ + The consequence as a receiver meets it. The caller handles its own + mistake inside its transaction and commits the rest of its work, and + a receiver that stored the id it was handed later asks for the + result. + """ + attempt, refused_with = refusal() + + with enqueued_ids() as seen, transaction.atomic(): + with suppress(refused_with): + attempt() + kept = tasks.default_step.enqueue("the caller's other work") + + # The backend's lookup by id, as a receiver that kept only the id + # would make it; Task.get_result would also check the task path. + missing = [] + for task_id in seen: + try: + task_backends["flows"].get_result(task_id) + except TaskResultDoesNotExist: + missing.append(task_id) + assert str(kept.id) in seen + assert missing == [] + + +# -- controls: these pass today ----------------------------------------- + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("inside_caller_atomic", [False, True]) +@pytest.mark.parametrize("refusal", REFUSALS) +def test_a_receiver_deferred_with_on_commit_sees_no_refused_add( + refusal, inside_caller_atomic +): + attempt, refused_with = refusal() + control = workflows.create("control") + + with enqueued_ids(deferred=True) as committed: + if inside_caller_atomic: + with transaction.atomic(): + with suppress(refused_with): + attempt() + good = control.add("ok", tasks.step, "ok") + else: + with pytest.raises(refused_with): + attempt() + good = control.add("ok", tasks.step, "ok") + + # The receiver is not silent: it reports the add that committed, and + # only that one. + assert committed == [str(good.id)] + + +@pytest.mark.django_db +def test_a_unique_task_refused_as_a_node_sends_no_task_enqueued(): + flow = workflows.create("unique") + flow.add("first", tasks.step, "k") + rows_before = OxTask.objects.count() + + with ( + enqueued_ids() as seen, + pytest.raises(ImproperlyConfigured, match="unique"), + ): + flow.add("second", unique_step, "k") + + assert (len(seen), OxTask.objects.count() - rows_before) == (0, 0) diff --git a/tests_oxpull/test_workflows_hazard_add_mysql.py b/tests_oxpull/test_workflows_hazard_add_mysql.py new file mode 100644 index 0000000..26b2362 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_add_mysql.py @@ -0,0 +1,265 @@ +""" +Workflow.add() on MySQL: a replaced client flag and an InnoDB deadlock. + +Each test asserts the behaviour `oxpull` 1.2.0 already gives Batch.add. +All but the guard catch workflow regressions in the cases below: + +* add() read its no-op OPEN-claim UPDATE's rowcount as "the workflow is + OPEN". On MySQL that rowcount counts matched rows only because Django + asks for CLIENT_FOUND_ROWS, and a project that sets + OPTIONS["client_flag"] replaces the flag. 1.2.0 fixed exactly this in + Batch._add_once, and Workflow.add now shares that code; the tests are + ports of tests_oxpull/test_batches.py's without_found_rows pair. +* add() was one atomic block with no reissue after an InnoDB deadlock + (1213) when it owns its transaction. 1.2.0's Batch.add reissues up to + MAX_ADD_ATTEMPTS, and Workflow.add now shares that loop. A port of + test_batches.py's deadlock_once test, which injects the 1213: no real + InnoDB deadlock inside add() was reached. The guard beside it pins that + a caller's own transaction is never reissued. +* Adjacent, same flag, still open: the reconciler's _cancel_one_level + reads a no-op SEALED-claim UPDATE's rowcount the same way, and batches + have no such statement. + +MySQL only, except the guard: the flag is a MySQL connection flag and 1213 +an InnoDB code. +""" + +from contextlib import contextmanager + +import pytest +from django.db import OperationalError, connections, transaction + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.backend import ER_LOCK_DEADLOCK +from oxpull.exceptions import WorkflowSealed +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .racing import race_all +from .test_batches import session_isolation +from .test_workflows import drain, flow_worker, reconcile, stored + +pytestmark = pytest.mark.django_db(transaction=True) + +# A value a project might really set, and zero. Neither carries the +# FOUND_ROWS bit (value 2); MULTI_STATEMENTS is bit 16 in both PyMySQL's +# and mysqlclient's constants. +WITHOUT_FOUND_ROWS = [ + pytest.param(0, id="client_flag=0"), + pytest.param(1 << 16, id="client_flag=MULTI_STATEMENTS"), +] + +# Read at collection: every expected failure below exists on MySQL only, and +# the tests skip elsewhere. +ON_MYSQL = connections["default"].vendor == "mysql" + + +def require_mysql(reason): + if connections["default"].vendor != "mysql": + pytest.skip(reason) + + +@contextmanager +def client_flag(flag): + """ + tests_oxpull.test_batches.without_found_rows with the flag value chosen, and + the OPTIONS restored before the block's caller reconnects, so no + later statement or test inherits a connection without the flag. + None leaves Django's own flags in place. + """ + connection = connections["default"] + require_mysql("CLIENT_FOUND_ROWS is a MySQL connection flag") + if flag is None: + yield + return + options = connection.settings_dict["OPTIONS"] + missing = object() + previous = options.get("client_flag", missing) + options["client_flag"] = flag + connection.close() + try: + yield + finally: + connection.close() + if previous is missing: + options.pop("client_flag", None) + else: + options["client_flag"] = previous + + +# -- item 4: the OPEN claim without CLIENT_FOUND_ROWS -------------------- + + +@pytest.mark.parametrize("flag", WITHOUT_FOUND_ROWS) +def test_add_on_an_open_workflow_does_not_depend_on_the_mysql_found_rows_flag(flag): + """ + Port of test_add_does_not_depend_on_the_mysql_found_rows_flag. + Without the fix, the first add() on a brand-new OPEN workflow raised + WorkflowSealed ("... is OPEN; membership is frozen."), so no workflow + could be built. + """ + with client_flag(flag): + flow = workflows.create("no-found-rows") + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + assert OxpullWorkflowEdge.objects.filter( + workflow_id=flow.id, + upstream_task_id=root.id, + downstream_task_id=child.id, + ).exists() + flow.seal() + with pytest.raises(WorkflowSealed, match="SEALED"): + flow.add("late", tasks.step, "late") + assert stored(flow).total_count == 2 + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 2 + assert OxTask.objects.count() == 2 + + +@pytest.mark.parametrize( + "flag", + [pytest.param(None, id="django-default-flags"), *WITHOUT_FOUND_ROWS], +) +def test_add_refused_by_a_committed_seal_names_the_status_it_locked(flag): + """ + Port of test_add_without_the_flag_reads_the_status_it_locked_not_its + _snapshot. Inside a caller's REPEATABLE READ transaction whose + snapshot says OPEN, a concurrent seal() commits; add() must refuse + and say SEALED. A zero-rowcount refusal alone is insufficient: + _not_open_error must not read the status with a plain SELECT, which + answers from the caller's snapshot and can report that the workflow + "is OPEN; membership is frozen". This also affects Django's own flags + whenever the session runs at REPEATABLE READ. A fallback that trusts + that snapshot read could instead admit a node into a sealed workflow. + """ + with client_flag(flag): + flow = workflows.create("stale-snapshot") + + def seal_and_commit(): + flow.seal() + return "sealed" + + with session_isolation("REPEATABLE READ"), transaction.atomic(): + # The caller's snapshot: the workflow is OPEN in it from here. + assert flow.status == OxpullWorkflow.Status.OPEN + assert race_all([seal_and_commit]) == (["sealed"], []) + with pytest.raises(WorkflowSealed, match="SEALED"): + flow.add("late", tasks.step, "late") + + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 0 + assert OxTask.objects.count() == 0 + assert stored(flow).total_count == 0 + + +# -- item 5: no reissue after an InnoDB deadlock ------------------------ + + +def node_insert_deadlocks_once(monkeypatch): + """ + tests_oxpull.test_batches.deadlock_once, on the node INSERT instead of the + membership INSERT, and for the reason given there: the error carries + MySQL's own code and text, and add()'s atomic block does the rollback + InnoDB would already have done. + """ + real_save = OxpullWorkflowNode.save + calls = [] + + def save_or_deadlock(self, *args, **kwargs): + calls.append(1) + if len(calls) == 1: + raise OperationalError( + ER_LOCK_DEADLOCK, + "Deadlock found when trying to get lock; try restarting transaction", + ) + return real_save(self, *args, **kwargs) + + monkeypatch.setattr(OxpullWorkflowNode, "save", save_or_deadlock) + return calls + + +def test_add_reissues_its_own_transaction_after_a_deadlock(monkeypatch): + """ + Port of test_add_reissues_its_own_transaction_after_a_deadlock. add() + owns its transaction, so losing a deadlock leaves nothing behind and + the whole add() (enqueue, workflow row claim, node, edge) can be + reissued instead of exposing the raw OperationalError(1213). + """ + require_mysql("the reissue is InnoDB's 1213") + flow = workflows.create("victim") + root = flow.add("root", tasks.step, "root") + calls = node_insert_deadlocks_once(monkeypatch) + + child = flow.add("child", tasks.step, "child", depends_on="root") + + assert len(calls) == 2 + assert OxTask.objects.count() == 2 + node = OxpullWorkflowNode.objects.get(workflow_id=flow.id, name="child") + assert str(node.task_id) == child.id + assert ( + OxpullWorkflowEdge.objects.filter( + upstream_task_id=root.id, downstream_task_id=child.id + ).count() + == 1 + ) + + +def test_add_inside_a_callers_transaction_leaves_the_deadlock_to_the_caller( + monkeypatch, +): + """ + Port of the test_batches test of the same name, and the guard on the + reissue above. Inside a caller's atomic block a deadlock has rolled + back the caller's work too, and only the caller can redo it, so add() + propagates the error after one attempt, on every database, and leaves + no task, node or edge of its own behind. A reissue that ignored who + owns the transaction passes the test above and fails this one. + """ + flow = workflows.create("ambient") + flow.add("root", tasks.step, "root") + calls = node_insert_deadlocks_once(monkeypatch) + + with pytest.raises(OperationalError), transaction.atomic(): + flow.add("child", tasks.step, "child", depends_on="root") + + assert len(calls) == 1 + assert OxTask.objects.count() == 1 + assert set(flow.node_ids()) == {"root"} + assert not OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).exists() + + +# -- adjacent: the reconciler's SEALED claim without CLIENT_FOUND_ROWS ---- + + +@pytest.mark.xfail( + ON_MYSQL, + strict=True, + reason=( + "On MySQL without CLIENT_FOUND_ROWS, the reconciler never cancels a " + "failed node's downstream nodes, so the workflow never completes" + ), +) +@pytest.mark.parametrize("flag", WITHOUT_FOUND_ROWS) +def test_the_reconciler_cancels_a_failed_cone_without_the_found_rows_flag(flag): + """ + _cancel_one_level opens with a no-op UPDATE under the SEALED + predicate. Without the found-rows flag, its rowcount is 0 even when + the predicate matches. Treating that as no match leaves a FAILED + node's cone uncancelled, the blocked child READY, and the workflow + incomplete. Build the workflow with Django's own flags to isolate + cancellation from add()'s handling of the flag. Both configurations + reach the same rows. + """ + flow = workflows.create("doomed-no-found-rows") + flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + + with client_flag(flag): + assert drain(flow_worker()) == 3 # root, three attempts + assert OxTask.objects.get(args=["root"]).status == OxTask.Status.FAILED + assert reconcile() == 1 + + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.DISCARDED + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert row.cancelled_count == 1 diff --git a/tests_oxpull/test_workflows_hazard_cancel_cas.py b/tests_oxpull/test_workflows_hazard_cancel_cas.py new file mode 100644 index 0000000..4a78df7 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_cancel_cas.py @@ -0,0 +1,401 @@ +""" +The cancellation UPDATE in +workflows._cancel_one_level discards on `status=READY` alone. + +What these tests found, stated before the tests so the reader knows what +fails and why: + +_cancel_one_level locks the upstream rows that are dead *when its +SELECT ... FOR UPDATE runs*, then reads the READY nodes blocked by a dead +upstream in a second, unlocked SELECT, then discards them. The docstring's +"There is no interleaving" argument holds only for an upstream that was +already dead at the locking read. An upstream that dies after the lock and +before the second SELECT is read as dead without being locked, so an +operator retry of it is not made to wait, and it can commit before the +discard UPDATE. The discard then lands on a node whose only upstream is no +longer dead. + +An (status, lease_epoch) compare-and-set on the discarded node, which is +what django_ox.actions.discard does, does not close that: retry() bumps +the upstream's epoch, not the downstream's, and the downstream is READY +before and after. It closes only the narrower variant in the second test, +where the downstream is also claimed and requeued inside the window. + +Each test asserts the correct behaviour, so each fails today on the +databases with row locks. Timing is forced with Django's public +connection.execute_wrapper on the reconciler's own connection: every +statement the reconciler issues is the production statement, and the +interleaved steps are the real worker, the real reaper and the real +operator actions, each on its own thread and connection. + +SQLite is skipped, not passed: the reconciler's transaction opens with a +write, SQLite serialises writers, so neither the upstream's failure write +nor the retry can commit inside it. +""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest +from django.db import connections +from django.tasks import task + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import flow_worker, order, reconcile, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +TASK_TABLE = OxTask._meta.db_table + +# Read at collection, as _row_locks_only reads it at run time: the expected +# failures exist where the database has row locks, and SQLite skips. +ROW_LOCKS = connections[oxpull_db_alias()].vendor != "sqlite" + + +# -- tasks ------------------------------------------------------------- +# +# A node that fails while its label is flagged and succeeds once the flag +# is cleared: the shape of an upstream that dies and then succeeds when an +# operator retries it. tests_oxpull/tasks.py has flaky tasks only on the default +# backend, which add() refuses for a workflow node. + + +def _flagged(label: str) -> str: + tasks._record(label) + if tasks.STATE.get(f"fail:{label}"): + raise ValueError(f"{label} fails while flagged") + return label + + +@task(backend="flows") +def flagged_step(label): + return _flagged(label) + + +@task(backend="flows_deferred") +def flagged_deferred_step(label): + return _flagged(label) + + +@dataclass(frozen=True) +class Shape: + """One workflow backend and the node tasks that live on it.""" + + alias: str + worker: Callable[[], Any] + dead: Any + flagged: Any + plain: Any + + +SHAPES = [ + pytest.param( + Shape("flows", flow_worker, tasks.failing_step, flagged_step, tasks.step), + id="flows", + ), + pytest.param( + Shape( + "flows_deferred", + deferring_worker, + tasks.failing_deferred_step, + flagged_deferred_step, + tasks.deferred_step, + ), + id="flows_deferred", + ), +] + + +@pytest.fixture(autouse=True) +def _row_locks_only(): + if connections[oxpull_db_alias()].vendor == "sqlite": + pytest.skip( + "SQLite serialises writers and the cancel transaction opens with a " + "write, so no upstream write can commit inside it" + ) + + +# -- harness ----------------------------------------------------------- + + +def in_thread(step: Callable[[], Any], timeout: float = 30) -> Any: + """Run `step` on its own thread and connection, as a separate process would.""" + out: list[Any] = [] + err: list[BaseException] = [] + + def run() -> None: + try: + out.append(step()) + except BaseException as exc: + err.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=timeout) + if thread.is_alive(): + raise RuntimeError( + "the interleaved step is waiting on the reconciler's transaction" + ) + if err: + raise err[0] + return out[0] + + +class Interleave: + """ + Pause the reconciler at two points inside _cancel_one_level: + + after_lock: after the SELECT ... FOR UPDATE on the dead upstreams, + before the unlocked SELECT of cancellable nodes; + after_read: after that SELECT, before whatever statement comes next. + Today that is the discard UPDATE. It is deliberately not + pinned to the UPDATE: a correct cancel pass may have + nothing to discard, and the step must still run. + + One-shot: later cancellation levels and the completion pass run + untouched. Errors are recorded, not raised, because _reconcile logs + and swallows every exception from a workflow's pass. + """ + + def __init__(self, after_lock, after_read): + self.steps = {"after_lock": after_lock, "after_read": after_read} + self.fired: list[str] = [] + self.before: dict[str, str] = {} + self.results: dict[str, Any] = {} + self.errors: list[tuple[str, BaseException]] = [] + self._locked = False + self._read = False + + def _fire(self, name: str, sql: str) -> None: + self.fired.append(name) + self.before[name] = " ".join(sql.split())[:48] + try: + # Never on the reconciler's own connection: that would run the + # step inside its transaction, which no other process can do. + self.results[name] = in_thread(self.steps[name]) + except BaseException as exc: + self.errors.append((name, exc)) + + def __call__(self, execute, sql, params, many, context): + if not self._locked: + result = execute(sql, params, many, context) + if "FOR UPDATE" in sql.upper() and TASK_TABLE in sql: + self._locked = True + return result + if not self.fired and sql.lstrip()[:6].upper() == "SELECT": + self._fire("after_lock", sql) + result = execute(sql, params, many, context) + self._read = True + return result + if self._read and self.fired == ["after_lock"]: + self._fire("after_read", sql) + return execute(sql, params, many, context) + + +def reconcile_interleaved(interleave: Interleave) -> int: + connection = connections[oxpull_db_alias()] + with connection.execute_wrapper(interleave): + completed = reconcile() + assert interleave.errors == [], interleave.errors + assert interleave.fired == ["after_lock", "after_read"], ( + f"the reconciler never reached both points: {interleave.fired}" + ) + return completed + + +def status(task_id) -> str: + return OxTask.objects.get(id=task_id).status + + +def run_until(worker, predicate, limit: int = 20) -> None: + """run_once until predicate() holds; a zero backoff is still a timestamp.""" + for _ in range(limit): + if predicate(): + return + if not worker.run_once(): + time.sleep(0.01) + assert predicate() + + +def sibling_dead_workflow(shape: Shape, name: str): + """ + dead (already FAILED) -> dead-child and upstream -> child + + `dead` and its child are what make the reconciler open its cancel + transaction and take a lock: a sealed workflow with a node that settled + without succeeding and a node that depends on it. The reconciler reads + first whether anything is cancellable, and a dead node with nothing + downstream of it opens no transaction. `upstream` is on its last attempt + and flagged to fail, and `child` depends on it alone. + """ + flow = workflows.create(name) + dead = flow.add("dead", shape.dead, "dead") + flow.add("dead-child", shape.plain, "dead-child", depends_on="dead") + upstream = flow.add("upstream", shape.flagged, "upstream") + child = flow.add("child", shape.plain, "child", depends_on="upstream") + flow.seal() + tasks.STATE["fail:upstream"] = True + + worker = shape.worker() + run_until(worker, lambda: status(dead.id) == OxTask.Status.FAILED) + # Two of the upstream's three attempts, so the next one is its last. + run_until(worker, lambda: OxTask.objects.get(id=upstream.id).attempts == 2) + assert status(upstream.id) == OxTask.Status.READY + assert status(child.id) == OxTask.Status.WAITING + assert "child" not in order() + return flow, dead.id, upstream.id, child.id + + +# -- the tests --------------------------------------------------------- + + +@pytest.mark.parametrize("shape", SHAPES) +def test_a_retry_of_an_upstream_that_died_after_the_lock_does_not_lose_its_child( + shape, +): + """ + 1. The reconciler locks the dead set, which is {dead}. + 2. The upstream's last attempt fails: a real worker, a real FAILED write. + 3. The reconciler reads the child as blocked by a dead upstream. + 4. An operator retries the upstream. Nothing makes it wait: the upstream + was not in the locked set. + 5. The reconciler discards the child on status=READY. + + Correct: a node is discarded only while an upstream is dead. At step 5 + the child's only upstream is READY, and once the retry succeeds the + child must run. + """ + flow, _dead, upstream, child = sibling_dead_workflow(shape, "retry-after-lock") + + def upstream_dies(): + shape.worker().run_once() + return OxTask.objects.get(id=upstream).status + + def operator_retries(): + return actions.retry(upstream) + + interleave = Interleave(upstream_dies, operator_retries) + reconcile_interleaved(interleave) + assert interleave.results["after_lock"] == OxTask.Status.FAILED + assert interleave.results["after_read"] is True + + child_after_pass = OxTask.objects.get(id=child) + upstream_after_pass = status(upstream) + + # The retry was meant to work. Let it. + tasks.STATE["fail:upstream"] = False + run_until(shape.worker(), lambda: status(upstream) == OxTask.Status.SUCCESSFUL) + for _ in range(3): + shape.worker().run_once() + reconcile() + row = stored(flow) + + facts = ( + f"retry landed before: {interleave.before['after_read']!r}; " + f"after the pass: child={child_after_pass.status} " + f"(run_after={child_after_pass.run_after}), upstream={upstream_after_pass}; " + f"after draining: upstream={status(upstream)}, child={status(child)}, " + f"child ran={'child' in order()}, workflow={row.status} " + f"failed_count={row.failed_count} cancelled_count={row.cancelled_count}" + ) + assert child_after_pass.status != OxTask.Status.DISCARDED, ( + "the reconciler discarded a node whose only upstream had been retried " + "before the discard: " + facts + ) + assert status(child) == OxTask.Status.SUCCESSFUL, facts + + +@pytest.mark.parametrize("shape", SHAPES) +def test_a_child_claimed_and_requeued_inside_the_window_is_not_discarded(shape): + """ + The narrower interleaving: the reaper requeues the discarded row + between the reconciler's SELECT and its UPDATE. + + For the child to be claimed at all inside the window its gate must + pass, so its upstream has to leave the dead set first: the same retry + as the test above, and then the upstream succeeds, the child is + claimed, its lease is expired (actions.expire_lease brings the + reaper's clock forward; the production trigger is the reconciler + stalling between two statements for longer than LOCK_TIMEOUT), and + the real reaper requeues it with a bumped epoch. + + The stalled first execution then finishes. Its outcome write is + refused by the epoch, as it should be. + + Correct: the child's upstream is SUCCESSFUL and it has attempts left, + so it is not discarded, and its second attempt runs. This is the one + variant a (status, lease_epoch) compare-and-set would catch. + """ + flow, _dead, upstream, child = sibling_dead_workflow(shape, "requeue-in-window") + + def upstream_dies(): + shape.worker().run_once() + return OxTask.objects.get(id=upstream).status + + def retry_succeed_claim_reap(): + assert actions.retry(upstream) is True + tasks.STATE["fail:upstream"] = False + worker = shape.worker() + assert worker.run_once() is True + assert OxTask.objects.get(id=upstream).status == OxTask.Status.SUCCESSFUL + claimed = worker.claim_one() + assert claimed is not None + assert str(claimed.id) == str(child) + assert actions.expire_lease(child) is True + assert worker.reap() == 1 + requeued = OxTask.objects.get(id=child) + assert requeued.status == OxTask.Status.READY + assert requeued.lease_epoch == claimed.lease_epoch + 1 + return claimed + + interleave = Interleave(upstream_dies, retry_succeed_claim_reap) + reconcile_interleaved(interleave) + assert interleave.results["after_lock"] == OxTask.Status.FAILED + stale = interleave.results["after_read"] + + child_after_pass = OxTask.objects.get(id=child) + # The discard leaves no lease behind: the reaper's requeue had already + # cleared it, and the cancel UPDATE clears locked_by and locked_at. + assert child_after_pass.locked_by is None + assert child_after_pass.locked_at is None + assert child_after_pass.lease_expires_at is None + + # The first execution was only stalled. It finishes now and is fenced. + shape.worker().execute(stale) + ran_before_drain = order().count("child") + fenced_status = status(child) + + for _ in range(3): + shape.worker().run_once() + reconcile() + row = stored(flow) + + facts = ( + f"requeue landed before: {interleave.before['after_read']!r}; " + f"after the pass: child={child_after_pass.status} " + f"epoch={child_after_pass.lease_epoch} attempts={child_after_pass.attempts}; " + f"stale execution ran the body {ran_before_drain}x and left child=" + f"{fenced_status}; after draining: child={status(child)}, " + f"child body runs={order().count('child')}, workflow={row.status} " + f"failed_count={row.failed_count} cancelled_count={row.cancelled_count}" + ) + assert child_after_pass.status != OxTask.Status.DISCARDED, ( + "the reconciler discarded a requeued node whose upstream had succeeded: " + + facts + ) + assert status(child) == OxTask.Status.SUCCESSFUL, facts + assert row.status == OxpullWorkflow.Status.COMPLETE, facts diff --git a/tests_oxpull/test_workflows_hazard_cancellation_locks_the_upstreams_that_are.py b/tests_oxpull/test_workflows_hazard_cancellation_locks_the_upstreams_that_are.py new file mode 100644 index 0000000..49b67c4 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_cancellation_locks_the_upstreams_that_are.py @@ -0,0 +1,541 @@ +""" +Cancellation locks the upstreams that are dead when its locking +read runs, but decides with a later read that treats any upstream dead +*now* as dead. + +workflows._cancel_one_level, in one transaction: + + 1. UPDATE the workflow row under SEALED (takes the workflow row lock); + 2. SELECT ... FOR UPDATE the workflow's dead task rows ("doomed"); + 3. SELECT the READY nodes blocked by a dead upstream ("cancellable"), + re-evaluating every upstream's current status, not only the locked + ids; a new statement, so a new snapshot at READ COMMITTED; + 4. UPDATE those nodes to DISCARDED; + 5. COMMIT. + +Its docstring argues "There is no interleaving": a retry either commits +before step 2 (the upstream is not dead) or waits on the step-2 lock +until step 5. That holds only for an upstream already dead at step 2. An +upstream X that is RUNNING at step 2 and writes its final FAILED before +step 3 is read as dead but was never locked, so django_ox.actions.retry(X) +does not wait, and it can commit anywhere between step 3 and step 5. + +What the test asserts, and why the precondition matters: + +The bare predicate "not (X READY and child DISCARDED)" is not by +itself something the design promises. When X is already dead at step 2, +retry(X) waits for step 5 and then lands, which also ends in X READY and +the child DISCARDED, and the existing +test_a_retry_cannot_interleave_with_the_cancellation_that_would_discard_it +accepts exactly that. What the design does promise, in the docstring and in +that test ("a torn state -- the parent successfully retried *and* the child +discarded in the same pass -- is not reachable"), is that a retry which +commits *while the pass's transaction is still open* does not lose its +child to that pass. So the test first proves, as checked preconditions, +that X was RUNNING when the dead rows were locked, that its FAILED write +committed inside the pass, and that the retry returned True without +waiting while the pass had not committed. Only then does it assert the +predicate, and after it the user-visible outcome: once X is retried +and succeeds its child runs, and with the other failure retried too the +workflow completes clean, with on_success. + +Every step is a production path: a real worker drives Y to FAILED, a real +worker claims X for its last attempt and runs it (claim_one + execute), +the operator action is actions.retry, and the pass is workflows._reconcile, +the scheduled task's body. No task row is written by hand. + +Timing is forced from outside the statements, located by what the cancel +transaction sends to the database rather than by which function sends it, +so the same test reaches the same points in a changed implementation. +The first point is before the first statement after the FOR UPDATE. +In the implementation that exposed the race, this was the cancellable +SELECT, equivalent to wrapping _blocked_by_dead_q; the retry's +"after the cancellable read" landing was the discard UPDATE, equivalent +to a shim on timezone.now(). The second landing is inside that +transaction's commit, after the discard UPDATE. Each interleaved step +runs on its own thread and connection. + +SQLite is skipped, not passed: the pass opens with a write and SQLite +serialises writers, so X's FAILED write cannot commit inside it. +""" + +from __future__ import annotations + +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import pytest +from django.db import connections +from django.tasks import task + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import flow_worker, order, reconcile, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +TASK_TABLE = OxTask._meta.db_table +WORKFLOW_TABLE = OxpullWorkflow._meta.db_table +LOCK = "SELECT dead task rows FOR UPDATE" + +# How long a hook lets an interleaved step run before the pass moves on. +# The steps here take milliseconds; a step still running after this is +# waiting on a lock the paused pass holds, and is recorded as such. +STEP_WAIT = 10.0 + + +# -- tasks ------------------------------------------------------------- +# +# A node that fails while its label is flagged and succeeds once the flag +# is cleared: an upstream that dies and then succeeds when retried. +# tests_oxpull/tasks.py has no such task on the workflow backends. + + +def _flagged(label: str) -> str: + tasks._record(label) + if tasks.STATE.get(f"fail:{label}"): + raise ValueError(f"{label} fails while flagged") + return label + + +@task(backend="flows") +def flagged_step(label): + return _flagged(label) + + +@task(backend="flows_deferred") +def flagged_deferred_step(label): + return _flagged(label) + + +@dataclass(frozen=True) +class Shape: + alias: str + worker: Callable[[], Any] + flagged: Any + plain: Any + + +SHAPES = [ + pytest.param(Shape("flows", flow_worker, flagged_step, tasks.step), id="flows"), + pytest.param( + Shape( + "flows_deferred", + deferring_worker, + flagged_deferred_step, + tasks.deferred_step, + ), + id="flows_deferred", + ), +] + +# Where the retry lands inside the pass's transaction. +AFTER_THE_CANCELLABLE_READ = "after_the_cancellable_read" +BEFORE_THE_COMMIT = "before_the_commit" +LANDINGS = [AFTER_THE_CANCELLABLE_READ, BEFORE_THE_COMMIT] + + +# Read at collection. The test skips itself on SQLite, so it is expected to +# fail only where the upstream's write can commit inside the pass. +HAS_ROW_LOCKS = connections[oxpull_db_alias()].vendor != "sqlite" + + +def skip_without_row_locks() -> None: + if connections[oxpull_db_alias()].vendor == "sqlite": + pytest.skip( + "SQLite serialises writers and the cancel transaction opens with a " + "write, so no upstream write can commit inside it" + ) + + +# -- harness ----------------------------------------------------------- + + +class Step: + """One interleaved step on its own thread and connection.""" + + def __init__(self, name: str, fn: Callable[[], Any]): + self.name = name + self.fn = fn + self.result: Any = None + self.error: BaseException | None = None + self.elapsed: float | None = None + self.finished_inside_pass: bool | None = None + self._thread = threading.Thread(target=self._run, name=name) + + def _run(self) -> None: + started = time.monotonic() + try: + self.result = self.fn() + except BaseException as exc: + self.error = exc + finally: + self.elapsed = time.monotonic() - started + connections.close_all() + + def run_inside_pass(self, wait: float) -> None: + self._thread.start() + self._thread.join(timeout=wait) + self.finished_inside_pass = not self._thread.is_alive() + + def join(self, timeout: float = 60) -> None: + if self._thread.ident is not None: + self._thread.join(timeout=timeout) + assert not self._thread.is_alive(), f"{self.name} never finished" + + def describe(self) -> str: + elapsed = None if self.elapsed is None else round(self.elapsed, 3) + return ( + f"{self.name}: result={self.result!r} error={self.error!r} " + f"elapsed={elapsed}s finished_inside_pass={self.finished_inside_pass}" + ) + + +@dataclass +class PassRecord: + events: list[str] = field(default_factory=list) + cancel_returned: Any = None + cancel_error: BaseException | None = None + reconcile_returned: Any = None + isolation: str = "" + + def timeline(self) -> str: + return " | ".join(self.events) + + +def isolation_level() -> str: + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SHOW transaction_isolation") + elif connection.vendor == "mysql": + cursor.execute("SELECT @@transaction_isolation") + else: + return connection.vendor + return f"{connection.vendor}:{cursor.fetchone()[0]}" + + +def reconcile_with_steps( + monkeypatch: pytest.MonkeyPatch, + *, + after_the_lock: Step | None, + retry: Step | None, + landing: str, + wait: float = STEP_WAIT, +) -> PassRecord: + """ + Run one real reconciler pass (workflows._reconcile) on this thread and + interleave up to two steps into the first transaction that locks dead + task rows: + + after_the_lock before the first statement after the locking read; + retry at AFTER_THE_CANCELLABLE_READ + before the second statement after it, or inside + the commit if no second statement comes; + retry at BEFORE_THE_COMMIT + inside that transaction's commit. + + Each fires once, on the reconciler's own connection only. On SQLite, + which has no FOR UPDATE, the locking read is the first task-table + SELECT after the workflow row UPDATE. + """ + record = PassRecord(isolation=isolation_level()) + connection = connections[oxpull_db_alias()] + sqlite = connection.vendor == "sqlite" + state = { + "workflow_update": False, + "locked": False, + "after_lock": 0, + "hook2": False, + "done": False, + } + + def log(entry: str) -> None: + record.events.append(entry) + + def fire(step: Step) -> None: + log(f"hook: {step.name} begins") + step.run_inside_pass(wait) + log( + f"hook: {step.name} " + + ("returned" if step.finished_inside_pass else "still waiting") + ) + + def classify(text: str) -> str: + upper = text.upper() + if upper.startswith("SELECT") and TASK_TABLE in text: + if "FOR UPDATE" in upper: + return LOCK + if sqlite and state["workflow_update"] and not state["locked"]: + return LOCK + if upper.startswith("UPDATE") and WORKFLOW_TABLE in text: + return "UPDATE workflow row" + if upper.startswith("UPDATE") and TASK_TABLE in text: + return "UPDATE task rows" + return text[:40] + + def statements(execute, sql, params, many, context): + label = classify(" ".join(sql.split())) + if state["locked"] and not state["done"]: + if state["after_lock"] == 0 and after_the_lock is not None: + log( + f"hook1 context: in_atomic_block={connection.in_atomic_block} " + f"next={label}" + ) + fire(after_the_lock) + elif ( + state["after_lock"] == 1 + and retry is not None + and landing == AFTER_THE_CANCELLABLE_READ + and not state["hook2"] + ): + state["hook2"] = True + log( + f"hook2 context: in_atomic_block={connection.in_atomic_block} " + f"next={label}" + ) + fire(retry) + state["after_lock"] += 1 + result = execute(sql, params, many, context) + log(f"sql: {label}") + if label == "UPDATE workflow row": + state["workflow_update"] = True + if label == LOCK and not state["done"]: + state["locked"] = True + return result + + original_commit = connection._commit + + def commit(): + if state["locked"] and not state["done"]: + state["done"] = True + if retry is not None and not state["hook2"]: + state["hook2"] = True + log("hook2 context: inside the commit") + fire(retry) + result = original_commit() + log("commit") + return result + + original_cancel = workflows._cancel_unreachable + + def cancel_unreachable(workflow_pk, db_alias): + try: + record.cancel_returned = original_cancel(workflow_pk, db_alias) + except BaseException as exc: + record.cancel_error = exc + raise + return record.cancel_returned + + monkeypatch.setattr(workflows, "_cancel_unreachable", cancel_unreachable) + monkeypatch.setattr(connection, "_commit", commit) + try: + with connection.execute_wrapper(statements): + record.reconcile_returned = reconcile() + finally: + monkeypatch.undo() + for step in (after_the_lock, retry): + if step is not None: + step.join() + assert record.cancel_error is None, record.cancel_error + return record + + +def status(task_id) -> str: + return OxTask.objects.get(id=task_id).status + + +def run_until(worker, predicate, limit: int = 30) -> None: + """run_once until predicate() holds; a zero backoff is still a timestamp.""" + for _ in range(limit): + if predicate(): + return + if not worker.run_once(): + time.sleep(0.01) + assert predicate() + + +@dataclass +class Torn: + flow: Any + y: Any + x: Any + d: Any + worker: Any + claimed: Any + + +def torn_workflow(shape: Shape, name: str) -> Torn: + """ + y (FAILED) -> ey x (RUNNING, last attempt) -> d (WAITING, blocked) + + y is the other dead node, and ey depends on it. Together they are what + make the pass open its cancel transaction at all: the reconciler reads + first whether anything is cancellable, and a dead node with nothing + downstream opens no transaction. x is claimed by a real worker on its + last attempt and flagged to fail, so its outcome write will be FAILED. + """ + flow = workflows.create( + name, + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + y = flow.add("y", shape.flagged, "y") + flow.add("ey", shape.plain, "ey", depends_on="y") + x = flow.add("x", shape.flagged, "x") + d = flow.add("d", shape.plain, "d", depends_on="x") + flow.seal() + tasks.STATE["fail:y"] = True + tasks.STATE["fail:x"] = True + + worker = shape.worker() + run_until(worker, lambda: status(y.id) == OxTask.Status.FAILED) + run_until(worker, lambda: OxTask.objects.get(id=x.id).attempts == 2) + assert status(x.id) == OxTask.Status.READY + claimed = worker.claim_one() + assert claimed is not None + assert str(claimed.id) == str(x.id) + row = OxTask.objects.get(id=x.id) + assert row.status == OxTask.Status.RUNNING + assert row.attempts == row.max_attempts == 3 + assert status(d.id) == OxTask.Status.WAITING + assert "d" not in order() + return Torn(flow, y, x, d, worker, claimed) + + +def upstream_last_attempt_fails(torn: Torn) -> Step: + def run(): + torn.worker.execute(torn.claimed) + return OxTask.objects.get(id=torn.x.id).status + + return Step("x's last attempt runs and writes its outcome", run) + + +def operator_retries_x(torn: Torn) -> Step: + def run(): + retried = actions.retry(torn.x.id) + return { + "retry": retried, + "x_seen_by_retry": status(torn.x.id), + "d_seen_by_retry": status(torn.d.id), + } + + return Step("operator retries x", run) + + +# -- the test ---------------------------------------------------------- + + +@pytest.mark.parametrize("landing", LANDINGS) +@pytest.mark.parametrize("shape", SHAPES) +def test_a_retry_that_commits_inside_the_pass_does_not_lose_its_child( + shape, landing, monkeypatch +): + """ + 1. The pass locks the dead set: {y}. x is RUNNING and not in it. + 2. x's last attempt fails: the real worker's FAILED write commits. + 3. The pass reads d as blocked by a dead upstream (x). + 4. The operator retries x. Nothing makes it wait: x was never locked. + It commits while the pass is still open (before the discard UPDATE, + or after it and before COMMIT). + 5. The pass discards d and commits. + + Correct: a retry that committed while the pass was open does not lose + its child to that pass; once x succeeds d runs, and with y retried too + the workflow completes. ey stays cancelled: y was dead when the pass + locked it, and nothing here revives a cancelled node, so the workflow + completes with that one cancellation and without on_success. + """ + skip_without_row_locks() + torn = torn_workflow(shape, f"torn-{landing}") + hook1 = upstream_last_attempt_fails(torn) + hook2 = operator_retries_x(torn) + + record = reconcile_with_steps( + monkeypatch, after_the_lock=hook1, retry=hook2, landing=landing + ) + + x_after_pass = status(torn.x.id) + d_after_pass = OxTask.objects.get(id=torn.d.id) + + # Preconditions: the interleaving really happened. If any of these + # fails the test did not reach the hazard and says nothing about it. + steps = f"{hook1.describe()}; {hook2.describe()}" + timeline = record.timeline() + assert hook1.error is None and hook2.error is None, steps + hook1_at = next( + i for i, e in enumerate(record.events) if e.startswith("hook1 context:") + ) + assert record.events[hook1_at - 1] == f"sql: {LOCK}", timeline + assert "in_atomic_block=True" in record.events[hook1_at], timeline + assert hook1.finished_inside_pass is True, steps + assert hook1.result == OxTask.Status.FAILED, steps + hook2_at = next( + i for i, e in enumerate(record.events) if e.startswith("hook2 context:") + ) + assert hook1_at < hook2_at < record.events.index("commit", hook1_at), timeline + assert hook2.finished_inside_pass is True, ( + "the retry waited on the pass, which is the designed interlock: " + steps + ) + assert hook2.result["retry"] is True, steps + assert hook2.result["x_seen_by_retry"] == OxTask.Status.READY, steps + # The pass had not committed when the retry did: the retry's own + # connection still reads d as WAITING, not DISCARDED. + assert hook2.result["d_seen_by_retry"] == OxTask.Status.WAITING, steps + assert x_after_pass == OxTask.Status.READY, timeline + + # Let the retry do what it was for, and retry the other failure too. + tasks.STATE["fail:x"] = False + tasks.STATE["fail:y"] = False + assert actions.retry(torn.y.id) is True + drainer = shape.worker() + run_until( + drainer, + lambda: ( + status(torn.x.id) == OxTask.Status.SUCCESSFUL + and status(torn.y.id) == OxTask.Status.SUCCESSFUL + ), + ) + for _ in range(3): + drainer.run_once() + reconcile() + row = stored(torn.flow) + + facts = ( + f"[{record.isolation}] timeline: {timeline}; steps: {steps}; " + f"cancel pass returned {record.cancel_returned}; " + f"after the pass: x={x_after_pass}, d={d_after_pass.status} " + f"(run_after={d_after_pass.run_after}); " + f"after retrying y and draining: x={status(torn.x.id)}, " + f"y={status(torn.y.id)}, d={status(torn.d.id)}, d ran={'d' in order()}, " + f"workflow={row.status} failed_count={row.failed_count} " + f"cancelled_count={row.cancelled_count} " + f"on_complete enqueued={row.callback_task_id is not None} " + f"on_success enqueued={row.success_task_id is not None}" + ) + # Printed as well as asserted: a passing run (a fixed implementation) + # still shows what it did, under pytest -rP. + print("FACTS", facts) # noqa: T201 + + assert not ( + x_after_pass == OxTask.Status.READY + and d_after_pass.status == OxTask.Status.DISCARDED + ), ( + "the pass discarded d although its only upstream was retried by a " + "commit that landed inside the pass: " + facts + ) + assert status(torn.d.id) == OxTask.Status.SUCCESSFUL, facts + assert row.status == OxpullWorkflow.Status.COMPLETE, facts + assert row.cancelled_count == 1, facts + assert row.success_task_id is None, facts diff --git a/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_does_not_take_nodes.py b/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_does_not_take_nodes.py new file mode 100644 index 0000000..944a06a --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_does_not_take_nodes.py @@ -0,0 +1,377 @@ +""" +Blocked nodes stay out of the claim scan. + +DEFER_BLOCKED_NODES used to leave a deferred node READY with run_after in the +year 3000 and its original enqueued_at. Neither ox_dequeue_idx nor +ox_dequeue_queue_idx holds run_after, so a claim walked READY rows in +(priority DESC, enqueued_at) order and rejected each deferred row with a row +filter. A blocked node is now WAITING, and status leads both indexes, so no +claim's key range reaches it. The steady state is ordinary: a root is +RUNNING, its cone was added before later work, and the later work sits +behind the cone. + +Each test asserts the property the waiting status delivers: + + * a claim's cost does not grow with the deferred nodes queued ahead of it + (all three databases, measured on the worker's real claim_one()); + * the shipped cost instrument counts the rows a plan filtered out + (PostgreSQL; tests/test_workflows_cost.rows_examined does not); + * on MySQL at REPEATABLE READ, a claim holds no row lock on a deferred + node it scanned past. + +The assertions are written against "blocked nodes" (an unsatisfied edge, not +RUNNING), not against run_after, so a fix that stops using run_after is +judged by the same tests. + +Shortcut, stated once: the MySQL lock test holds the claim's own +`_ready_queryset().select_for_update(skip_locked=True).first()` open inside +atomic(). In production that transaction lasts for the scan plus the claim +UPDATE; holding it open makes the lock set observable deterministically +instead of by racing a scan. +""" + +from __future__ import annotations + +import contextlib +import threading + +import pytest +from django.db import connection, connections, transaction + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows_cost import build, claim_candidates, postgres_only, rows_examined +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +mysql_only = pytest.mark.skipif( + connection.vendor != "mysql", reason="InnoDB row locks are MySQL's" +) + +SMALL = 50 +LARGE = 4000 +LOCK_SIZE = 400 + + +def refresh_statistics() -> None: + """ + Fresh planner statistics, as the shipped cost tests take them. + + MySQL's ANALYZE TABLE commits implicitly, which is why every test here + is transaction=True: inside a rolled-back test transaction it would + leak rows into the next test. SQLite is left as Django leaves it. + """ + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("ANALYZE django_ox_oxtask") + cursor.execute("ANALYZE oxpull_oxpullworkflowedge") + elif connection.vendor == "mysql": + for table in ("django_ox_oxtask", "oxpull_oxpullworkflowedge"): + cursor.execute(f"ANALYZE TABLE {table}") + cursor.fetchall() + + +def _pg_task_rows_read() -> int: + with connection.cursor() as cursor: + cursor.execute( + "SELECT COALESCE(seq_tup_read, 0) + COALESCE(idx_tup_fetch, 0) " + "FROM pg_stat_xact_user_tables WHERE relname = 'django_ox_oxtask'" + ) + row = cursor.fetchone() + return int(row[0]) if row else 0 + + +def _mysql_handler_reads() -> int: + with connection.cursor() as cursor: + cursor.execute("SHOW SESSION STATUS LIKE 'Handler_read%'") + return sum(int(value) for _, value in cursor.fetchall()) + + +def claim_cost(claim): + """ + Run `claim` and return (its result, what it cost server-side). + + PostgreSQL: task-table heap tuples read (seq_tup_read + idx_tup_fetch) + in this transaction, which counts rows a filter rejected. MySQL: the + session's Handler_read_* delta, less what reading the counter costs. + SQLite: VM instructions, from the progress handler. None of the three + moves with machine load. + """ + if connection.vendor == "postgresql": + with transaction.atomic(): + before = _pg_task_rows_read() + result = claim() + return result, _pg_task_rows_read() - before + if connection.vendor == "mysql": + idle_first = _mysql_handler_reads() + overhead = _mysql_handler_reads() - idle_first + before = _mysql_handler_reads() + result = claim() + return result, _mysql_handler_reads() - before - overhead + connection.ensure_connection() + raw = connection.connection + steps = 0 + + def count() -> int: + nonlocal steps + steps += 1 + return 0 + + raw.set_progress_handler(count, 1) + try: + result = claim() + finally: + raw.set_progress_handler(None, 1) + return result, steps + + +def full_rows_examined(plan) -> int: + """ + Rows every plan node touched: returned plus removed by a filter. + + PostgreSQL reports both per loop, so both are multiplied by loops. + """ + if isinstance(plan, str): + import json + + plan = json.loads(plan) + root = plan[0]["Plan"] if isinstance(plan, list) else plan["Plan"] + total = 0 + + def walk(node: dict) -> None: + nonlocal total + loops = int(node.get("Actual Loops", 1)) + total += int(node.get("Actual Rows", 0)) * loops + for removed in ( + "Rows Removed by Filter", + "Rows Removed by Join Filter", + "Rows Removed by Index Recheck", + ): + total += int(node.get(removed, 0)) * loops + for child in node.get("Plans", []): + walk(child) + + walk(root) + return total + + +def blocked_nodes(): + """ + Task rows the gate holds back and no worker is running: the rows a + claim has no reason to touch, however a release marks them. + """ + return OxTask.objects.exclude(workflows.runnable_filter_q()).exclude( + status=OxTask.Status.RUNNING + ) + + +def cone_behind_a_running_root(size: int, later: str | None, task=None) -> None: + """ + The steady state: a root claimed by a real worker, `size` children + waiting on it, and (if `later` is given) ordinary work enqueued after. + """ + task = task or tasks.deferred_step + build(size, task) + claimed = OxpullWorker(backend_alias=task.backend, backoff_initial=0).claim_one() + assert claimed is not None and claimed.args == ["root"], claimed + if later is not None: + task.enqueue(later) + + +# -- cost ---------------------------------------------------------------- + + +def test_a_claim_behind_deferred_nodes_costs_no_more_as_they_pile_up(): + """ + Blocked nodes are outside the candidate set, so a claim behind 4,000 of + them costs what a claim behind 50 does. Measured on the worker's real + claim, which on PostgreSQL is the single-statement UPDATE ... SKIP LOCKED. + """ + cone_behind_a_running_root(SMALL, "later-small") + refresh_statistics() + assert blocked_nodes().count() == SMALL + assert claim_candidates(gated=True).first().args == ["later-small"] + small_claim, small = claim_cost(deferring_worker().claim_one) + assert small_claim.args == ["later-small"] + + # A second workflow in flight, its root claimed the same way, so the + # blocked nodes ahead of the next ordinary task are now LARGE. + cone_behind_a_running_root(LARGE - SMALL, "later-large") + refresh_statistics() + assert blocked_nodes().count() == LARGE + assert ( + OxTask.objects.filter(args=["root"], status=OxTask.Status.RUNNING).count() == 2 + ) + assert claim_candidates(gated=True).first().args == ["later-large"] + large_claim, large = claim_cost(deferring_worker().claim_one) + assert large_claim.args == ["later-large"] + + print( # noqa: T201 + f"\n [s2 cost] {connection.vendor}: claim behind {SMALL} deferred={small}, " + f"behind {LARGE} deferred={large} ({large / max(small, 1):.1f}x)" + ) + assert large <= 2 * small, ( + f"a claim behind {LARGE} deferred nodes cost {large} against {small} " + f"behind {SMALL}; deferred nodes are still in the claim's scan" + ) + + +@postgres_only +@pytest.mark.xfail( + connection.vendor == "postgresql", + strict=True, + reason=( + "The shared cost helper rows_examined leaves out rows a plan read and " + "then filtered out, so it under-reports what a claim touched" + ), +) +@pytest.mark.parametrize("root", ["claimed", "as-shipped"]) +def test_the_cost_instrument_counts_rows_the_plan_filtered_out(root): + """ + tests/test_workflows_cost.rows_examined sums Actual Rows only, so a + plan that reads a row and rejects it with a filter reports nothing for + it. Both metrics are taken from one and the same plan. "as-shipped" is + the deferred-flatness test's own fixture (root never claimed). + """ + if root == "claimed": + cone_behind_a_running_root(LOCK_SIZE, "later") + else: + build(LOCK_SIZE, tasks.deferred_step) + # A waiting node is filtered by nothing: the claim's index range never + # reaches it. Put back to READY, the blocked nodes are rows the plan reads + # and the gate rejects, which is what the instrument has to count. + OxTask.objects.filter(status=OxTask.Status.WAITING).update( + status=OxTask.Status.READY + ) + refresh_statistics() + with transaction.atomic(): + plan = claim_candidates(gated=True).explain(analyze=True, format="JSON") + + class SamePlan: + def explain(self, **kwargs): + return plan + + shipped = rows_examined(SamePlan()) + touched = full_rows_examined(plan) + if touched == shipped: + pytest.fail( + "the claim's plan removed no rows by a filter, so it cannot show " + f"whether the instrument counts them (rows_examined={shipped}, " + f"with removed rows={touched})" + ) + print( # noqa: T201 + f"\n [s2 instrument] root={root}: rows_examined={shipped}, " + f"with rows removed by filter={touched}" + ) + assert shipped == touched, ( + f"the shipped instrument reported {shipped} rows on a plan that " + f"touched {touched}" + ) + + +# -- MySQL locks ----------------------------------------------------------- + + +@contextlib.contextmanager +def holding_a_claim_scan(alias: str, level: str): + """ + Another connection, at `level`, runs the claim's own candidate select + inside atomic() and holds the transaction until the block exits. + Yields {"thread": performance_schema thread id, "candidate": OxTask|None}. + """ + ready = threading.Event() + release = threading.Event() + info: dict = {} + + def hold() -> None: + try: + session = connections["default"] + with session.cursor() as cursor: + cursor.execute(f"SET SESSION TRANSACTION ISOLATION LEVEL {level}") + cursor.execute("SELECT PS_CURRENT_THREAD_ID()") + info["thread"] = cursor.fetchone()[0] + worker = OxpullWorker(backend_alias=alias, backoff_initial=0) + with transaction.atomic(): + info["candidate"] = ( + worker._ready_queryset().select_for_update(skip_locked=True).first() + ) + ready.set() + release.wait(60) + except BaseException as exc: + info["error"] = exc + ready.set() + finally: + connections.close_all() + + thread = threading.Thread(target=hold) + thread.start() + try: + assert ready.wait(60), "the holder never ran its scan" + assert "error" not in info, info.get("error") + yield info + finally: + release.set() + thread.join(60) + assert not thread.is_alive() + + +def record_locks(thread_id: int) -> list[tuple[str, str, str]]: + with connection.cursor() as cursor: + cursor.execute( + "SELECT INDEX_NAME, LOCK_MODE, LOCK_DATA FROM performance_schema.data_locks" + " WHERE THREAD_ID = %s AND OBJECT_NAME = 'django_ox_oxtask'" + " AND LOCK_TYPE = 'RECORD'", + [thread_id], + ) + return list(cursor.fetchall()) + + +@mysql_only +def test_a_claim_holds_no_row_lock_on_the_deferred_nodes_it_scanned_past(): + """ + SKIP LOCKED claims one row; the deferred nodes it rejected on the way + are not its business. At REPEATABLE READ InnoDB keeps a lock on every + row a locking read scanned, so each deferred node ahead of the claimed + row is one more lock the claim holds until it commits, and any write to + that node (a release, a cancellation, an operator's discard) waits. + + Only locks on live blocked nodes count. The supremum pseudo-record and + delete-marked rows a previous test's flush left for purge are not + blocked nodes, so they do not fail this test. + + Not asserted here, and why: + * READ COMMITTED, deferred: InnoDB already releases the rejected rows. + * The gated backend (no deferral): its rows stay locked at both + levels, but no fix to deferral can change that. + * An enqueue waiting on the scan's gap locks: a priority-5 enqueue + waits behind a REPEATABLE READ claim with no blocked nodes at all, + so a deterministic test cannot tell this hazard from InnoDB. + """ + cone_behind_a_running_root(LOCK_SIZE, "later") + refresh_statistics() + blocked = {pk.hex for pk in blocked_nodes().values_list("pk", flat=True)} + assert len(blocked) == LOCK_SIZE + + with holding_a_claim_scan("flows_deferred", "REPEATABLE READ") as info: + candidate = info["candidate"] + assert candidate is not None and candidate.args == ["later"] + locks = record_locks(info["thread"]) + + on_blocked = [ + lock for lock in locks if any(f"'{pk}'" in (lock[2] or "") for pk in blocked) + ] + shape: dict[tuple[str, str], int] = {} + for index, mode, _ in on_blocked: + shape[(index, mode)] = shape.get((index, mode), 0) + 1 + print( # noqa: T201 + f"\n [s2 locks] flows_deferred at REPEATABLE READ: {len(locks)} record " + f"locks, {len(on_blocked)} on the {LOCK_SIZE} blocked nodes: {shape}" + ) + assert on_blocked == [], ( + f"the claim holds {len(on_blocked)} record locks on deferred nodes it " + f"did not take ({shape})" + ) diff --git a/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_is_decided_by_the_node.py b/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_is_decided_by_the_node.py new file mode 100644 index 0000000..a2b7e56 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_defer_blocked_nodes_is_decided_by_the_node.py @@ -0,0 +1,283 @@ +""" +A waiting node is released by the `oxpull.worker.OxpullWorker` that ran its upstream, +whatever backend alias that worker is on. + +add() used to park a blocked node at a far-future run_after only when the +node's own backend declared DEFER_BLOCKED_NODES, and only an OxpullWorker +whose own backend declared the option released it. A claim is scoped to +queue names, not to backend aliases, so a parent could be claimed and +succeeded by an OxpullWorker whose alias did not defer, and nothing +released the children until the reconciler's sweep, which released at most +a thousand per tick, install-wide. + +Every node with dependencies now waits in django-ox's WAITING status, and +the option is gone. Nothing runs out of order either way: this is liveness +only. What the tests assert is the fast path: the `oxpull.worker.OxpullWorker` +that wrote the parent's SUCCESSFUL releases the child, and live worker loops +finish a chain with no reconciler tick. + +A worker on a backend that declares WORKFLOWS releases, and so does a worker +on a backend that does not, "limited", once any backend on the database +declares it. The test_control_* tests show the red is not vacuous: +the same sequence with the workflow backend's own worker releases at once, +the reconciler releases what the peer left, and live loops finish the chain +well inside WAIT. +""" + +import threading +import time + +import pytest +from django.core import checks +from django.db import connections +from django.tasks import task_backends + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import flow_worker, reconcile +from .test_workflows_deferred import deferring_worker + +# Committed state throughout, so a worker loop on its own thread and +# connection sees exactly what a worker process would. +pytestmark = pytest.mark.django_db(transaction=True) + +WAIT = 10.0 + +WAITING = OxTask.Status.WAITING + +# OxpullWorker aliases that share the "default" queue with flows_deferred. flows +# declares WORKFLOWS; limited declares none, only a rate limit on tasks this +# test never enqueues. +PEERS = { + "flows": flow_worker, + "limited": lambda: OxpullWorker( + backend_alias="limited", backoff_initial=0, poll_interval=0.01 + ), +} + + +def _task(result): + return OxTask.objects.get(pk=result.id) + + +def _waiting(ids): + return OxTask.objects.filter(pk__in=ids, status=WAITING) + + +def _errors(alias): + return [ + message.id + for message in task_backends[alias].check() + if message.level >= checks.ERROR + ] + + +def _chain_on_the_deferring_backend(name, length): + flow = workflows.create(name) + results, previous = [], None + for i in range(length): + label = f"n{i}" + depends_on = None if previous is None else previous + results.append( + flow.add(label, tasks.deferred_step, label, depends_on=depends_on) + ) + previous = label + flow.seal() + return results + + +# -- one alias per workflow, a peer `oxpull.worker.OxpullWorker` runs the parent --- + + +@pytest.mark.parametrize("peer", ["flows", "limited"]) +def test_a_deferred_child_is_released_by_the_oxpull_worker_that_ran_its_parent(peer): + """ + Every node of this workflow is on flows_deferred. The user mixed + nothing inside the workflow; two backends merely share a queue name, + which is Django's default (QUEUES defaults to ["default"]). + """ + assert _errors("flows_deferred") == [] + assert _errors(peer) == [] + parent, child = _chain_on_the_deferring_backend(f"one-alias-{peer}", 2) + assert _task(child).status == WAITING + + worker = PEERS[peer]() + assert worker.run_once() is True + ran = _task(parent) + assert ran.status == OxTask.Status.SUCCESSFUL + assert worker.worker_id in ran.worker_ids, "the peer did not run the parent" + + parked = _task(child).status + claimed = deferring_worker().claim_one() + assert claimed is not None and str(claimed.pk) == str(child.id), ( + f"parent SUCCESSFUL, written by an OxpullWorker on {peer!r}, and its " + f"child is not claimable: it is {parked}; only the reconciler will " + "release it" + ) + + +def test_control_the_deferring_worker_releases_the_same_child_at_once(): + parent, child = _chain_on_the_deferring_backend("one-alias-control", 2) + assert _task(child).status == WAITING + worker = deferring_worker() + assert worker.run_once() is True + assert _task(parent).status == OxTask.Status.SUCCESSFUL + claimed = deferring_worker().claim_one() + assert claimed is not None and str(claimed.pk) == str(child.id) + + +def test_control_the_reconciler_releases_what_the_peer_left(): + parent, child = _chain_on_the_deferring_backend("one-alias-reconciled", 2) + assert PEERS["limited"]().run_once() is True + assert _task(parent).status == OxTask.Status.SUCCESSFUL + # Whether the child is still waiting here is deliberately not asserted. + # One reconciler pass must leave it claimable either way. + reconcile() + assert _task(child).status == OxTask.Status.READY + claimed = deferring_worker().claim_one() + assert claimed is not None and str(claimed.pk) == str(child.id) + + +# -- one workflow spanning two workflow backends ------------------------------ + + +def test_a_fan_out_on_a_deferring_backend_is_released_when_its_root_succeeds(): + """ + The root is on flows, the children on flows_deferred, and add() accepts + all of them into one workflow. Wider than the reconciler's release page, + so the failure message can report how many ticks the backstop needs. + """ + width = workflows.BACKSTOP_PAGE + 500 + flow = workflows.create("mixed-aliases") + root = flow.add("root", tasks.step, "root") + children = [ + flow.add(f"c{i:04d}", tasks.deferred_step, f"c{i:04d}", depends_on="root") + for i in range(width) + ] + flow.seal() + ids = [child.id for child in children] + assert _waiting(ids).count() == width + + worker = flow_worker() + assert worker.run_once() is True + ran = _task(root) + assert ran.status == OxTask.Status.SUCCESSFUL + assert worker.worker_id in ran.worker_ids + + still_parked = _waiting(ids).count() + claimable_now = deferring_worker().claim_one() is not None + ticks, per_tick = 0, [] + while _waiting(ids).count() and ticks < 10: + reconcile() + ticks += 1 + per_tick.append(_waiting(ids).count()) + assert still_parked == 0, ( + f"root SUCCESSFUL, written by an OxpullWorker on 'flows'; {still_parked} " + f"of {width} children still WAITING; a worker on flows_deferred could " + f"claim one: {claimable_now}; still waiting after each reconciler pass: " + f"{per_tick} ({ticks} passes to release them all)" + ) + + +# -- live worker loops, no reconciler ---------------------------------------- + + +class Loop: + """A worker's production run() loop on its own thread and connection.""" + + def __init__(self, worker): + self.worker = worker + self.errors = [] + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + try: + self.worker.run() + except BaseException as exc: + self.errors.append(exc) + finally: + connections.close_all() + + def start(self): + self.thread.start() + return self + + def stop(self): + self.worker.request_stop() + self.thread.join(timeout=WAIT + 30) + assert not self.thread.is_alive() + assert not self.errors, self.errors + + +def _wait_for(predicate, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.05) + return predicate() + + +def _statuses(results): + return [ + (row.args[0], row.status, row.run_after) + for row in OxTask.objects.filter(pk__in=[r.id for r in results]).order_by( + "args" + ) + ] + + +def _all_succeeded(results): + return OxTask.objects.filter( + pk__in=[r.id for r in results], status=OxTask.Status.SUCCESSFUL + ).count() == len(results) + + +def _run_chain_under_live_loops(name, parent_runner): + """ + Start `parent_runner`'s loop alone until it has run n0, then a loop on + flows_deferred beside it, and give the chain WAIT seconds. Neither flows + nor flows_deferred schedules anything, so no reconciler tick can happen. + """ + chain = _chain_on_the_deferring_backend(name, 3) + loops = [Loop(parent_runner()).start()] + try: + assert _wait_for( + lambda: _task(chain[0]).status == OxTask.Status.SUCCESSFUL, WAIT + ), _statuses(chain) + assert loops[0].worker.worker_id in _task(chain[0]).worker_ids + loops.append(Loop(deferring_worker()).start()) + started = time.monotonic() + finished = _wait_for(lambda: _all_succeeded(chain), WAIT) + elapsed = time.monotonic() - started + finally: + for loop in loops: + loop.stop() + return chain, finished, elapsed + + +def test_a_chain_on_a_deferring_backend_finishes_under_live_loops_without_a_tick(): + """ + Two ox_worker-shaped loops share the "default" queue: one on flows and + one on flows_deferred. The flows loop happens to claim the first node + (here, by starting first; in production, by polling first or by the + other process restarting). With no reconciler dispatched, the rest of + the chain must still run. + """ + chain, finished, elapsed = _run_chain_under_live_loops("live-peer", flow_worker) + assert finished, ( + f"n0 run by the 'flows' loop; after {elapsed:.1f}s with a " + f"flows_deferred loop polling beside it, the chain reads " + f"{_statuses(chain)}" + ) + + +def test_control_the_same_chain_finishes_when_the_deferring_loop_runs_it(): + chain, finished, elapsed = _run_chain_under_live_loops( + "live-control", deferring_worker + ) + assert finished, _statuses(chain) + assert elapsed < WAIT / 2, elapsed diff --git a/tests_oxpull/test_workflows_hazard_deleting_a_workflow_row_cascades_its_edges.py b/tests_oxpull/test_workflows_hazard_deleting_a_workflow_row_cascades_its_edges.py new file mode 100644 index 0000000..35aabde --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_deleting_a_workflow_row_cascades_its_edges.py @@ -0,0 +1,244 @@ +""" +Deleting a workflow row used to cascade its edges away, and the gate then +let every blocked node through. + +OxpullWorkflowNode.workflow and OxpullWorkflowEdge.workflow were +on_delete=CASCADE, and the claim gate was NOT EXISTS over edge rows, so a +node with no edges left was runnable. Workflows ships no cancel API and no +admin, and the reconciler sweeps SEALED workflows only. That makes an ORM +delete of the parent row the obvious cleanup for an abandoned or stuck +workflow. Both foreign keys are now PROTECT, so the delete is refused while +any node or edge remains, and a node that waits for an upstream is WAITING, +which no claim reads. + +Every test asserts the promise that should survive that cleanup: a node +never runs ahead of an upstream that has not succeeded. A fix that +refuses the delete, or discards the workflow's READY nodes as part of +it, passes these tests. + +Real paths throughout. A real OxpullWorker makes the attempts, the +failure write, the retry backoff and the claim. The reconciler runs as +the scheduled task's body. The cleanup is the ORM delete a support +engineer would type into `manage.py shell`. +""" + +import pytest +from django.db.models import ProtectedError, RestrictedError + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, flow_worker, order, reconcile +from .test_workflows_deferred import deferring_worker, run_after + +pytestmark = pytest.mark.django_db + + +def status_of(task_id): + return OxTask.objects.filter(id=task_id).values_list("status", flat=True).first() + + +def operator_deletes(flow): + """ + The cleanup: delete the workflow row through the ORM. + + A refused delete is a correct outcome, so it is reported rather than + failed on. A delete that goes through must really remove the row, so + no test below can pass because the delete matched nothing. + """ + try: + OxpullWorkflow.objects.filter(pk=flow.id).delete() + except (ProtectedError, RestrictedError): + return False + assert not OxpullWorkflow.objects.filter(pk=flow.id).exists() + return True + + +def aftermath(flow, **nodes): + statuses = {name: status_of(node.id) for name, node in nodes.items()} + try: + workflows.get(flow.id) + record = "workflows.get() still finds it" + except OxpullWorkflow.DoesNotExist: + record = "workflows.get() raises DoesNotExist" + callbacks = OxTask.objects.filter( + task_path="tests_oxpull.tasks.record_callback" + ).count() + edges = OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count() + members = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() + return ( + f"statuses {statuses}; execution order {order()}; " + f"edge rows left {edges}, node rows left {members}; " + f"{record}; callbacks enqueued {callbacks}" + ) + + +def assert_dependency_order(ran, depends_on): + """Every node that ran did so after each of its upstreams had run.""" + for position, name in enumerate(ran): + for upstream in depends_on.get(name, []): + assert upstream in ran[:position], ( + f"{name!r} ran before its upstream {upstream!r} (execution order {ran})" + ) + + +# -- an upstream that FAILED ------------------------------------------- + + +def test_deleting_an_open_workflow_does_not_run_a_child_whose_upstream_failed(): + """ + The builder died between add() and seal(). The workflow stays OPEN, + its nodes run gated anyway, and no reconciler tick ever cancels the + dead cone, so the child is blocked for good. Deleting the row is the + cleanup, and it must not turn "blocked for good" into "runs". + """ + flow = workflows.create("abandoned-open") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert status_of(root.id) == OxTask.Status.FAILED + reconcile() # a healthy tick; it does not look at OPEN workflows + assert worker.run_once() is False # the gate holds the child + assert order() == ["root", "root", "root"] + + deleted = operator_deletes(flow) + reconcile() + drain(worker) + + assert "child" not in order(), ( + f"a gating worker ran 'child' although its only upstream FAILED; " + f"delete went through: {deleted}; {aftermath(flow, root=root, child=child)}" + ) + + +def test_deleting_a_sealed_workflow_before_cancellation_does_not_run_the_cone(): + """ + SEALED, and the root has just FAILED. No reconciler tick has cancelled + the cone yet: either the delete lands inside the tick interval, or + nothing schedules the reconciler (oxpull.W003, a warning only). + """ + flow = workflows.create( + "sealed", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + root = flow.add("root", tasks.failing_step, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + leaf = flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.seal() + + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert status_of(root.id) == OxTask.Status.FAILED + assert worker.run_once() is False # the gate holds mid and leaf + + deleted = operator_deletes(flow) + reconcile() + drain(worker) + + downstream = [name for name in order() if name != "root"] + assert downstream == [], ( + f"a gating worker ran {downstream} downstream of a FAILED root; " + f"delete went through: {deleted}; " + f"{aftermath(flow, root=root, mid=mid, leaf=leaf)}" + ) + + +# -- an upstream that has not finished --------------------------------- + + +def test_deleting_a_workflow_waiting_on_a_retry_does_not_run_the_child_early(): + """ + A healthy reconciler cannot help here: nothing is dead, so there is + nothing to cancel. The root's first attempt failed and its retry is + due in ten minutes. The workflow looks stuck and is deleted, and the + child must still wait for a root that has not succeeded. + """ + flow = workflows.create("waiting") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + + worker = OxpullWorker( + backend_alias="flows", backoff_initial=600, poll_interval=0.01 + ) + assert worker.run_once() is True # attempt 1 fails; retry backs off 600s + assert status_of(root.id) == OxTask.Status.READY + assert reconcile() == 0 + assert worker.run_once() is False # root not due yet, child gated + + deleted = operator_deletes(flow) + reconcile() + drain(worker) + + root_row = OxTask.objects.get(id=root.id) + assert "child" not in order(), ( + f"a gating worker ran 'child' while its upstream was {root_row.status} " + f"with attempts {root_row.attempts}/{root_row.max_attempts} and a retry " + f"due at {root_row.run_after}; delete went through: {deleted}; " + f"{aftermath(flow, root=root, child=child)}" + ) + + +def test_deleting_an_abandoned_open_chain_keeps_whatever_runs_in_dependency_order(): + """ + The priorities make claim order the reverse of dependency order, as in + test_workflows.diamond, so only the gate can put this chain in order. + """ + flow = workflows.create("abandoned-chain") + flow.add("extract", tasks.step.using(priority=0), "extract") + flow.add( + "transform", tasks.step.using(priority=5), "transform", depends_on="extract" + ) + flow.add("load", tasks.step.using(priority=10), "load", depends_on="transform") + # never sealed: the builder crashed after its last add() + + operator_deletes(flow) + reconcile() + drain(flow_worker()) + + assert_dependency_order(order(), {"transform": ["extract"], "load": ["transform"]}) + + +# -- DEFER_BLOCKED_NODES ----------------------------------------------- + + +def test_deferred_the_reconciler_does_not_release_a_child_whose_upstream_failed(): + """ + On the backend that still carries the leftover DEFER_BLOCKED_NODES + option, which changes nothing now. The child waits, so the delete alone + could not make it claimable. The reconciler's release backstop runs on + the next tick, and it must still not release the child. + """ + flow = workflows.create("abandoned-deferred") + root = flow.add("root", tasks.failing_deferred_step, "root") + child = flow.add("child", tasks.deferred_step, "child", depends_on="root") + + worker = deferring_worker() + for _ in range(3): + assert worker.run_once() is True + assert status_of(root.id) == OxTask.Status.FAILED + reconcile() + assert worker.run_once() is False + + deleted = operator_deletes(flow) + deferred_after_delete = run_after("child") + claimed_before_tick = worker.run_once() + reconcile() # the scheduled reconciler's next tick + drain(worker) + + assert "child" not in order(), ( + f"'child' ran although its only upstream FAILED. Right after the " + f"delete its run_after was {deferred_after_delete} and a claim " + f"returned {claimed_before_tick}; one reconciler tick later it ran. " + f"delete went through: {deleted}; {aftermath(flow, root=root, child=child)}" + ) diff --git a/tests_oxpull/test_workflows_hazard_discard_promise.py b/tests_oxpull/test_workflows_hazard_discard_promise.py new file mode 100644 index 0000000..08343c6 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_discard_promise.py @@ -0,0 +1,1804 @@ +""" +What oxpull writes on django-ox's tables. + +The package's promise, as its backend module and README are to state it: +oxpull adds no table, column, index, constraint or migration to a django-ox +table, and it writes django-ox task rows in exactly four ways, which it +names. + +- **Cancellation.** A READY or WAITING node whose upstream failed or was + discarded is set to DISCARDED. The UPDATE matches status READY or WAITING + and the lease epoch the reconciler read, and writes only columns + django_ox.actions.discard writes. +- **The waiting insert.** A node with dependencies is inserted WAITING + through django_ox._waiting.enqueue, the row django-ox's own enqueue + would insert in every other column. +- **Release.** A WAITING node moves to READY, matched on status WAITING and + the lease epoch the release read, and its run_after becomes the later of + now and the time its caller asked for. Never NULL, and nothing else. +- **Revival.** Workflow.retry() puts a cancelled node back to WAITING with a + new lease epoch. Only columns django_ox.actions.retry writes. A FAILED + upstream goes back through django-ox's own retry or through the same + READY write, so a revival may leave run_after NULL the way that retry + does, but only on a node nothing blocks. + +Any other statement oxpull issues against a django-ox table is a write the +package does not document, and fails here: another column, another status, +a discard that is not epoch-checked, a release not pinned to its epoch, a +run_after cleared to NULL, another INSERT, a DELETE, a table lock, or DDL. + +How. Every statement on every connection is captured and attributed to the +package whose code issued it: the innermost stack frame in either package, +with one exception. django-ox code that oxpull code calls issues oxpull's +statements, unless oxpull reached it from the class that subclasses it: +OxpullBackend's enqueue calling django-ox's, or OxpullWorker's claim and +outcome methods calling Worker's. So django-ox's enqueue INSERT, claim +UPDATE and outcome UPDATE are django-ox's, even when the claim carries an +oxpull conjunct. On PostgreSQL OxpullWorker runs a copy of django-ox's +single-statement claim with one read column added, the node flag. That +statement counts as django-ox's only when its text is django-ox's statement +byte for byte once the flag is taken out. A QuerySet.update() called from +oxpull/workflows.py is oxpull's, and so are the INSERT and UPDATEs of +django_ox._waiting that oxpull code calls, and the UPDATE of a +django_ox.actions.discard or retry that oxpull code calls: a discard made +through django-ox's operator API is held to the same READY-only, +epoch-checked shape as one oxpull writes itself. Each write is then checked +twice: as a statement (which columns it sets, what its WHERE clause +matches), and as a change to the node rows (which columns moved, and to +what). + +The tests marked xfail fail on this branch for the reason each gives. The +tests named test_control_* pass: the inventory tells the documented shapes +from every other shape in this database's own SQL, and add, the gated +claim, oxpull_prune, the released unique and batch paths and every oxpull +migration write nothing undocumented. Every test prints the statements it +saw against a django-ox table, so the log is the inventory per database. + +Known limits. The revival test calls Workflow.retry() with no arguments, +the workflow-wide form: the method does not exist yet and its signature is +not settled. If it lands taking a required argument, the call raises +TypeError, which the marker does not accept, so the test fails loudly and +the call needs adapting. The documentation test reads the backend module's +docstring and README.md, not the pages under docs/. +""" + +from __future__ import annotations + +import re +import threading +import time +import traceback +import uuid +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from io import StringIO +from pathlib import Path +from typing import Any + +import pytest +from django.apps import apps +from django.core.management import call_command +from django.db import DatabaseError, connection, connections, transaction +from django.db.models import Case, DateTimeField, F, Value, When +from django.utils import timezone + +import django_ox +import oxpull +import oxpull.backend +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.models import ( + OxpullBatch, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, +) +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_upgrade import _latest_migration +from .test_workflows import drain, flow_worker, free_worker, reconcile +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +OXPULL_DIR = Path(oxpull.__file__).resolve().parent +OX_DIR = Path(django_ox.__file__).resolve().parent +TASK_TABLE = OxTask._meta.db_table +EDGE_TABLE = OxpullWorkflowEdge._meta.db_table +OX_TABLES = frozenset( + model._meta.db_table for model in apps.get_app_config("django_ox").get_models() +) +PREVIOUS_RELEASE_MIGRATION = "0003_oxpullratewindow" +MIGRATIONS = sorted( + path.stem for path in (OXPULL_DIR / "migrations").glob("[0-9][0-9][0-9][0-9]_*.py") +) + +READY = OxTask.Status.READY +WAITING = OxTask.Status.WAITING +DISCARDED = OxTask.Status.DISCARDED +FAILED = OxTask.Status.FAILED +SUCCESSFUL = OxTask.Status.SUCCESSFUL + + +def _column(name: str) -> str: + return OxTask._meta.get_field(name).column + + +STATUS = _column("status") +RUN_AFTER = _column("run_after") +LEASE_EPOCH = _column("lease_epoch") + +# What django_ox.actions.discard writes. A cancelled node looks like an +# operator discard in django-ox's admin, and writes nothing more. +DISCARD_COLUMNS = frozenset( + _column(name) + for name in ("status", "finished_at", "locked_by", "locked_at", "lease_expires_at") +) +# What django_ox.actions.retry writes: status, a new lease_epoch, +# max_attempts, run_after and the lock and finish fields. +REVIVE_COLUMNS = frozenset( + _column(name) + for name in ( + "status", + "lease_epoch", + "max_attempts", + "run_after", + "finished_at", + "locked_by", + "locked_at", + "lease_expires_at", + ) +) + +CANCELLATION = "cancellation" +HOLD = "waiting insert" +RELEASE = "release" +REVIVAL = "revival" +REVIVAL_UNBLOCKED = "revival of a node nothing blocks" + +PK = OxTask._meta.pk.column + +# A release writes the later of now and the requested time. "Now" is the +# database's clock or the worker's; either is within minutes of the test's. +CLOCK_SLACK = timedelta(minutes=5) + +_NAME = r'[`"]?(\w+)[`"]?' +_WRITE_TARGETS = tuple( + re.compile(pattern, re.IGNORECASE | re.DOTALL) + for pattern in ( + rf"^\s*UPDATE\s+{_NAME}", + rf"^\s*DELETE\s+(?:FROM\s+)?{_NAME}", + rf"^\s*(?:INSERT|REPLACE)\s+(?:IGNORE\s+)?INTO\s+{_NAME}", + rf"^\s*(?:ALTER|DROP|TRUNCATE)\s+TABLE\s+(?:IF\s+EXISTS\s+)?{_NAME}", + rf"^\s*CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?{_NAME}", + ( + r"^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?" + rf'(?:IF\s+NOT\s+EXISTS\s+)?[`"]?\w+[`"]?\s+ON\s+{_NAME}' + ), + rf"^\s*CREATE\s+(?:OR\s+REPLACE\s+)?TRIGGER\s+.*?\bON\s+{_NAME}", + rf"^\s*COMMENT\s+ON\s+(?:TABLE|COLUMN)\s+{_NAME}", + rf"^\s*LOCK\s+TABLES?\s+{_NAME}", + ) +) +_SCHEMA_CHANGE = re.compile( + r"^\s*(?:ALTER|CREATE|DROP|TRUNCATE|COMMENT)\b", re.IGNORECASE +) + + +# -- the instrument ---------------------------------------------------- + + +# django-ox modules oxpull extends by subclassing, each from the oxpull +# module of the same name: OxpullWorker(Worker) and OxpullBackend(OxBackend). +SUBCLASSED = frozenset({"worker.py", "backend.py"}) + + +def _issuer() -> str: + """ + Who issued this statement: the innermost frame in oxpull or django_ox, + with one exception. django-ox code called from oxpull code issues the + statement on oxpull's behalf, so it is oxpull's, unless oxpull called + it from the subclass of that django-ox class: django_ox/worker.py + reached from oxpull/worker.py, or django_ox/backend.py from + oxpull/backend.py. + """ + innermost_ox = None + nearest_ox_path = None + for frame in reversed(traceback.extract_stack()): + path = Path(frame.filename).resolve() + if path.is_relative_to(OXPULL_DIR): + innermost_oxpull = f"oxpull/{path.relative_to(OXPULL_DIR)}:{frame.name}" + if innermost_ox is None: + return innermost_oxpull + if ( + nearest_ox_path.parent == OX_DIR + and path.parent == OXPULL_DIR + and nearest_ox_path.name == path.name + and path.name in SUBCLASSED + ): + return innermost_ox + return f"{innermost_oxpull} via {innermost_ox}" + if path.is_relative_to(OX_DIR): + if innermost_ox is None: + innermost_ox = f"django_ox/{path.relative_to(OX_DIR)}:{frame.name}" + nearest_ox_path = path + return innermost_ox or "test" + + +@dataclass(frozen=True) +class Statement: + sql: str + params: Any + issuer: str + thread: str + many: bool = False + + @property + def write_target(self) -> str | None: + """The table a write or DDL statement acts on, or None for a read.""" + for pattern in _WRITE_TARGETS: + match = pattern.match(self.sql) + if match: + return match.group(1) + return None + + @property + def is_schema_change(self) -> bool: + return self.write_target is not None and bool(_SCHEMA_CHANGE.match(self.sql)) + + @property + def locking_read_target(self) -> str | None: + """The outer FROM table of a SELECT ... FOR UPDATE, or None.""" + if not re.match(r"^\s*SELECT\b", self.sql, re.IGNORECASE): + return None + if "FOR UPDATE" not in self.sql.upper(): + return None + match = re.search(rf"\bFROM\s+{_NAME}", self.sql, re.IGNORECASE) + return match.group(1) if match else None + + @property + def by_oxpull(self) -> bool: + return self.issuer.startswith("oxpull/") + + def names(self, table: str) -> bool: + return re.search(rf"\b{re.escape(table)}\b", self.sql) is not None + + +# OxpullWorker's copy of django-ox's single-statement PostgreSQL claim. +COPIED_CLAIM = "oxpull/worker.py:_claim_one_postgresql" + + +def is_django_ox_claim_with_the_node_flag(sql: str) -> bool: + """ + Is this django-ox's POSTGRES_CLAIM_SQL, with the node flag after RETURNING * + and nothing else changed? + + The queue and extra clauses are django-ox's own hook points and may hold + anything. Every other byte must be django-ox's. + """ + from django_ox.worker import POSTGRES_CLAIM_SQL + + flag = workflows.node_flag_sql() + if sql.count(flag) != 1 or not sql.rstrip().endswith(f"RETURNING *{flag}"): + return False + unflagged = sql.replace(flag, "") + for lease_clock in ("STATEMENT_TIMESTAMP()", "%(lease_now)s"): + template = re.escape( + POSTGRES_CLAIM_SQL.format( + table=TASK_TABLE, + queue_clause="\x00QUEUE\x00", + extra_clause="\x00EXTRA\x00", + lease_clock=lease_clock, + ) + ) + pattern = template.replace(re.escape("\x00QUEUE\x00"), "(.*?)").replace( + re.escape("\x00EXTRA\x00"), "(.*?)" + ) + if re.fullmatch(pattern, unflagged, re.DOTALL): + return True + return False + + +@contextmanager +def statements_on(): + """ + Record every statement this thread runs on any database alias, with + its issuer. + + Connections are thread-local, so a worker thread opens its own capture + inside the thread. + """ + seen: list[Statement] = [] + + def wrapper(execute, sql, params, many, context): + issuer = _issuer() + if issuer == COPIED_CLAIM and is_django_ox_claim_with_the_node_flag(sql): + issuer = f"django_ox/worker.py:_claim_one_postgresql, copied as {issuer}" + seen.append( + Statement(sql, params, issuer, threading.current_thread().name, many) + ) + return execute(sql, params, many, context) + + with ExitStack() as stack: + for alias in connections: + stack.enter_context(connections[alias].execute_wrapper(wrapper)) + yield seen + + +# -- reading an UPDATE ------------------------------------------------- + + +class _Expression: + """A SET or WHERE operand that is SQL rather than one bound value.""" + + def __repr__(self) -> str: + return "" + + +EXPRESSION = _Expression() + + +def _scan(text: str): + """(index, depth) of every character outside a quoted string or name.""" + depth = 0 + quote = "" + for index, char in enumerate(text): + if quote: + if char == quote: + quote = "" + continue + if char in "'\"`": + quote = char + continue + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + yield index, depth + + +def _placeholders(text: str) -> int: + return sum(1 for match in re.finditer(r"%%|%s", text) if match.group() == "%s") + + +def _split_top_level(text: str) -> list[str]: + parts, start = [], 0 + for index, depth in _scan(text): + if depth == 0 and text[index] == ",": + parts.append(text[start:index]) + start = index + 1 + parts.append(text[start:]) + return parts + + +def _top_level_where(text: str) -> int | None: + for index, depth in _scan(text): + if ( + depth == 0 + and text[index] in "Ww" + and re.match(r"WHERE\b", text[index:], re.IGNORECASE) + and (index == 0 or not re.match(r"\w", text[index - 1])) + ): + return index + return None + + +def _parenthesised(text: str) -> str: + """The inside of the parenthesis `text` starts with.""" + for index, depth in _scan(text): + if depth == 0 and text[index] == ")": + return text[1:index] + return text[1:] + + +@dataclass(frozen=True) +class Assignment: + column: str + expression: str + params: tuple[Any, ...] + + @property + def value(self) -> Any: + """The bound or literal value, None for NULL, or EXPRESSION.""" + text = self.expression.strip() + if text == "%s" and len(self.params) == 1: + return self.params[0] + if text.upper() == "NULL": + return None + literal = re.fullmatch(r"'([^']*)'", text) + if literal: + return literal.group(1) + return EXPRESSION + + +@dataclass(frozen=True) +class Update: + table: str + sets: dict[str, Assignment] + where: str + where_params: tuple[Any, ...] + + def where_values(self, column: str) -> list[Any]: + """ + What the WHERE clause compares this table's `column` to with = or + IN, in order. A subquery or a column reference is EXPRESSION. + """ + pattern = re.compile( + rf'(?:[`"]{re.escape(self.table)}[`"]\.|(? bool: + """Does the WHERE clause pin `column` to a bound value, alone or in a row?""" + if any(value is not EXPRESSION for value in self.where_values(column)): + return True + return bool( + re.search( + rf'[`"]{re.escape(column)}[`"]\s*\)\s*IN\s*\(\s*\(', + self.where, + re.IGNORECASE, + ) + ) + + +_UPDATE_HEAD = re.compile(rf"^\s*UPDATE\s+{_NAME}\s+SET\s+", re.IGNORECASE) + + +def parse_update(sql: str, params: Any) -> Update | None: + head = _UPDATE_HEAD.match(sql) + if head is None: + return None + body = sql[head.end() :] + cut = _top_level_where(body) + set_text, where = (body, "") if cut is None else (body[:cut], body[cut + 5 :]) + bound = tuple(params or ()) + used = 0 + sets: dict[str, Assignment] = {} + for part in _split_top_level(set_text): + match = re.match(r'^\s*[`"]?(\w+)[`"]?\s*=\s*(.*?)\s*$', part, re.DOTALL) + if match is None: + return None + count = _placeholders(match.group(2)) + sets[match.group(1)] = Assignment( + match.group(1), match.group(2), bound[used : used + count] + ) + used += count + return Update(head.group(1), sets, where, bound[used:]) + + +def _instant(value: Any) -> datetime | None: + if isinstance(value, datetime): + moment = value + elif isinstance(value, str): + try: + moment = datetime.fromisoformat(value) + except ValueError: + return None + else: + return None + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +# -- the inventory ----------------------------------------------------- + + +def _epoch_pinned(update: Update) -> bool: + """Does the WHERE clause pin the lease epoch, by value or per row by CASE?""" + if update.matches_a_value_of(LEASE_EPOCH): + return True + return bool( + re.search(rf'[`"]{re.escape(LEASE_EPOCH)}[`"]\s*=\s*CASE\b', update.where, re.I) + ) + + +def classify(statement: Statement) -> tuple[str | None, str]: + """ + Which documented write this statement is, or None and why it is none + of them. + """ + target = statement.write_target + if statement.is_schema_change: + return None, f"changes the schema of {target}" + verb = statement.sql.split(None, 1)[0].upper() + if verb == "INSERT" and target == TASK_TABLE: + values = statement.params if isinstance(statement.params, (list, tuple)) else () + if "django_ox/_waiting.py:enqueue" in statement.issuer and WAITING in values: + return HOLD, "inserts a node born WAITING through django_ox._waiting" + return None, ( + "an INSERT into the task table that is not a node born WAITING " + "through django_ox._waiting" + ) + update = None if statement.many else parse_update(statement.sql, statement.params) + if update is None: + if verb == "UPDATE": + return None, "an UPDATE whose SET clause this inventory cannot read" + return None, ( + f"a {verb} on {target}; every documented write is the waiting insert " + "or an UPDATE of existing task rows" + ) + if update.table != TASK_TABLE: + return None, f"an UPDATE of {update.table}, not of the task table" + + columns = frozenset(update.sets) + status = update.sets.get(STATUS) + guards = update.where_values(STATUS) + if status is None: + return None, f"writes {sorted(columns)}, which no documented write sets" + + if status.value == DISCARDED: + outside = sorted(columns - DISCARD_COLUMNS) + if outside: + return None, ( + f"discards and also writes {outside}, which " + "django_ox.actions.discard never writes" + ) + if not guards or any(guard not in (READY, WAITING) for guard in guards): + return None, ( + "discards a row without matching status READY or WAITING alone" + ) + if not _epoch_pinned(update): + return None, ( + "discards by id and status without matching the lease epoch " + "the decision was made on" + ) + return ( + CANCELLATION, + "discards a READY or WAITING row at the lease epoch it read", + ) + + if status.value == READY and guards and all(guard == WAITING for guard in guards): + if columns != {STATUS, RUN_AFTER}: + return None, ( + f"releases a WAITING row writing {sorted(columns)}, where a " + "release writes status and run_after and nothing else" + ) + if update.sets[RUN_AFTER].value is None: + return None, ( + "releases a WAITING row with run_after NULL, where a release " + "writes the later of now and the time the caller asked for" + ) + if not _epoch_pinned(update): + return None, ( + "releases a WAITING row without matching the lease epoch the " + "release decided on" + ) + return RELEASE, "moves a WAITING row to READY at the lease epoch it read" + + if status.value == READY: + outside = sorted(columns - REVIVE_COLUMNS) + if outside: + return None, ( + f"puts a row back to READY and also writes {outside}, which " + "django_ox.actions.retry never writes" + ) + if LEASE_EPOCH not in columns: + return None, "puts a row back to READY without a new lease epoch" + return REVIVAL_UNBLOCKED, ( + "puts a row back to READY with a new lease epoch, which a worker " + "without the dependency gate claims at once, so it is documented " + "only on a node nothing blocks" + ) + + if status.value == WAITING: + outside = sorted(columns - REVIVE_COLUMNS) + if outside: + return None, ( + f"revives a row and also writes {outside}, which " + "django_ox.actions.retry never writes" + ) + if not guards or any(guard != DISCARDED for guard in guards): + return None, "moves a row to WAITING from something other than DISCARDED" + if LEASE_EPOCH not in columns: + return None, "revives a row without a new lease epoch" + return REVIVAL, "revives a DISCARDED row to WAITING with a new lease epoch" + + return None, ( + f"writes status {status.value!r}; the documented writes set only " + "DISCARDED, READY and WAITING" + ) + + +def undocumented(statements: list[Statement]) -> list[tuple[Statement, str]]: + """oxpull's writes, and anyone's schema changes, on a django-ox table + that are none of the documented writes.""" + found = [] + for statement in statements: + if statement.write_target not in OX_TABLES: + continue + if not (statement.by_oxpull or statement.is_schema_change): + continue + kind, reason = classify(statement) + if kind is None: + found.append((statement, reason)) + elif kind == REVIVAL_UNBLOCKED: + problem = _revived_while_blocked(statement) + if problem: + found.append((statement, problem)) + return found + + +def _revived_while_blocked(statement: Statement) -> str | None: + """ + A revival to READY is documented only on rows nothing blocks. Read which + rows its WHERE clause names by id, and whether any of them has an + upstream that is absent or not SUCCESSFUL, now. + """ + update = parse_update(statement.sql, statement.params) + ids = update.where_values(PK) if update else [] + if not ids or any(value is EXPRESSION for value in ids): + return ( + "puts rows back to READY without naming them by id, so nothing " + "shows that no upstream blocks them" + ) + succeeded = OxTask.objects.filter(status=SUCCESSFUL).values("pk") + blocked = sorted( + str(value) + for value in ids + if OxpullWorkflowEdge.objects.filter(downstream_task_id=value) + .exclude(upstream_task_id__in=succeeded) + .exists() + ) + if not blocked: + return None + return ( + f"puts {blocked} back to READY while an upstream of each has not " + "succeeded, so a worker without the dependency gate claims it at once; " + "a blocked node is revived WAITING" + ) + + +def oxpull_writes_to_django_ox(statements: list[Statement]) -> list[Statement]: + return [s for s in statements if s.by_oxpull and s.write_target in OX_TABLES] + + +def _one_line(sql: str, limit: int = 600) -> str: + flat = " ".join(sql.split()) + return flat if len(flat) <= limit else flat[:limit] + " ..." + + +def _params(params: Any, limit: int = 240) -> str: + text = repr(params) + return text if len(text) <= limit else text[:limit] + " ..." + + +def _emit(text: str) -> None: + """Write to the test's stdout, which -rA carries into the log.""" + print(text) # noqa: T201 + + +def inventory(title: str, statements: list[Statement]) -> None: + """Print every statement that writes, locks or reads a django-ox table.""" + _emit(f"\n=== {title} [{connection.vendor}] ===") + shown = 0 + for s in statements: + if s.write_target in OX_TABLES: + if s.by_oxpull or s.is_schema_change: + documented, reason = classify(s) + offences = undocumented([s]) + if offences: + documented, reason = None, offences[0][1] + kind = f"WRITE {documented or 'UNDOCUMENTED'}" + else: + kind, reason = "WRITE", "" + elif s.locking_read_target in OX_TABLES: + kind, reason = "LOCKING READ", "" + elif s.write_target is not None and any(s.names(t) for t in OX_TABLES): + kind, reason = f"write {s.write_target}, reads ox", "" + elif any(s.names(table) for table in OX_TABLES): + kind, reason = "read", "" + else: + continue + shown += 1 + owner = "OXPULL" if s.by_oxpull else s.issuer.split("/", 1)[0] + _emit(f"- {kind:<20} by {owner:<9} {s.issuer} [{s.thread}]") + if reason: + _emit(f" why: {reason}") + _emit(f" sql: {_one_line(s.sql)}") + _emit(f" params: {_params(s.params)}") + _emit( + f"({shown} statement(s) touching a django-ox table, " + f"of {len(statements)} captured)" + ) + + +def documented_problem(statements: list[Statement], where: str) -> str | None: + offenders = undocumented(statements) + if not offenders: + return None + return ( + f"{where}: {len(offenders)} write(s) to a django-ox table on " + f"{connection.vendor} that are not among the writes oxpull documents:\n" + + "\n".join( + f" {s.issuer}: {reason}\n" + f" {_one_line(s.sql, 300)} params={_params(s.params, 160)}" + for s, reason in offenders + ) + ) + + +def assert_only_documented_writes(statements: list[Statement], where: str) -> None: + problem = documented_problem(statements, where) + assert problem is None, problem + + +def assert_no_oxpull_writes(statements: list[Statement], where: str) -> None: + """For the paths no documented write belongs to at all.""" + offenders = oxpull_writes_to_django_ox(statements) + assert offenders == [], ( + f"{where}: oxpull issued {len(offenders)} write(s) to a django-ox table " + f"on {connection.vendor}, and no documented write belongs on this path:\n" + + "\n".join( + f" {s.issuer}: {_one_line(s.sql, 300)} params={_params(s.params, 160)}" + for s in offenders + ) + ) + + +# -- rows -------------------------------------------------------------- + + +def row(label: str) -> OxTask: + return OxTask.objects.get(args=[label]) + + +def snapshot(*labels: str) -> dict[str, dict[str, Any]]: + return { + label: OxTask.objects.filter(args=[label]).values().get() for label in labels + } + + +def assert_changed_only(before, after, label: str, allowed, write: str) -> None: + changed = sorted(f for f, v in after[label].items() if before[label][f] != v) + outside = [f for f in changed if f not in allowed] + assert not outside, ( + f"{write} changed {outside} on the {label!r} node's task row, which the " + f"documented write never touches (it may change only {sorted(allowed)})" + ) + + +def released_problem(after, label: str, requested, started, finished) -> str | None: + """run_after is the later of now and the time the caller asked for.""" + value = after[label]["run_after"] + if value is None: + return ( + f"the release left the {label!r} node's run_after NULL, not the later " + "of now and the time its caller asked for" + ) + if requested is None: + if not started - CLOCK_SLACK <= value <= finished + CLOCK_SLACK: + return ( + f"the {label!r} node was released to {value}, not to now " + f"(between {started} and {finished})" + ) + elif value != requested: + return ( + f"the {label!r} node was released to {value}, losing the run_after " + f"{requested} its caller asked for" + ) + return None + + +def release_problems(seen, where, after, held, later, started, finished): + """What the documented waiting insert and release require of the children.""" + problems = [ + documented_problem(seen, where), + released_problem(after, "child", None, started, finished), + ] + if held != WAITING: + problems.append( + f"the 'scheduled' node, whose caller asked for run_after {later}, " + f"was {held} when it was added, not WAITING, so while its upstream " + "was unfinished nothing in its row held it back once that time came" + ) + else: + problems.append(released_problem(after, "scheduled", later, started, finished)) + return [problem for problem in problems if problem] + + +def failed_root_and_waiting_child(name: str): + """root -> child on the gating backend, with root FAILED for good.""" + flow = workflows.create(name) + flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert row("root").status == FAILED + assert row("child").status == WAITING + return flow, worker + + +def parent_and_waiting_children(name: str) -> tuple[datetime, Any]: + """ + parent -> {child, scheduled} on the second workflow backend. `scheduled` + carries a run_after its caller chose, an hour out. Returns that time, + and the status `scheduled` has once added, WAITING when the waiting + insert covers a node given its own time. + """ + later = timezone.now() + timedelta(hours=1) + flow = workflows.create(name) + flow.add("parent", tasks.deferred_step, "parent") + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.add( + "scheduled", + tasks.deferred_step.using(run_after=later), + "scheduled", + depends_on="parent", + ) + flow.seal() + assert row("child").status == WAITING + return later, row("scheduled").status + + +# -- the documented writes, on the Workflows runtime paths ------------- + + +def test_the_reconciler_cancels_a_doomed_node_with_only_the_documented_discard(): + """ + A node fails for good; the reconciler, run by a real worker as a + scheduled task, cancels the node downstream of it. + + Production sequence: a sealed workflow root -> child, the root + exhausts its attempts, the once-a-minute reconcile schedule fires and + a worker runs it. The only departure is that the reconcile task is + enqueued here rather than by the schedule dispatcher. + """ + assert oxpull_db_alias() == "default" + _flow, worker = failed_root_and_waiting_child("doomed") + before = snapshot("child") + + tick = workflows.reconcile.enqueue() + with statements_on() as seen: + assert worker.run_once() is True # claims and runs the reconciler + after = snapshot("child") + + # The run provoked what it is meant to observe. + assert OxTask.objects.get(id=tick.id).status == SUCCESSFUL + assert after["child"]["status"] == DISCARDED + + inventory("reconciler cancels a doomed node (worker runs reconcile)", seen) + assert_changed_only(before, after, "child", DISCARD_COLUMNS, "the cancellation") + assert_only_documented_writes(seen, "the reconciler's cancellation") + + +# Upper bound on the placements below: far more statements than one +# reconciler tick issues for a two-node workflow. +MAX_PLACEMENTS = 400 + + +@dataclass(frozen=True) +class Placement: + before_statement: int + statements: int + claimed: bool + new_epoch: int + first_discard: str | None + child_after_it: tuple[Any, ...] | None + child_at_the_end: tuple[Any, ...] + + +def _claimed_then_failed_back(n: int) -> Placement | None: + """ + One sealed workflow root -> child with root FAILED for good, and one + reconciler tick run by a worker, with the child turned into a new + incarnation in two steps on the reconciler's own connection. + + Before the n-th statement of the tick, a worker claims the child if it + is still READY or WAITING, as a release and a claim together can: + RUNNING, lease epoch + 1, one more attempt. Just before the first + statement after that which sets DISCARDED on the task table, the attempt + fails back to READY under the same new epoch, as a failed attempt with + retries left does. Returns None when the tick issued fewer than n + statements. + """ + _flow, worker = failed_root_and_waiting_child(f"stale-discard-{n}") + child = row("child") + state: dict[str, Any] = { + "count": 0, + "claimable": None, + "claimed": False, + "busy": False, + "first_discard": None, + "after_it": None, + } + + def place(execute, sql, params, many, context): + if state["busy"]: + return execute(sql, params, many, context) + state["busy"] = True + try: + state["count"] += 1 + if state["count"] == n: + now = timezone.now() + state["claimed"] = state["claimable"] = bool( + OxTask.objects.filter( + pk=child.pk, status__in=[READY, WAITING] + ).update( + status=OxTask.Status.RUNNING, + lease_epoch=F("lease_epoch") + 1, + attempts=F("attempts") + 1, + locked_by="another-worker", + locked_at=now, + started_at=now, + lease_expires_at=now + timedelta(minutes=5), + ) + ) + update = None if many else parse_update(sql, params) + sets_discarded = ( + update is not None + and update.table == TASK_TABLE + and STATUS in update.sets + and update.sets[STATUS].value == DISCARDED + ) + if state["first_discard"] or not (state["claimed"] and sets_discarded): + return execute(sql, params, many, context) + OxTask.objects.filter(pk=child.pk).update( + status=READY, locked_by=None, locked_at=None, lease_expires_at=None + ) + state["first_discard"] = _one_line(sql, 300) + result = execute(sql, params, many, context) + state["after_it"] = tuple( + OxTask.objects.filter(pk=child.pk) + .values_list("status", "lease_epoch") + .get() + ) + return result + finally: + state["busy"] = False + + workflows.reconcile.enqueue() + with connection.execute_wrapper(place): + ran = worker.run_once() + if ran is not True: + pytest.fail(f"setup: the worker ran nothing at placement {n}") + if state["count"] < n: + return None + return Placement( + before_statement=n, + statements=state["count"], + claimed=state["claimable"], + new_epoch=child.lease_epoch + 1, + first_discard=state["first_discard"], + child_after_it=state["after_it"], + child_at_the_end=tuple( + OxTask.objects.filter(pk=child.pk) + .values_list("status", "lease_epoch") + .get() + ), + ) + + +def test_a_discard_decided_on_one_lease_epoch_does_not_land_on_the_next(): + """ + Between the reconciler deciding to cancel a node and its write, the + node becomes a new incarnation: a worker claims it (RUNNING, lease + epoch + 1) and the attempt fails back to READY under that new epoch. + That incarnation is not the one the decision was about, and the + discard must not land on it. + + Where the decision falls among a tick's statements depends on the + implementation, so the claim is placed before each statement of the + tick in turn: one fresh workflow and one tick per placement, rolled + back afterwards, until a tick runs out of statements first. The + fail-back to READY comes just before the first statement after the + claim that sets DISCARDED on the task table. Both changes are made on + the reconciler's own connection, so it sees them whatever transaction + it holds. + + Required at every placement where the claim took (the node was still + READY): that DISCARDED statement leaves the node READY at the new + epoch, and a claimed node that no such statement followed is not + DISCARDED at its new epoch at the end. What is asserted is that + statement's own effect; a later statement that decides again on the + new epoch is not ruled out. A placement after the discard finds the + node no longer READY and claims nothing, and one whose claim the + reconciler's own rollback undoes ends at the old epoch; neither is + judged. At least one placement must be followed by a DISCARDED + statement, or nothing was provoked. + + A discard that binds the epoch it read with its decision passes every + placement: placed before the decision, the claim shows the node RUNNING + and it is not chosen; placed after it, the epoch no longer matches. + Rejected: an UPDATE that matches no epoch, as today's does, at every + placement before it; and an implementation that decides, then re-reads + the epoch just before writing, at the placement between the two. + """ + placements: list[Placement] = [] + for n in range(1, MAX_PLACEMENTS + 1): + with transaction.atomic(): + placement = _claimed_then_failed_back(n) + transaction.set_rollback(True) + if placement is None: + break + placements.append(placement) + else: + pytest.fail(f"setup: a tick issued more than {MAX_PLACEMENTS} statements") + + report = "\n".join( + f" claim before statement {p.before_statement} of {p.statements} " + f"({'took' if p.claimed else 'found the child no longer READY'}): " + f"child after the first discard that followed: {p.child_after_it} " + f"({p.first_discard or 'no discard followed'}); " + f"at the end: {p.child_at_the_end}" + for p in placements + ) + _emit(f"\nplacements [{connection.vendor}]:\n{report}") + provoked = [p for p in placements if p.child_after_it is not None] + if not provoked: + pytest.fail( + "no placement was followed by a statement that sets DISCARDED, so " + "nothing was provoked:\n" + report + ) + if any(p.child_after_it[1] != p.new_epoch for p in provoked): + pytest.fail("setup: the new incarnation's epoch did not take:\n" + report) + landed = [p.before_statement for p in provoked if p.child_after_it[0] != READY] + discarded_while_running = [ + p.before_statement + for p in placements + if p.claimed + and p.child_after_it is None + and p.child_at_the_end == (DISCARDED, p.new_epoch) + ] + assert (landed, discarded_while_running) == ([], []), ( + f"on {connection.vendor}, with the claim placed before statements " + f"{landed}, the discard decided on the child's earlier epoch landed on " + f"its new incarnation at epoch {provoked[0].new_epoch}; placements that " + f"left it RUNNING and still discarded it: {discarded_while_running}\n" + f"{report}" + ) + + +@pytest.mark.django_db(transaction=True) +def test_a_worker_that_finishes_a_parent_writes_only_the_documented_release(): + """ + A real worker on its own thread and connection runs the parent; its + outcome write lands and the worker releases the children. + Production sequence: exactly this, in a worker process. + """ + later, held = parent_and_waiting_children("released") + before = snapshot("child", "scheduled") + seen: list[Statement] = [] + errors: list[BaseException] = [] + + def run(): + try: + worker = deferring_worker() + with statements_on() as statements: + ran = worker.run_once() + seen.extend(statements) + assert ran is True + except BaseException as exc: + errors.append(exc) + finally: + connections.close_all() + + started = timezone.now() + thread = threading.Thread(target=run, name="oxpull-worker-thread") + thread.start() + thread.join(timeout=60) + finished = timezone.now() + assert not thread.is_alive() + assert not errors, errors + after = snapshot("child", "scheduled") + + assert row("parent").status == SUCCESSFUL + assert after["child"]["status"] == READY, ( + "the worker released nothing, so nothing was provoked" + ) + assert all(s.thread == "oxpull-worker-thread" for s in seen) + + inventory("worker thread writes SUCCESSFUL and releases the children", seen) + for label in ("child", "scheduled"): + assert_changed_only(before, after, label, {STATUS, RUN_AFTER}, "the release") + problems = release_problems( + seen, "the worker's release", after, held, later, started, finished + ) + assert not problems, "\n".join(problems) + + +def test_the_reconcilers_release_writes_only_the_documented_release(): + """ + The backstop: a release no worker made is made by the reconciler. + + Production sequence: a worker without the release, django-ox's own + Worker in a fleet still being upgraded, never claims the waiting + children, runs the parent and writes SUCCESSFUL, and releases nothing. The next + reconcile tick, run by an oxpull worker, releases the children. + """ + later, held = parent_and_waiting_children("backstop") + assert free_worker().run_once() is True # the parent, and no release + assert row("parent").status == SUCCESSFUL + assert row("child").status == WAITING + before = snapshot("child", "scheduled") + + workflows.reconcile.enqueue() + started = timezone.now() + with statements_on() as seen: + assert deferring_worker().run_once() is True # claims and runs reconcile + finished = timezone.now() + after = snapshot("child", "scheduled") + + assert after["child"]["status"] == READY, ( + "the reconciler released nothing, so nothing was provoked" + ) + inventory("reconciler release after a parent no oxpull worker ran", seen) + for label in ("child", "scheduled"): + assert_changed_only(before, after, label, {STATUS, RUN_AFTER}, "the release") + problems = release_problems( + seen, "the reconciler's release", after, held, later, started, finished + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "Workflow.retry() does not exist yet, so nothing puts a cancelled node " + "back to WAITING with a new lease epoch, and django-ox's own retry " + "never moves a DISCARDED row" + ), +) +def test_workflow_retry_revives_a_cancelled_node_with_only_the_documented_write(): + """ + Production sequence: root fails for good, a reconcile tick cancels + child, and the operator calls Workflow.retry(). + + root, which nothing blocks, may come back READY with run_after NULL, the + way django-ox's own retry leaves it; child, still blocked by root, must + come back WAITING. The call takes no arguments, the + workflow-wide form. A Workflow.retry() that requires one raises + TypeError, which the marker does not accept, so this fails loudly and + the call needs adapting. + """ + flow, worker = failed_root_and_waiting_child("revived") + workflows.reconcile.enqueue() + assert worker.run_once() is True + assert row("child").status == DISCARDED + before = snapshot("root", "child") + + retry = getattr(workflows.get(flow.id), "retry", None) + assert callable(retry), "Workflow.retry() does not exist" + with statements_on() as seen: + retry() + after = snapshot("root", "child") + + assert after["child"]["status"] == WAITING, "Workflow.retry() revived nothing" + inventory("Workflow.retry() revives a cancelled node", seen) + for label in ("root", "child"): + assert_changed_only(before, after, label, REVIVE_COLUMNS, "the revival") + assert after["child"]["lease_epoch"] > before["child"]["lease_epoch"], ( + "the revived node kept its lease epoch, so a discard decided on its " + "cancelled incarnation can still land on it" + ) + assert_only_documented_writes(seen, "Workflow.retry()") + + +# The schema promise names all five. A sentence that names only these, and +# no rows or state, is about schema. +SCHEMA_NOUNS = ("table", "column", "index", "constraint", "migration") +_NEGATION = re.compile(r"\b(no|not|never|nothing|none|without|cannot)\b|n't\b", re.I) +_WRITE_VERB = re.compile(r"\b(writ|modif|touch|chang|alter|updat|mutat|edit)\w*", re.I) +_ROW_NOUN = re.compile(r"\b(rows?|state|data|records?|values?)\b", re.I) +_SCHEMA_NOUN = re.compile( + r"\b(tables?|columns?|index(es)?|indices|constraints?|migrations?|schema)\b", re.I +) +_SUBJECT = re.compile(r"\b(oxpull|this module|this package|the package)\b", re.I) +_RESTRICTION = re.compile(r"\b(other|else|except|beyond|besides|only)\b", re.I) +_BROAD_DENIAL = re.compile( + r"nothing (owned by|belonging to) django-ox|no django-ox (rows?|state|data)", re.I +) + + +def _paragraphs(text: str) -> list[str]: + return [" ".join(block.split()) for block in re.split(r"\n\s*\n", text) if block] + + +def _sentences(paragraph: str) -> list[str]: + return [s for s in re.split(r"(?<=[.!?])\s+(?=[A-Z`*(\[])", paragraph) if s] + + +def promises_no_schema_change(paragraphs: list[str]) -> bool: + """One sentence: no table, column, index, constraint or migration on django-ox.""" + return any( + "django-ox" in sentence.lower() + and _NEGATION.search(sentence) + and all(re.search(rf"\b{noun}", sentence, re.I) for noun in SCHEMA_NOUNS) + for paragraph in paragraphs + for sentence in _sentences(paragraph) + ) + + +def names_the_row_writes(paragraphs: list[str]) -> bool: + """One paragraph naming DISCARDED, run_after, the release and revival.""" + return any( + "DISCARDED" in paragraph + and "run_after" in paragraph + and re.search(r"releas", paragraph, re.I) + and re.search(r"reviv|Workflow\.retry", paragraph, re.I) + for paragraph in paragraphs + ) + + +def denials_of_row_writes(paragraphs: list[str]) -> list[str]: + """ + Sentences saying oxpull does not write django-ox: "nothing owned by + django-ox", or oxpull, this module or the package with a negation and a + write verb, unless the sentence is only about schema or only restricts + the writes to some ("no other", "except"). + """ + found = [] + for paragraph in paragraphs: + for sentence in _sentences(paragraph): + if "django-ox" not in sentence.lower(): + continue + if _BROAD_DENIAL.search(sentence): + found.append(sentence) + continue + if not ( + _SUBJECT.search(sentence) + and _NEGATION.search(sentence) + and _WRITE_VERB.search(sentence) + ): + continue + only_schema = _SCHEMA_NOUN.search(sentence) and not _ROW_NOUN.search( + sentence + ) + if not (only_schema or _RESTRICTION.search(sentence)): + found.append(sentence) + return found + + +def test_the_package_names_its_row_writes_instead_of_denying_them(): + """ + The backend module's docstring and README.md must each: + + - state the schema promise in one sentence: no table, column, index, + constraint or migration on a django-ox table; + - name the row writes in one paragraph: DISCARDED, run_after, the + release, and revival or Workflow.retry(); + - deny row writes nowhere. "Nothing owned by django-ox" is a denial, and + so is a sentence in which oxpull, this module or the package does not + write, modify, touch or change django-ox, unless it is only about + schema or only rules out writes beyond the named ones. + + A denial reworded, or the writes named in one place while the other + still denies them, fails here. + """ + readme = (OXPULL_DIR.parent.parent / "README.md").read_text(encoding="utf-8") + places = { + "the backend module's docstring": oxpull.backend.__doc__ or "", + "README.md": readme, + } + problems = [] + for where, text in places.items(): + paragraphs = _paragraphs(text) + if not promises_no_schema_change(paragraphs): + problems.append( + f"{where} has no sentence promising no table, column, index, " + "constraint or migration on a django-ox table" + ) + if not names_the_row_writes(paragraphs): + problems.append( + f"{where} has no paragraph naming the DISCARDED, run_after, " + "release and revival writes" + ) + problems.extend( + f"{where} still denies writing django-ox: {sentence!r}" + for sentence in denials_of_row_writes(paragraphs) + ) + assert not problems, "\n".join(problems) + + +# -- controls: the instrument, and the paths with nothing to hide (green) -- + + +def test_control_the_inventory_accepts_the_documented_shapes_and_nothing_else(): + """ + The classifier on real statements from this database's own compiler, + so an inventory that accepted everything, or nothing, shows here + first. Every UPDATE matches no row, so nothing changes. One names a + real node that is still blocked, so a revival to READY is shown to be + caught by the rows it names. + """ + flow = workflows.create("inventory-control") + flow.add("up", tasks.step, "up") + blocked = OxTask.objects.filter( + pk=flow.add("down", tasks.step, "down", depends_on="up").id + ) + task = OxTask.objects.filter(pk=uuid.uuid4()) + now = timezone.now() + discard = { + "status": DISCARDED, + "finished_at": now, + "locked_by": None, + "locked_at": None, + "lease_expires_at": None, + } + epoch_bump = F("lease_epoch") + 1 + later_of_now = Case( + When(run_after__gt=now, then=F("run_after")), + default=Value(now, output_field=DateTimeField()), + ) + cases = [ + ( + "epoch-checked READY discard", + CANCELLATION, + lambda: task.filter(status=READY, lease_epoch=0).update(**discard), + ), + ( + "epoch-checked READY or WAITING discard", + CANCELLATION, + lambda: task.filter(status__in=[READY, WAITING], lease_epoch=0).update( + **discard + ), + ), + ( + "discard by id and status", + None, + lambda: task.filter(status=READY).update(**discard), + ), + ( + "discard of any status", + None, + lambda: task.filter(lease_epoch=0).update(**discard), + ), + ( + "discard of READY or FAILED", + None, + lambda: task.filter(status__in=[READY, FAILED], lease_epoch=0).update( + **discard + ), + ), + ( + "discard against its own epoch", + None, + lambda: task.filter(status=READY, lease_epoch=F("lease_epoch")).update( + **discard + ), + ), + ( + "discard writing errors", + None, + lambda: task.filter(status=READY, lease_epoch=0).update( + errors=["cancelled"], **discard + ), + ), + ( + "release pinned to its epoch", + RELEASE, + lambda: task.filter(status=WAITING, lease_epoch=0).update( + status=READY, run_after=later_of_now + ), + ), + ( + "release to a bound time", + RELEASE, + lambda: task.filter(status=WAITING, lease_epoch=0).update( + status=READY, run_after=now + ), + ), + ( + "release to NULL", + None, + lambda: task.filter(status=WAITING, lease_epoch=0).update( + status=READY, run_after=None + ), + ), + ( + "release without its epoch", + None, + lambda: task.filter(status=WAITING).update( + status=READY, run_after=later_of_now + ), + ), + ( + "release that leaves run_after alone", + None, + lambda: task.filter(status=WAITING, lease_epoch=0).update(status=READY), + ), + ("run_after moved off anything", None, lambda: task.update(run_after=now)), + ( + "revival", + REVIVAL, + lambda: task.filter(status=DISCARDED).update( + status=WAITING, lease_epoch=epoch_bump, finished_at=None + ), + ), + ( + "revival to WAITING with no new epoch", + None, + lambda: task.filter(status=DISCARDED).update(status=WAITING), + ), + ( + "revival leaving run_after", + REVIVAL_UNBLOCKED, + lambda: task.filter(status=DISCARDED).update( + status=READY, lease_epoch=epoch_bump + ), + ), + ( + "revival clearing run_after", + REVIVAL_UNBLOCKED, + lambda: task.filter(status=DISCARDED).update( + status=READY, run_after=None, lease_epoch=epoch_bump + ), + ), + ( + "revival clearing a blocked node's run_after", + REVIVAL_UNBLOCKED, + lambda: blocked.filter(status=DISCARDED).update( + status=READY, run_after=None, lease_epoch=epoch_bump + ), + ), + ( + "revival clearing run_after, rows not named by id", + REVIVAL_UNBLOCKED, + lambda: OxTask.objects.filter( + task_path="no.such.task", status=DISCARDED + ).update(status=READY, run_after=None, lease_epoch=epoch_bump), + ), + ( + "revival with no new epoch", + None, + lambda: task.filter(status=DISCARDED).update(status=READY), + ), + ( + "revival resetting attempts", + None, + lambda: task.filter(status=DISCARDED).update( + status=READY, lease_epoch=epoch_bump, attempts=0 + ), + ), + ("attempts", None, lambda: task.update(attempts=0)), + ("FAILED", None, lambda: task.update(status=FAILED)), + ("DELETE", None, lambda: task._raw_delete(connection.alias)), + ] + # Classified as a revival that leaves run_after NULL, and undocumented + # all the same, because the rows it names are blocked or not named. + undocumented_by_its_rows = { + "revival clearing a blocked node's run_after", + "revival clearing run_after, rows not named by id", + } + wrong = [] + for label, expected, run in cases: + with statements_on() as seen: + run() + writes = [s for s in seen if s.write_target == TASK_TABLE] + assert len(writes) == 1, (label, seen) + as_oxpull = replace(writes[0], issuer="oxpull/control") + kind, reason = classify(as_oxpull) + offences = undocumented([as_oxpull]) + _emit(f"- {label:<48} -> {kind or 'UNDOCUMENTED'}: {reason}") + if offences: + _emit(f" undocumented: {offences[0][1]}") + _emit( + f" {_one_line(as_oxpull.sql, 300)} params={_params(as_oxpull.params)}" + ) + if kind != expected: + wrong.append((label, expected, kind, reason, as_oxpull.sql)) + documented = expected is not None and label not in undocumented_by_its_rows + if (offences == []) != documented: + wrong.append((label, documented, offences, reason, as_oxpull.sql)) + + quote = connection.ops.quote_name + for sql in ( + f"CREATE INDEX {quote('ox_extra')} ON {quote(TASK_TABLE)} ({quote(RUN_AFTER)})", + f"ALTER TABLE {quote(TASK_TABLE)} ADD COLUMN {quote('cause')} varchar(8)", + " ".join(("INSERT INTO", quote(TASK_TABLE), f"({quote('id')}) VALUES (%s)")), + f"LOCK TABLES {quote(TASK_TABLE)} WRITE", + ): + statement = Statement(sql, (), "oxpull/control", "MainThread") + assert statement.write_target == TASK_TABLE, sql + kind, reason = classify(statement) + _emit(f"- {sql:<60} -> {kind or 'UNDOCUMENTED'}: {reason}") + if kind is not None: + wrong.append((sql, None, kind, reason, sql)) + # Anyone's DDL on a django-ox table counts, not only oxpull's. + assert undocumented( + [Statement(f"ALTER TABLE {quote(TASK_TABLE)} ADD x int", (), "test", "t")] + ) + assert wrong == [] + + +def test_control_adding_blocked_nodes_writes_only_the_waiting_insert(): + """ + add() of a blocked node, and of a blocked node whose caller chose its + own run_after. Production sequence: exactly these calls. + """ + later = timezone.now() + timedelta(hours=1) + flow = workflows.create("deferred-add") + flow.add("parent", tasks.deferred_step, "parent") + with statements_on() as seen: + flow.add("child", tasks.deferred_step, "child", depends_on="parent") + flow.add( + "scheduled", + tasks.deferred_step.using(run_after=later), + "scheduled", + depends_on="parent", + ) + + assert row("child").status == WAITING + assert any( + s.write_target == TASK_TABLE and "django_ox/_waiting.py:enqueue" in s.issuer + for s in seen + ), "the waiting insert was not seen; the instrument is blind" + + inventory("add() of blocked nodes", seen) + assert_only_documented_writes(seen, "add() of blocked nodes") + + +def test_control_adding_nodes_on_the_gating_backend_writes_nothing_undocumented(): + with statements_on() as seen: + flow = workflows.create("plain") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + + inserts = [s for s in seen if s.write_target == TASK_TABLE] + by_django_ox = [s for s in inserts if s.issuer.startswith("django_ox/")] + waiting = [s for s in inserts if "django_ox/_waiting.py:enqueue" in s.issuer] + assert (len(by_django_ox), len(waiting), len(inserts)) == (1, 1, 2), ( + "expected django-ox's enqueue INSERT for a and the waiting insert for b" + ) + inventory("create/add/seal on the gating backend", seen) + assert_only_documented_writes(seen, "create/add/seal on the gating backend") + + +def test_control_the_gated_claim_is_a_django_ox_write_carrying_an_oxpull_read(): + flow = workflows.create("claimed") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + with statements_on() as seen: + claimed = flow_worker().claim_one() + + assert claimed is not None and claimed.args == ["a"] + claim_writes = [s for s in seen if s.write_target == TASK_TABLE] + assert claim_writes + assert all(s.issuer.startswith("django_ox/") for s in claim_writes) + # The gate reaches django-ox's own statement: the edge table and a + # second reference to the task table sit inside it. + assert any(s.issuer.startswith("django_ox/") and s.names(EDGE_TABLE) for s in seen) + inventory("one gated claim", seen) + assert_no_oxpull_writes(seen, "the gated claim") + + +def test_control_oxpull_prune_writes_no_django_ox_table(): + """ + Setup shortcut: completed_at is moved back 30 days, which stands in + for waiting past --older-than. Everything else is the real worker, + reconciler and management command. + """ + flow = workflows.create("old") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + flow.seal() + assert drain(flow_worker()) == 2 + assert reconcile() == 1 + OxpullWorkflow.objects.filter(pk=flow.id).update( + completed_at=timezone.now() - timedelta(days=30) + ) + + with statements_on() as seen: + call_command("oxpull_prune", stdout=StringIO()) + + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 0 + assert OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count() == 0 + assert OxTask.objects.count() == 2 + inventory("oxpull_prune", seen) + assert [s for s in seen if s.write_target in OX_TABLES] == [] + + +def test_control_the_released_unique_and_batch_paths_write_no_django_ox_row(worker): + """ + 1.2.0's features on the same instrument: a unique task whose stale + lock is released by a second enqueue, and a batch completed by its + reconciler. No documented write belongs on either path. + """ + with statements_on() as seen: + first = tasks.unique_echo.enqueue("key") + assert worker.run_once() is True + second = tasks.unique_echo.enqueue("key") + batch = batches.create("released") + batch.add(tasks.plain_echo, "member") + batch.seal() + assert drain(worker) == 2 + assert batches._reconcile() == 1 + + assert second.id != first.id + assert OxpullBatch.objects.get(pk=batch.id).status == OxpullBatch.Status.COMPLETE + inventory("1.2.0 unique stale-lock release and batch completion", seen) + assert_no_oxpull_writes(seen, "released unique and batch paths") + + +# transaction=True: SQLite's schema editor, which sqlmigrate drives, refuses +# to start inside the atomic block a plain django_db test runs in. +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("migration", MIGRATIONS) +def test_control_no_oxpull_migration_names_a_django_ox_table(migration): + out = StringIO() + call_command("sqlmigrate", "oxpull", migration, stdout=out) + sql = out.getvalue() + _emit(f"\n=== sqlmigrate oxpull {migration} [{connection.vendor}] ===\n{sql}") + assert sql.strip() + named = sorted(t for t in OX_TABLES if re.search(rf"\b{t}\b", sql)) + assert named == [] + + +def _schema(tables): + with connection.cursor() as cursor: + return { + table: ( + [ + tuple(c) + for c in connection.introspection.get_table_description( + cursor, table + ) + ], + connection.introspection.get_constraints(cursor, table), + ) + for table in sorted(tables) + } + + +@pytest.mark.django_db(transaction=True) +def test_control_migrating_from_the_last_release_changes_no_django_ox_table(): + """ + The real `migrate` command from the previous release's schema to the + newest migration, with a django-ox task row present so a data + migration would have something to touch, every statement captured, + and the django-ox tables' columns, constraints and indexes + introspected before and after. + """ + latest = _latest_migration() + call_command("migrate", "oxpull", PREVIOUS_RELEASE_MIGRATION, verbosity=0) + try: + assert EDGE_TABLE not in connection.introspection.table_names() + OxTask.objects.create( + task_path="tests_oxpull.tasks.plain_echo", + backend_name="default", + enqueued_at=timezone.now(), + ) + before = _schema(OX_TABLES) + with statements_on() as seen: + call_command("migrate", "oxpull", latest, verbosity=0) + after = _schema(OX_TABLES) + assert EDGE_TABLE in connection.introspection.table_names() + finally: + call_command("migrate", "oxpull", latest, verbosity=0) + + _emit( + f"\n=== migrate oxpull {PREVIOUS_RELEASE_MIGRATION} -> {latest} " + f"[{connection.vendor}] ===" + ) + ddl = [s for s in seen if s.write_target is not None] + for s in ddl: + _emit(f"- {s.write_target:<32} {_one_line(s.sql, 400)}") + _emit(f"({len(ddl)} write/DDL statement(s) of {len(seen)} captured)") + for table in sorted(OX_TABLES): + _emit(f"{table}: {sorted(after[table][1])}") + + assert ddl, "the migrate command ran no DDL; the instrument saw nothing" + assert [s for s in seen if s.write_target in OX_TABLES] == [] + assert [s for s in seen if any(s.names(t) for t in OX_TABLES)] == [] + assert before == after + oxpull_tables = [ + name + for name in connection.introspection.table_names() + if name.startswith("oxpull_") + ] + foreign_keys = { + (table, name): info["foreign_key"] + for table, (_columns, constraints) in _schema(oxpull_tables).items() + for name, info in constraints.items() + if info["foreign_key"] + } + _emit(f"foreign keys on oxpull tables: {foreign_keys}") + assert not [fk for fk in foreign_keys.values() if fk[0] in OX_TABLES] + + +@pytest.mark.django_db(transaction=True) +def test_control_migrating_from_the_last_release_waits_on_no_task_table_lock(): + """ + A second session holds the strongest lock the database has on + django-ox's task table, as a long ALTER or LOCK on a big live table + would. The migrating session gives up after three seconds of waiting. + A probe that does read the table must time out (the lock is real); + the migration must not. + + The second session keeps its lock until the migration has returned, so + a statement of the migration that waits on it is never let through: it + ends at the three-second timeout and the migration raises. That error + is the whole verdict. How long the migration's own DDL takes is the + server's speed, and is printed rather than judged. + """ + vendor = connection.vendor + if vendor not in ("postgresql", "mysql"): + pytest.skip( + "SQLite has one database-wide write lock and no table locks, so " + "there is no per-table lock for a migration to wait on." + ) + quoted = connection.ops.quote_name(TASK_TABLE) + latest = _latest_migration() + call_command("migrate", "oxpull", PREVIOUS_RELEASE_MIGRATION, verbosity=0) + blocker = connection.copy() + probe_error = None + migrate_error = None + seen: list[Statement] = [] + try: + blocker.ensure_connection() + if vendor == "postgresql": + blocker.set_autocommit(False) + with blocker.cursor() as cursor: + cursor.execute(f"LOCK TABLE {quoted} IN ACCESS EXCLUSIVE MODE") + with connection.cursor() as cursor: + cursor.execute("SET lock_timeout = '3s'") + else: + with blocker.cursor() as cursor: + cursor.execute(f"LOCK TABLES {quoted} WRITE") + with connection.cursor() as cursor: + cursor.execute("SET SESSION lock_wait_timeout = 3") + + started = time.monotonic() + try: + with connection.cursor() as cursor: + # The name comes from the model and is quoted by the backend. + cursor.execute(f"SELECT COUNT(*) FROM {quoted}") # noqa: S608 + except DatabaseError as exc: + probe_error = exc + probe_seconds = time.monotonic() - started + + started = time.monotonic() + try: + with statements_on() as seen: + call_command("migrate", "oxpull", latest, verbosity=0) + except DatabaseError as exc: + migrate_error = exc + migrate_seconds = time.monotonic() - started + finally: + if vendor == "postgresql": + blocker.rollback() + else: + with blocker.cursor() as cursor: + cursor.execute("UNLOCK TABLES") + blocker.close() + with connection.cursor() as cursor: + if vendor == "postgresql": + cursor.execute("RESET lock_timeout") + else: + cursor.execute("SET SESSION lock_wait_timeout = DEFAULT") + call_command("migrate", "oxpull", latest, verbosity=0) + + _emit( + f"\n=== migrate under a held lock on {TASK_TABLE} [{vendor}] ===\n" + f"probe SELECT on the locked table: {probe_seconds:.2f}s, " + f"error={probe_error!r}\n" + f"migrate {PREVIOUS_RELEASE_MIGRATION} -> {latest}: {migrate_seconds:.2f}s, " + f"error={migrate_error!r}, {len(seen)} statement(s)" + ) + assert probe_error is not None, "the probe did not wait: the lock is not real" + assert migrate_error is None, f"migrate waited on the locked {TASK_TABLE}" diff --git a/tests_oxpull/test_workflows_hazard_django_ox_stats_ox_health_and_the_prometheus.py b/tests_oxpull/test_workflows_hazard_django_ox_stats_ox_health_and_the_prometheus.py new file mode 100644 index 0000000..4b75eb8 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_django_ox_stats_ox_health_and_the_prometheus.py @@ -0,0 +1,318 @@ +""" +django_ox.stats, ox_health and the /ox/metrics gauges count +workflow nodes that no worker can claim as eligible backlog. + +django-ox documents ready_count() as "READY tasks eligible to run now, +mirroring the worker's dequeue predicate ... This is the backlog number", +oldest_ready_age() as measured "from when it became eligible", and tells +operators to alert on those two numbers through ox_health --max-backlog / +--max-age and the django_ox_ready_tasks / django_ox_oldest_ready_age_seconds +gauges. stats._ready() is READY and due. A workflow node whose upstream +has not succeeded used to be READY behind a dependency gate stats cannot +see. It is WAITING now, which none of these surfaces counts as backlog, and +its release writes a run_after of the moment it became eligible. + +Every test asserts what the documented surface promises, read through the +real public entry points (stats functions, call_command("ox_health"), a GET +on django_ox.urls). The upstream is held RUNNING by a real OxpullWorker +executing it on its own thread, and ages come from real elapsed time, not +from backdated rows. Two controls at the bottom pass today and show the +same harness can reach the passing branch and does see real backlog. +""" + +from __future__ import annotations + +import io +import re +import threading +import time + +import pytest +from django.core.management import CommandError, call_command +from django.db import connections +from django.db.models import Q +from django.tasks import task +from django.test import Client, override_settings +from django.urls import include, path +from django.utils import timezone + +from django_ox import stats +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import flow_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +# django_ox.urls mounted the way docs/monitoring.md tells a project to. +urlpatterns = [path("ox/", include("django_ox.urls"))] + +QUEUE = "default" +FAN_OUT = 20 + +# Released by the test once it has read the surfaces. A module-level Event is +# shared by the worker thread and the test because both are one process. +HOLD = threading.Event() + + +@task(backend="flows") +def held_extract(label): + """An upstream that stays RUNNING until the test releases it.""" + HOLD.wait(timeout=60) + return label + + +@task(backend="flows_deferred") +def held_extract_deferred(label): + HOLD.wait(timeout=60) + return label + + +NODE_TASKS = { + "flows": (held_extract, tasks.step), + "flows_deferred": (held_extract_deferred, tasks.deferred_step), +} + + +@pytest.fixture(autouse=True) +def hold_reset(): + HOLD.clear() + yield + HOLD.set() + + +def gating_worker(alias): + if alias == "flows": + return flow_worker() + return OxpullWorker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + + +def fan_out_flow(alias, name): + """extract -> FAN_OUT parts -> load, sealed. Returns (extract id, part ids).""" + upstream, step = NODE_TASKS[alias] + flow = workflows.create(name) + extract = flow.add("extract", upstream.using(priority=0), "extract") + parts = [f"part-{i:02d}" for i in range(FAN_OUT)] + part_ids = [ + flow.add(part, step.using(priority=5), part, depends_on="extract").id + for part in parts + ] + flow.add("load", step.using(priority=10), "load", depends_on=parts) + flow.seal() + return extract.id, part_ids + + +class RunningUpstream: + """ + A real OxpullWorker on its own thread, claiming and executing the held + extract through run_once(): a real lease, a real renewal thread, and the + real outcome write, and the release that follows it, when the test lets + it finish. + """ + + def __init__(self, alias, extract_id): + self.alias = alias + self.extract_id = extract_id + self.result = [] + self.errors = [] + self.thread = threading.Thread(target=self._work) + + def _work(self): + try: + self.result.append(gating_worker(self.alias).run_once()) + except BaseException as exc: + self.errors.append(exc) + finally: + connections.close_all() + + def __enter__(self): + self.thread.start() + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if OxTask.objects.filter( + pk=self.extract_id, status=OxTask.Status.RUNNING + ).exists(): + return self + time.sleep(0.02) + HOLD.set() + self.thread.join(timeout=30) + raise AssertionError(f"extract never went RUNNING; errors={self.errors}") + + def finish(self): + HOLD.set() + self.thread.join(timeout=30) + assert not self.thread.is_alive() + assert self.errors == [], self.errors + assert self.result == [True], self.result + assert OxTask.objects.get(pk=self.extract_id).status == OxTask.Status.SUCCESSFUL + + def __exit__(self, *exc_info): + if self.thread.is_alive() or not HOLD.is_set(): + HOLD.set() + self.thread.join(timeout=30) + return False + + +def health(*flags): + """manage.py ox_health as a cron line or probe sees it: OK text or the failure.""" + out = io.StringIO() + try: + call_command("ox_health", "--queue", QUEUE, *flags, stdout=out) + except CommandError as exc: + return f"FAIL: {exc}" + return out.getvalue().strip() + + +@override_settings(ROOT_URLCONF=__name__) +def scrape(): + """GET /ox/metrics through django_ox's own view, parsed for QUEUE.""" + response = Client().get("/ox/metrics") + assert response.status_code == 200 + body = response.content.decode() + samples = {} + for name in ("django_ox_ready_tasks", "django_ox_oldest_ready_age_seconds"): + match = re.search( + rf'^{name}{{queue="{QUEUE}"}} (\S+)$', body, flags=re.MULTILINE + ) + samples[name] = None if match is None else float(match.group(1)) + return samples + + +def readings(max_backlog, max_age): + age = stats.oldest_ready_age(QUEUE) + return { + "ready_count": stats.ready_count(QUEUE), + "oldest_ready_age_s": None if age is None else round(age.total_seconds(), 2), + "ox_health": health( + "--max-backlog", str(max_backlog), "--max-age", str(max_age) + ), + "metrics": scrape(), + } + + +def nothing_is_claimable(alias, workers=3): + """The autoscaler's extra workers: every gating claim comes back empty.""" + return [gating_worker(alias).claim_one() for _ in range(workers)] + + +# -- the hazard ---------------------------------------------------------- + + +def test_nodes_behind_a_running_upstream_are_not_reported_as_backlog(): + """ + extract is RUNNING on a real worker. Its 20 parts and the load behind + them wait for it, and no worker can claim any of them. After a real + 1.5 s wait, the backlog surfaces must read an empty backlog: ready_count + 0, no oldest age, ox_health OK at --max-backlog 10 --max-age 1, gauge 0 + and no age sample. + """ + extract_id, _ = fan_out_flow("flows", "backlog") + with RunningUpstream("flows", extract_id) as upstream: + assert nothing_is_claimable("flows") == [None, None, None] + time.sleep(1.5) + seen = readings(max_backlog=10, max_age=1) + assert nothing_is_claimable("flows") == [None, None, None] + upstream.finish() + + observed = ( + seen["ready_count"], + seen["oldest_ready_age_s"], + seen["ox_health"].startswith("OK: backlog=0 oldest_age=none"), + seen["metrics"], + ) + correct = ( + 0, + None, + True, + {"django_ox_ready_tasks": 0.0, "django_ox_oldest_ready_age_seconds": None}, + ) + assert observed == correct, f"no worker could claim any of these; observed {seen}" + + +@pytest.mark.parametrize("alias", ["flows", "flows_deferred"]) +def test_a_released_node_ages_from_when_it_became_eligible(alias): + """ + django-ox measures age "from when it became eligible ... so a task + deferred by a week does not read as a week of backlog". A part becomes + eligible when extract turns SUCCESSFUL. extract runs for 3 s on a real + worker; the moment it finishes, the parts are the oldest waiting work, + and they have been eligible for milliseconds. --max-age 2 must pass. + + Run on the backend that carries the leftover DEFER_BLOCKED_NODES option + as well, which must change nothing. A release that set run_after back + to NULL would make Coalesce(run_after, enqueued_at) fall back to + enqueued_at. + """ + extract_id, part_ids = fan_out_flow(alias, f"aging-{alias}") + with RunningUpstream(alias, extract_id) as upstream: + time.sleep(3.0) + upstream.finish() + age = stats.oldest_ready_age(QUEUE) + status = health("--max-age", "2") + gauge = scrape()["django_ox_oldest_ready_age_seconds"] + + # Premise: the parts really are claimable now: READY, due (whatever a + # release writes to run_after, it is not in the future), and through + # the claim's own gate. + released = ( + OxTask.objects.filter(pk__in=part_ids, status=OxTask.Status.READY) + .filter(Q(run_after__isnull=True) | Q(run_after__lte=timezone.now())) + .filter(workflows.runnable_filter_q()) + ) + assert released.count() == FAN_OUT + + seen = { + "oldest_ready_age_s": None if age is None else round(age.total_seconds(), 2), + "ox_health": status, + "gauge_s": gauge, + } + assert age is not None and age.total_seconds() < 2.0, seen + assert status.startswith("OK"), seen + assert gauge is not None and gauge < 2.0, seen + + +# -- controls: pass today ------------------------------------------------ + + +def test_control_the_same_harness_reads_empty_backlog_on_a_deferring_backend(): + """ + The scenario of the first test on flows_deferred, the backend with the + leftover option. The same readings, the same expected values, so the + option changes nothing a monitor reads. + """ + extract_id, _ = fan_out_flow("flows_deferred", "backlog-deferred") + with RunningUpstream("flows_deferred", extract_id) as upstream: + assert nothing_is_claimable("flows_deferred") == [None, None, None] + time.sleep(1.5) + seen = readings(max_backlog=10, max_age=1) + upstream.finish() + + assert seen["ready_count"] == 0, seen + assert seen["oldest_ready_age_s"] is None, seen + assert seen["ox_health"].startswith("OK: backlog=0 oldest_age=none"), seen + assert seen["metrics"] == { + "django_ox_ready_tasks": 0.0, + "django_ox_oldest_ready_age_seconds": None, + }, seen + + +def test_control_the_same_flags_page_on_real_stale_backlog(): + """ + The instrument sees real backlog: 11 claimable tasks with no worker, + after the same 1.5 s, fail the same ox_health flags and show in the + gauges. Without this, a green first test could be a blind probe. + """ + for i in range(11): + tasks.default_step.enqueue(f"plain-{i}") + time.sleep(1.5) + seen = readings(max_backlog=10, max_age=1) + + assert seen["ready_count"] == 11, seen + assert seen["oldest_ready_age_s"] >= 1.5, seen + status = seen["ox_health"] + assert status.startswith("FAIL: backlog is 11, over --max-backlog 10"), seen + assert "over --max-age 1s" in status, seen + assert seen["metrics"]["django_ox_ready_tasks"] == 11.0, seen + assert seen["metrics"]["django_ox_oldest_ready_age_seconds"] >= 1.5, seen diff --git a/tests_oxpull/test_workflows_hazard_every_reconciler_tick_row_locks_every_dead.py b/tests_oxpull/test_workflows_hazard_every_reconciler_tick_row_locks_every_dead.py new file mode 100644 index 0000000..fb0509c --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_every_reconciler_tick_row_locks_every_dead.py @@ -0,0 +1,503 @@ +""" +Every reconciler tick row-locks every dead node of a sealed +workflow that still has pending work. + +The state: a SEALED workflow with one FAILED node, the cone below it +already DISCARDED by an earlier tick, and one unrelated node still RUNNING +(a long or rate-limited branch), so the workflow cannot complete. Nothing +in it is cancellable any more. Every tick from then on is idle for this +workflow, and every such tick should cost what an idle tick costs: no +write, no row lock, nothing another connection has to wait for. + +What the shipped reconciler does instead, per tick, for as long as the +workflow stays SEALED: _has_dead_node() answers only "is any node dead", +which stays true forever, so _cancel_one_level() opens a transaction, +takes the workflow row with a no-op UPDATE, and runs SELECT ... FOR UPDATE +over every FAILED, LOST and DISCARDED node row, including every row its +own earlier cancellation discarded. Only then does it find nothing +cancellable and commit. + +Every test asserts the idle-tick property, whatever the fix: + + * an idle tick issues no write statement (all three databases); + * an idle tick issues no locking read (PostgreSQL, MySQL); + * an idle tick leaves the dead node tuples' xmax untouched (PostgreSQL, + where a row lock is a tuple-header write); + * an idle tick holds a bounded number of InnoDB record locks on the task + table, independent of how many nodes are dead (MySQL); + * an operator write on a dead node, from another connection, is not + blocked at any point inside an idle tick (PostgreSQL, MySQL): the real + ox_prune over the old DISCARDED cone, and actions.retry on the FAILED + node. Both run inside a transaction that is rolled back, so each probe + leaves the state exactly as it found it. + +A fix that keeps the lock for a tick that does cancel something passes all +of them. So does one that never opens a transaction on an idle tick: with +no transaction, no lock outlives a statement, and the mid-tick probes have +nowhere to fire. + +Real paths: nodes are enqueued through Workflow.add(); the RUNNING node is +held by a real OxpullWorker.claim_one() lease; the root fails through a +real worker's run_once() until its attempts are exhausted; the cone is +cancelled by a reconciler pass; ox_prune is the django-ox management +command and actions.retry is the django-ox action. + +Shortcuts, each named where it is taken: + + * The reconciler pass is workflows._reconcile() called inline, as every + workflow test calls it. The scheduled task runs the same function. + * The prune probe backdates the cone's finished_at by eight days. It + stands in for the RUNNING branch keeping the workflow SEALED for over a + week, which is what makes the cone old enough for default ox_prune. + * The mid-tick probes fire from a connection execute wrapper, after a + statement returns and before the next one runs, while the reconciler's + connection is inside a transaction. That pause lengthens the tick; it + does not add a lock. The lock timeout (700 ms on PostgreSQL, 1 s on + MySQL) is what turns a wait into a failure the test can see. Without + one, production waits for the tick to commit instead. +""" + +import io +import os +import re +import threading +import time +from contextlib import contextmanager +from datetime import timedelta + +import pytest +from django.core.management import call_command +from django.db import OperationalError, connections, transaction +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import flow_worker, reconcile, stored + +# transaction=True throughout: each reconciler pass must be its own real +# transaction, as it is in production. Inside the test-case transaction a +# second FOR UPDATE by the same transaction id writes nothing new to the +# tuple, and the PostgreSQL xmax test would pass for the wrong reason. +pytestmark = pytest.mark.django_db(transaction=True) + +# Two sizes by default, so the logs show the cost growing with the dead set. +# A measurement run sets OX_LOCK_WIDTHS=20000. +WIDTHS = [int(w) for w in os.environ.get("OX_LOCK_WIDTHS", "40,400").split(",")] + +# The most InnoDB record locks on the task table an idle tick may hold at +# any instant. A constant, independent of the dead set's size. +IDLE_TASK_ROW_LOCK_BOUND = 4 + +# How many statements inside the tick's transactions get a probe after them. +PROBE_LIMIT = 6 + + +def vendor(): + return connections["default"].vendor + + +needs_row_locks = pytest.mark.skipif( + "vendor() == 'sqlite'", + reason="SQLite has no row locks; the no-write test covers its database lock", +) + + +# -- the state --------------------------------------------------------- + + +def sealed_with_a_dead_cone(name, width): + """ + root (fails) -> `width` children, plus an unrelated node `long` held + RUNNING by a real worker's lease. Sealed. One reconciler pass then + discards the whole cone, and the workflow stays SEALED because `long` + is still running. + + Returns (flow, node ids, the first tick's captured queries). + """ + flow = workflows.create(name) + root = flow.add("root", tasks.failing_step, "root") + long = flow.add("long", tasks.step.using(priority=10), "long") + for i in range(width): + flow.add(f"c{i:05d}", tasks.step, f"c{i:05d}", depends_on="root") + flow.seal() + + # `long` outranks the root, so the first claim takes it and holds it. + holder = flow_worker() + held = holder.claim_one() + assert held is not None and str(held.id) == str(long.id), held + assert OxTask.objects.get(id=long.id).status == OxTask.Status.RUNNING + + # The root exhausts its three attempts in a real worker. + worker = flow_worker() + for _ in range(3): + worker.run_once() + root_row = OxTask.objects.get(id=root.id) + assert root_row.status == OxTask.Status.FAILED, root_row.status + assert root_row.attempts == root_row.max_attempts == 3 + + with CaptureQueriesContext(connections["default"]) as first_tick: + assert reconcile() == 0 + ids = flow.node_ids() + assert discarded(ids) == width, "the first tick did not cancel the cone" + assert stored(flow).status == OxpullWorkflow.Status.SEALED + return flow, ids, first_tick.captured_queries + + +def discarded(ids): + return OxTask.objects.filter( + id__in=ids.values(), status=OxTask.Status.DISCARDED + ).count() + + +def dead_ids(ids): + return list( + OxTask.objects.filter( + id__in=ids.values(), status__in=workflows.DEAD_STATUSES + ).values_list("pk", flat=True) + ) + + +def assert_the_tick_was_idle(flow, ids, width): + """Preconditions: the tick under test cancelled and completed nothing.""" + assert discarded(ids) == width + assert OxTask.objects.get(id=ids["root"]).status == OxTask.Status.FAILED + assert OxTask.objects.get(id=ids["long"]).status == OxTask.Status.RUNNING + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + +def locking_reads(queries): + return [q["sql"] for q in queries if "FOR UPDATE" in q["sql"].upper()] + + +def writes(queries): + return [ + q["sql"] + for q in queries + if q["sql"].lstrip().split(None, 1)[0].upper() in {"UPDATE", "INSERT", "DELETE"} + ] + + +def short(sql, n=160): + return " ".join(sql.split())[:n] + + +# -- probes from another connection ------------------------------------ + + +class _RolledBack(Exception): + """Raised to roll a probe's transaction back once it has done its write.""" + + +def on_another_connection(action, *, lock_wait=True): + """ + Run `action` on its own thread and connection, inside a transaction + that is always rolled back. With lock_wait, the session's lock wait is + shortened first, so a write blocked by the tick fails instead of + waiting for it. + """ + outcome = {} + + def run(): + try: + conn = connections["default"] + if lock_wait: + with conn.cursor() as cursor: + if conn.vendor == "postgresql": + cursor.execute("SET lock_timeout = '700ms'") + elif conn.vendor == "mysql": + cursor.execute("SET SESSION innodb_lock_wait_timeout = 1") + started = time.monotonic() + try: + with transaction.atomic(): + outcome["result"] = action() + raise _RolledBack + except _RolledBack: + outcome["ok"] = True + except OperationalError as exc: + outcome["ok"] = False + cause = exc.__cause__ or exc + outcome["error"] = f"{type(cause).__name__}: {short(str(exc), 90)}" + outcome["seconds"] = round(time.monotonic() - started, 3) + except BaseException as exc: # a crash is reported, not swallowed + outcome["ok"] = False + outcome["crash"] = repr(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=60) + assert not thread.is_alive(), "the probe thread did not finish" + assert "crash" not in outcome, outcome + return outcome + + +@contextmanager +def after_each_statement_inside_a_transaction(callback, limit=PROBE_LIMIT): + """ + Call `callback` after each statement the default connection runs while + it is inside an atomic block, up to `limit` times, and collect what it + returns. The statement has returned and its locks are held; the next + statement has not started. + """ + conn = connections["default"] + fired = [] + busy = [] + + def wrapper(execute, sql, params, many, context): + result = execute(sql, params, many, context) + if not busy and conn.in_atomic_block and len(fired) < limit: + busy.append(True) + try: + fired.append({"after": short(sql, 70), **callback()}) + finally: + busy.clear() + return result + + with conn.execute_wrapper(wrapper): + yield fired + + +def prune_old_rows(): + """The real ox_prune, default arguments. Returns how many task rows it deleted.""" + out = io.StringIO() + call_command("ox_prune", stdout=out) + match = re.search(r"Deleted (\d+) \S+ task row", out.getvalue()) + assert match, out.getvalue() + return int(match.group(1)) + + +# -- an idle tick writes nothing --------------------------------------- + + +@pytest.mark.parametrize("width", WIDTHS[:1]) +def test_an_idle_tick_over_a_workflow_with_dead_nodes_writes_nothing(width): + """ + The property test_an_idle_sealed_workflow_costs_no_write_on_a_reconciler_tick + pins for a workflow with no dead node, extended to one whose dead + nodes are all already dealt with. A write here is a write transaction + per such workflow per tick, for as long as it stays SEALED; on SQLite + that is the database write lock, serialised against every worker. + """ + flow, ids, _ = sealed_with_a_dead_cone("idle-writes", width) + + with CaptureQueriesContext(connections["default"]) as tick: + assert reconcile() == 0 + assert_the_tick_was_idle(flow, ids, width) + + written = writes(tick.captured_queries) + assert written == [], ( + f"an idle tick over a SEALED workflow with {width + 1} dead nodes and " + f"nothing cancellable issued {len(written)} write(s), every tick: " + + " | ".join(short(sql) for sql in written) + ) + + +# -- an idle tick locks nothing ---------------------------------------- + + +@needs_row_locks +@pytest.mark.parametrize("width", WIDTHS) +def test_an_idle_tick_issues_no_locking_read(width): + flow, ids, first_tick = sealed_with_a_dead_cone("idle-locking-read", width) + + started = time.monotonic() + with CaptureQueriesContext(connections["default"]) as tick: + assert reconcile() == 0 + elapsed_ms = round((time.monotonic() - started) * 1000) + assert_the_tick_was_idle(flow, ids, width) + + locking = locking_reads(tick.captured_queries) + assert locking == [], ( + f"an idle tick ({elapsed_ms} ms) issued {len(locking)} locking read(s) " + f"over a workflow with {width + 1} dead nodes and nothing cancellable " + f"(the cancelling tick before it issued {len(locking_reads(first_tick))}): " + + " | ".join(short(sql, 400) for sql in locking) + ) + + +def _xmax(pks): + table = OxTask._meta.db_table + with connections["default"].cursor() as cursor: + cursor.execute( + f'SELECT id::text, xmax::text FROM "{table}" WHERE id = ANY(%s)', # noqa: S608 + [list(pks)], + ) + return dict(cursor.fetchall()) + + +def _explain_wal(sql): + """ + EXPLAIN (ANALYZE, WAL) of one captured locking read, in a transaction + that is rolled back. Diagnostic only: it goes into the failure message. + """ + try: + with transaction.atomic(): + with connections["default"].cursor() as cursor: + cursor.execute("EXPLAIN (ANALYZE, WAL, BUFFERS, FORMAT JSON) " + sql) + plan = cursor.fetchone()[0] + raise _RolledBack + except _RolledBack: + pass + except Exception as exc: # the message must not hide the assertion + return f"EXPLAIN failed: {exc!r}" + top = plan[0]["Plan"] if isinstance(plan, list) else plan + return ( + f"{top['Node Type']} rows={top.get('Actual Rows')} " + f"WAL records={top.get('WAL Records')} bytes={top.get('WAL Bytes')} " + f"FPI={top.get('WAL FPI')} shared dirtied={top.get('Shared Dirtied Blocks')} " + f"time={round(plan[0].get('Execution Time', 0), 1)} ms" + ) + + +@pytest.mark.skipif( + "vendor() != 'postgresql'", + reason="xmax is PostgreSQL's tuple header; MySQL is measured by data_locks", +) +@pytest.mark.parametrize("width", WIDTHS) +def test_an_idle_tick_leaves_the_dead_node_tuples_unwritten_on_postgresql(width): + """ + On PostgreSQL a row lock is written into the tuple header: xmax takes + the locking transaction's id, the page is dirtied and the lock is + WAL-logged. An idle tick should leave every dead node's xmax as it was. + """ + flow, ids, _ = sealed_with_a_dead_cone("idle-xmax", width) + dead = dead_ids(ids) + assert len(dead) == width + 1 + + before = _xmax(dead) + started = time.monotonic() + with CaptureQueriesContext(connections["default"]) as tick: + assert reconcile() == 0 + elapsed_ms = round((time.monotonic() - started) * 1000) + after = _xmax(dead) + assert_the_tick_was_idle(flow, ids, width) + + relocked = [pk for pk, xmax in before.items() if after[pk] != xmax] + if relocked: + wal = [_explain_wal(sql) for sql in locking_reads(tick.captured_queries)] + assert relocked == [], ( + f"an idle tick ({elapsed_ms} ms) wrote a new xmax into {len(relocked)} of " + f"{len(dead)} dead node tuples, e.g. {before[str(relocked[0])]} -> " + f"{after[str(relocked[0])]}. The tick's locking read, re-run under " + f"EXPLAIN (ANALYZE, WAL): {wal}" + ) + + +def _data_locks(thread_id): + table = OxTask._meta.db_table + with connections["default"].cursor() as cursor: + cursor.execute( + "SELECT OBJECT_NAME, INDEX_NAME, LOCK_TYPE, LOCK_MODE, COUNT(*)" + " FROM performance_schema.data_locks WHERE THREAD_ID = %s" + " GROUP BY OBJECT_NAME, INDEX_NAME, LOCK_TYPE, LOCK_MODE", + [thread_id], + ) + rows = cursor.fetchall() + return { + "task_row_locks": sum( + n for obj, _i, kind, _m, n in rows if obj == table and kind == "RECORD" + ), + "locks": [ + f"{obj}.{idx} {kind} {mode} x{n}" for obj, idx, kind, mode, n in rows + ], + } + + +@pytest.mark.skipif( + "vendor() != 'mysql'", reason="performance_schema.data_locks is MySQL's" +) +@pytest.mark.parametrize("width", WIDTHS) +def test_an_idle_tick_holds_a_bounded_number_of_row_locks_on_mysql(width): + """ + Read from another connection while the tick is inside its transaction: + the InnoDB record locks the reconciler's thread holds on the task table + must not grow with the number of dead nodes. + """ + flow, ids, _ = sealed_with_a_dead_cone("idle-data-locks", width) + with connections["default"].cursor() as cursor: + cursor.execute("SELECT PS_CURRENT_THREAD_ID(), @@transaction_isolation") + thread_id, isolation = cursor.fetchone() + + with after_each_statement_inside_a_transaction( + lambda: on_another_connection(lambda: _data_locks(thread_id), lock_wait=False)[ + "result" + ] + ) as fired: + assert reconcile() == 0 + assert_the_tick_was_idle(flow, ids, width) + + peak = max((f["task_row_locks"] for f in fired), default=0) + assert peak <= IDLE_TASK_ROW_LOCK_BOUND, ( + f"{isolation}: mid idle tick the reconciler held {peak} record locks on " + f"{OxTask._meta.db_table} for a workflow with {width + 1} dead nodes " + f"(bound {IDLE_TASK_ROW_LOCK_BOUND}). Per statement: " + + " || ".join(f"after [{f['after']}]: {f['locks']}" for f in fired) + ) + + +# -- an idle tick blocks no operator write ----------------------------- + + +def _prune_probe(ids, width): + # Shortcut: the cone was discarded eight days ago, which is what a + # RUNNING branch holding the workflow SEALED for over a week produces. + cone = [pk for name, pk in ids.items() if name.startswith("c")] + OxTask.objects.filter(id__in=cone, status=OxTask.Status.DISCARDED).update( + finished_at=timezone.now() - timedelta(days=8) + ) + return prune_old_rows, width + + +def _retry_probe(ids, width): + root = ids["root"] + return (lambda: int(actions.retry(root))), 1 + + +PROBES = {"ox_prune": _prune_probe, "actions.retry": _retry_probe} + + +@needs_row_locks +@pytest.mark.parametrize("probe", list(PROBES)) +@pytest.mark.parametrize("width", WIDTHS[:1]) +def test_an_idle_tick_blocks_no_operator_write_on_a_dead_node(probe, width): + """ + From another connection, at every statement boundary inside the idle + tick's transactions: the operator write succeeds. ox_prune deletes the + old DISCARDED cone; actions.retry requeues the FAILED root. Both are + rolled back, so every probe sees the same state. + """ + flow, ids, _ = sealed_with_a_dead_cone(f"idle-blocks-{probe}", width) + action, expected = PROBES[probe](ids, width) + + # Control: with no tick in flight the probe succeeds and does its whole + # write. Without this a probe that touched nothing would pass below. + control = on_another_connection(action) + assert control.get("ok") and control["result"] == expected, control + + with after_each_statement_inside_a_transaction( + lambda: on_another_connection(action) + ) as fired: + assert reconcile() == 0 + assert_the_tick_was_idle(flow, ids, width) + + blocked = [f for f in fired if not f.get("ok")] + assert blocked == [], ( + f"{probe} from another connection was blocked at {len(blocked)} of " + f"{len(fired)} statement boundaries inside an idle tick over " + f"{width + 1} dead nodes: " + + " || ".join( + f"after [{f['after']}]: " + + ( + f"blocked {f['seconds']} s, {f['error']}" + if not f.get("ok") + else f"ok, result {f['result']}" + ) + for f in fired + ) + ) diff --git a/tests_oxpull/test_workflows_hazard_following_w003_s_hint_verbatim_stops_workers.py b/tests_oxpull/test_workflows_hazard_following_w003_s_hint_verbatim_stops_workers.py new file mode 100644 index 0000000..d872537 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_following_w003_s_hint_verbatim_stops_workers.py @@ -0,0 +1,376 @@ +""" +Following oxpull.W003's hint verbatim stops workers starting. + +W003 is emitted once per WORKFLOWS backend, and its hint is one fixed +mapping: a schedule named "oxpull-workflow-reconcile" with no backend and +no queue_name. Two common multi-backend shapes turn that paste into an +Error-level configuration: + + (a) two WORKFLOWS backends. Each warns, each hint says the same thing, + and pasting each into its own backend defines one schedule name on + two backends: django_ox.E003. + (b) a WORKFLOWS backend with its own QUEUES. The reconciler Task is + built on the default backend's queue, and schedules_from_options + rebinds it to this backend and revalidates: django_ox.E002. + +Every step runs in a fresh interpreter, the way an operator meets it: +`manage.py check` prints the hint, the hint is parsed out of that output +and pasted where it points, and then `manage.py check`, `manage.py +migrate`, the worker construction `ox_worker` performs, and `ox_worker` +itself run against the pasted settings. The subprocesses use this leg's +test database, so the same file runs on SQLite, PostgreSQL and MySQL. + +Each test asserts the correct behaviour: a configuration that was +warning-only and ran its workers before the paste still runs them after +it. Today both fail. + +CHECK_SCRIPT in test_workflows.py is the shape this follows. It is not +imported because its TASKS are fixed; these tests need their own. +""" + +import ast +import copy +import json +import os +import re +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from django.db import connection + +import oxpull +from oxpull import workflows + +# transaction=True: the subprocesses share this leg's test database, so +# nothing here may hold a transaction open while they run, and whatever +# a started worker writes is flushed afterwards. +pytestmark = pytest.mark.django_db(transaction=True) + +# How long a started `ox_worker` must stay up to count as running. A +# configuration error exits within the interpreter's startup; the grace +# only has to outlast startup under load. +STARTUP_GRACE = 10.0 + +RUNNER = """ +import json +import os +import sys + +try: + import MySQLdb # noqa: F401 +except ModuleNotFoundError: + try: + import pymysql + + pymysql.install_as_MySQLdb() + except ModuleNotFoundError: + pass + +import django +from django.conf import settings + +settings.configure( + SECRET_KEY="check-only", + USE_TZ=True, + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + INSTALLED_APPS=["django_ox", "oxpull"], + DATABASES={"default": json.loads(os.environ["OX_HAZARD_DATABASE"])}, + TASKS=json.loads(os.environ["OX_HAZARD_TASKS"]), +) +django.setup() + +import oxpull + +if os.path.dirname(oxpull.__file__) != os.environ["OX_HAZARD_OXPULL_DIR"]: + sys.exit(f"wrong oxpull imported: {oxpull.__file__}") + +if sys.argv[1] == "construct": + # The line ox_worker's handle() runs, minus the command around it. + from django_ox.worker import worker_class + + alias = sys.argv[2] + try: + worker_class(alias)(backend_alias=alias) + except Exception as exc: + print(f"{type(exc).__name__}: {exc}") + sys.exit(1) + print("constructed") +else: + from django.core.management import execute_from_command_line + + execute_from_command_line(["manage.py", *sys.argv[1:]]) +""" + +BATCH_RECONCILE = { + "oxpull-batch-reconcile": {"task": "oxpull.batches.reconcile", "cron": "* * * * *"}, +} + + +def _default_backend(): + """ + The project's default backend, with the batch reconciler scheduled. + + Scheduled so oxpull.W002 stays silent and W003 is the only thing the + base configuration warns about. + """ + return { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": {"SCHEDULES": copy.deepcopy(BATCH_RECONCILE)}, + } + + +def _workflow_backend(queues): + """A correctly declared workflow backend: WORKFLOWS and a gating worker.""" + return { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": queues, + "OPTIONS": { + "WORKER_CLASS": "oxpull.worker.OxpullWorker", + "WORKFLOWS": True, + }, + } + + +# -- running the operator's commands ----------------------------------- + + +def _database(): + """This leg's test database, as a subprocess's settings need it.""" + source = connection.settings_dict + database = { + key: source[key] + for key in ("ENGINE", "NAME", "USER", "PASSWORD", "HOST", "PORT", "OPTIONS") + if key in source + } + if source["ENGINE"].endswith("sqlite3"): + database["NAME"] = str(Path(source["NAME"]).resolve()) + return database + + +def _env(tasks): + env = {k: v for k, v in os.environ.items() if k != "DJANGO_SETTINGS_MODULE"} + env["OX_HAZARD_TASKS"] = json.dumps(tasks) + env["OX_HAZARD_DATABASE"] = json.dumps(_database()) + env["OX_HAZARD_OXPULL_DIR"] = str(Path(oxpull.__file__).parent) + return env + + +def _run(tasks, *argv): + """One command in a fresh interpreter; returns (returncode, output).""" + result = subprocess.run( # noqa: S603 + [sys.executable, "-c", RUNNER, *argv], + capture_output=True, + text=True, + env=_env(tasks), + check=False, + ) + return result.returncode, result.stdout + result.stderr + + +def _say(text): + """Printed, so a failing run's captured stdout carries the whole sequence.""" + print(text) # noqa: T201 + + +def _check_ids(output, level): + return re.findall(rf"\((\w+\.{level}\d{{3}})\)", output) + + +def _start_workers(tasks, aliases, logdir): + """ + `manage.py ox_worker --backend ALIAS` for every alias, concurrently. + + "running" when the process is still up after STARTUP_GRACE and logged + no traceback; otherwise how it exited. Survivors get SIGTERM, which is + how an operator stops one. + """ + logdir.mkdir(parents=True, exist_ok=True) + env = _env(tasks) + procs = {} + for alias in aliases: + log = (logdir / f"{alias}.log").open("w") + procs[alias] = ( + subprocess.Popen( # noqa: S603 + [sys.executable, "-c", RUNNER, "ox_worker", "--backend", alias], + stdout=log, + stderr=subprocess.STDOUT, + text=True, + env=env, + ), + log, + ) + deadline = time.monotonic() + STARTUP_GRACE + while time.monotonic() < deadline and any( + proc.poll() is None for proc, _ in procs.values() + ): + time.sleep(0.2) + outcomes, outputs = {}, {} + for alias, (proc, log) in procs.items(): + survived = proc.poll() is None + if survived: + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + log.close() + output = (logdir / f"{alias}.log").read_text() + outputs[alias] = output + if survived and "Traceback" not in output: + outcomes[alias] = "running" + elif survived: + outcomes[alias] = "running, but logged a traceback" + else: + errors = _check_ids(output, "E") + last = output.strip().splitlines()[-1:] or [""] + outcomes[alias] = f"exited rc={proc.returncode} {errors or last}" + return outcomes, outputs + + +# -- following the hint ------------------------------------------------ + +W003_BLOCK = re.compile( + r"^\?: \(oxpull\.W003\) (?P.*?)\n\s*HINT: (?P.*?)(?=\n\?: |\n\n|\Z)", + re.S | re.M, +) + + +def _mapping_in(hint): + """The first brace-balanced literal in the hint, as Python data.""" + start = hint.index("{") + depth = 0 + for index, char in enumerate(hint[start:], start): + depth += char == "{" + depth -= char == "}" + if depth == 0: + return ast.literal_eval(hint[start : index + 1]) + raise AssertionError(f"no complete mapping in hint: {hint!r}") + + +def _where_the_hint_points(message, hint): + """ + The backend whose OPTIONS the operator pastes into. + + The backend the hint names, if it names one (TASKS['x']); otherwise + the backend the warning is about, which is the only one today's text + names; otherwise the default backend. + """ + named = re.search(r"TASKS\[['\"]([^'\"]+)['\"]\]", hint) or re.search( + r"[Bb]ackend '([^']+)'", message + ) + return named.group(1) if named else "default" + + +def _follow_every_w003_hint(tasks, check_output): + pasted = copy.deepcopy(tasks) + pastes = [] + for block in W003_BLOCK.finditer(check_output): + alias = _where_the_hint_points(block["msg"], block["hint"]) + mapping = _mapping_in(block["hint"]) + options = pasted[alias].setdefault("OPTIONS", {}) + options.setdefault("SCHEDULES", {}).update(mapping) + pastes.append((alias, mapping)) + return pasted, pastes + + +def _follow_the_hint_and_start_everything(tasks, tmp_path): + """ + The operator's sequence, returning what each command did afterwards. + + Premises are asserted on the way: the base configuration is + warning-only, it prints W003, and every backend's worker starts. + """ + _say(f"oxpull: {oxpull.__file__}") + _say(f"database: {connection.vendor} {_database()['NAME']}") + + rc, base_output = _run(tasks, "check") + _say(f"== manage.py check, before the paste (rc={rc})\n{base_output}") + assert rc == 0, base_output + assert _check_ids(base_output, "E") == [], base_output + assert "oxpull.W003" in base_output, base_output + + base_workers, base_logs = _start_workers(tasks, list(tasks), tmp_path / "base") + _say(f"== ox_worker, before the paste: {base_workers}") + assert base_workers == dict.fromkeys(tasks, "running"), base_logs + + pasted, pastes = _follow_every_w003_hint(tasks, base_output) + _say(f"== pasted, one entry per W003 printed: {pastes}") + assert pastes, base_output + + check_rc, check_output = _run(pasted, "check") + _say(f"== manage.py check, after the paste (rc={check_rc})\n{check_output}") + migrate_rc, migrate_output = _run(pasted, "migrate", "--noinput") + _say(f"== manage.py migrate, after (rc={migrate_rc})\n{migrate_output}") + constructed = {} + for alias in pasted: + _, output = _run(pasted, "construct", alias) + constructed[alias] = output.strip() + _say(f"== worker_class(alias)(backend_alias=alias), after: {constructed}") + workers, worker_logs = _start_workers(pasted, list(pasted), tmp_path / "pasted") + _say(f"== ox_worker, after the paste: {workers}") + for alias, output in worker_logs.items(): + _say(f"-- ox_worker --backend {alias} output:\n{output}") + + return { + "check rc": check_rc, + "check errors": _check_ids(check_output, "E"), + "W003 still printed": "oxpull.W003" in check_output, + "migrate rc": migrate_rc, + "worker construction": constructed, + "ox_worker": workers, + } + + +def _started_cleanly(tasks): + return { + "check rc": 0, + "check errors": [], + "W003 still printed": False, + "migrate rc": 0, + "worker construction": dict.fromkeys(tasks, "constructed"), + "ox_worker": dict.fromkeys(tasks, "running"), + } + + +# -- the hazards ------------------------------------------------------- + + +def test_pasting_each_w003_hint_into_its_backend_keeps_two_workflow_backends_running( + tmp_path, +): + """ + (a) Two WORKFLOWS backends, each warned about, each hint pasted into + the backend its warning names. Today: django_ox.E003 on both, rc=1, + and every command that runs the checks refuses, ox_worker included, + for every backend, the default one too. + """ + tasks = { + "default": _default_backend(), + "flows": _workflow_backend(["default"]), + "flows_b": _workflow_backend(["default"]), + } + outcome = _follow_the_hint_and_start_everything(tasks, tmp_path) + assert outcome == _started_cleanly(tasks) + + +def test_pasting_the_w003_hint_into_a_backend_with_its_own_queues_keeps_it_running( + tmp_path, +): + """ + (b) The workflow backend has its own QUEUES, and the default backend + does not share them. The hint gives no queue_name. Today: + django_ox.E002 ("Queue 'default' is not valid for backend"), rc=1. + """ + tasks = { + "default": _default_backend(), + "flows": _workflow_backend(["pipelines"]), + } + assert workflows.RECONCILE_TASK_PATH == "oxpull.workflows.reconcile" + outcome = _follow_the_hint_and_start_everything(tasks, tmp_path) + assert outcome == _started_cleanly(tasks) diff --git a/tests_oxpull/test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost.py b/tests_oxpull/test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost.py new file mode 100644 index 0000000..3dfdd26 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost.py @@ -0,0 +1,533 @@ +""" +In-flight fan-in edges make the dependency gate cost O(edges). + +The gate is NOT EXISTS(edge JOIN upstream task WHERE upstream.status <> +SUCCESSFUL), conjoined into every claim on a WORKFLOWS backend and into the +eager release's SELECT. Three legs, each asserting the property a claim path +needs, so each fails while the hazard is present: + + 1. A claim of a task that is not in any workflow must not get more expensive + as a fan-in in flight gets wider (the fan-in node waits, so it is not a + candidate; only its edges are in the gate's inner relation). + 2. A fan-in node ranked ahead of its upstreams (priority=10), put back to + READY as a django-ox retry leaves it, must not make each claim re-walk + every edge that is already satisfied. + 3. The eager release a worker runs after each parent's SUCCESSFUL must not + re-scan every parent of the shared child, which makes a map/reduce drain + O(K^2). + +**Work, not wall-clock.** The machine is shared and loaded, so every assertion +is on work the database reports, measured around the real call: + + * PostgreSQL: auto_explain (log_analyze, JSON) captures the plan of every + statement the call runs, and work is the sum over plan nodes of + (Actual Rows + Rows Removed by Filter/Join Filter/Index Recheck) x Loops. + * MySQL: the delta of the session's Handler_read_* counters, less the cost + of reading them. + * SQLite: SQLite VM instructions, counted with a progress handler. + +The units differ, so every assertion is a ratio of two readings taken with the +same instrument on the same database. Wall-clock is printed, never asserted. + +**Real paths.** Workflows are built with Workflow.add() and seal(); claims go +through OxpullWorker.claim_one() and run_once(); the release is the one +OxpullWorker._write_outcome() makes. Two shortcuts, both bulk status writes, +both marked where they happen: maps put into RUNNING (what a fleet claiming +them produces; claiming 20,000 through the gate is the hazard itself) and a +finished fan-in settled to SUCCESSFUL between two measurements. +""" + +from __future__ import annotations + +import contextlib +import json +import time +from dataclasses import dataclass, field +from datetime import timedelta + +import pytest +from django.db import connection, transaction +from django.db.models import F +from django.utils import timezone + +from django_ox.models import OxTask +from oxpull import worker as oxpull_worker +from oxpull import workflows +from oxpull.models import OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import flow_worker +from .test_workflows_deferred import deferring_worker + +# transaction=True, not the default wrapper: MySQL's ANALYZE TABLE commits +# implicitly, which would end the test's own transaction halfway through a +# measurement. Every leg runs the same way so the three databases are compared +# on the same sequence of commits. +pytestmark = pytest.mark.django_db(transaction=True) + + +# -- the instrument ---------------------------------------------------- + + +@dataclass +class Reading: + work: int = 0 + seconds: float = 0.0 + statements: list[tuple[int, str]] = field(default_factory=list) + + def __str__(self) -> str: + return f"{self.work} ({self.seconds * 1000:.1f} ms)" + + +def _plan_work(node: dict) -> int: + loops = int(node.get("Actual Loops", 1)) + rows = float(node.get("Actual Rows", 0)) + removed = sum( + float(node.get(key, 0)) + for key in ( + "Rows Removed by Filter", + "Rows Removed by Join Filter", + "Rows Removed by Index Recheck", + ) + ) + total = round((rows + removed) * loops) + for child in node.get("Plans", []): + total += _plan_work(child) + return total + + +class Meter: + """Database-reported work done while a block runs, on this connection.""" + + def __init__(self) -> None: + self.vendor = connection.vendor + self._raw = None + self._notes: list[str] = [] + self._capturing = False + self._mysql_overhead = 0 + + def _ensure(self) -> None: + connection.ensure_connection() + raw = connection.connection + if raw is self._raw: + return + self._raw = raw + if self.vendor == "postgresql": + raw.add_notice_handler(self._on_notice) + with connection.cursor() as cursor: + cursor.execute("LOAD 'auto_explain'") + for setting in ( + "auto_explain.log_analyze = on", + "auto_explain.log_timing = off", + "auto_explain.log_format = 'json'", + "auto_explain.log_level = 'notice'", + "auto_explain.log_nested_statements = on", + "auto_explain.log_min_duration = -1", + "client_min_messages = 'notice'", + ): + cursor.execute(f"SET {setting}") + elif self.vendor == "mysql": + overheads = [] + for _ in range(5): + before = self._handler_reads() + overheads.append(self._handler_reads() - before) + self._mysql_overhead = min(overheads) + + def _on_notice(self, diagnostic) -> None: + if self._capturing: + self._notes.append(diagnostic.message_primary or "") + + def _handler_reads(self) -> int: + with connection.cursor() as cursor: + cursor.execute("SHOW SESSION STATUS LIKE 'Handler_read%'") + return sum(int(value) for _, value in cursor.fetchall()) + + @contextlib.contextmanager + def measure(self): + self._ensure() + reading = Reading() + if self.vendor == "postgresql": + with connection.cursor() as cursor: + cursor.execute("SET auto_explain.log_min_duration = 0") + self._notes = [] + self._capturing = True + started = time.monotonic() + try: + yield reading + finally: + reading.seconds = time.monotonic() - started + self._capturing = False + with connection.cursor() as cursor: + cursor.execute("SET auto_explain.log_min_duration = -1") + for note in self._notes: + if "plan:" not in note: + continue + plan = json.loads(note.split("plan:", 1)[1]) + work = _plan_work(plan["Plan"]) + text = " ".join(plan.get("Query Text", "").split())[:90] + reading.statements.append((work, text)) + reading.work += work + elif self.vendor == "mysql": + before = self._handler_reads() + started = time.monotonic() + try: + yield reading + finally: + reading.seconds = time.monotonic() - started + after = self._handler_reads() + reading.work = after - before - self._mysql_overhead + else: + raw = connection.connection + steps = [0] + + def tick() -> int: + steps[0] += 1 + return 0 + + raw.set_progress_handler(tick, 10) + started = time.monotonic() + try: + yield reading + finally: + reading.seconds = time.monotonic() - started + raw.set_progress_handler(None, 10) + reading.work = steps[0] * 10 + + +def analyze() -> None: + """ + Fresh statistics before a measurement, as autovacuum / InnoDB's + auto-recalc give a running install. + + SQLite too, though Django never runs ANALYZE there: SQLite's own guidance + is PRAGMA optimize on long-lived connections, which gathers the same + statistics. It matters. Without statistics SQLite probes + oxpull_wf_edge_down_idx per candidate and leg 1 stays flat; with them, + and one 20,000-way fan-in in the table, it plans the probe as a full scan + of the edge table (measured 390 -> 400,380 VM instructions per claim). + """ + tables = [OxTask._meta.db_table, OxpullWorkflowEdge._meta.db_table] + with connection.cursor() as cursor: + for table in tables: + if connection.vendor == "postgresql": + cursor.execute(f'ANALYZE "{table}"') + elif connection.vendor == "mysql": + cursor.execute(f"ANALYZE TABLE `{table}`") + cursor.fetchall() + else: + cursor.execute(f'ANALYZE "{table}"') + + +def unit() -> str: + return { + "postgresql": "plan rows", + "mysql": "Handler_read_*", + "sqlite": "VM instructions", + }[connection.vendor] + + +def report(line: str) -> None: + print(f"\n [s2 {connection.vendor}] {line}", end="") # noqa: T201 + + +# -- fixtures built through the public API ----------------------------- + + +def fan_in(name: str, width: int, task, *, reduce_task=None): + """ + `width` map nodes and one reduce node depending on every one of them, + through Workflow.add(). One outer transaction so twenty thousand adds + cost one commit; add() inside a caller's transaction is supported. + """ + flow = workflows.create(name) + names = [f"{name}-map-{i:05d}" for i in range(width)] + with transaction.atomic(): + for node in names: + flow.add(node, task, node) + flow.add("reduce", reduce_task or task, f"{name}-reduce", depends_on=names) + flow.seal() + return flow + + +def map_ids(flow): + return ( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id) + .exclude(name="reduce") + .values("task_id") + ) + + +def mark_maps_running(flow) -> int: + """ + SHORTCUT: the state a fleet leaves after claiming every map. The real + sequence is one OxpullWorker.claim_one() per map; at 20,000 maps that + sequence pays the very cost under test once per map. + """ + now = timezone.now() + return OxTask.objects.filter( + pk__in=map_ids(flow), status=OxTask.Status.READY + ).update( + status=OxTask.Status.RUNNING, + locked_by="fleet", + locked_at=now, + lease_expires_at=now + timedelta(hours=1), + lease_epoch=F("lease_epoch") + 1, + attempts=F("attempts") + 1, + started_at=now, + last_attempted_at=now, + ) + + +def settle(flow) -> None: + """ + SHORTCUT: every node of a fan-in SUCCESSFUL, as workers running the maps + and then the released reduce would leave it. + """ + now = timezone.now() + ids = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values("task_id") + OxTask.objects.filter(pk__in=ids).update( + status=OxTask.Status.SUCCESSFUL, + run_after=None, + finished_at=now, + locked_by=None, + locked_at=None, + lease_expires_at=None, + ) + + +def claim_cost(meter: Meter, worker, *, label_prefix: str, samples: int = 3) -> Reading: + """ + The median of `samples` real claims, each asserted to have taken a task + whose label starts with `label_prefix`, so the reading is of the claim + the fixture meant to measure. Each claimed task is executed, as the + worker would, so the next sample sees the table a worker would. + """ + readings = [] + for _ in range(samples): + with meter.measure() as reading: + claimed = worker.claim_one() + assert claimed is not None, "nothing was claimable" + assert str(claimed.args[0]).startswith(label_prefix), ( + f"the claim took {claimed.args[0]!r}, not a {label_prefix!r} task" + ) + worker.execute(claimed, inline=True) + readings.append(reading) + readings.sort(key=lambda r: r.work) + return readings[len(readings) // 2] + + +# -- leg 1: unrelated claims pay for a fan-in in flight ---------------- + + +def test_an_in_flight_fan_in_does_not_tax_the_claim_of_an_unrelated_task(): + """ + A 2,000-way and then a 20,000-way fan-in are in flight: every map claimed + and running, the reduce waiting, so nothing of the workflow is a claim + candidate. The claim takes a plain task that is in + no workflow. Its cost must not scale with the fan-in's width. + """ + meter = Meter() + worker = deferring_worker() + with transaction.atomic(): + for i in range(1000): + tasks.deferred_step.enqueue(f"unrelated-{i:04d}") + + analyze() + baseline = claim_cost(meter, worker, label_prefix="unrelated") + + small = fan_in("small-fan-in", 2000, tasks.deferred_step) + reduce_small = OxTask.objects.get( + pk=OxpullWorkflowNode.objects.get(workflow_id=small.id, name="reduce").task_id + ) + assert reduce_small.status == OxTask.Status.WAITING + analyze() + small_ready = claim_cost(meter, worker, label_prefix="unrelated") + assert mark_maps_running(small) == 2000 + analyze() + small_running = claim_cost(meter, worker, label_prefix="unrelated") + + settle(small) + analyze() + settled = claim_cost(meter, worker, label_prefix="unrelated") + + big = fan_in("big-fan-in", 20000, tasks.deferred_step) + analyze() + big_ready = claim_cost(meter, worker, label_prefix="unrelated") + assert mark_maps_running(big) == 20000 + analyze() + big_running = claim_cost(meter, worker, label_prefix="unrelated") + + report(f"unit={unit()}") + report(f"no workflow rows: {baseline}") + report(f"2,000-way fan-in, maps READY: {small_ready}") + report(f"2,000-way fan-in, maps RUNNING: {small_running}") + report(f"2,000-way fan-in settled: {settled}") + report(f"20,000-way fan-in, maps READY: {big_ready}") + report(f"20,000-way fan-in, maps RUNNING: {big_running}") + for work, text in big_running.statements: + report(f" stmt {work}: {text}") + print() # noqa: T201 + + growth = big_running.work / max(small_running.work, 1) + assert growth < 3, ( + f"an unrelated claim cost {small_running.work} {unit()} beside a 2,000-way " + f"fan-in in flight and {big_running.work} beside a 20,000-way one " + f"({growth:.1f}x); with no workflow rows it cost {baseline.work}" + ) + + +# -- leg 2: a candidate fan-in node re-walks its satisfied edges ------- + + +def _priority_fan_in(name: str, width: int): + """`width` upstreams at priority 0 and a priority-10 node on all of them.""" + flow = workflows.create(name) + names = [f"{name}-up-{i:05d}" for i in range(width)] + with transaction.atomic(): + for node in names: + flow.add(node, tasks.step, node) + flow.add( + "load", tasks.step.using(priority=10), f"{name}-load", depends_on=names + ) + flow.seal() + return flow + + +def _half_drained_claim_cost(meter: Meter, worker, name: str, width: int) -> Reading: + _priority_fan_in(name, width) + for _ in range(width // 2): + assert worker.run_once() is True + assert tasks.STATE["order"][-1] == f"{name}-up-{width // 2 - 1:05d}", ( + "the upstreams did not drain in insertion order" + ) + # The load node waits for the other half. Put back to READY, as a + # django-ox retry leaves a row, it is a candidate the gate has to judge, + # with half its edges marked satisfied. + assert ( + OxTask.objects.filter( + args=[f"{name}-load"], status=OxTask.Status.WAITING + ).update(status=OxTask.Status.READY) + == 1 + ) + analyze() + reading = claim_cost(meter, worker, label_prefix=f"{name}-up-") + # Finish the workflow through the same worker, load last, so the next + # width starts from settled history rather than a second live fan-in. + while worker.run_once(): + pass + assert tasks.STATE["order"][-1] == f"{name}-load" + return reading + + +def test_a_high_priority_fan_in_node_does_not_rewalk_its_satisfied_edges(): + """ + The fan-in node outranks its upstreams, so every claim tests its gate + first. With half the upstreams done, the claim must + cost about the same at width 4,000 as at width 500. + """ + meter = Meter() + worker = flow_worker() + small = _half_drained_claim_cost(meter, worker, "p500", 500) + large = _half_drained_claim_cost(meter, worker, "p4000", 4000) + + report(f"unit={unit()}") + report(f"W=500, 250 upstreams done: {small}") + report(f"W=4000, 2000 upstreams done: {large}") + for work, text in large.statements: + report(f" stmt {work}: {text}") + print() # noqa: T201 + + assert large.work <= 2 * small.work, ( + f"with half the upstreams done, one claim cost {small.work} {unit()} at " + f"W=500 and {large.work} at W=4000 ({large.work / max(small.work, 1):.1f}x)" + ) + + +# -- leg 3: the eager release re-scans every parent -------------------- + + +@pytest.mark.xfail( + connection.vendor == "postgresql", + strict=True, + reason=( + "The release run after each parent succeeds re-reads every parent of " + "the shared child, so draining a fan-in costs its width squared. " + "SQLite's plan skips the satisfied edges; PostgreSQL's does not" + ), +) +@pytest.mark.xfail( + connection.vendor == "mysql", + strict=False, + reason=( + "The release run after each parent succeeds re-reads every parent of " + "the shared child; on MySQL the narrow baseline swings with what the " + "earlier tests left in the tables, so the result is timing-dependent" + ), +) +@pytest.mark.xfail( + connection.vendor == "sqlite", + strict=False, + reason=( + "The release run after each parent succeeds can re-read every parent " + "of the shared child on SQLite. Run alone the drain grows 10x for 10x " + "the width, and run after the file before it the same drain grew 94x, " + "so the result depends on the plan SQLite picks" + ), +) +def test_draining_a_wide_fan_in_does_not_cost_its_width_squared(monkeypatch): + """ + Each map's SUCCESSFUL marks its edge in OxpullWorker._write_outcome, and + then runs release_downstream on the shared reduce. The total work + those releases do over a whole drain must grow about linearly with the + fan-in's width: 10x the width, well under 20x the work. + """ + meter = Meter() + releases: list[Reading] = [] + real_release = oxpull_worker.release_downstream + + def metered_release(task_ids, db_alias): + with meter.measure() as reading: + released = real_release(task_ids, db_alias) + releases.append(reading) + return released + + monkeypatch.setattr(oxpull_worker, "release_downstream", metered_release) + + results = {} + for width in (200, 2000): + releases.clear() + fan_in(f"r{width}", width, tasks.deferred_step) + analyze() + worker = deferring_worker() + ran = 0 + while worker.run_once(): + ran += 1 + assert ran == width + 1 + assert tasks.STATE["order"][-1] == f"r{width}-reduce", ( + "the reduce did not run last" + ) + # One release per map, then the reduce's own (it has no dependents). + assert len(releases) == width + 1 + maps = releases[:width] + results[width] = { + "points": {i: maps[i] for i in (0, 100, width // 2, width - 1)}, + "last": maps[-1], + "total": sum(r.work for r in maps), + "seconds": sum(r.seconds for r in maps), + } + + report(f"unit={unit()}") + for width, r in results.items(): + points = ", ".join(f"i={i}: {reading}" for i, reading in r["points"].items()) + report(f"K={width}: release of parent {points}") + report(f"K={width}: whole drain {r['total']} ({r['seconds']:.2f} s)") + for work, text in results[2000]["last"].statements: + report(f" stmt {work}: {text}") + print() # noqa: T201 + + growth = results[2000]["total"] / max(results[200]["total"], 1) + assert growth < 20, ( + f"releasing every parent of a 200-way fan-in cost {results[200]['total']} " + f"{unit()} and of a 2,000-way one {results[2000]['total']} ({growth:.0f}x " + "for 10x the width)" + ) diff --git a/tests_oxpull/test_workflows_hazard_lost_straggler.py b/tests_oxpull/test_workflows_hazard_lost_straggler.py new file mode 100644 index 0000000..f675992 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_lost_straggler.py @@ -0,0 +1,568 @@ +""" +A workflow node the reaper marks LOST while its execution is still running +must hold its downstream nodes, not discard them, and that execution's late +SUCCESSFUL must release them. + +django-ox 1.2.0 lets the late write land on purpose. WRITABLE_STATUSES is +(RUNNING, LOST), and the reaper's exhausted branch leaves lease_epoch alone +so the holder of the lease can still supply the outcome. LOST says the +outcome was never observed, not that the work failed. So a workflow treats +it as unknown: the nodes downstream of it wait, gated, until the execution +reports or someone acts on the node or the workflow. FAILED and DISCARDED +still discard what depends on them; that is not what these tests cover. + +Everything below runs through the production path: a real OxpullWorker +executing a node whose body blocks, a lease that really expires, the real +reaper, the reconciler run as the task a SCHEDULES entry enqueues and a +worker claims, and the straggler's own outcome write. Two ways of losing +the lease are covered: + +- renewals_fail: the worker's full run() loop with a one-second lease, + its renewal UPDATE raising as it does when that worker cannot reach the + database, and the loop's own reaper taking the row. The fault is + injected at renew_leases, so the real _renewal_loop handles it. +- operator_expires_lease: django_ox.actions.expire_lease on a live + worker's task, which that function documents as "It does not stop the + task", then another worker's reaper. + +The earlier attempts fail for real so that the blocking attempt is the +last one, which is what sends the reaper down its LOST branch. No status, +lease_epoch or other state column is written by hand. The one hand-written +change is a move of timestamps into the past, described in phase 1; the +straggler's outcome write matches on status and lease_epoch, never on a +timestamp, so the move cannot change whether it lands. + +Each test runs root -> child -> grandchild in three phases: + +1. While root is LOST and its execution is alive, five reconciler passes + run: one at real time, then one each with Django's clock + (django.utils.timezone.now) an hour, eight days and 400 days ahead, + then one more at real time after every timestamp on the workflow's + rows has been moved 400 days into the past in the database: the node + task rows (all but run_after), the workflow row, and its node and edge + rows. The last pass is what a hold measured on the database clock sees + 400 days later. After every pass root is still LOST, child and + grandchild have neither finished nor started, the workflow has not + completed, and no callback has been enqueued. +2. The clock goes back to real time and the straggler writes SUCCESSFUL. +3. Reconciler passes and drains run until the workflow completes, then one + more of each. All three nodes are SUCCESSFUL and ran once, in order. + The workflow is COMPLETE with nothing failed or cancelled. on_complete + and on_success were each enqueued exactly once and each ran once. + Nothing here retries the workflow, so the whole run is one generation, + and a second on_complete is a defect. + +Every check is collected before the one assertion, so a failure names +each rule it broke. + +Implementations these tests reject: + +- Discard on LOST, as the reconciler does today: the first pass in phase 1 + sees child DISCARDED and the workflow complete. +- Discard the LOST node itself first, so its late write matches nothing: + phase 1 sees root no longer LOST, and phase 3 sees the chain unrun. +- Hold for a grace period, then discard. A pass that treats a LOST row as + dead once its finished_at is older than some interval sees child + DISCARDED in a later pass of phase 1, whichever clock it reads: against + timezone.now() the passes with Django's clock ahead catch it, and + against the database's own now (Now() in a query) or a now() bound at + import the pass after the timestamps move catches it. The same holds for + an interval measured on any other timestamp of those rows. Counting + passes instead fails the same way if it gives up within five. +- Discard on LOST, then put the discarded nodes back and reopen the + workflow when the late SUCCESSFUL lands: phase 1 sees the discard, and + phase 3 counts two on_complete callbacks. +- Treat LOST as satisfied: child starts during phase 1. + +What the passes cannot reach: a hold that gives up only after more than +five passes, or one that expires on a clock neither move touches, such as +a monotonic timer kept in the reconciler's process. Such a hold is caught +only if it expires within the few seconds phase 1 really takes. +""" + +import threading +import time +from datetime import timedelta + +import pytest +from django.db import OperationalError, connections +from django.db.models import DateTimeField, F +from django.tasks import task +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import drain, flow_worker, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +WAIT = 30.0 + +LEASE_LOSS = ["renewals_fail", "operator_expires_lease"] + +# The reconciler passes that run while root is LOST, and how far Django's +# clock is ahead of real time for each. +HOLD_PASSES = ( + ("at real time", timedelta(0)), + ("with the clock an hour ahead", timedelta(hours=1)), + ("with the clock eight days ahead", timedelta(days=8)), + ("with the clock 400 days ahead", timedelta(days=400)), +) + +# How far the last pass of phase 1 moves the workflow's timestamps back. +AGED_BY = timedelta(days=400) + +FINISHED = ( + OxTask.Status.SUCCESSFUL, + OxTask.Status.FAILED, + OxTask.Status.LOST, + OxTask.Status.DISCARDED, +) + +# label -> the stalling node's controls. Module state rather than +# tasks.STATE because a worker thread resolves the task by its dotted path +# and must reach the same Events the test holds. +_CONTROL: dict[str, dict] = {} + + +@pytest.fixture(autouse=True) +def _control(): + _CONTROL.clear() + yield + for ctl in _CONTROL.values(): + ctl["release"].set() + _CONTROL.clear() + + +class Clock: + """ + django.utils.timezone.now, moved ahead on demand. + + Replaced on the module, so every caller that looks it up when it calls + it sees the move, django-ox included. django-ox's leases run on the + database clock, which this does not move, so moving it neither expires + nor extends a lease. + """ + + def __init__(self, monkeypatch): + self.ahead = timedelta(0) + real_now = timezone.now + monkeypatch.setattr(timezone, "now", lambda: real_now() + self.ahead) + + +@pytest.fixture +def clock(monkeypatch): + return Clock(monkeypatch) + + +def _stall(label): + """ + Fail every attempt before the last, then block on the last until the + test releases it, then succeed. + """ + ctl = _CONTROL[label] + ctl["calls"] += 1 + if ctl["calls"] < ctl["stall_on"]: + raise RuntimeError(f"{label}: attempt {ctl['calls']} fails") + ctl["started"].set() + ctl["released"] = ctl["release"].wait(timeout=60) + tasks._record(label) + return label + + +@task(backend="flows") +def stalling_step(label): + return _stall(label) + + +@task(backend="flows_deferred") +def stalling_deferred_step(label): + return _stall(label) + + +class Straggler: + """One node execution that outlives its lease, then finishes.""" + + def __init__(self, factory, root_id, label, how): + self.factory = factory + self.root_id = root_id + self.how = how + self.ctl = _CONTROL[label] = { + "calls": 0, + "stall_on": OxTask.objects.get(id=root_id).max_attempts, + "started": threading.Event(), + "release": threading.Event(), + "released": None, + } + self.renewals_refused = 0 + self.errors: list[BaseException] = [] + self.thread: threading.Thread | None = None + self.executor = None + self.claimed_epoch = None + self.lost_epoch = None + + def _renewal_cannot_reach_the_database(self): + self.renewals_refused += 1 + raise OperationalError( + "injected: this worker's renewal cannot reach the database" + ) + + def _run(self, target): + try: + target() + except BaseException as exc: + self.errors.append(exc) + finally: + connections.close_all() + + def start(self): + if self.how == "renewals_fail": + # The whole poll loop: claiming, retrying, renewing and reaping + # are all the worker's own. + self.executor = self.factory(lock_timeout=1.0) + self.executor.renew_leases = self._renewal_cannot_reach_the_database + target = self.executor.run + else: + self.executor = self.factory() + for _ in range(1, self.ctl["stall_on"]): + assert self.executor.run_once() is True + assert OxTask.objects.get(id=self.root_id).status == OxTask.Status.READY + target = self.executor.run_once + self.thread = threading.Thread(target=self._run, args=(target,), daemon=True) + self.thread.start() + assert self.ctl["started"].wait(WAIT), "the node's last attempt never started" + row = OxTask.objects.get(id=self.root_id) + assert row.status == OxTask.Status.RUNNING + assert row.attempts == row.max_attempts + # A 1.2.0 lease: stored expiry and a bumped epoch, not the + # NULL-expiry fallback the existing crash tests reap. + assert row.lease_expires_at is not None + assert row.lease_epoch >= 1 + self.claimed_epoch = row.lease_epoch + + def lose_lease(self): + if self.how == "renewals_fail": + deadline = time.monotonic() + WAIT + while OxTask.objects.get(id=self.root_id).status == OxTask.Status.RUNNING: + assert time.monotonic() < deadline, ( + "the worker's reaper never took the row" + ) + time.sleep(0.05) + assert self.renewals_refused >= 1 + else: + assert actions.expire_lease(self.root_id) is True + assert self.factory().reap() == 1 + row = OxTask.objects.get(id=self.root_id) + assert row.status == OxTask.Status.LOST + assert self.thread.is_alive(), "the execution must outlive its row's LOST" + self.lost_epoch = row.lease_epoch + + def finish(self): + self.ctl["release"].set() + if self.how == "renewals_fail": + # run() drains the in-flight attempt, outcome write included, + # before it returns. + self.executor.request_stop() + self.thread.join(WAIT) + assert not self.thread.is_alive() + assert self.errors == [] + assert self.ctl["released"] is True, "the body timed out rather than finishing" + + def close(self): + self.ctl["release"].set() + if self.executor is not None: + self.executor.request_stop() + if self.thread is not None: + self.thread.join(WAIT) + + +def chain(name, step_task, stalling_task): + """root -> child -> grandchild, with both callbacks.""" + flow = workflows.create( + name, + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + root = flow.add("root", stalling_task, "root") + flow.add("child", step_task, "child", depends_on="root") + flow.add("grandchild", step_task, "grandchild", depends_on="child") + flow.seal() + return flow, root + + +def _timestamps(model, *, keep=()): + return [ + field.name + for field in model._meta.concrete_fields + if isinstance(field, DateTimeField) and field.name not in keep + ] + + +def age_every_timestamp(flow): + """ + Move every timestamp on this workflow's rows AGED_BY into the past, in + the database: the node task rows (all but run_after, which decides + claims), the workflow row, and its node and edge rows. NULL stays + NULL. Nothing but timestamps is written. + """ + task_ids = list(flow.node_ids().values()) + targets = ( + (OxTask, {"id__in": task_ids}, ("run_after",)), + (OxpullWorkflow, {"pk": flow.id}, ()), + (OxpullWorkflowNode, {"workflow_id": flow.id}, ()), + (OxpullWorkflowEdge, {"workflow_id": flow.id}, ()), + ) + for model, match, keep in targets: + rows = model.objects.filter(**match) + fields = _timestamps(model, keep=keep) + assert rows.exists() and fields, model + rows.update(**{name: F(name) - AGED_BY for name in fields}) + + +def reconciler_tick(factory): + """ + One reconciler tick, run the way a SCHEDULES entry runs it: enqueued + as the task, then claimed and executed by a worker. + """ + result = workflows.reconcile.enqueue() + runner = factory() + deadline = time.monotonic() + WAIT + while OxTask.objects.get(id=result.id).status in ( + OxTask.Status.READY, + OxTask.Status.RUNNING, + ): + assert time.monotonic() < deadline, "no worker ran the reconciler" + if not runner.run_once(): + time.sleep(0.01) + row = OxTask.objects.get(id=result.id) + assert row.status == OxTask.Status.SUCCESSFUL, row.errors + return row.return_value + + +def settle(factory, flow, at_least=3, at_most=8): + """ + Give the late success every chance to take effect: reconciler ticks + and drains until the workflow completes, then one more of each, so a + workflow that reopens gets the chance to complete and call back again. + """ + for done in range(1, at_most + 1): + reconciler_tick(factory) + drain(factory()) + if done >= at_least and stored(flow).status == OxpullWorkflow.Status.COMPLETE: + break + reconciler_tick(factory) + drain(factory()) + + +def node_rows(flow): + ids = flow.node_ids() + rows = {row.id: row for row in OxTask.objects.filter(id__in=list(ids.values()))} + return {name: rows[tid] for name, tid in ids.items()} + + +def callback_rows(label): + """Every record_callback task row enqueued with this label.""" + return [ + row + for row in OxTask.objects.filter(task_path=tasks.record_callback.module_path) + if row.args[:1] == [label] + ] + + +def describe(flow): + nodes = { + name: ( + f"{row.status} attempts={row.attempts} epoch={row.lease_epoch} " + f"run_after={row.run_after and row.run_after.isoformat()}" + ) + for name, row in node_rows(flow).items() + } + row = stored(flow) + fields = ("status", "failed_count", "cancelled_count") + return { + "nodes": nodes, + "workflow": {field: getattr(row, field, None) for field in fields}, + "callbacks enqueued": { + label: len(callback_rows(label)) for label in ("complete", "success") + }, + } + + +def held_problems(flow): + """What the workflow must never show while root is LOST.""" + rows = node_rows(flow) + problems = [] + if rows["root"].status != OxTask.Status.LOST: + problems.append( + f"root is {rows['root'].status}, although only its own " + "still-running execution may settle it" + ) + for name in ("child", "grandchild"): + row = rows[name] + if row.status in FINISHED: + problems.append( + f"{name!r} is {row.status}; it must wait for its upstream " + "to report, not finish" + ) + if row.status == OxTask.Status.RUNNING or row.attempts: + problems.append( + f"{name!r} started (status={row.status}, " + f"attempts={row.attempts}) before its upstream reported" + ) + if stored(flow).status == OxpullWorkflow.Status.COMPLETE: + problems.append("the workflow completed") + for label in ("complete", "success"): + enqueued = len(callback_rows(label)) + if enqueued: + problems.append(f"on_{label} was enqueued {enqueued} time(s)") + return problems + + +def settled_problems(flow): + """What must hold once the late success has been given every chance.""" + rows = node_rows(flow) + problems = [ + f"{name!r} ended {row.status}; its upstream succeeded, so it must " + "have run and succeeded" + for name, row in rows.items() + if row.status != OxTask.Status.SUCCESSFUL + ] + ran = [label for label in tasks.STATE.get("order", []) if label in rows] + if ran != ["root", "child", "grandchild"]: + problems.append(f"the nodes ran as {ran}, not once each in order") + row = stored(flow) + if row.status != OxpullWorkflow.Status.COMPLETE: + problems.append(f"the workflow did not complete: {row.status}") + elif row.failed_count or row.cancelled_count: + problems.append( + f"COMPLETE with failed_count={row.failed_count} and " + f"cancelled_count={row.cancelled_count}, although nothing failed " + "and nothing needed cancelling" + ) + for label in ("complete", "success"): + enqueued = len(callback_rows(label)) + if enqueued != 1: + problems.append( + f"on_{label} was enqueued {enqueued} time(s) in one " + "generation; it must be enqueued exactly once" + ) + executed = tasks.STATE.get(label, 0) + if executed != 1: + problems.append( + f"on_{label} ran {executed} time(s); it must run exactly once" + ) + return problems + + +def _scenario(how, factory, step_task, stalling_task, name, clock): + flow, root = chain(name, step_task, stalling_task) + straggler = Straggler(factory, root.id, "root", how) + problems: list[str] = [] + seen: set[str] = set() + passes: list[str] = [] + + def hold_pass(when): + completed = reconciler_tick(factory) + passes.append( + f"pass {when} completed {completed} workflow(s): {describe(flow)}" + ) + # Each rule reported once, at the first pass that broke it. + for problem in held_problems(flow): + if problem not in seen: + seen.add(problem) + problems.append(f"while root is LOST, after the pass {when}: {problem}") + + try: + straggler.start() + straggler.lose_lease() + for when, ahead in HOLD_PASSES: + clock.ahead = ahead + hold_pass(when) + clock.ahead = timedelta(0) + age_every_timestamp(flow) + assert OxTask.objects.get(id=root.id).status == OxTask.Status.LOST + hold_pass("at real time with the workflow's timestamps 400 days in the past") + straggler.finish() + finally: + clock.ahead = timedelta(0) + straggler.close() + after_write = describe(flow) + settle(factory, flow) + problems.extend( + f"after the late SUCCESSFUL and settling: {problem}" + for problem in settled_problems(flow) + ) + report = "\n".join( + [ + f"lease lost via {how}", + *problems, + ( + f"epoch at claim={straggler.claimed_epoch}, " + f"at LOST={straggler.lost_epoch}, " + f"renewals refused={straggler.renewals_refused}" + ), + *passes, + f"after the straggler's write: {after_write}", + f"after settling: {describe(flow)}", + ( + f"root's body ran {straggler.ctl['calls']} time(s); " + f"order={tasks.STATE.get('order')}; " + f"on_complete ran {tasks.STATE.get('complete', 0)}, " + f"on_success ran {tasks.STATE.get('success', 0)}" + ), + ] + ) + return problems, report + + +@pytest.mark.xfail( + strict=True, + reason=( + "When a node is marked LOST while its execution is still running, " + "the reconciler discards the nodes downstream of it and completes the " + "workflow instead of holding them until that execution reports. The " + "late success then lands, and the discarded nodes never run." + ), +) +@pytest.mark.parametrize("how", LEASE_LOSS) +def test_a_late_success_over_lost_never_leaves_its_cone_discarded(how, clock): + problems, report = _scenario( + how, flow_worker, tasks.step, stalling_step, f"late-success-{how}", clock + ) + assert not problems, report + + +@pytest.mark.xfail( + strict=True, + reason=( + "On a backend that defers blocked nodes, when a node is marked LOST " + "while its execution is still running, the reconciler discards the " + "nodes downstream of it and completes the workflow instead of holding " + "them until that execution reports. The late success then lands, and " + "there is nothing left for it to release." + ), +) +@pytest.mark.parametrize("how", LEASE_LOSS) +def test_a_late_success_over_lost_never_leaves_its_deferred_cone_discarded(how, clock): + """ + On a backend that defers blocked nodes, the downstream nodes wait + behind a far-future run_after, and the late SUCCESSFUL releases them + through the worker's outcome write. They must still be there to + release. + """ + problems, report = _scenario( + how, + deferring_worker, + tasks.deferred_step, + stalling_deferred_step, + f"late-deferred-success-{how}", + clock, + ) + assert not problems, report diff --git a/tests_oxpull/test_workflows_hazard_mariadb_after_a_server_upgrade_from_10_6.py b/tests_oxpull/test_workflows_hazard_mariadb_after_a_server_upgrade_from_10_6.py new file mode 100644 index 0000000..74bc4d8 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mariadb_after_a_server_upgrade_from_10_6.py @@ -0,0 +1,275 @@ +""" +MariaDB, after a server upgrade from 10.6 to 10.7 or later. + +Django picks a UUIDField's column type from the server version when a +migration runs: char(32) below MariaDB 10.7, native uuid from 10.7 on. It +never alters an existing column. It also picks how to bind a UUID +parameter from the server version, but it does that on every connection: +hex below 10.7, a hyphenated string from 10.7 on. + +A project that migrated django_ox on 10.6 and then upgraded the server +keeps django_ox_oxtask.id as char(32). `migrate oxpull` run after the +upgrade creates the workflow edge and node columns as uuid. The hazard as +written asks what the workflow joins then do across the two types. + +These tests only mean something against a database built that way: +mariadb:10.6 on a volume, `migrate django_ox` and `migrate oxpull 0003`, +then mariadb:10.11 on the same volume, `mariadb-upgrade`, then +`migrate oxpull`, with a settings module that points at that server. +Run with --reuse-db, or pytest-django replaces that database with a fresh +one where every column is uuid. Controls where every test should pass: a +fresh 10.11 database and a fresh 10.6 server. The suite's other legs skip +the file, because MySQL has no native uuid type and PostgreSQL and SQLite +use one type on both sides. + +Two of these tests hold UUID binding to hex, which is how a pre-10.7 +connection wrote its rows (see `hex_binding`). Each one says why. +""" + +import json +import re +from contextlib import contextmanager + +import pytest +from django.db import connection +from django.tasks import default_task_backend + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import flow_worker, reconcile, stored +from .test_workflows_cost import claim_candidates + +pytestmark = pytest.mark.skipif( + connection.vendor != "mysql", + reason="MariaDB only: the hazard needs native uuid beside char(32)", +) + +BLOCKED = 400 + + +@pytest.fixture(autouse=True) +def mariadb(): + if not connection.mysql_is_mariadb: + pytest.skip("MariaDB only: MySQL has no native uuid column type") + + +def column_type(table: str, column: str) -> str: + with connection.cursor() as cursor: + cursor.execute( + "SELECT column_type FROM information_schema.columns" + " WHERE table_schema = DATABASE() AND table_name = %s" + " AND column_name = %s", + [table, column], + ) + return cursor.fetchone()[0] + + +def join_columns() -> dict[str, str]: + """Every column the gate, the counts and the release compare.""" + edge = OxpullWorkflowEdge._meta + node = OxpullWorkflowNode._meta + return { + "oxtask.id": column_type(OxTask._meta.db_table, OxTask._meta.pk.column), + "edge.upstream_task_id": column_type(edge.db_table, "upstream_task_id"), + "edge.downstream_task_id": column_type(edge.db_table, "downstream_task_id"), + "node.task_id": column_type(node.db_table, "task_id"), + } + + +def premise() -> str: + return ( + f"server {connection.mysql_version}, " + f"native uuid binding {connection.features.has_native_uuid_field}, " + f"columns {join_columns()}" + ) + + +@contextmanager +def hex_binding(): + """ + Bind UUID parameters as 32 hex digits, as Django does below MariaDB 10.7. + + This is a shortcut. The production sequence that writes rows this way + is django-ox enqueueing on 10.6 before the upgrade. + It is also how every write would + bind if django-ox adopted Django 5.0's documented remedy for existing + UUIDFields on MariaDB 10.7+ (a char(32) subclass). That remedy is the + only way to get workflow rows beside char(32) task ids on 10.7+, + because an unpatched connection cannot insert a task at all. + """ + features = connection.features + original = features.has_native_uuid_field + features.__dict__["has_native_uuid_field"] = False + try: + yield + finally: + features.__dict__["has_native_uuid_field"] = original + + +def root_and_child(name: str): + """ + root -> child, with priorities that put the child first in claim order. + The only thing that can hold the child back is the gate. + """ + flow = workflows.create(name) + root = flow.add("root", tasks.step.using(priority=0), "root") + child = flow.add("child", tasks.step.using(priority=10), "child", depends_on="root") + flow.seal() + return flow, root, child + + +def assert_child_is_held(flow, root, child) -> None: + claimed = flow_worker().claim_one() + assert claimed is not None, premise() + # TaskResult.id is a str; OxTask.pk is a UUID. + assert str(claimed.pk) == str(root.id), ( + f"claimed {claimed.pk} ({claimed.args}), expected the root; {premise()}" + ) + assert flow_worker().claim_one() is None, premise() + counts = workflows._node_task_counts(flow.id, "default") + assert counts == {"RUNNING": 1, "READY": 1}, f"{counts}; {premise()}" + assert reconcile() == 0, premise() + assert stored(flow).status == OxpullWorkflow.Status.SEALED, premise() + + +# -- real path: the schema and the public API ------------------------------ + + +@pytest.mark.django_db +def test_the_gate_join_columns_have_one_type(): + """ + The root of the hazard, read from the schema that the real migrate + commands left behind. Every column the gate compares should have one type. + """ + columns = join_columns() + assert len(set(columns.values())) == 1, premise() + + +@pytest.mark.django_db +def test_a_task_can_be_enqueued_after_the_upgrade(): + """ + Below the workflows code: the `django_ox.backend.OxBackend`'s own enqueue. On the + upgraded schema Django binds a 36-character id into char(32), and strict + mode refuses it with error 1406. + """ + result = tasks.plain_echo.enqueue("after-the-upgrade") + assert default_task_backend.get_result(result.id).status == "READY", premise() + + +@pytest.mark.django_db +def test_a_workflow_built_after_the_upgrade_holds_its_child_behind_a_running_root(): + """ + The upgrade scenario, driven through the public API: create, add, + seal, and a real claim sets the root RUNNING. The child must stay + unclaimable, the counts must read one RUNNING and one READY, and the + reconciler must leave the workflow SEALED. + """ + flow, root, child = root_and_child("after-upgrade") + assert_child_is_held(flow, root, child) + + +# -- shortcut: rows written with pre-10.7 binding ---------------------------- + + +@pytest.mark.django_db +def test_a_task_enqueued_before_the_upgrade_is_still_found_by_id(): + """ + Written with the binding the 10.6 server used (hex), read back with the + binding the upgraded server uses (hyphenated). get_result on a task + enqueued before a database upgrade should still find it. + """ + with hex_binding(): + result = tasks.plain_echo.enqueue("before-the-upgrade") + assert OxTask.objects.count() == 1, premise() + assert default_task_backend.get_result(result.id).status == "READY", premise() + + +@pytest.mark.django_db +def test_the_gate_compares_char32_task_ids_with_uuid_edges_by_value(): + """ + The hazard's outcome (b): if MariaDB compared the two columns as strings, + a hyphenated uuid would never equal bare hex, every upstream would read as + absent and so satisfied, and the child would be claimed first. + + Binding is held to hex for the whole test, so that the enqueue failure + above cannot hide the answer. Nothing inside the gate, the counts or the + reconciler's completion check binds a task id: they compare columns with + columns, and that comparison is the question. + """ + with hex_binding(): + flow, root, child = root_and_child("by-value") + assert_child_is_held(flow, root, child) + + +def _plan_tables(node, out: list[dict]) -> list[dict]: + if isinstance(node, dict): + if "table_name" in node: + out.append(node) + for value in node.values(): + _plan_tables(value, out) + elif isinstance(node, list): + for value in node: + _plan_tables(value, out) + return out + + +@pytest.mark.django_db(transaction=True) +def test_the_gated_claim_probes_each_upstream_by_primary_key(): + """ + The hazard's outcome (a): the comparison is right, but the inner lookup + `U.id = edge.upstream_task_id` casts the char(32) side, so it cannot use + PRIMARY and scans django_ox_oxtask once for every edge the claim checks. + + transaction=True because ANALYZE TABLE commits implicitly on MariaDB. + The rows are written with hex binding, for the reason `hex_binding` gives. + """ + with hex_binding(): + flow = workflows.create("cost") + flow.add("root", tasks.step.using(priority=0), "root") + for i in range(BLOCKED): + flow.add( + f"b{i:04d}", + tasks.step.using(priority=10), + f"b{i:04d}", + depends_on="root", + ) + flow.seal() + with connection.cursor() as cursor: + cursor.execute( + "ANALYZE TABLE django_ox_oxtask, oxpull_oxpullworkflowedge," + " oxpull_oxpullworkflownode" + ) + cursor.fetchall() + + queryset = claim_candidates(gated=True) + sql = str(queryset.query) + # The outer FROM is followed by WHERE; the gate's inner lookup has an alias. + upstream_alias = re.search(r"FROM `django_ox_oxtask` (?!WHERE\b)(\w+)", sql).group( + 1 + ) + plan = json.loads(queryset.explain(format="json", analyze=True)) + if isinstance(plan, str): + # Django's MySQL backend hands MariaDB's JSON back quoted as a string. + plan = json.loads(plan) + tables = _plan_tables(plan, []) + upstream = [t for t in tables if t["table_name"] == upstream_alias] + assert len(upstream) == 1, tables + probe = upstream[0] + read = probe.get("r_loops", 0) * probe.get("r_rows", 0) + summary = { + t["table_name"]: ( + t.get("access_type"), + t.get("key"), + t.get("r_loops"), + t.get("r_rows"), + ) + for t in tables + } + assert (probe.get("access_type"), probe.get("key")) == ("eq_ref", "PRIMARY"), ( + f"upstream probe {upstream_alias}: {probe.get('access_type')} on " + f"{probe.get('key')}, ~{read:.0f} rows read over {BLOCKED} blocked nodes; " + f"plan {summary}; {premise()}" + ) diff --git a/tests_oxpull/test_workflows_hazard_mixed_fleet_migration.py b/tests_oxpull/test_workflows_hazard_mixed_fleet_migration.py new file mode 100644 index 0000000..86bbd77 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mixed_fleet_migration.py @@ -0,0 +1,896 @@ +""" +A mixed fleet, and migration order. + +A workflow node that depends on something unfinished can be held back in two +places, and this file asserts each one separately. + + * In the node's own row. A blocked node waits in django-ox's WAITING + status. Every django-ox claim reads only READY rows, so every claimer + honours it: the `django_ox.worker.Worker`, oxpull 1.2.0's worker, and this + version's. A worker that knows nothing about workflows never releases the + node; the reconciler does, one tick later. + * In the claim. OxpullWorker adds a dependency check to its claim. That is + the only thing holding a node whose row has been put back to READY by + something other than oxpull's release: django-ox's retry writes that + shape on the rows it moves, and so does an application's own UPDATE. + +A django-ox claim filters on status, run_after and queue name. It never reads +the backend alias, and a worker started without --queues takes every queue its +backend lists (every queue at all, when that list is empty). So workers +configured for other backends claim workflow nodes too. + +Part (i-a), the row. Claimers that can never check dependencies (the +`django_ox.worker.Worker` in process, and oxpull 1.2.0 as a real ox_worker +process from the released source) run the same DAG, with claim order the exact +reverse of dependency order. No node may run before every node it depends on. +That holds whether or not the caller passed its own run_after: one already due, +one that comes due while the parent is still unfinished, and one an hour out. +Every blocked row is WAITING before anything runs, and a requested time still +in the future survives the release. The in-process `django_ox.worker.Worker`'s +claim adds no condition of its own (claim_filter_q() is None), which each of +these tests checks first, so nothing but the row can be what holds the node. +Both workflow backends hold the row the same way; "flows_deferred" carries the +removed DEFER_BLOCKED_NODES option, which changes nothing. + +Part (i-b), the claim. Every `oxpull.worker.OxpullWorker` on a database where +some backend declares workflows checks dependencies in its claim, whatever its +own backend declares and whatever queues it lists, including after a deploy has +moved the workflow backend off the queue its waiting nodes carry. The blocked +rows are put back to READY with no run_after first, so only the claim can hold +them. The workflow backend's own worker is the control and passes. + +Part (ii) rewinds oxpull to the migration before workflows (0003) with +tests_oxpull.test_upgrade's helpers. It asks what an operator sees when a workflow +worker, a plain `oxpull.worker.OxpullWorker`, or the new prune runs before the +workflow migration (0004) is applied. The plain `oxpull.worker.OxpullWorker`'s +case passes today, and stays as a guard: once every +`oxpull.worker.OxpullWorker` checks dependencies, its claim meets the missing +edge table too. + +Each test with an expected-failure marker asserts the correct behaviour, and +still fails. +""" + +from __future__ import annotations + +import contextlib +import copy +import io +import os +import re +import signal +import subprocess +import sys +import time +from datetime import timedelta +from pathlib import Path + +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import DatabaseError, connection +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullUniqueLock, OxpullWorkflowEdge +from oxpull.worker import OxpullWorker + +from . import tasks +from ._historical import extract_released_oxpull +from .subprocesses import django_ox_path +from .test_upgrade import _latest_migration, _migrate +from .test_workflows import drain, flow_worker, free_worker, order, reconcile + +REPO_ROOT = Path(__file__).resolve().parent.parent +BEFORE_WORKFLOWS = "0003_oxpullratewindow" + +NODES = ["extract", "transform-a", "transform-b", "load"] +EDGES = [ + ("extract", "transform-a"), + ("extract", "transform-b"), + ("transform-a", "load"), + ("transform-b", "load"), +] +BLOCKED_NODES = ["transform-a", "transform-b", "load"] +NODE_TASKS = {"flows": tasks.step, "flows_deferred": tasks.deferred_step} + +# Drain-then-reconcile rounds a DAG gets before the test reads what ran. +ROUNDS = 10 + +# How long the pre-0004 worker is watched before anything else happens. +STALL_SECONDS = 5.0 + + +def inverted_diamond(name, task, **using): + """ + tests_oxpull.test_workflows.diamond for any node task: priorities make claim + order the exact reverse of dependency order, so only a block can stop + `load` from running first. + """ + flow = workflows.create(name) + flow.add("extract", task.using(priority=0, **using), "extract") + for branch in ("transform-a", "transform-b"): + flow.add(branch, task.using(priority=5, **using), branch, depends_on="extract") + flow.add( + "load", + task.using(priority=10, **using), + "load", + depends_on=["transform-a", "transform-b"], + ) + flow.seal() + return flow + + +def out_of_order(sequence, edges=EDGES): + """Edges whose downstream was claimed before, or without, its upstream.""" + first: dict[str, int] = {} + for index, label in enumerate(sequence): + first.setdefault(label, index) + return [ + (upstream, downstream) + for upstream, downstream in edges + if downstream in first + and (upstream not in first or first[upstream] > first[downstream]) + ] + + +def drain_and_reconcile(worker, expected): + """ + Drain, then one reconciler pass as the schedule would run it, until every + label in `expected` has run. The pass releases what a worker that knows + nothing about workflows never releases. + """ + for _ in range(ROUNDS): + drain(worker) + reconcile() + if set(expected) <= set(order()): + break + return order() + + +def run_after_of(ids): + """Each node's run_after as the database holds it now, for messages.""" + rows = dict( + OxTask.objects.filter(pk__in=ids.values()).values_list("pk", "run_after") + ) + return { + name: (rows[task_id].isoformat() if rows.get(task_id) else None) + for name, task_id in ids.items() + } + + +def _check_ids(*aliases): + from django.tasks import task_backends + + return {alias: [m.id for m in task_backends[alias].check()] for alias in aliases} + + +def premise(condition, message): + """ + A fact the test stands on, not the behaviour under test. pytest.fail is + not an AssertionError, and strict markers that expect an assertion + never absorb it. + """ + if not condition: + pytest.fail(f"premise: {message}") + + +def require_the_free_worker_checks_nothing(): + """Only the row can hold a node back from the `django_ox.worker.Worker`.""" + condition = free_worker().claim_filter_q() + premise( + condition is None, + f"the `django_ox.worker.Worker`'s claim carries a condition of its own: " + f"{condition!r}", + ) + + +# -- (i-a) the row holds the node: claimers that never check dependencies -- + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "backend", + ["flows", "flows_deferred"], +) +def test_the_free_worker_runs_no_node_before_its_parent(backend): + """ + The `django_ox.worker.Worker` is not this package's code and can never check + dependencies, so only the node's own row can hold a node back from it. + """ + require_the_free_worker_checks_nothing() + flow = inverted_diamond(f"free-worker-{backend}", NODE_TASKS[backend]) + enqueued = run_after_of(flow.node_ids()) + + ran = drain_and_reconcile(free_worker(), NODES) + + assert out_of_order(ran) == [], ( + f"the `django_ox.worker.Worker` on backend {backend!r} ran {ran}; " + f"run_after of each node as enqueued: {enqueued}" + ) + assert sorted(ran) == sorted(NODES), ( + f"ran {ran} after {ROUNDS} rounds; run_after now: " + f"{run_after_of(flow.node_ids())}" + ) + + +OWN_TIME_EDGES = [("parent", "due"), ("parent", "soon"), ("parent", "scheduled")] +OWN_TIME_BLOCKED = ["due", "soon", "scheduled"] + +# How far out "soon" asks to run. The test's clock stands still while the +# nodes are added and is moved past this before anything is drained, so the +# figure is never waited out. It is minutes rather than seconds so that a +# hold deciding by the database's clock, which the test does not move, would +# still read `soon` as not yet due however long the adds take. +SOON = timedelta(minutes=10) + + +class HeldClock: + """ + django.utils.timezone.now, held at one instant until the test moves it. + + Replaced on the module, so every caller that looks it up when it calls + it reads the held instant: add(), the release, and the run_after cutoff + of django-ox's claim, which is the worker's clock on every database and + never the server's. django-ox's leases run on the database clock, which + this does not move, so moving it neither expires nor extends a lease. + """ + + def __init__(self, monkeypatch): + self.at = timezone.now() + monkeypatch.setattr(timezone, "now", lambda: self.at) + + +@pytest.fixture +def held_clock(monkeypatch): + return HeldClock(monkeypatch) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "backend", + ["flows", "flows_deferred"], +) +def test_a_node_with_its_own_run_after_waits_for_its_parent_and_keeps_its_time( + backend, held_clock +): + """ + Three children of one parent, each given the caller's own run_after: + `due`, already due when it is added; `soon`, ten minutes out when it is + added and due by the time anything runs; and `scheduled`, an hour away. + While the parent is unfinished all three are blocked, and the row must + hold them like any other node, whatever time the caller asked for and + whether or not that time had passed when add() ran: before anything + runs, each of their rows is WAITING, which no claim reads. Once the + parent succeeds, `due` and `soon` run, and `scheduled` keeps the time the + caller asked for instead of losing it to the release. + + A hold applied only to nodes whose own time has already come when they + are added lets `soon` through: its time passes while the parent waits. + """ + require_the_free_worker_checks_nothing() + task = NODE_TASKS[backend] + now = held_clock.at + due = now - timedelta(hours=1) + soon = now + SOON + scheduled = now + timedelta(hours=1) + flow = workflows.create(f"own-run-after-{backend}") + flow.add("parent", task.using(priority=0), "parent") + flow.add("due", task.using(priority=10, run_after=due), "due", depends_on="parent") + flow.add( + "soon", task.using(priority=10, run_after=soon), "soon", depends_on="parent" + ) + flow.add( + "scheduled", + task.using(priority=10, run_after=scheduled), + "scheduled", + depends_on="parent", + ) + flow.seal() + ids = flow.node_ids() + enqueued = run_after_of(ids) + rows = dict( + OxTask.objects.filter( + pk__in=[ids[name] for name in OWN_TIME_BLOCKED] + ).values_list("pk", "status") + ) + unfenced = { + name: rows[ids[name]] + for name in OWN_TIME_BLOCKED + if rows[ids[name]] != OxTask.Status.WAITING + } + # soon's time passes while the parent is still unfinished. Every add() + # above read the clock at `now`, before soon; every claim below reads it + # after. + held_clock.at = soon + timedelta(seconds=1) + + ran = drain_and_reconcile(free_worker(), ["parent", "due", "soon"]) + + assert out_of_order(ran, OWN_TIME_EDGES) == [], ( + f"the `django_ox.worker.Worker` on backend {backend!r} ran {ran}; " + f"the caller asked for due={due.isoformat()}, soon={soon.isoformat()} " + f"and scheduled={scheduled.isoformat()}; run_after as enqueued: {enqueued}" + ) + assert sorted(ran) == ["due", "parent", "soon"], ( + f"expected parent, due and soon to run and scheduled to wait for " + f"{scheduled.isoformat()}; ran {ran}; run_after now: {run_after_of(ids)}" + ) + kept = OxTask.objects.get(pk=ids["scheduled"]).run_after + assert kept is not None and abs(kept - scheduled) < timedelta(seconds=1), ( + f"scheduled asked for run_after={scheduled.isoformat()} and holds " + f"{kept} after its parent succeeded" + ) + assert unfenced == {}, ( + f"while their parent was unfinished, these blocked rows were not " + f"WAITING, so nothing in the row held them back: {unfenced}" + ) + + +# -- (i-a) the row holds the node: oxpull 1.2.0 as a real ox_worker process -- + + +@pytest.fixture(scope="module") +def released_oxpull(tmp_path_factory): + """The released package's source, read from the vendored archive.""" + root = tmp_path_factory.mktemp("oxpull-released") + return extract_released_oxpull(root) + + +def process_env(tmp_path, src, *, schedule_reconciler): + """ + The environment for a separate process on this session's test + database: `src` first on the path, and the session's settings as an + operator would have them. That means DEBUG django_ox logs (the + "Claimed task" line is the claim order). `flows_ungated` is a + deliberate oxpull.E007 fixture that stops `ox_worker` at its system + checks. Its alias stays, because tests_oxpull.tasks declares a task on it, but + it stops declaring WORKFLOWS. Optionally the workflow reconciler is + scheduled, so the checks are otherwise silent. The session's settings + schedule it already, and oxpull 1.2.0 cannot import it, so without + the option it is taken out. + """ + schedule = ( + "TASKS['default']['OPTIONS']['SCHEDULES']['oxpull-workflow-reconcile'] = " + "{'task': 'oxpull.workflows.reconcile', 'cron': '* * * * *'}\n" + if schedule_reconciler + else "TASKS['default']['OPTIONS']['SCHEDULES'].pop(" + "'oxpull-workflow-reconcile', None)\n" + ) + (tmp_path / "fleet_settings.py").write_text( + "import copy\n" + f"from {settings.SETTINGS_MODULE} import * # noqa: F403\n" + "TASKS = copy.deepcopy(TASKS)\n" + "TASKS['flows_ungated']['OPTIONS']['WORKFLOWS'] = False\n" + f"{schedule}" + "LOGGING = {\n" + " 'version': 1,\n" + " 'disable_existing_loggers': False,\n" + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},\n" + " 'loggers': {'django_ox': {'handlers': ['console'], 'level': 'DEBUG'}},\n" + "}\n" + ) + name = connection.settings_dict["NAME"] + if connection.vendor == "sqlite": + name = Path(name).resolve() + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "fleet_settings", + # The current source runs against this process's django-ox. A released + # oxpull runs against the django-ox the interpreter has installed. + "PYTHONPATH": os.pathsep.join( + [str(src)] + + ([django_ox_path()] if src == REPO_ROOT / "src" else []) + + [str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(name), + } + + +def run_python(env, *args): + return subprocess.run( # noqa: S603 + [sys.executable, *args], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def assert_imports(env, src, version): + """Prove which oxpull the process runs before believing anything it does.""" + probe = run_python( + env, "-c", "import oxpull; print(oxpull.__version__); print(oxpull.__file__)" + ) + lines = probe.stdout.split() + assert probe.returncode == 0, probe.stderr + assert lines[0] == version, lines + assert lines[1].startswith(str(src)), lines + + +def start_ox_worker(env, log, backend, interval): + return subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "django", + "ox_worker", + "--backend", + backend, + "--interval", + str(interval), + ], + env=env, + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + + +def stop(process): + """SIGTERM, as a deploy does; the worker drains and exits.""" + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return process.returncode + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "backend", + ["flows", "flows_deferred"], +) +def test_a_released_oxpull_worker_process_runs_no_node_before_its_parent( + backend, released_oxpull, tmp_path +): + """ + oxpull 1.2.0 does not know the WORKFLOWS key, so its claim never checks + dependencies. The test process plays the reconciler's schedule. + """ + env = process_env(tmp_path, released_oxpull, schedule_reconciler=False) + assert_imports(env, released_oxpull, "1.2.0") + check = run_python(env, "-m", "django", "check") + + flow = inverted_diamond(f"released-{backend}", NODE_TASKS[backend]) + ids = flow.node_ids() + enqueued = run_after_of(ids) + label_of = {str(task_id): name for name, task_id in ids.items()} + + log_path = tmp_path / "ox_worker.log" + with log_path.open("w") as log: + worker = start_ox_worker(env, log, backend, interval=0.05) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and worker.poll() is None: + reconcile() + unfinished = OxTask.objects.filter(pk__in=ids.values()).exclude( + status=OxTask.Status.SUCCESSFUL + ) + if not unfinished.exists(): + break + time.sleep(0.1) + finally: + code = stop(worker) + text = log_path.read_text() + claimed = [ + label_of[task_id] + for task_id in re.findall(r"Claimed task id=([0-9a-f-]{36}) ", text) + if task_id in label_of + ] + + assert out_of_order(claimed) == [], ( + f"oxpull 1.2.0 `ox_worker --backend {backend}` claimed {claimed}; " + f"run_after of each node as enqueued: {enqueued}; `manage.py check` " + f"in that process: {(check.stdout + check.stderr).strip()!r}" + ) + assert sorted(claimed) == sorted(NODES), f"exit {code}\n{text[-4000:]}" + + +# -- (i-b) the claim holds the node: every `oxpull.worker.OxpullWorker` on the database + +PIPELINES = "pipelines" + + +def pipelines_tasks(): + """ + The suite's TASKS with every backend that declares WORKFLOWS moved to + its own queue, "pipelines", plus a plain `oxpull.backend.OxpullBackend`, + "catchall", whose empty QUEUES accepts every queue name. After the move no + plain alias's queue list shares a name with any workflow backend's, so a + worker that checks dependencies only where queue lists overlap does not + check here. + """ + configured = copy.deepcopy(settings.TASKS) + for config in configured.values(): + if config.get("OPTIONS", {}).get("WORKFLOWS") is True: + config["QUEUES"] = [PIPELINES] + configured["catchall"] = { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": [], + "OPTIONS": {"WORKER_CLASS": "oxpull.worker.OxpullWorker"}, + } + return configured + + +def oxpull_worker(alias, **kwargs): + """This version's OxpullWorker on `alias`, as its ox_worker would build it.""" + return OxpullWorker( + backend_alias=alias, backoff_initial=0, poll_interval=0.01, **kwargs + ) + + +# claimer -> (TASKS for the test or None for the suite's, the nodes' queue, +# the worker). None of the plain aliases declares WORKFLOWS; every one shares +# the database with "flows", which does. +CLAIMERS = { + "control-the-workflow-backends-own-worker": (None, "default", flow_worker), + "control-the-workflow-backends-own-worker-on-its-own-queue": ( + pipelines_tasks, + PIPELINES, + flow_worker, + ), + "plain-alias-sharing-the-queue-name": ( + None, + "default", + lambda: oxpull_worker("default"), + ), + "plain-alias-with-empty-queues": ( + pipelines_tasks, + PIPELINES, + lambda: oxpull_worker("catchall"), + ), + "plain-alias-started-with-queues-naming-the-workflow-queue": ( + pipelines_tasks, + PIPELINES, + lambda: oxpull_worker("default", queues=[PIPELINES]), + ), +} + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "claimer", + [ + "control-the-workflow-backends-own-worker", + "control-the-workflow-backends-own-worker-on-its-own-queue", + pytest.param( + "plain-alias-sharing-the-queue-name", + marks=pytest.mark.xfail( + strict=True, + reason=( + "A `oxpull.worker.OxpullWorker` on a backend without workflows " + "that shares a queue name with a workflow backend checks no " + "dependencies in its claim, so it runs a node before its upstream " + "once the node's row no longer holds it back" + ), + ), + ), + pytest.param( + "plain-alias-with-empty-queues", + marks=pytest.mark.xfail( + strict=True, + reason=( + "A `oxpull.worker.OxpullWorker` on a backend with an empty queue " + "list and no workflows checks no dependencies in its claim, so it " + "runs a node from a workflow backend's queue before its upstream " + "once the node's row no longer holds it back" + ), + ), + ), + pytest.param( + "plain-alias-started-with-queues-naming-the-workflow-queue", + marks=pytest.mark.xfail( + strict=True, + reason=( + "A `oxpull.worker.OxpullWorker` started with --queues naming a " + "workflow backend's queue, on a backend whose own queue list " + "shares nothing with it, checks no dependencies in its claim and " + "runs a node before its upstream once the node's row no longer " + "holds it back" + ), + ), + ), + ], +) +def test_every_oxpull_worker_on_the_database_holds_a_node_its_row_no_longer_holds( + claimer, +): + """ + The blocked nodes are put back to READY, the shape django-ox's retry + writes on the rows it moves and an application's own UPDATE writes. + Nothing in those rows holds them now, and `extract` has not run, so only + a dependency check in the claim can keep `load` from running first. No + reconciler pass runs, so nothing can put a block back. + """ + layout, queue, make_worker = CLAIMERS[claimer] + overridden = ( + override_settings(TASKS=layout()) if layout else contextlib.nullcontext() + ) + with overridden: + flow = inverted_diamond(f"row-cleared-{claimer}", tasks.step, queue_name=queue) + ids = flow.node_ids() + OxTask.objects.filter(pk__in=[ids[name] for name in BLOCKED_NODES]).update( + status=OxTask.Status.READY, run_after=None + ) + # The premise: every node is READY with no run_after, extract included. + rows = OxTask.objects.filter(pk__in=ids.values()) + assert {(row.status, row.run_after) for row in rows} == { + (OxTask.Status.READY, None) + } + + worker = make_worker() + drain(worker) + ran = order() + + assert out_of_order(ran) == [], ( + f"{claimer}: OxpullWorker(backend_alias={worker.backend.alias!r}, " + f"queues={worker.queues}) ran {ran}; check ids: " + f"{_check_ids(worker.backend.alias, 'flows')}" + ) + assert sorted(ran) == sorted(NODES), ran + + +@pytest.mark.xfail( + strict=True, + reason=( + "After the workflow backends move to their own queue, a " + "`oxpull.worker.OxpullWorker` on the old queue checks no dependencies in its " + "claim, so it claims the nodes still waiting there before their upstream once " + "their rows no longer hold them back" + ), +) +@pytest.mark.django_db +def test_an_oxpull_worker_holds_nodes_left_on_a_queue_the_workflow_backends_moved_off(): + """ + The nodes are enqueued under the suite's settings, on "default". Then a + deploy moves every workflow backend to its own queue, "pipelines", while + they wait. A plain `oxpull.worker.OxpullWorker` on "default" now lists no + queue that any workflow backend lists, and it still claims those nodes, + because a claim reads the row's queue name, not the backend that wrote it. + + Every `oxpull.worker.OxpullWorker` on the database checks dependencies, so + the only node this one may claim is `extract`, and while `extract` runs it + claims nothing else. A worker that checks only where its queue list meets a + workflow backend's current one does not check here, and fails. + + Claims are read, not executions: after the move "default" is not a queue + the workflow backend accepts, so executing one of these nodes would fail + task validation, which is a separate matter from the order of claims. + """ + flow = inverted_diamond("queue-moved", tasks.step, queue_name="default") + ids = flow.node_ids() + label_of = {str(task_id): name for name, task_id in ids.items()} + OxTask.objects.filter(pk__in=[ids[name] for name in BLOCKED_NODES]).update( + status=OxTask.Status.READY, run_after=None + ) + rows = OxTask.objects.filter(pk__in=ids.values()) + premise( + {(row.status, row.run_after, row.queue_name) for row in rows} + == {(OxTask.Status.READY, None, "default")}, + "every node is READY on 'default' with no run_after", + ) + + with override_settings(TASKS=pipelines_tasks()): + worker = oxpull_worker("default") + premise( + PIPELINES not in worker.queues and "default" in worker.queues, + f"the worker on 'default' lists {worker.queues}", + ) + claimed = [] + for _ in NODES: + task_row = worker.claim_one() + if task_row is None: + break + claimed.append(label_of[str(task_row.pk)]) + + assert claimed == ["extract"], ( + f"OxpullWorker(backend_alias='default', queues={worker.queues}) " + f"claimed {claimed} from 'default' after the workflow backends moved " + f"to {PIPELINES!r}; only extract has no unfinished upstream. Check " + f"ids: {_check_ids('default', 'flows')}" + ) + + +# -- (ii) migration order -------------------------------------------------- + + +@pytest.fixture +def before_workflows_migration(): + """oxpull at 0003 for one test, restored to the leaf even on failure.""" + latest = _latest_migration() + _migrate(BEFORE_WORKFLOWS) + try: + yield + finally: + _migrate(latest) + + +@pytest.mark.django_db(transaction=True) +def test_the_rewound_schema_has_no_workflow_tables(before_workflows_migration): + """Pins the premise of every test below.""" + tables = connection.introspection.table_names() + assert OxpullWorkflowEdge._meta.db_table not in tables + + +@pytest.mark.django_db(transaction=True) +def test_before_0004_a_claim_without_a_dependency_check_still_runs_the_queue( + before_workflows_migration, +): + """ + Control: the task row and the queue are fine at 0003, so the stall below + belongs to the claim that checks dependencies. The `django_ox.worker.Worker` + never checks them, whatever any backend declares. + """ + loose = tasks.step.enqueue("loose") + assert free_worker().run_once() is True + assert OxTask.objects.get(pk=loose.id).status == OxTask.Status.SUCCESSFUL + + +@pytest.mark.xfail( + strict=True, + reason=( + "Before the workflow migration (0004) is applied, a workflow worker's " + "claim raises a raw database error about the missing edge table " + "instead of claiming the plain task or saying to run migrate" + ), +) +@pytest.mark.django_db(transaction=True) +def test_before_0004_a_workflows_claim_returns_the_plain_task_or_names_the_migration( + before_workflows_migration, +): + """ + No workflow can exist without the edge table, so the check has nothing + to hold. The claim either returns the plain task or refuses in words + that say `migrate`. + """ + loose = tasks.step.enqueue("loose") + worker = flow_worker() + claimed = raised = None + try: + claimed = worker.claim_one() + except (DatabaseError, ImproperlyConfigured) as exc: + raised = exc + + if raised is None: + assert claimed is not None + assert str(claimed.pk) == loose.id + return + kind = f"{type(raised).__module__}.{type(raised).__qualname__}" + assert isinstance(raised, ImproperlyConfigured), f"{kind}: {raised}" + assert "migrate" in str(raised), f"{kind}: {raised}" + + +@pytest.mark.django_db(transaction=True) +def test_before_0004_a_plain_oxpull_worker_claims_the_plain_task_or_names_the_migration( + before_workflows_migration, +): + """ + Guard, green today. A `oxpull.worker.OxpullWorker` on a backend without + workflows, on the database where "flows" declares them. Once every + `oxpull.worker.OxpullWorker` there checks dependencies in its claim, this + claim meets the missing edge table exactly as the workflow backend's own + does above, and it must behave the same way: return the plain task, or + refuse in words that say `migrate`. Today it checks nothing and returns the + task. + """ + loose = tasks.step.enqueue("loose") + worker = oxpull_worker("default") + claimed = raised = None + try: + claimed = worker.claim_one() + except (DatabaseError, ImproperlyConfigured) as exc: + raised = exc + + if raised is None: + assert claimed is not None + assert str(claimed.pk) == loose.id + return + kind = f"{type(raised).__module__}.{type(raised).__qualname__}" + assert isinstance(raised, ImproperlyConfigured), f"{kind}: {raised}" + assert "migrate" in str(raised), f"{kind}: {raised}" + + +@pytest.mark.xfail( + strict=True, + reason=( + "Before the workflow migration (0004) is applied, a workflow ox_worker " + "stays up, runs no plain task and logs a database error every pass, " + "instead of running the task or exiting with a message naming migrate" + ), +) +@pytest.mark.django_db(transaction=True) +def test_before_0004_a_workflows_ox_worker_runs_plain_tasks_or_refuses_to_start( + before_workflows_migration, tmp_path +): + """ + The operator's view: the real `ox_worker` for the workflow backend, + this version's code, the schema one migration behind. Correct is either + the plain task running or the process exiting non-zero with `migrate` + in what it printed. Recorded along the way: how the loop behaves, what + it logs, and whether it recovers once `migrate` runs underneath it. + """ + src = REPO_ROOT / "src" + env = process_env(tmp_path, src, schedule_reconciler=True) + assert_imports(env, src, __import__("oxpull").__version__) + check = run_python(env, "-m", "django", "check") + loose = tasks.step.enqueue("loose") + + log_path = tmp_path / "ox_worker.log" + recovered_after = None + with log_path.open("w") as log: + worker = start_ox_worker(env, log, "flows", interval=0.2) + try: + deadline = time.monotonic() + STALL_SECONDS + while time.monotonic() < deadline and worker.poll() is None: + time.sleep(0.1) + alive = worker.poll() is None + status = OxTask.objects.get(pk=loose.id).status + stalled_text = log_path.read_text() + if alive and status != OxTask.Status.SUCCESSFUL: + # Apply the missing migrations under the running worker. + started = time.monotonic() + _migrate(_latest_migration()) + while time.monotonic() - started < 30: + row = OxTask.objects.get(pk=loose.id) + if row.status == OxTask.Status.SUCCESSFUL: + recovered_after = round(time.monotonic() - started, 2) + break + time.sleep(0.1) + finally: + code = stop(worker) + + poll_failures = stalled_text.count("could not reach the database this pass") + error = re.search(r"^django\.db\.utils\.\w+: .*$", stalled_text, re.MULTILINE) + ran = status == OxTask.Status.SUCCESSFUL + refused = not alive and code != 0 and "migrate" in stalled_text + assert ran or refused, ( + f"schema at {BEFORE_WORKFLOWS}; `ox_worker --backend flows` watched " + f"{STALL_SECONDS}s: alive={alive}, plain task {status}, " + f"{poll_failures} pass(es) logged 'could not reach the database this " + f"pass', error={error.group(0) if error else None!r}; after migrate " + f"under the running worker the task ran in {recovered_after}s " + f"(None = never); exit code after SIGTERM {code}; `manage.py check`: " + f"{(check.stdout + check.stderr).strip()!r}" + ) + + +@pytest.mark.django_db(transaction=True) +def test_before_0004_oxpull_prune_completes_or_names_the_migration( + before_workflows_migration, +): + """ + The new prune also works on the workflow tables. Before 0004 it must + either finish, or stop with a CommandError that says `migrate`. Deleting + stale unique locks first is independent, correct work, so whether that + already happened is recorded in the message and not judged. + """ + tasks.unique_echo.enqueue("prune-me") + assert free_worker().run_once() is True + assert OxpullUniqueLock.objects.count() == 1 + + out = io.StringIO() + raised = None + try: + call_command("oxpull_prune", stdout=out) + except (DatabaseError, CommandError) as exc: + raised = exc + locks_left = OxpullUniqueLock.objects.count() + + if raised is None: + assert locks_left == 0, out.getvalue() + return + kind = f"{type(raised).__module__}.{type(raised).__qualname__}" + detail = ( + f"{kind}: {raised}; printed before it: {out.getvalue()!r}; " + f"stale locks left {locks_left} of 1" + ) + assert isinstance(raised, CommandError), detail + assert "migrate" in str(raised), detail diff --git a/tests_oxpull/test_workflows_hazard_mysql_a_gated_claim_whose_scan_spans_an.py b/tests_oxpull/test_workflows_hazard_mysql_a_gated_claim_whose_scan_spans_an.py new file mode 100644 index 0000000..d3da42f --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_a_gated_claim_whose_scan_spans_an.py @@ -0,0 +1,361 @@ +""" +A gated claim whose scan spans an add() commit. + +The gate's invariant (Workflow.add's docstring) is that a worker cannot +observe a node's task row without also observing every edge that gates +it. add() keeps its half: the task row, the node and the edges commit +together. The claim is where it can break on MySQL. There the claim is +one SELECT ... FOR UPDATE SKIP LOCKED with the gate as nested NOT EXISTS +subqueries. The locking clause makes InnoDB read the task table as a +current read, which sees rows committed while the scan runs. The +subqueries over the edge table stay consistent reads, answered from a +read view fixed when the statement first reads them (READ COMMITTED) or +at the transaction's first consistent read (REPEATABLE READ). A node +whose add() commits after that view began, and whose row the outer scan +reaches afterwards, is READY and unlocked to the outer read and has no +edge to the subquery. NOT EXISTS is true and the node is claimed while +its upstream is still running. + +Both tests build one workflow with two roots. "doomed" is claimed and +held RUNNING through the window, then fails. "done" has already +succeeded. What a fix has to do to pass: + + * No child of doomed ever starts: no claim takes one while doomed is + running, and nothing runs one after doomed has failed, the + reconciler's pass included. The fix inserts every blocked node + WAITING in the transaction that adds it, so the claim's own locking + read, which reads only READY rows, excludes the node whatever its + subqueries can see. + * Claims keep working. Children of done are free to run, and a worker + must run every one of them: during the race in the first test, and + from inside the pinned read view in the second. They sit at a lower + priority than every child of doomed, so a claim reaches them only + after passing every blocked node, which keeps the window as wide as + the blocked nodes alone make it. A fix that stops claiming, blocks + every workflow node, or returns nothing once the first candidate it + meets is blocked fails here. + +Both tests failed on MySQL before blocked nodes were inserted WAITING. +Only the checks that a child of doomed never started are plain asserts. +Everything else calls pytest.fail: a setup step that went wrong (the held +claim, done's run, doomed still running, a thread that raised, the edge +count), a drain that never settles, and every check that claims keep +working. An implementation that stops claiming therefore fails loudly. +PostgreSQL (one snapshot per statement) and SQLite (serialised writers) +are the controls. +""" + +from __future__ import annotations + +import threading +import time + +import pytest +from django.db import connections, transaction + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflowEdge + +from . import tasks +from .racing import race_all +from .test_batches import session_isolation +from .test_workflows import flow_worker, order, reconcile + +pytestmark = pytest.mark.django_db + +# Blocked READY nodes ahead of each late node in claim order. The scan +# walks all of them before it reaches the late node, so this is what +# widens the window. 4000 costs about 17 s of real add() calls on MySQL, +# and a claim over it takes about 20 ms there. +BACKLOG = 4000 +CLAIMERS = 2 +# How long the adds race the claims. The race ends early at the first +# child of doomed that runs. +RACE_SECONDS = 10.0 +# Bound on draining the workflow afterwards, so a claim that never +# settles fails the test rather than hanging it. +SETTLE_SECONDS = 180.0 + +# Node names, which are also the label each node records when it runs. +# Children of doomed carry the first prefix, children of done the second. +BLOCKED = "blocked-" +RUNNABLE = "runnable-" +# Below the default priority of every child of doomed. The claim orders +# by priority first, so it passes every blocked node before it reaches a +# runnable one. +RUNNABLE_PRIORITY = -1 +# Ids per query when reading nodes back, under SQLite's bound-variable +# limit however many nodes the race added. +CHUNK = 500 + + +def require(condition, message): + """ + A check that is not the defect under test. pytest.fail raises + Failed, which is not an AssertionError, so the MySQL marker never + turns it into an expected failure. + """ + if not condition: + pytest.fail(message) + + +def doomed_is_running(held): + return OxTask.objects.get(id=held.id).status == OxTask.Status.RUNNING + + +def two_roots(name): + """ + An OPEN workflow with two roots, and the worker holding one of them. + + "doomed" is claimed and held, so it stays RUNNING until settle() lets + it fail. "done" is added after that claim and run to SUCCESSFUL. + """ + flow = workflows.create(name) + flow.add("doomed", tasks.failing_step, "doomed") + worker = flow_worker() + held = worker.claim_one() + require( + held is not None and held.args == ["doomed"], + f"setup: the worker's first claim returned " + f"{None if held is None else held.args}, not doomed, the only READY node", + ) + flow.add("done", tasks.step, "done") + require( + worker.run_once() is True and order() == ["done"], + f"setup: the worker did not run done, a root with no upstream; ran {order()}", + ) + require(doomed_is_running(held), "setup: doomed is not RUNNING") + return flow, worker, held + + +def add_runnable(flow, name): + flow.add( + name, + tasks.step.using(priority=RUNNABLE_PRIORITY), + name, + depends_on="done", + ) + + +def tasks_named(flow, prefix): + """The task rows of every node whose name starts with prefix.""" + ids = sorted( + str(task_id) + for name, task_id in flow.node_ids().items() + if name.startswith(prefix) + ) + found = [] + for start in range(0, len(ids), CHUNK): + found.extend(OxTask.objects.filter(id__in=ids[start : start + CHUNK])) + return found + + +def ran(prefix): + return [label for label in order() if label.startswith(prefix)] + + +def blocked_children_that_started(): + """Children of doomed that any worker ran, in the order they ran.""" + return ran(BLOCKED) + + +def blocked_children_claimed(flow): + """Children of doomed whose task row shows a claim, with that row's state.""" + return sorted( + (row.args[0], row.status, row.attempts) + for row in tasks_named(flow, BLOCKED) + if row.attempts or row.started_at is not None + ) + + +def run_until_idle(worker, held, deadline): + """Run whatever the worker claims until it claims nothing and doomed is settled.""" + while True: + require( + time.monotonic() < deadline, + f"the worker was still finding work after {SETTLE_SECONDS} s", + ) + if worker.run_once(): + continue + if OxTask.objects.get(id=held.id).status != OxTask.Status.READY: + return + # doomed's retry is written with no backoff, and on SQLite it can + # take a millisecond to become eligible. + time.sleep(0.01) + + +def settle(flow, worker, held): + """ + Let doomed fail on the worker that holds it, then run everything a + worker will run, seal, reconcile once, and run again. + + Every attempt of doomed fails, so it ends FAILED. The reconciler's + pass comes after that failure, so a fix whose backstop hands a child + of a failed node back to the claim is caught by the second drain. + """ + deadline = time.monotonic() + SETTLE_SECONDS + worker.execute(held) + run_until_idle(worker, held, deadline) + flow.seal() + reconcile() + run_until_idle(worker, held, deadline) + status = OxTask.objects.get(id=held.id).status + require( + status == OxTask.Status.FAILED, + f"setup: doomed ended {status}, not FAILED", + ) + + +def require_runnable_children_all_ran(flow): + """Claims keep working: every child of done ran to SUCCESSFUL.""" + rows = tasks_named(flow, RUNNABLE) + require(rows, "setup: the test added no child of done") + names = sorted(row.args[0] for row in rows) + not_successful = sorted( + (row.args[0], row.status) + for row in rows + if row.status != OxTask.Status.SUCCESSFUL + ) + require( + not_successful == [] and sorted(ran(RUNNABLE)) == names, + f"{len(not_successful)} of {len(names)} nodes whose only upstream, " + f"done, had succeeded did not run to SUCCESSFUL: " + f"{not_successful[:5]}. Ran: {len(ran(RUNNABLE))}", + ) + + +@pytest.mark.django_db(transaction=True) +def test_a_node_added_while_workers_scan_a_blocked_backlog_waits_for_its_upstream(): + """ + Adding to an OPEN workflow while workers drain it, with the backend's + default options. + + Real worker claims (run_once, which also executes what it claims) on + CLAIMERS threads, and real add() calls on one more, over a backlog of + BACKLOG children of doomed built with add(). The adding thread adds + pairs: a child of doomed, then a child of done. doomed stays RUNNING + for the whole race and fails afterwards, so a correct gate never + starts a child of it, and the children of done give the workers real + work throughout. + """ + flow, worker, held = two_roots("fan-out-while-draining") + for i in range(BACKLOG): + name = f"{BLOCKED}{i:05d}" + flow.add(name, tasks.step, name, depends_on="doomed") + + stop = threading.Event() + + def claimer(): + claiming = flow_worker() + while not stop.is_set(): + if claiming.run_once() and blocked_children_that_started(): + stop.set() + return "claimer" + + def adder(): + deadline = time.monotonic() + RACE_SECONDS + pairs = 0 + while not stop.is_set() and time.monotonic() < deadline: + blocked = f"{BLOCKED}late{pairs:05d}" + flow.add(blocked, tasks.step, blocked, depends_on="doomed") + add_runnable(flow, f"{RUNNABLE}{pairs:05d}") + pairs += 1 + stop.set() + return pairs + + results, errors = race_all([claimer] * CLAIMERS + [adder]) + + require(errors == [], f"a thread raised: {errors[:2]}") + pairs = [result for result in results if result != "claimer"] + require( + len(results) == CLAIMERS + 1 and len(pairs) == 1, + f"setup: the race returned {results}", + ) + ran_while_adding = ran(RUNNABLE) + require(doomed_is_running(held), "setup: doomed stopped running during the race") + + settle(flow, worker, held) + + assert blocked_children_that_started() == [], ( + f"after {pairs[0]} pairs of add() calls racing {CLAIMERS} workers over " + f"{BACKLOG} blocked nodes, a worker ran " + f"{blocked_children_that_started()} although their only upstream, " + f"doomed, never succeeded. Claimed rows (name, status, attempts): " + f"{blocked_children_claimed(flow)}" + ) + assert blocked_children_claimed(flow) == [] + require( + ran_while_adding, + f"no worker ran a child of done while {pairs[0]} pairs of add() calls " + f"raced them: claims stopped", + ) + require_runnable_children_all_ran(flow) + + +@pytest.mark.django_db(transaction=True) +def test_a_gated_claim_does_not_admit_a_node_committed_after_its_read_view_began(): + """ + The same window, made deterministic at REPEATABLE READ. + + Under REPEATABLE READ the transaction's first consistent read fixes + the read view the gate's subqueries are answered from. The edge read + below fixes it, a concurrent add() then commits a child of doomed and + its edge, and a worker's own claim_one() runs. The outer locking read + sees that child, and the subquery does not see its edge. A child of + done, committed before the view began and behind the late child in + claim order, is what the claim must return instead. + + Pinning the view with an earlier statement stands in for the scan + itself: at READ COMMITTED the first subquery probe fixes the view, + and the outer scan keeps walking the backlog after it. The first test + reproduces that form through real worker threads. + """ + if connections["default"].vendor == "sqlite": + pytest.skip( + "a reading SQLite transaction holds its shared lock, so the " + "concurrent add() could not commit under it" + ) + flow, worker, held = two_roots("late-under-a-read-view") + runnable = f"{RUNNABLE}early" + add_runnable(flow, runnable) + late = f"{BLOCKED}late" + + def add_and_commit(): + flow.add(late, tasks.step, late, depends_on="doomed") + return "added" + + claiming = flow_worker() + with session_isolation("REPEATABLE READ"), transaction.atomic(): + # The transaction's first consistent read; the read view starts here. + edges_seen = OxpullWorkflowEdge.objects.values_list("pk").count() + require(edges_seen == 1, f"setup: the read view saw {edges_seen} edges") + results, errors = race_all([add_and_commit]) + require( + (results, errors) == (["added"], []), + f"setup: the concurrent add() returned {results}, raised {errors[:2]}", + ) + claimed = claiming.claim_one() + + edges = OxpullWorkflowEdge.objects.count() + require(edges == 2, f"setup: {edges} edges after the add, not 2") + require(doomed_is_running(held), "setup: doomed is not RUNNING") + assert claimed is None or claimed.args != [late], ( + f"claim_one() returned {late} with status {claimed.status} while its " + f"only upstream, doomed, was RUNNING" + ) + require( + claimed is not None and claimed.args == [runnable], + f"claim_one() returned {None if claimed is None else claimed.args} " + f"although {runnable}, whose only upstream had succeeded, was READY", + ) + + claiming.execute(claimed) + settle(flow, worker, held) + + assert blocked_children_that_started() == [] + assert blocked_children_claimed(flow) == [], ( + f"a child of doomed shows a claim although doomed never succeeded: " + f"{blocked_children_claimed(flow)}" + ) + require_runnable_children_all_ran(flow) diff --git a/tests_oxpull/test_workflows_hazard_mysql_edge_columns_created_on_a_different.py b/tests_oxpull/test_workflows_hazard_mysql_edge_columns_created_on_a_different.py new file mode 100644 index 0000000..c601a81 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_edge_columns_created_on_a_different.py @@ -0,0 +1,555 @@ +""" +MySQL: workflow task-id columns on a different collation from +django_ox_oxtask.id make every gated claim raise error 1267. + +How a project gets there. `migrate` creates a table on the database's default +collation at that moment, and mysqldump writes an explicit COLLATE= into every +table it restores. Restore django-ox's tables from a 5.7-era database +(utf8mb4_unicode_ci or utf8mb4_general_ci) into a database created on 8.x +(default utf8mb4_0900_ai_ci), and task ids keep the old collation while +migration 0004 creates the edge and node tables on the new one. ALTER DATABASE +... COLLATE produces the same split, and so does a DBA converting django-ox's +tables to the new default after 0004 has run. + +The gate compares the edge columns with django_ox_oxtask.id inside the claim. +MySQL refuses to compare two IMPLICIT columns of one character set on +different collations, and it refuses while resolving the statement, before a +row is read. So by reading, every claim on a WORKFLOWS backend fails, whether +or not a workflow exists. + +Every test asserts the correct behaviour. The in-process tests admit either +fix direction: + * the gate works whatever the columns' collations, so the claim returns the + right task, or + * a system check run against the database (`manage.py check --database + default`, which `migrate` also runs) names the collation mismatch. +The ox_worker process test is stricter, and says why at the test. + +Migration 0004 now creates the four task-id columns on OxTask.id's +collation, oxpull.E009 reports a later split from `check --database`, and +an OxpullWorker on a WORKFLOWS backend refuses to start on one. + +MySQL only. PostgreSQL stores a UUIDField as uuid, which has no collation, and +SQLite never raises on a collation mix. +""" + +import os +import re +import signal +import subprocess +import sys +import time +from contextlib import contextmanager +from pathlib import Path + +import pytest +from django.conf import settings +from django.core import checks +from django.core.management import call_command +from django.db import DatabaseError, connection + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .subprocesses import django_ox_path +from .test_upgrade import _latest_migration, _migrate +from .test_workflows import drain, flow_worker, order + +REPO_ROOT = Path(__file__).resolve().parent.parent +SRC = REPO_ROOT / "src" + +# The migration before Workflows. Rewinding to it drops the workflow tables. +BEFORE_WORKFLOWS = "0003_oxpullratewindow" + +# MySQL 8.x's default collation for utf8mb4, and 5.7's. +NEW_DEFAULT = "utf8mb4_0900_ai_ci" +OLD_DEFAULT = "utf8mb4_general_ci" + +TASK_TABLE = OxTask._meta.db_table +TASK_ID = OxTask._meta.pk.column +EDGE_TABLE = OxpullWorkflowEdge._meta.db_table +NODE_TABLE = OxpullWorkflowNode._meta.db_table + +# The two columns the claim's gate compares with a task id. Neither carries a +# foreign key, so moving their collation leaves the schema otherwise valid. +EDGE_TASK_ID_COLUMNS = [ + (EDGE_TABLE, OxpullWorkflowEdge._meta.get_field("upstream_task_id").column), + (EDGE_TABLE, OxpullWorkflowEdge._meta.get_field("downstream_task_id").column), +] +# Reported in failure messages: 0004 creates this one on the default as well. +NODE_TASK_ID_COLUMN = (NODE_TABLE, OxpullWorkflowNode._meta.get_field("task_id").column) + + +@pytest.fixture(autouse=True) +def mysql_only(): + if connection.vendor != "mysql": + pytest.skip( + "MySQL only: a uuid column has no collation on PostgreSQL, and " + "SQLite does not raise on a collation mix" + ) + + +# -- schema helpers ------------------------------------------------------ + + +def column_info(table, column): + with connection.cursor() as cursor: + cursor.execute( + "SELECT COLUMN_TYPE, IS_NULLABLE, COLLATION_NAME" + " FROM information_schema.COLUMNS" + " WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s" + " AND COLUMN_NAME = %s", + [table, column], + ) + row = cursor.fetchone() + assert row is not None, f"no column {table}.{column}" + return row + + +def collation_of(table, column): + return column_info(table, column)[2] + + +def collations(): + """The task id and every workflow task-id column, for failure messages.""" + return { + f"{table}.{column}": collation_of(table, column) + for table, column in [ + (TASK_TABLE, TASK_ID), + *EDGE_TASK_ID_COLUMNS, + NODE_TASK_ID_COLUMN, + ] + } + + +def set_collation(table, column, collation): + column_type, nullable, _ = column_info(table, column) + charset = collation.split("_", 1)[0] + null = "NULL" if nullable == "YES" else "NOT NULL" + with connection.cursor() as cursor: + cursor.execute( + f"ALTER TABLE `{table}` MODIFY `{column}` {column_type}" + f" CHARACTER SET {charset} COLLATE {collation} {null}" + ) + + +@contextmanager +def edge_columns_on(collation): + """Move both edge task-id columns to `collation`, and back afterwards.""" + original = {(t, c): collation_of(t, c) for t, c in EDGE_TASK_ID_COLUMNS} + try: + for table, column in EDGE_TASK_ID_COLUMNS: + set_collation(table, column, collation) + # About this setup only: the DDL did what it says. + assert {collation_of(t, c) for t, c in EDGE_TASK_ID_COLUMNS} == {collation} + yield + finally: + for (table, column), before in original.items(): + set_collation(table, column, before) + + +def a_collation_other_than_the_task_ids(preferred): + task_collation = collation_of(TASK_TABLE, TASK_ID) + if preferred == task_collation: + pytest.skip(f"task ids are already on {preferred}; nothing to split") + return preferred + + +def schema_default(): + with connection.cursor() as cursor: + cursor.execute( + "SELECT DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA" + " WHERE SCHEMA_NAME = DATABASE()" + ) + return cursor.fetchone()[0] + + +def set_schema_default(collation): + name = connection.settings_dict["NAME"] + with connection.cursor() as cursor: + cursor.execute(f"ALTER DATABASE `{name}` COLLATE {collation}") + + +def collation_reported(): + """ + What `manage.py check --database default` says about collations, at + Warning level or above. `migrate` runs the same check with the same + database before it applies anything. + + oxpull's database-tagged check is called directly, not through + checks.run_checks(), whose other registered checks depend on which + modules earlier tests imported. A test that puts django.contrib.auth + in INSTALLED_APPS registers that app's checks for the rest of the + process, and they raise once it is out of INSTALLED_APPS again. + E009 is the only collation message either way. + """ + from oxpull.checks import check_workflow_schema + + return [ + f"{message.id}: {message.msg}" + for message in check_workflow_schema(databases=["default"]) + if message.level >= checks.WARNING + and "collation" in f"{message.msg} {message.hint or ''}".lower() + ] + + +def add_root_and_child(name): + """ + root(priority 0) -> child(priority 10). An ungated claim reaches the child + first, so only the gate can put root first. + """ + flow = workflows.create(name) + flow.add("root", tasks.step.using(priority=0), "root") + flow.add("child", tasks.step.using(priority=10), "child", depends_on="root") + flow.seal() + return flow + + +def describe(error, stage): + if error is None: + return "no error" + return f"{type(error).__name__} at {stage}: {error}" + + +# -- the claim ----------------------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("collation", [NEW_DEFAULT, OLD_DEFAULT]) +def test_a_gated_claim_runs_the_workflow_whatever_collation_the_edge_columns_are_on( + collation, +): + """Edge columns on another collation, then a real gated drain.""" + collation = a_collation_other_than_the_task_ids(collation) + error = None + stage = "create" + with edge_columns_on(collation): + try: + add_root_and_child(f"split-{collation}") + stage = "claim" + drain(flow_worker()) + except DatabaseError as exc: + error = exc + reported = collation_reported() + snapshot = collations() + + assert reported or (error is None and order() == ["root", "child"]), ( + f"OxpullWorker(backend_alias='flows') ran {order()} with " + f"{describe(error, stage)}. Columns: {snapshot}. " + f"Collation checks from `check --database default`: {reported}" + ) + + +@pytest.mark.django_db(transaction=True) +def test_a_plain_task_on_a_workflows_backend_runs_despite_the_split(): + """ + No workflow exists at all. The gate still sits in every claim on a + WORKFLOWS backend, so a plain task there is exposed as well. + """ + collation = a_collation_other_than_the_task_ids(NEW_DEFAULT) + error = None + ran = False + with edge_columns_on(collation): + # About this setup only: nothing workflow-shaped is in the database. + assert not OxpullWorkflowEdge.objects.exists() + tasks.step.enqueue("plain") + try: + ran = flow_worker().run_once() + except DatabaseError as exc: + error = exc + reported = collation_reported() + snapshot = collations() + + assert reported or (error is None and ran and order() == ["plain"]), ( + f"a plain task on 'flows' with no workflow in the database: ran={ran}, " + f"order={order()}, {describe(error, 'claim')}. Columns: {snapshot}. " + f"Collation checks: {reported}" + ) + + +# -- the production sequence: migrate after the default changed ----------- + + +@pytest.mark.django_db(transaction=True) +def test_migrate_after_the_default_collation_changed_leaves_claims_working(): + """ + Not an ALTER on oxpull's own columns: the real `migrate` creates the + workflow tables on a database whose default collation is no longer the one + django-ox's tables carry, the state a mysqldump restore into a database + created on 8.x leaves behind. + """ + task_collation = collation_of(TASK_TABLE, TASK_ID) + new_default = NEW_DEFAULT if task_collation != NEW_DEFAULT else OLD_DEFAULT + original_default = schema_default() + latest = _latest_migration() + error = None + stage = "migrate" + _migrate(BEFORE_WORKFLOWS) + try: + set_schema_default(new_default) + # About this setup only: django-ox's table kept its collation, and the + # database now defaults to another one. + assert collation_of(TASK_TABLE, TASK_ID) == task_collation + assert schema_default() == new_default + # skip_checks: the suite's settings carry flows_ungated, a deliberate + # oxpull.E007 fixture that would stop `migrate` before it applied + # anything. The checks are run below, against the migrated schema. + call_command("migrate", "oxpull", skip_checks=True, verbosity=0) + snapshot = collations() + try: + stage = "create" + add_root_and_child("restored") + stage = "claim" + drain(flow_worker()) + except DatabaseError as exc: + error = exc + reported = collation_reported() + finally: + # Migration 0004 refuses to unapply while a workflow row exists, so + # the rows this test made go first, children before their workflow. + for model in (OxpullWorkflowEdge, OxpullWorkflowNode, OxpullWorkflow): + model.objects.all().delete() + _migrate(BEFORE_WORKFLOWS) + set_schema_default(original_default) + _migrate(latest) + + assert reported or (error is None and order() == ["root", "child"]), ( + f"after `migrate oxpull` on a database defaulting to {new_default} with " + f"task ids on {task_collation}: ran {order()} with " + f"{describe(error, stage)}. Columns: {snapshot}. " + f"Collation checks: {reported}" + ) + + +# -- a real ox_worker process -------------------------------------------- + + +def process_env(tmp_path): + """ + The suite's settings for a separate process on this test database, with + DEBUG django_ox logs (the "Claimed task" lines). flows_ungated stops + declaring WORKFLOWS: it is the suite's deliberate oxpull.E007 fixture and + would stop ox_worker at start-up, which has nothing to do with this hazard. + """ + (tmp_path / "collation_settings.py").write_text( + "import copy\n" + f"from {settings.SETTINGS_MODULE} import * # noqa: F403\n" + "TASKS = copy.deepcopy(TASKS) # noqa: F405\n" + "TASKS['flows_ungated']['OPTIONS']['WORKFLOWS'] = False\n" + "LOGGING = {\n" + " 'version': 1,\n" + " 'disable_existing_loggers': False,\n" + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},\n" + " 'loggers': {'django_ox': {'handlers': ['console'], 'level': 'DEBUG'}},\n" + "}\n" + ) + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "collation_settings", + "PYTHONPATH": os.pathsep.join( + [str(SRC), django_ox_path(), str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(connection.settings_dict["NAME"]), + } + + +def stop(process): + """SIGTERM, as a deploy does; the worker drains and exits.""" + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return process.returncode + + +@pytest.mark.django_db(transaction=True) +def test_an_ox_worker_process_runs_its_tasks_or_refuses_to_start( + tmp_path, +): + """ + What a user's worker does on the split schema. + + Correct is: it runs the workflow in order and the plain task, or it exits + non-zero on its own and names the collation. A check that only `migrate` + or `check --database` prints does not pass this test, deliberately: + ox_worker runs no database-tagged check, and the first `migrate` runs its + checks before 0004 has created the table there is to inspect. + """ + collation = a_collation_other_than_the_task_ids(NEW_DEFAULT) + env = process_env(tmp_path) + probe = subprocess.run( + [ + sys.executable, + "-c", + ( + "import django; django.setup(); import oxpull; " + "from django_ox.worker import worker_class; " + "cls = worker_class('flows'); print(oxpull.__file__); " + "print(cls.__module__ + '.' + cls.__qualname__)" + ), + ], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + lines = probe.stdout.split() + assert len(lines) == 2, probe.stdout + probe.stderr + assert lines[0].startswith(str(SRC)), lines + assert lines[1] == "oxpull.worker.OxpullWorker", lines + + log_path = tmp_path / "ox_worker.log" + with edge_columns_on(collation): + # Evidence only, not asserted: what an operator's own check says about + # this schema, in a fresh interpreter. + check = subprocess.run( + [sys.executable, "-m", "django", "check", "--database", "default"], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + flow = add_root_and_child("process") + plain = tasks.step.enqueue("plain") + ids = {**flow.node_ids(), "plain": plain.id} + label_of = {str(task_id): label for label, task_id in ids.items()} + with log_path.open("w") as log: + worker = subprocess.Popen( + [ + sys.executable, + "-m", + "django", + "ox_worker", + "--backend", + "flows", + "--concurrency", + "1", + "--interval", + "0.05", + ], + env=env, + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and worker.poll() is None: + unfinished = OxTask.objects.filter(pk__in=ids.values()).exclude( + status=OxTask.Status.SUCCESSFUL + ) + if not unfinished.exists(): + break + time.sleep(0.2) + alive_at_the_end = worker.poll() is None + finally: + code = stop(worker) + statuses = { + label: OxTask.objects.get(pk=task_id).status + for label, task_id in ids.items() + } + snapshot = collations() + + text = log_path.read_text() + claimed = [ + label_of[task_id] + for task_id in re.findall(r"Claimed task id=([0-9a-f-]{36}) ", text) + if task_id in label_of + ] + poll_failures = text.count("could not reach the database this pass") + first_mix = next( + (line for line in text.splitlines() if "Illegal mix of collations" in line), + None, + ) + + ran_in_order = ( + set(statuses.values()) == {OxTask.Status.SUCCESSFUL} + and {"root", "child"} <= set(claimed) + and claimed.index("root") < claimed.index("child") + ) + refused_to_start = ( + not alive_at_the_end and code != 0 and "collation" in text.lower() + ) + assert ran_in_order or refused_to_start, ( + f"`ox_worker --backend flows` on columns {snapshot}: statuses {statuses}, " + f"claimed {claimed}, still running at the deadline: {alive_at_the_end}, " + f"exit {code} after SIGTERM. {poll_failures} poll passes logged " + f"'could not reach the database this pass'; first 1267 line: {first_mix!r}. " + f"`check --database default` in that environment: exit {check.returncode}, " + f"{(check.stdout + check.stderr).strip()!r}" + ) + + +# -- the other workflow reads that carry the gate --------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_progress_and_the_reconciler_work_whatever_collation_the_edge_columns_are_on( + caplog, +): + """ + progress() counts blocked nodes with the gate, and the reconciler's + release backstop reads the edges on every tick. On a sealed root -> + child with root still READY, correct is blocked == 1, and a reconcile + pass that completes nothing, raises nothing and logs no failure. + """ + collation = a_collation_other_than_the_task_ids(NEW_DEFAULT) + flow = add_root_and_child("reads") + errors = {} + progress = completed = None + with edge_columns_on(collation): + try: + progress = flow.progress() + except DatabaseError as exc: + errors["progress"] = f"{type(exc).__name__}: {exc}" + try: + completed = workflows._reconcile() + except DatabaseError as exc: + errors["reconcile"] = f"{type(exc).__name__}: {exc}" + reported = collation_reported() + snapshot = collations() + failures_logged = [ + record.getMessage() + for record in caplog.records + if getattr(record, "event", "") == "workflow_reconcile_failed" + ] + + correct = ( + not errors + and progress is not None + and progress.blocked == 1 + and completed == 0 + and not failures_logged + ) + assert reported or correct, ( + f"progress={progress}, reconcile completed={completed}, errors={errors}, " + f"reconcile failures logged={failures_logged}. Columns: {snapshot}. " + f"Collation checks: {reported}" + ) + + +# -- control ------------------------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("target", ["task-id-collation", "utf8mb4_bin"]) +def test_control_the_same_alter_to_a_compatible_collation_keeps_the_gate_working( + target, +): + """ + Passes today. The same DDL, to the task ids' own collation and to a binary + collation MySQL lets win a comparison, leaves the gated drain correct. So + the failures above are the collation split, not the ALTER or the fixture. + """ + collation = collation_of(TASK_TABLE, TASK_ID) if target != "utf8mb4_bin" else target + with edge_columns_on(collation): + add_root_and_child(f"control-{target}") + drain(flow_worker()) + assert order() == ["root", "child"] diff --git a/tests_oxpull/test_workflows_hazard_mysql_node_name_uniqueness_follows_the_column.py b/tests_oxpull/test_workflows_hazard_mysql_node_name_uniqueness_follows_the_column.py new file mode 100644 index 0000000..5499c46 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_node_name_uniqueness_follows_the_column.py @@ -0,0 +1,156 @@ +""" +A node's name must not be identified by the column's collation on MySQL. + +A unique index on (workflow, name) used to enforce node-name uniqueness, so +the column's collation decided which names counted as the same. PostgreSQL +and SQLite compare bytes. MySQL's server default (utf8mb4_0900_ai_ci on 8.x) +folds case and accents, and utf8mb4_unicode_ci, which tests_oxpull/settings_mysql.py +creates, also pads trailing spaces. Uniqueness and depends_on now go through +name_key, the SHA-256 digest of the name's UTF-8 bytes, so a name is its +exact spelling on every database. + +Every test here asserts the behaviour PostgreSQL and SQLite give, and runs +unskipped on every database, so the MySQL leg is the one that matters. +""" + +import pytest +from django.db import connection + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.exceptions import UnknownWorkflowNode, WorkflowNodeExists +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import drain, flow_worker, order, reconcile, stored + +pytestmark = pytest.mark.django_db + +# Spellings that Python's == and a byte comparison keep apart. The first +# pair differs by case, the second by accents, the third by a trailing +# space, which only a PAD SPACE collation folds. +PAIRS = [ + pytest.param("Extract", "extract", id="case"), + pytest.param("resume", "résumé", id="accent"), + pytest.param("load", "load ", id="trailing-space"), +] + + +def name_collation(): + """What decides node-name identity on this connection, for messages.""" + if connection.vendor != "mysql": + return f"{connection.vendor}: bytes" + with connection.cursor() as cursor: + cursor.execute( + "SELECT c.collation_name, k.pad_attribute" + " FROM information_schema.columns c" + " JOIN information_schema.collations k" + " ON k.collation_name = c.collation_name" + " WHERE c.table_schema = DATABASE()" + " AND c.table_name = %s AND c.column_name = 'name'", + [OxpullWorkflowNode._meta.db_table], + ) + ((collation, pad),) = cursor.fetchall() + return f"mysql: {collation} ({pad})" + + +@pytest.mark.parametrize(("first", "second"), PAIRS) +def test_two_names_python_tells_apart_are_two_nodes(first, second): + """ + add() of a second name that differs from the first by case, an accent + or a trailing space admits a second node, as it does on PostgreSQL. + + On MySQL a unique index on the name column made the node INSERT hit + oxpull_wf_node_name_uniq, and add() turned the IntegrityError into + WorkflowNodeExists. + """ + flow = workflows.create("fold") + a = flow.add(first, tasks.step, "a") + try: + b = flow.add(second, tasks.step, "b") + except WorkflowNodeExists as exc: + pytest.fail( + f"[{name_collation()}] add({second!r}) after add({first!r}) " + f"raised WorkflowNodeExists: {exc} " + f"(OxTask rows: {OxTask.objects.count()}, " + f"nodes: {sorted(flow.node_ids())})" + ) + assert {name: str(task_id) for name, task_id in flow.node_ids().items()} == { + first: str(a.id), + second: str(b.id), + } + assert OxTask.objects.count() == 2 + + +@pytest.mark.parametrize(("first", "second"), PAIRS) +def test_a_name_add_calls_taken_is_a_name_depends_on_resolves(first, second): + """ + The unique index and depends_on must agree about what a name names. + + add(second) either admits a node named `second`, in which case + depends_on=second points at that node, or refuses with + WorkflowNodeExists ("already has a node named ..."), in which case the + name is taken and depends_on=second must point at the node that took + it. Refusing the name as existing and then refusing it again as + missing is the one answer that is wrong under every design. + + On MySQL that was the answer: the IN filter fetched the row stored as + `first`, and the exact membership test in _resolve_dependencies + discarded it and raised UnknownWorkflowNode. + """ + flow = workflows.create("fold-resolve") + flow.add(first, tasks.step, first) + try: + flow.add(second, tasks.step, second) + except WorkflowNodeExists as exc: + refused_as_existing = str(exc) + else: + refused_as_existing = None + try: + downstream = flow.add("downstream", tasks.step, "d", depends_on=second) + except UnknownWorkflowNode as exc: + pytest.fail( + f"[{name_collation()}] add({second!r}) was refused as existing " + f"({refused_as_existing}), then depends_on={second!r} was refused " + f"as missing: {exc}" + ) + nodes = flow.node_ids() + edge = OxpullWorkflowEdge.objects.get(downstream_task_id=downstream.id) + named = second if refused_as_existing is None else first + assert edge.upstream_task_id == nodes[named] + + +@pytest.mark.parametrize(("first", "second"), PAIRS) +def test_a_dag_named_from_data_builds_runs_and_completes(first, second): + """ + A builder that names one node per input (a file, a customer, a key) + and then seals gets a workflow that runs every node and completes. + + On MySQL the build used to die at the second name. The node already + added was committed, and because nodes of an OPEN workflow run, a real + worker ran it. The reconciler sweeps only SEALED workflows, so the + workflow stayed OPEN, never completed and never fired a callback. + """ + names = [first, second] + flow = workflows.create("from-data") + refused = None + try: + for name in names: + flow.add(name, tasks.step, name) + flow.seal() + except WorkflowNodeExists as exc: + refused = exc + drain(flow_worker()) + reconcile() + reconcile() + ran = sorted(order()) + status = stored(flow).status + assert (refused, ran, status) == ( + None, + sorted(names), + OxpullWorkflow.Status.COMPLETE, + ), ( + f"[{name_collation()}] build refused: {refused!r}; nodes that ran: " + f"{ran}; workflow status: {status}; " + f"node rows: {OxpullWorkflowNode.objects.count()}" + ) diff --git a/tests_oxpull/test_workflows_hazard_mysql_release_dependents_and_the_cancellation.py b/tests_oxpull/test_workflows_hazard_mysql_release_dependents_and_the_cancellation.py new file mode 100644 index 0000000..9719d63 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_release_dependents_and_the_cancellation.py @@ -0,0 +1,510 @@ +""" +MySQL: a wide fan-out or cone goes to the database as one +literal IN list, and past the range optimizer's memory budget that +statement reads the whole task table. + +The release (the worker, right after a SUCCESSFUL write lands) +and _cancel_one_level (the reconciler, inside the transaction holding the +workflow row lock and FOR UPDATE on the dead upstreams) each put every id +of the fan-out or cone into a single `id IN (...)`. django-ox's own bulk +actions chunk at UPDATE_CHUNK_SIZE = 1000 "so that one statement does not +hold a long lock over a large table". MySQL charges every IN element +against range_optimizer_max_mem_size (8 MiB by default); past it the +optimizer raises warning 3170 and plans a scan of django_ox_oxtask, so +the cost follows the retained table, not the cone. + +The MySQL tests assert the behaviour a wide workflow needs: the +statements that carry the cone read it by primary-key range, and the +eager release reads fewer than three rows per released node by table scan. +The release reads the fan-out's edges to find the children and writes them +to mark them satisfied, and where one chunk of primary keys covers the +whole edge table MySQL scans it for each, which is two rows per node that +follow the fan-out, not the retained task table. Rows are +asserted as well, and on MySQL they end up correct; there it is a cost +cliff, not a wrong answer. + +On PostgreSQL and SQLite the same single list meets a hard cap on bound +parameters instead, and past it the statement is refused, not slow. The +last two tests assert that a fan-out or cone wider than the cap is still +released by the worker and cancelled by the reconciler. + +The release and the cancellation now send the ids in chunks of at most +workflows.ID_CHUNK_SIZE, fewer where the database's own parameter limit +is lower, and every test here passes. + +Shortcuts, each producing the rows a real sequence produces: +- The first REAL_ADDS children are added with Workflow.add(); the rest + are bulk-enqueued with django_ox.bulk.enqueue_many, moved to WAITING, + and given the node and edge rows add() writes, and the workflow row's + node_count is raised by the same number, as add() raises it. _build asserts + every bulk child has the same stored shape as the real ones, and that seal() + counted every child. Real add() is about 6 ms + per node on MySQL, which is four minutes per 40,000-node leg. +- The retained history is enqueue_many plus an UPDATE to SUCCESSFUL, + standing in for a week of default-backend tasks that ran and that + ox_prune has not yet deleted. +- The failing root's retry backoff is cleared between attempts, as + test_workflows_deferred does, instead of waiting it out. +- The parameter cap is applied to the test's own connection around the + worker or reconciler call: a PostgreSQL connection with + OPTIONS["server_side_binding"] = True swapped in for "default", or + setlimit() on SQLite's. A project gets the first from its DATABASES + setting and the second from a SQLite built with the default + SQLITE_MAX_VARIABLE_NUMBER. +""" + +from __future__ import annotations + +import contextlib +import logging +import re +import sqlite3 +import time +import uuid + +import pytest +from django.db import DEFAULT_DB_ALIAS, connection, connections +from django.db.models import F +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox.bulk import enqueue_many +from django_ox.compat import task_finished +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from .test_workflows import flow_worker +from .test_workflows_deferred import deferring_worker + +mysql_only = pytest.mark.skipif( + connection.vendor != "mysql", + reason="range_optimizer_max_mem_size and warning 3170 are MySQL's", +) + +pytestmark = pytest.mark.django_db(transaction=True) + +WIDTHS = [1000, 10000, 40000] +RETAINED_HISTORY = 60000 +REAL_ADDS = 100 +CHUNK = 1000 + +TASK_TABLE = OxTask._meta.db_table +EDGE_TABLE = OxpullWorkflowEdge._meta.db_table +NODE_TABLE = OxpullWorkflowNode._meta.db_table + +# A literal list of task ids, not a subquery: `django_ox_oxtask`.`id` IN ('..', ..) +LITERAL_ID_LIST = re.compile(rf"`{TASK_TABLE}`\.`id` IN \((?!SELECT)([^)]*)\)") +SHAPE = ( + "status", + "run_after", + "backend_name", + "queue_name", + "priority", + "max_attempts", + "attempts", + "lease_epoch", + "task_path", +) + + +def _children(flow): + return OxTask.objects.filter( + id__in=OxpullWorkflowNode.objects.filter(workflow_id=flow.id) + .exclude(name="root") + .values("task_id") + ) + + +def _build(width, *, root_task, child_task): + """root -> `width` children, sealed. Returns (flow, root id).""" + flow = workflows.create(f"wide-{width}") + flow.add("root", root_task, "root") + labels = [f"c{i:06d}" for i in range(width)] + for label in labels[:REAL_ADDS]: + flow.add(label, child_task, label, depends_on="root") + root_id = uuid.UUID(str(flow.node_ids()["root"])) + + rest = labels[REAL_ADDS:] + results = enqueue_many(child_task, [((label,), {}) for label in rest]) + bulk_ids = [uuid.UUID(str(r.id)) for r in results] + for i in range(0, len(bulk_ids), CHUNK): + OxTask.objects.filter(pk__in=bulk_ids[i : i + CHUNK]).update( + status=OxTask.Status.WAITING + ) + now = timezone.now() + OxpullWorkflowNode.objects.bulk_create( + [ + OxpullWorkflowNode( + workflow_id=flow.id, + name=label, + name_key=workflows.name_key(label), + task_id=tid, + created_at=now, + ) + for label, tid in zip(rest, bulk_ids, strict=True) + ], + batch_size=2000, + ) + OxpullWorkflowEdge.objects.bulk_create( + [ + OxpullWorkflowEdge( + workflow_id=flow.id, + upstream_task_id=root_id, + downstream_task_id=tid, + created_at=now, + ) + for tid in bulk_ids + ], + batch_size=2000, + ) + OxpullWorkflow.objects.filter(pk=flow.id).update( + node_count=F("node_count") + len(rest) + ) + flow.seal() + assert OxpullWorkflow.objects.get(pk=flow.id).total_count == width + 1 + + real = set( + OxTask.objects.filter( + id__in=OxpullWorkflowNode.objects.filter( + workflow_id=flow.id, name__in=labels[:REAL_ADDS] + ).values("task_id") + ) + .values_list(*SHAPE) + .distinct() + ) + every = set(_children(flow).values_list(*SHAPE).distinct()) + assert len(real) == 1, f"add() wrote children of more than one shape: {real}" + assert every == real, f"bulk children differ from add()'s: {every - real}" + assert _children(flow).count() == width + assert OxpullWorkflowEdge.objects.filter(upstream_task_id=root_id).count() == width + return flow, root_id + + +def _retained_history(count=RETAINED_HISTORY): + """Settled default-backend rows, as ox_prune's 7-day retention keeps them.""" + for start in range(0, count, 20000): + results = enqueue_many( + tasks.default_step, + [((f"h{i}",), {}) for i in range(start, min(count, start + 20000))], + ) + ids = [r.id for r in results] + for i in range(0, len(ids), CHUNK): + OxTask.objects.filter(pk__in=ids[i : i + CHUNK]).update( + status=OxTask.Status.SUCCESSFUL, + finished_at=timezone.now(), + attempts=1, + ) + + +def _analyze(): + with connection.cursor() as c: + c.execute(f"ANALYZE TABLE {TASK_TABLE}, {EDGE_TABLE}, {NODE_TABLE}") + c.fetchall() + + +def _rnd_next(): + with connection.cursor() as c: + c.execute("SHOW SESSION STATUS LIKE 'Handler_read_rnd_next'") + return int(c.fetchone()[1]) + + +def _status_read_overhead(): + """What reading the counter itself adds, so it can be subtracted.""" + first = _rnd_next() + return _rnd_next() - first + + +def _explain_id_lists(captured): + """ + EXPLAIN every captured statement that carries a literal list of task + ids. Returns (examined, problems): one entry per such statement, and + the ones that scan the task table or drew warning 3170. + """ + examined, problems = [], [] + for query in captured: + sql = query["sql"] + match = LITERAL_ID_LIST.search(sql) + if match is None: + continue + ids = match.group(1).count(",") + 1 + with connection.cursor() as c: + c.execute("EXPLAIN " + sql) + columns = [d[0] for d in c.description] + plan = [dict(zip(columns, row, strict=True)) for row in c.fetchall()] + c.execute("SHOW WARNINGS") + warned = [w[2] for w in c.fetchall() if int(w[1]) == 3170] + access = [ + (p["type"], p["key"], p["rows"]) for p in plan if p["table"] == TASK_TABLE + ] + entry = { + "statement": sql.split(" ", 1)[0], + "ids": ids, + "access": access, + "secs": query["time"], + } + examined.append(entry) + if warned or any(kind in ("ALL", "index") for kind, _, _ in access): + problems.append({**entry, "warning": [w[:90] for w in warned]}) + return examined, problems + + +@mysql_only +@pytest.mark.parametrize("width", WIDTHS) +def test_a_worker_finishing_a_wide_root_releases_its_children_by_range(width): + """ + One root, `width` waiting children, and the retained history ox_prune + keeps. A real worker runs the root; its + outcome write releases the children before task_finished is sent. + """ + flow, root_id = _build( + width, + root_task=tasks.deferred_step, + child_task=tasks.deferred_step, + ) + children = _children(flow) + assert children.filter(status=OxTask.Status.WAITING).count() == width + _retained_history() + _analyze() + + worker = deferring_worker() + signalled = [] + + def on_finished(sender, task_result, **kwargs): + signalled.append(time.perf_counter()) + + task_finished.connect(on_finished, weak=False) + try: + overhead = _status_read_overhead() + before = _rnd_next() + started = time.perf_counter() + with CaptureQueriesContext(connection) as ctx: + ran = worker.run_once() + after = _rnd_next() + finally: + task_finished.disconnect(on_finished) + + assert ran is True + assert tasks.STATE.get("order") == ["root"] + assert OxTask.objects.get(pk=root_id).status == OxTask.Status.SUCCESSFUL + released = children.filter(status=OxTask.Status.READY).count() + assert released == width, "the rows are wrong, not just slow" + + rnd_next = after - before - overhead + examined, problems = _explain_id_lists(ctx.captured_queries) + waited = (signalled[0] - started) if signalled else None + print( # noqa: T201 + f"\n[release] width={width} retained={RETAINED_HISTORY} " + f"table_rows~{OxTask.objects.count()} released={released} " + f"Handler_read_rnd_next={rnd_next} " + f"run_once_to_task_finished_s={waited and round(waited, 3)} " + f"id_list_statements={examined}" + ) + assert examined, "no statement carried the fan-out; the instrument saw nothing" + failures = [] + if problems: + failures.append(f"statements over the fan-out scan {TASK_TABLE}: {problems}") + if rnd_next >= 3 * width: + failures.append( + f"releasing {width} children read {rnd_next} rows by table scan " + f"(bound {3 * width}) in a table of ~{OxTask.objects.count()} rows" + ) + assert not failures, "\n".join(failures) + + +@mysql_only +@pytest.mark.parametrize("width", WIDTHS) +def test_the_reconciler_cancels_a_wide_cone_by_range(width): + """ + A root that fails every attempt on a real worker, `width` WAITING + children blocked on it, and one real reconciler tick. + """ + flow, root_id = _build(width, root_task=tasks.failing_step, child_task=tasks.step) + children = _children(flow) + _retained_history() + _analyze() + + worker = flow_worker() + for _ in range(10): + if not worker.run_once(): + break + OxTask.objects.filter(pk=root_id, status=OxTask.Status.READY).update( + run_after=None + ) + assert OxTask.objects.get(pk=root_id).status == OxTask.Status.FAILED + assert tasks.STATE.get("order") == ["root", "root", "root"] + assert children.filter(status=OxTask.Status.WAITING).count() == width + + overhead = _status_read_overhead() + before = _rnd_next() + with CaptureQueriesContext(connection) as ctx: + workflows._reconcile() + after = _rnd_next() + + cancelled = children.filter(status=OxTask.Status.DISCARDED).count() + assert cancelled == width, "the rows are wrong, not just slow" + + examined, problems = _explain_id_lists(ctx.captured_queries) + print( # noqa: T201 + f"\n[cancel] width={width} retained={RETAINED_HISTORY} " + f"table_rows~{OxTask.objects.count()} cancelled={cancelled} " + f"tick_Handler_read_rnd_next={after - before - overhead} " + f"id_list_statements={examined}" + ) + assert examined, "no statement carried the cone; the instrument saw nothing" + assert not problems, ( + f"statements over the cone scan {TASK_TABLE} inside the cancelling " + f"transaction: {problems}" + ) + + +# -- past a bind-parameter cap: PostgreSQL and SQLite ------------------ +# +# PostgreSQL's protocol carries at most 65,535 bound parameters. Django's +# psycopg backend binds client-side by default, where no cap applies, and +# server-side under OPTIONS["server_side_binding"] = True. SQLite's +# compile-time SQLITE_MAX_VARIABLE_NUMBER defaults to 32,766 since 3.32.0; +# this build raises it to 250,000, so the test lowers it to the default. + +PARAMETER_CAPS = {"postgresql": 65535, "sqlite": 32766} + +capped_vendors = pytest.mark.skipif( + connection.vendor not in PARAMETER_CAPS, + reason="MySQL's drivers interpolate parameters client-side; no bind cap", +) + + +@contextlib.contextmanager +def _parameter_cap(capped): + """ + Put the vendor's bind-parameter cap on the default connection. Yields + a check that the cap is still in force, so a reconnect inside the + worker cannot turn this into a quiet pass. + """ + if not capped: + yield lambda: None + return + if connection.vendor == "postgresql": + original = connections[DEFAULT_DB_ALIAS] + # Unpooled: Django keeps one pool per alias, built from the OPTIONS + # it first saw, so a checkout from it would bind client-side + # whatever these say. + options = { + name: value + for name, value in original.settings_dict["OPTIONS"].items() + if name != "pool" + } + settings_dict = { + **original.settings_dict, + "OPTIONS": {**options, "server_side_binding": True}, + } + swapped = original.__class__(settings_dict, alias=DEFAULT_DB_ALIAS) + connections[DEFAULT_DB_ALIAS] = swapped + + def still_capped(): + assert connections[DEFAULT_DB_ALIAS] is swapped + with connection.cursor() as c: + assert type(c.cursor).__name__ == "ServerBindingCursor" + + try: + yield still_capped + finally: + swapped.close() + connections[DEFAULT_DB_ALIAS] = original + return + connection.ensure_connection() + raw = connection.connection + limit = sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER + previous = raw.setlimit(limit, PARAMETER_CAPS["sqlite"]) + + def still_capped(): + assert connection.connection is raw + assert raw.getlimit(limit) == PARAMETER_CAPS["sqlite"] + + try: + yield still_capped + finally: + raw.setlimit(limit, previous) + + +def _logged_failures(caplog): + return [ + (r.getMessage()[:70], str(r.exc_info[1])[:90] if r.exc_info else None) + for r in caplog.records + if r.name.startswith("oxpull") and r.levelno >= logging.WARNING + ] + + +@capped_vendors +@pytest.mark.parametrize( + "capped", + [pytest.param(False, id="uncapped"), pytest.param(True, id="capped")], +) +def test_a_worker_releases_a_fan_out_wider_than_the_parameter_cap(capped, caplog): + caplog.set_level(logging.WARNING, logger="oxpull") + width = PARAMETER_CAPS[connection.vendor] + 5000 + flow, root_id = _build( + width, + root_task=tasks.deferred_step, + child_task=tasks.deferred_step, + ) + children = _children(flow) + worker = deferring_worker() + with _parameter_cap(capped) as still_capped: + ran = worker.run_once() + still_capped() + + assert ran is True + assert OxTask.objects.get(pk=root_id).status == OxTask.Status.SUCCESSFUL + released = children.filter(status=OxTask.Status.READY).count() + logged = _logged_failures(caplog) + print( # noqa: T201 + f"\n[cap-release] vendor={connection.vendor} capped={capped} " + f"width={width} released={released} logged={logged}" + ) + assert released == width, ( + f"the worker that finished root released {released} of {width} children " + f"and logged {logged}; the rest wait for the reconciler, " + f"{workflows.BACKSTOP_PAGE} per tick" + ) + + +@capped_vendors +@pytest.mark.parametrize( + "capped", + [pytest.param(False, id="uncapped"), pytest.param(True, id="capped")], +) +def test_the_reconciler_cancels_a_cone_wider_than_the_parameter_cap(capped, caplog): + caplog.set_level(logging.WARNING, logger="oxpull") + width = PARAMETER_CAPS[connection.vendor] + 5000 + flow, root_id = _build(width, root_task=tasks.failing_step, child_task=tasks.step) + children = _children(flow) + worker = flow_worker() + for _ in range(10): + if not worker.run_once(): + break + OxTask.objects.filter(pk=root_id, status=OxTask.Status.READY).update( + run_after=None + ) + assert OxTask.objects.get(pk=root_id).status == OxTask.Status.FAILED + caplog.clear() + + ticks = 3 + with _parameter_cap(capped) as still_capped: + for _ in range(ticks): + workflows._reconcile() + still_capped() + + cancelled = children.filter(status=OxTask.Status.DISCARDED).count() + status = OxpullWorkflow.objects.get(pk=flow.id).status + logged = _logged_failures(caplog) + print( # noqa: T201 + f"\n[cap-cancel] vendor={connection.vendor} capped={capped} " + f"width={width} ticks={ticks} cancelled={cancelled} workflow={status} " + f"logged={logged}" + ) + assert (cancelled, status) == (width, OxpullWorkflow.Status.COMPLETE), ( + f"after {ticks} reconciler ticks {cancelled} of {width} doomed nodes are " + f"cancelled and the workflow is {status}; logged {logged}" + ) diff --git a/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_inside_a_caller_s.py b/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_inside_a_caller_s.py new file mode 100644 index 0000000..6195163 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_inside_a_caller_s.py @@ -0,0 +1,291 @@ +""" +Workflow.add() inside a caller's transaction reads a stale view. + +add() takes the workflow row lock with an UPDATE, which is a current read, +and then resolves depends_on and reads its upstreams' outcomes. Under +REPEATABLE READ a plain SELECT inside a caller's transaction answers from +the read view that transaction's first consistent read fixed, not from the +state under the lock. A node, or an upstream outcome, committed after that +first read and before add() is invisible to it: + +* _resolve_dependencies would raise UnknownWorkflowNode for a node that + exists. +* add() would leave a node WAITING whose upstream has already succeeded and + already ran its release, so the node would wait for a reconciler tick. + +Django's MySQL and PostgreSQL backends both default to READ COMMITTED, where +each statement reads current state. A project reaches REPEATABLE READ by +setting DATABASES[...]["OPTIONS"]["isolation_level"], or None on MySQL for +the server's default. The level is set on the caller's session here, as +tests_oxpull/test_batches.py and tests_oxpull/test_unique.py do, and READ COMMITTED runs +the same sequence as the control. + +add() resolves depends_on with a locking read, which is a current read on +MySQL, and when it leaves a node WAITING it checks the node's upstreams +again once its transaction commits. add() also locks the upstream's node +row, and a worker recording that upstream's outcome locks the same row. So +in the last test, an upstream that finishes while the adding transaction is +still open waits for its commit, and then marks the new edge and releases +the node. + +Every test asserts the correct behaviour, and every test passes. +""" + +import threading +from contextlib import contextmanager + +import pytest +from django.db import OperationalError, connections, transaction + +import oxpull.worker as oxpull_worker +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.backend import is_serialization_failure +from oxpull.exceptions import UnknownWorkflowNode +from oxpull.models import OxpullWorkflowEdge + +from . import isolation, tasks +from .racing import race_all +from .test_workflows import order +from .test_workflows_deferred import deferring_worker, status +from .test_workflows_hazard_postgresql_repeatable_read_concurrent_add import ( + wait_for_lock_waiters, +) + +pytestmark = pytest.mark.django_db(transaction=True) + +LEVELS = ["READ COMMITTED", "REPEATABLE READ"] + +VENDOR = connections["default"].vendor + + +def skip_where_no_concurrent_commit_fits(): + if connections["default"].vendor == "sqlite": + pytest.skip( + "a reading SQLite transaction holds its shared lock, so the " + "concurrent commit this hazard needs cannot land under it" + ) + + +@contextmanager +def caller_session_at(level): + """ + The caller's session at `level` for the block, on MySQL or PostgreSQL. + The connection is closed afterwards so the next use reopens it at the + configured level. + """ + connection = connections["default"] + if connection.vendor == "mysql": + statement = f"SET SESSION TRANSACTION ISOLATION LEVEL {level}" + else: + statement = ( + f"SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL {level}" + ) + isolation.outlives_the_transaction(level) + with connection.cursor() as cursor: + cursor.execute(statement) + try: + yield + finally: + connection.close() + + +@pytest.mark.parametrize("level", LEVELS) +def test_add_resolves_a_dependency_committed_after_the_callers_first_read(level): + """ + _resolve_dependencies promises that a concurrent add() of the named + node "commits before this transaction gets the lock, or after this one + releases it, so the name either resolves or does not". Here stage-2 + commits before this add() takes the lock, so it must resolve. + + The caller's first read is workflows.get(), what a view running under + ATOMIC_REQUESTS does before it adds a node. Another connection adds + stage-2 and commits. The caller then adds stage-3 depending on it. + + PostgreSQL REPEATABLE READ never reaches the stale lookup: the other + add() rewrote the workflow row after the caller's snapshot, so the + caller's lock UPDATE raises a serialization failure, which is that + level's documented and retryable refusal. + """ + skip_where_no_concurrent_commit_fits() + if connections["default"].vendor == "postgresql" and level == "REPEATABLE READ": + pytest.skip( + "PostgreSQL REPEATABLE READ refuses the workflow-row UPDATE with a " + "serialization failure before any name is looked up" + ) + flow = workflows.create("resolve") + flow.add("root", tasks.step, "root") + + def another_request_adds_stage_2(): + return ( + workflows.get(flow.id) + .add("stage-2", tasks.step, "stage-2", depends_on="root") + .id + ) + + with caller_session_at(level), transaction.atomic(): + # The caller's first consistent read. Under REPEATABLE READ the + # read view is fixed here. + handle = workflows.get(flow.id) + results, errors = race_all([another_request_adds_stage_2]) + assert errors == [] + (stage_2_id,) = results + try: + stage_3 = handle.add("stage-3", tasks.step, "stage-3", depends_on="stage-2") + except UnknownWorkflowNode as exc: + pytest.fail( + f"add() raised {exc!r}, but stage-2 (task {stage_2_id}) had " + "committed before add() took the workflow row lock" + ) + + edge = OxpullWorkflowEdge.objects.get(downstream_task_id=stage_3.id) + assert str(edge.upstream_task_id) == stage_2_id + + +@pytest.mark.parametrize("level", LEVELS) +def test_a_node_whose_upstream_succeeded_behind_the_callers_snapshot_is_not_deferred( + level, monkeypatch +): + """ + add() releases a node whose upstreams have all succeeded + (test_a_node_whose_parent_already_succeeded_is_released_as_it_is_added). + That matters because the one release an upstream makes happens when its + outcome lands. A node left WAITING after that waits for a reconciler + tick, and without a scheduled reconciler it waits forever. + + The caller reads first. A real worker on another connection then runs + the root to SUCCESSFUL and makes its release, which finds nothing to + release because the child does not exist yet. Then the caller adds the + child. + """ + skip_where_no_concurrent_commit_fits() + releases = [] + real_release = oxpull_worker.release_downstream + + def recording_release(task_ids, db_alias): + released = real_release(task_ids, db_alias) + releases.append((sorted(str(task_id) for task_id in task_ids), released)) + return released + + monkeypatch.setattr(oxpull_worker, "release_downstream", recording_release) + + flow = workflows.create("defer") + root = flow.add("root", tasks.deferred_step, "root") + + def a_worker_runs_the_root(): + return deferring_worker().run_once() + + refusals = [] + with caller_session_at(level): + for attempt in range(2): + try: + with transaction.atomic(): + handle = workflows.get(flow.id) + if attempt == 0: + results, errors = race_all([a_worker_runs_the_root]) + assert (results, errors) == ([True], []) + handle.add("child", tasks.deferred_step, "child", depends_on="root") + break + except OperationalError as exc: + if attempt or not is_serialization_failure(exc, "default"): + raise + refusals.append(exc) + + # The root's outcome rewrote its node row, which add() locks as the + # child's upstream. At PostgreSQL REPEATABLE READ that row changed after + # the caller's snapshot, so add() refuses with a serialization failure, + # which is that level's documented and retryable refusal. The caller runs + # its transaction again, on a snapshot that sees the root succeeded. + refused = VENDOR == "postgresql" and level == "REPEATABLE READ" + assert len(refusals) == (1 if refused else 0), refusals + + # The upstream had succeeded and made its one release before add() ran. + # No edge pointed out of it then, so there was nothing to release. + assert OxTask.objects.get(id=root.id).status == OxTask.Status.SUCCESSFUL + assert releases == [([], 0)] + assert status("child") == OxTask.Status.READY, ( + "the child was left WAITING behind an upstream that had already " + "succeeded and released; only a reconciler tick can release it now" + ) + assert deferring_worker().run_once() is True + assert order() == ["root", "child"] + + +@pytest.mark.parametrize("level", LEVELS) +def test_a_node_whose_upstream_succeeds_before_the_adding_transaction_commits_runs( + level, monkeypatch +): + """ + The same liveness at any isolation level. The caller adds the child + while the root is READY, so add() rightly leaves it WAITING. A real + worker then runs the root while the caller's transaction is still open. + add() locked the root's node row, so the root's outcome waits for the + caller's commit and then marks the child's edge. The child must be + released when the caller commits, not a reconciler tick later. + + Two releases run once the caller commits, and either may move the child: + the root worker's, after its outcome commits, and add()'s check after its + own commit, which releases the child if it reads the edge marked. Which + one wins is timing. The test asserts that the worker released after its + outcome, and that exactly one of the two moved the child. + """ + skip_where_no_concurrent_commit_fits() + releases = [] + checks = [] + real_release = oxpull_worker.release_downstream + + def recording_release(task_ids, db_alias): + released = real_release(task_ids, db_alias) + releases.append((sorted(str(task_id) for task_id in task_ids), released)) + return released + + def recording_check(task_ids, db_alias): + released = real_release(task_ids, db_alias) + checks.append((sorted(str(task_id) for task_id in task_ids), released)) + return released + + monkeypatch.setattr(oxpull_worker, "release_downstream", recording_release) + monkeypatch.setattr(workflows, "release_downstream", recording_check) + + flow = workflows.create("defer-late") + root = flow.add("root", tasks.deferred_step, "root") + + def a_worker_runs_the_root(): + return deferring_worker().run_once() + + box = {} + + def run_the_root_in_the_background(): + try: + box["ran"] = a_worker_runs_the_root() + except BaseException as exc: + box["error"] = exc + finally: + connections.close_all() + + runner = threading.Thread(target=run_the_root_in_the_background) + with caller_session_at(level), transaction.atomic(): + child = workflows.get(flow.id).add( + "child", tasks.deferred_step, "child", depends_on="root" + ) + assert status("child") == OxTask.Status.WAITING, "the root had not run yet" + runner.start() + # Polled from its own connection: a snapshot of lock waits taken + # inside the caller's transaction would not change while it is open. + (waited,), poll_errors = race_all([lambda: wait_for_lock_waiters(1)]) + assert poll_errors == [] + runner.join(60) + + assert not runner.is_alive() + assert box == {"ran": True}, box + assert waited >= 1, "the root's outcome never waited on the adding transaction" + assert OxTask.objects.get(id=root.id).status == OxTask.Status.SUCCESSFUL + assert [ids for ids, _ in releases] == [[str(child.id)]] + assert [ids for ids, _ in checks] == [[str(child.id)]] + assert sum(moved for _, moved in releases + checks) == 1, (releases, checks) + assert status("child") == OxTask.Status.READY, ( + "the child stayed WAITING after its upstream succeeded and the adding " + "transaction committed; only a reconciler tick can release it now" + ) + assert deferring_worker().run_once() is True + assert order() == ["root", "child"] diff --git a/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_the_cancellation_s.py b/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_the_cancellation_s.py new file mode 100644 index 0000000..94cbadd --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_mysql_repeatable_read_the_cancellation_s.py @@ -0,0 +1,934 @@ +""" +On MySQL the cancellation's locking read locks rows it does not +return, so live work waits on, or is skipped because of, a pass that is not +about it. + +workflows._cancel_one_level, one transaction per level: + + 1. UPDATE the workflow row under SEALED; + 2. SELECT pk FROM task WHERE id IN (the workflow's node task ids) + AND status IN (FAILED, LOST, DISCARDED) ORDER BY pk FOR UPDATE; + 3. SELECT the READY or WAITING nodes blocked by a dead upstream; + 4. UPDATE those nodes to DISCARDED; + 5. COMMIT. + +The docstring's guard is "only terminal statuses are locked, which no worker +ever claims". InnoDB does not keep to it. Which rows it locks depends on the +plan it picks for step 2, and the plan depends on what the task table holds: + +- Node-driven: the node table read for the workflow's id, by index or by a + filtered scan, then a PRIMARY lookup per node, with the status test above + the storage engine. Picked here whenever the table holds a few hundred + DISCARDED rows, from history or from the workflow's own cancelled cone. + At REPEATABLE READ every node row of the workflow stays X-locked to + COMMIT, RUNNING and READY included. At READ COMMITTED InnoDB releases + rows that fail the filter, except the last row the scan reads, which + stays locked; with a workflow of mostly live nodes that is a live node. +- A range on ox_reaper_idx (status, locked_at) over the dead statuses, + picked when dead rows are few. At REPEATABLE READ this takes next-key and + gap locks over that index range, table-wide, so any task, in a workflow + or not, whose status moves into FAILED, LOST or DISCARDED waits for the + level to commit. + +Who runs at REPEATABLE READ: Django's MySQL backend sets READ COMMITTED on +every connection unless DATABASES[...]["OPTIONS"]["isolation_level"] says +otherwise; a project that sets "repeatable read", or None for InnoDB's +server default, runs at REPEATABLE READ. oxpull already treats that as a +supported configuration (OxpullBackend._reads_a_snapshot, the batch seal +tests). The fixture sets the level the way a project does, through OPTIONS, +and every test checks the level of the pass's own open transaction, read +while it is held. + +Every step is a production path. 600 unrelated tasks are enqueued and +discarded with django_ox.actions.discard_many, where a cell asks for that +history. A real worker drives the root through three failing attempts to +FAILED. A second real worker claims the "running" node and executes it +(run_once), its body parked on an Event, so the row holds a real lease and +sits in that worker's in-flight set. The bystander, a task outside any +workflow, is failed twice by a real worker and claimed for its last attempt. +OPTIMIZE TABLE then rebuilds the tables and refreshes their statistics. A +test that ran before this one can leave tens of thousands of deleted task +rows that InnoDB has not purged yet. Its estimate of the dead-status range +then falls to a few dozen rows, and it picks the other plan. Each cell names +the plan it is about, and the pass's own optimizer trace says which plan the +locking read ran. The EXPLAIN printed with each result can disagree with it +while InnoDB is still purging. A pass that ran another plan fails the test as +not provoked, so it counts neither as the expected failure nor as a fix. + +The pass is workflows._reconcile, the scheduled task's body, on its own +thread and connection. The probes are the running worker's +renew_leases(), that worker finishing its task (_write_outcome), another +worker claiming everything it can, and the bystander's worker executing its +last attempt. No task row is written by hand. + +The pass is held open at one of two statements, found by the SQL the +reconciler's connection sends: just after the locking read (step 2), and just +after the discard UPDATE (step 4, the last statement before COMMIT). The pass +stays held until the probe is done, and no verdict compares wall-clock +intervals: the claim probe is a hundred transactions, and how long they take +is the server's speed, not a wait. While the probe runs, the database is +asked who waits on a lock the pass's connection holds (data_lock_waits on +MySQL, pg_blocking_pids on PostgreSQL). A wait found there is the verdict; the +pass is then released, so the waiter finishes, and the wait is reported. When +the probe is done the pass's transaction is read once more, still open, so +everything the probe did happened while the pass held its locks. The plan +InnoDB chose for the locking read is printed with every result. + +Every test asserts the correct behaviour, so each cell here fails today on +MySQL. The cells where the same assertion holds today (the other plan, the +other isolation level) are the controls, run from a separate file. +PostgreSQL runs the probe tests too and is not expected to be exposed: its +FOR UPDATE locks only the rows the filter returns. The data_locks test is +InnoDB's own. SQLite is skipped: it has no row locks, its writers serialise, +and the pass opens with a write, so every write waits on it by design. +""" + +from __future__ import annotations + +import json +import re +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import pytest +from django.db import connections, transaction +from django.tasks import task + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflowNode +from oxpull.routing import oxpull_db_alias + +from . import isolation, tasks +from .test_workflows import flow_worker, order, reconcile + +pytestmark = pytest.mark.django_db(transaction=True) + +TASK_TABLE = OxTask._meta.db_table.upper() + +# The cells in this file fail on MySQL only; PostgreSQL runs them green. +ON_MYSQL = connections[oxpull_db_alias()].vendor == "mysql" + +AFTER_THE_LOCKING_READ = "after_the_locking_read" +AFTER_THE_DISCARD = "after_the_discard" +POINTS = [AFTER_THE_LOCKING_READ, AFTER_THE_DISCARD] +LABEL_AT = { + AFTER_THE_LOCKING_READ: "locking read", + AFTER_THE_DISCARD: "discard update", +} + +REPEATABLE_READ = "repeatable read" +READ_COMMITTED = "read committed" + +# What the task table holds before the workflow is built. It decides the +# plan: see the module docstring. +NO_HISTORY = "no_history" +DISCARDED_HISTORY = "discarded_history" +HISTORY_WIDTH = 600 + +# The plan each history is there to provoke on MySQL. The bystander cells +# take whichever plan InnoDB picks, so they name none. +PLAN_OF_HISTORY = {DISCARDED_HISTORY: "node-driven"} + + +class PlanNotProvoked(Exception): + """ + The locking read ran under another plan than the cell is about. + + Not an AssertionError. The xfail markers expect one, so this is reported + as a failure: a pass under the wrong plan proves neither the hazard nor + its fix. + """ + + +# Claimable READY nodes of the workflow, so that "the last row the scan +# reads" is almost always a live one. +LIVE_READY = 100 + +# How often the database is asked who waits on the pass while a probe runs. +WAIT_POLL = 0.05 +# Safety limits, never reached on a passing run. A probe that waits on the pass +# is reported by the database within a poll or two. PROBE_LIMIT is what a probe +# held up by anything the database does not report as a lock wait runs into, +# and it ends inside HOLD_LIMIT, so the pass is still held when it does. +PASS_REACH_LIMIT = 20.0 +PROBE_LIMIT = 20.0 +HOLD_LIMIT = 30.0 +GATE_LIMIT = 60.0 + + +def cell(history: str, level: str): + return pytest.param(history, level, id=f"{history}-{level.replace(' ', '_')}") + + +# The cells that fail today. The controls file runs every other cell. +LIVE_NODE_CELLS = [cell(DISCARDED_HISTORY, REPEATABLE_READ)] +CLAIMABLE_CELLS = [ + cell(DISCARDED_HISTORY, REPEATABLE_READ), + cell(DISCARDED_HISTORY, READ_COMMITTED), +] +BYSTANDER_CELLS = [cell(NO_HISTORY, REPEATABLE_READ)] + + +# -- the parked node --------------------------------------------------- + + +@task(backend="flows") +def held_step(label): + """A node whose body runs until the test releases it.""" + tasks._record(label) + if not tasks.STATE["gate"].wait(GATE_LIMIT): + raise TimeoutError(f"{label} was never released") + return label + + +# -- configuration ----------------------------------------------------- + + +@pytest.fixture +def project_isolation(): + """The isolation level the whole project is configured at.""" + return REPEATABLE_READ + + +@pytest.fixture +def history(): + return DISCARDED_HISTORY + + +def session_facts(connection) -> tuple[str, Any]: + """This connection's isolation level, normalised, and its server id.""" + with connection.cursor() as cursor: + if connection.vendor == "mysql": + cursor.execute("SELECT @@transaction_isolation, CONNECTION_ID()") + else: + cursor.execute( + "SELECT current_setting('transaction_isolation'), pg_backend_pid()" + ) + level, ident = cursor.fetchone() + return str(level).lower().replace("-", " ").replace("_", " "), ident + + +def transaction_level(connection) -> str: + """ + The isolation level the connection's open transaction runs at, read on + the raw DB-API cursor so no execute wrapper sees it. On PostgreSQL the + configured level applies from BEGIN, so only a read inside the + transaction shows it. + """ + if connection.vendor == "mysql": + query = "SELECT @@transaction_isolation" + else: + query = "SELECT current_setting('transaction_isolation')" + cursor = connection.connection.cursor() + try: + cursor.execute(query) + (level,) = cursor.fetchone() + finally: + cursor.close() + return str(level).lower().replace("-", " ").replace("_", " ") + + +@pytest.fixture +def isolated_project(project_isolation, monkeypatch): + """ + DATABASES OPTIONS["isolation_level"], as a project sets it, for every + connection opened during the test. Threads open their own connections + from these settings; the main thread's is reopened. + """ + alias = oxpull_db_alias() + connection = connections[alias] + if connection.vendor == "sqlite": + pytest.skip( + "SQLite has no row locks: writers serialise and the pass opens " + "with a write, so every write waits on it by design" + ) + if connection.vendor == "mysql": + value: Any = project_isolation + else: + from psycopg import IsolationLevel + + value = IsolationLevel[project_isolation.upper().replace(" ", "_")] + targets = { + id(options): options + for options in ( + connections.settings[alias]["OPTIONS"], + connection.settings_dict["OPTIONS"], + ) + } + isolation.outlives_the_transaction(project_isolation) + for options in targets.values(): + monkeypatch.setitem(options, "isolation_level", value) + connection.close() + yield project_isolation + connection.close() + + +# -- threads that record what they send -------------------------------- + + +class Recorded: + """A callable on its own thread and connection, timing each statement.""" + + def __init__(self, name: str, fn: Callable[[], Any]) -> None: + self.fn = fn + self.result: Any = None + self.error: BaseException | None = None + self.elapsed: float | None = None + self.statements: list[tuple[str, float]] = [] + # Set once fn has returned or raised, with its connection still open. + self.done = threading.Event() + self._started = 0.0 + self.thread = threading.Thread(target=self._run, name=name) + + def _record(self, execute, sql, params, many, context): + started = time.monotonic() + try: + return execute(sql, params, many, context) + finally: + text = " ".join(sql.split()) + self.statements.append((text[:70], time.monotonic() - started)) + + def _run(self) -> None: + connection = connections[oxpull_db_alias()] + try: + with connection.execute_wrapper(self._record): + self.result = self.fn() + except BaseException as exc: + self.error = exc + finally: + self.elapsed = time.monotonic() - self._started + self.done.set() + connections.close_all() + + def start(self) -> None: + self._started = time.monotonic() + self.thread.start() + + def slowest(self) -> str: + if not self.statements: + return "no statements recorded" + text, took = max(self.statements, key=lambda item: item[1]) + return f"slowest statement {took:.3f}s: {text}" + + +# -- the workflow ------------------------------------------------------ + + +@dataclass +class Scenario: + ids: dict[str, Any] + live_ready: list[Any] + running_worker: Any + runner: Recorded + bystander_worker: Any + bystander: OxTask + history: str + + @property + def live(self) -> set[Any]: + return {self.ids["running"], *self.live_ready} + + def status(self, pk: Any) -> str: + return OxTask.objects.get(pk=pk).status + + def name_of(self, lock_data: str) -> str: + value = lock_data.strip().strip("'") + if value == self.bystander.pk.hex: + return "bystander" + for name, pk in self.ids.items(): + if value == pk.hex: + return name + return f"other:{lock_data}" + + +def analyze_tables() -> None: + connection = connections[oxpull_db_alias()] + tables = [ + OxTask._meta.db_table, + OxpullWorkflowNode._meta.db_table, + "oxpull_oxpullworkflowedge", + ] + with connection.cursor() as cursor: + if connection.vendor == "mysql": + # A rebuild as well as the statistics: see the module docstring. + cursor.execute("OPTIMIZE TABLE " + ", ".join(tables)) + cursor.fetchall() + else: + for table in tables: + cursor.execute(f"ANALYZE {table}") + + +@pytest.fixture +def workflow(isolated_project, history): + """ + root (FAILED) -> child (WAITING, blocked); running (RUNNING, a real + lease); LIVE_READY claimable READY nodes with no edges. Sealed, so the + reconciler sweeps it, and never complete while live nodes are pending. + Beside it, the bystander: no workflow's node, RUNNING its last attempt. + """ + alias = oxpull_db_alias() + tasks.STATE["gate"] = threading.Event() + if history == DISCARDED_HISTORY: + results = [tasks.plain_echo.enqueue(i) for i in range(HISTORY_WIDTH)] + changed, _ = actions.discard_many([result.id for result in results]) + assert changed == HISTORY_WIDTH + + flow = workflows.create("cancellation-locks") + with transaction.atomic(using=alias): + flow.add("root", tasks.failing_step.using(priority=10), "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.add("running", held_step.using(queue_name="emails", priority=5), "running") + for index in range(LIVE_READY): + name = f"zz-ready-{index:03d}" + flow.add(name, tasks.step.using(queue_name="emails", priority=0), name) + flow.seal() + ids = dict( + OxpullWorkflowNode.objects.using(alias) + .filter(workflow_id=flow.id) + .values_list("name", "task_id") + ) + live_ready = [pk for name, pk in ids.items() if name.startswith("zz-ready-")] + + setup = flow_worker(queues=["default"]) + for _ in range(3): + assert setup.run_once() is True + assert order() == ["root"] * 3, order() + assert setup.run_once() is False, "the child ran past its FAILED root" + + tasks.failing_step.using(priority=10).enqueue("bystander") + bystander_worker = flow_worker(queues=["default"]) + for _ in range(2): + assert bystander_worker.run_once() is True + bystander = bystander_worker.claim_one() + assert bystander is not None and bystander.args == ["bystander"] + assert bystander.attempts == bystander.max_attempts + + running_worker = flow_worker(queues=["emails"]) + runner = Recorded("runner", running_worker.run_once) + runner.start() + scenario = Scenario( + ids=ids, + live_ready=live_ready, + running_worker=running_worker, + runner=runner, + bystander_worker=bystander_worker, + bystander=bystander, + history=history, + ) + try: + deadline = time.monotonic() + 10 + while scenario.status(ids["running"]) != OxTask.Status.RUNNING: + assert time.monotonic() < deadline, "the runner never claimed its node" + time.sleep(0.02) + assert scenario.status(ids["root"]) == OxTask.Status.FAILED + assert scenario.status(ids["child"]) == OxTask.Status.WAITING + assert scenario.status(bystander.pk) == OxTask.Status.RUNNING + assert ( + OxTask.objects.filter(pk__in=live_ready, status=OxTask.Status.READY).count() + == LIVE_READY + ) + # The renewal probe works when nothing is in its way. + assert running_worker.renew_leases() == 1 + analyze_tables() + yield scenario + finally: + tasks.STATE["gate"].set() + runner.thread.join(GATE_LIMIT + 30) + + +# -- the held pass ----------------------------------------------------- + + +class HeldPass: + """One real reconciler pass on its own thread, paused at one statement.""" + + def __init__(self, point: str) -> None: + self.point = point + self.reached = threading.Event() + self.resume = threading.Event() + self.events: list[str] = [] + self.level: str | None = None + self.session_level: str | None = None + self.session: Any = None + self.in_atomic_block: bool | None = None + self.held_for: float | None = None + self.returned: Any = None + self.error: BaseException | None = None + self.locking_sql: str | None = None + self.locking_params: Any = None + self.join_order: list[str] | None = None + self._fired = False + self.thread = threading.Thread(target=self._run, name="reconciler") + + def _run(self) -> None: + connection = connections[oxpull_db_alias()] + try: + self.session_level, self.session = session_facts(connection) + with connection.execute_wrapper(self._statement): + self.returned = reconcile() + except BaseException as exc: + self.error = exc + finally: + connections.close_all() + + def _statement(self, execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + on_tasks = TASK_TABLE in text + locking = on_tasks and text.startswith("SELECT") and "FOR UPDATE" in text + # MySQL's optimizer trace of the locking read itself says which plan + # this execution ran. An EXPLAIN run afterwards can disagree. + trace = ON_MYSQL and locking and self.locking_sql is None + connection = connections[oxpull_db_alias()] + if trace: + raw_statement(connection, "SET SESSION optimizer_trace = 'enabled=on'") + result = execute(sql, params, many, context) + if trace: + self.join_order = traced_join_order(connection) + raw_statement(connection, "SET SESSION optimizer_trace = 'enabled=off'") + if locking: + label = "locking read" + if self.locking_sql is None: + self.locking_sql, self.locking_params = sql, params + elif on_tasks and text.startswith("UPDATE") and self.locking_sql: + label = "discard update" + else: + label = text[:48] + self.events.append(label) + if not self._fired and label == LABEL_AT[self.point]: + self._fired = True + self.in_atomic_block = connections[oxpull_db_alias()].in_atomic_block + self.level = transaction_level(connections[oxpull_db_alias()]) + started = time.monotonic() + self.reached.set() + self.resume.wait(HOLD_LIMIT) + self.held_for = time.monotonic() - started + return result + + +def raw_statement(connection, sql: str) -> list[tuple[Any, ...]]: + """One statement on the raw DB-API cursor, where no execute wrapper sees it.""" + cursor = connection.connection.cursor() + try: + cursor.execute(sql) + return list(cursor.fetchall()) + finally: + cursor.close() + + +def traced_join_order(connection) -> list[str]: + """ + The tables of the statement this connection just traced, in the join order + it ran, from refine_plan in MySQL's optimizer trace. + """ + rows = raw_statement( + connection, "SELECT TRACE FROM information_schema.OPTIMIZER_TRACE" + ) + pending = [json.loads(rows[0][0])] if rows else [] + while pending: + item = pending.pop(0) + if isinstance(item, dict): + if isinstance(item.get("refine_plan"), list): + return [str(entry.get("table", "")) for entry in item["refine_plan"]] + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + return [] + + +def executed_plan(held: HeldPass) -> str: + """node-driven when the traced locking read read the node table first.""" + if not held.join_order: + return "unknown" + if OxpullWorkflowNode._meta.db_table in held.join_order[0]: + return "node-driven" + return "task-driven" + + +def plan_of(held: HeldPass) -> tuple[str, str]: + """EXPLAIN of the statement the pass locked with; it takes no locks.""" + if held.locking_sql is None: + return "none", "" + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "mysql": + cursor.execute( + "EXPLAIN FORMAT=TREE " + held.locking_sql, held.locking_params + ) + text = str(cursor.fetchone()[0]) + else: + cursor.execute("EXPLAIN " + held.locking_sql, held.locking_params) + text = "\n".join(str(row[0]) for row in cursor.fetchall()) + # Node-driven: the node table, read for the workflow's id by index or by a + # filtered scan, drives the join, and the task table follows by primary key. + nodes = re.search( + r"lookup on U0 using oxpull_wf_node_\w+ \(workflow_id='\w+'" + r"|Filter: \(U0\.workflow_id = '\w+'\)[^\n]*\n\s*-> Table scan on U0", + text, + ) + tasks_by_pk = text.find(f"on {OxTask._meta.db_table} using PRIMARY") + if "using ox_reaper_idx" in text: + kind = "reaper-index-range" + elif nodes is not None and tasks_by_pk > nodes.start(): + kind = "node-driven" + else: + kind = "other" + return kind, text + + +def waits_on_the_pass(held: HeldPass) -> list[tuple[Any, ...]]: + """ + The lock waits the database reports, right now, behind the pass's + connection: what each waiter asks for and the statement it is running. + + A lock the pass holds is held until the pass commits, and the pass is not + released while a probe runs, so a probe waiting on the pass is still + waiting at the next poll, whenever that comes. + """ + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "mysql": + cursor.execute( + "SELECT l.INDEX_NAME, l.LOCK_MODE, l.LOCK_DATA, " + "LEFT(waiter.PROCESSLIST_INFO, 70) " + "FROM performance_schema.data_lock_waits w " + "JOIN performance_schema.threads holder " + "ON holder.THREAD_ID = w.BLOCKING_THREAD_ID " + "JOIN performance_schema.threads waiter " + "ON waiter.THREAD_ID = w.REQUESTING_THREAD_ID " + "JOIN performance_schema.data_locks l " + "ON l.ENGINE_LOCK_ID = w.REQUESTING_ENGINE_LOCK_ID " + "WHERE holder.PROCESSLIST_ID = %s", + [held.session], + ) + else: + cursor.execute( + "SELECT wait_event, pid, LEFT(query, 70) FROM pg_stat_activity " + "WHERE wait_event_type = 'Lock' AND %s = ANY(pg_blocking_pids(pid))", + [held.session], + ) + return list(cursor.fetchall()) + + +def open_transaction_of(held: HeldPass) -> tuple[Any, ...] | None: + """ + The pass's transaction as the database has it, or None when its connection + has none open: on MySQL InnoDB's state for it and the rows it has locked, + on PostgreSQL the session's state and its transaction id. + """ + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "mysql": + cursor.execute( + "SELECT trx_state, trx_rows_locked " + "FROM information_schema.INNODB_TRX " + "WHERE trx_mysql_thread_id = %s AND trx_rows_locked > 0", + [held.session], + ) + else: + cursor.execute( + "SELECT state, backend_xid FROM pg_stat_activity WHERE pid = %s " + "AND state = 'idle in transaction' AND backend_xid IS NOT NULL", + [held.session], + ) + return cursor.fetchone() + + +@dataclass +class Outcome: + held: HeldPass + probe: Recorded + inside: bool | None + plan: str = "none" + plan_text: str = "" + snapshot: Any = None + waits: list[tuple[Any, ...]] = field(default_factory=list) + open_transaction: tuple[Any, ...] | None = None + notes: list[str] = field(default_factory=list) + + def describe(self) -> str: + elapsed = self.probe.elapsed + held_for = self.held.held_for + return ( + f"level={self.held.level} plan={self.plan} " + f"ran={executed_plan(self.held)} {self.held.join_order} " + f"point={self.held.point} " + f"probe finished inside the pass={self.inside} " + f"lock waits on the pass={self.waits} " + f"the pass's transaction after the probe={self.open_transaction} " + f"probe elapsed={None if elapsed is None else round(elapsed, 3)}s " + f"({self.probe.slowest()}) error={self.probe.error!r} " + f"pass held={None if held_for is None else round(held_for, 3)}s" + ) + + +def during_the_held_pass( + point: str, + probe_fn: Callable[[], Any], + before_probe: Callable[[HeldPass], Any] | None = None, +) -> Outcome: + """ + Run the probe while the pass is held, and release the pass only once the + probe is done, the database reports a wait on the pass, or PROBE_LIMIT is + up. + + inside is the verdict. It is True only when the probe finished, nothing was + seen waiting on the pass, and after the probe finished the hold had not + been released and the database still showed the pass's transaction open + with its locks. + """ + held = HeldPass(point) + held.thread.start() + probe = Recorded("probe", probe_fn) + outcome = Outcome(held=held, probe=probe, inside=None) + # The pass is released whatever a read in here raises, so a failure is + # reported at once and not after HOLD_LIMIT, with the pass's locks still + # in the way of the next test. + try: + if held.reached.wait(PASS_REACH_LIMIT): + outcome.plan, outcome.plan_text = plan_of(held) + if before_probe is not None: + outcome.snapshot = before_probe(held) + probe.start() + deadline = time.monotonic() + PROBE_LIMIT + while not probe.done.wait(WAIT_POLL): + outcome.waits = waits_on_the_pass(held) + if outcome.waits or time.monotonic() >= deadline: + break + finished = probe.done.is_set() + outcome.open_transaction = open_transaction_of(held) + outcome.inside = ( + finished + and not outcome.waits + and held.held_for is None + and outcome.open_transaction is not None + ) + finally: + held.resume.set() + held.thread.join(60) + if probe.thread.ident is not None: + probe.thread.join(GATE_LIMIT + 30) + return outcome + + +def assert_the_pass_was_real_and_held( + outcome: Outcome, scenario: Scenario, project_isolation: str +) -> None: + held = outcome.held + assert held.reached.is_set(), ( + f"the pass never reached {held.point}: {held.events} error={held.error!r}" + ) + assert held.level == project_isolation, held.level + assert held.in_atomic_block is True + assert not held.thread.is_alive(), "the pass never finished" + assert held.error is None, repr(held.error) + assert not outcome.probe.thread.is_alive(), "the probe never finished" + # The pass did its real work and completed nothing: live nodes were + # pending when it read them. + assert held.returned == 0 + assert scenario.status(scenario.ids["child"]) == OxTask.Status.DISCARDED + print(f"\nPLAN {outcome.plan}:\n{outcome.plan_text}") # noqa: T201 + expected = PLAN_OF_HISTORY.get(scenario.history) + # Only a locking read that names the node table can run a node-driven + # plan. One that names task rows by primary key alone has no such plan to + # provoke, and the cell then tests that read as it runs. + joins_nodes = ( + OxpullWorkflowNode._meta.db_table.upper() + in " ".join((held.locking_sql or "").split()).upper() + ) + if ( + ON_MYSQL + and expected is not None + and joins_nodes + and executed_plan(held) != expected + ): + raise PlanNotProvoked( + f"the {scenario.history} cell is about the {expected} plan, and the " + f"pass ran {executed_plan(held)}: " + outcome.describe() + ) + + +# -- the checks, shared with the controls file -------------------------- + + +def check_renewal(workflow: Scenario, project_isolation: str, point: str) -> None: + outcome = during_the_held_pass(point, workflow.running_worker.renew_leases) + assert_the_pass_was_real_and_held(outcome, workflow, project_isolation) + print("RESULT renewal", outcome.describe()) # noqa: T201 + + assert outcome.inside, ( + "the RUNNING node's lease renewal waited on the cancellation pass: " + + outcome.describe() + ) + assert outcome.probe.error is None + assert outcome.probe.result == 1 + + +def check_outcome_write(workflow: Scenario, project_isolation: str, point: str) -> None: + runner = workflow.runner + + def finish(): + tasks.STATE["gate"].set() + runner.thread.join(GATE_LIMIT) + return runner.result + + outcome = during_the_held_pass(point, finish) + assert_the_pass_was_real_and_held(outcome, workflow, project_isolation) + print("RESULT outcome", outcome.describe(), "runner", runner.slowest()) # noqa: T201 + + assert outcome.inside, ( + "the RUNNING node's outcome write waited on the cancellation pass: " + + outcome.describe() + + "; runner " + + runner.slowest() + ) + assert runner.error is None, repr(runner.error) + assert runner.result is True + assert workflow.status(workflow.ids["running"]) == OxTask.Status.SUCCESSFUL + + +def check_claimable(workflow: Scenario, project_isolation: str, point: str) -> None: + claimer = flow_worker(queues=["emails"]) + + def claim_everything(): + claimed = [] + while (row := claimer.claim_one()) is not None: + claimed.append(row.pk) + return claimed + + outcome = during_the_held_pass(point, claim_everything) + assert_the_pass_was_real_and_held(outcome, workflow, project_isolation) + claimed = set(outcome.probe.result or []) + passed_over = set(workflow.live_ready) - claimed + names = sorted(name for name, pk in workflow.ids.items() if pk in passed_over) + print( # noqa: T201 + "RESULT claim", + outcome.describe(), + f"claimed={len(claimed)} passed over={len(passed_over)} {names[:5]}", + ) + + assert outcome.probe.error is None, repr(outcome.probe.error) + assert not passed_over, ( + f"a worker claiming during the cancellation pass passed over " + f"{len(passed_over)} of {LIVE_READY} claimable READY nodes {names[:5]}: " + + outcome.describe() + ) + assert claimed == set(workflow.live_ready) + assert outcome.inside, outcome.describe() + + +def locks_held_by_the_pass(held: HeldPass) -> list[tuple[Any, ...]]: + with connections[oxpull_db_alias()].cursor() as cursor: + cursor.execute( + "SELECT l.INDEX_NAME, l.LOCK_MODE, l.LOCK_STATUS, l.LOCK_DATA " + "FROM performance_schema.data_locks l " + "JOIN performance_schema.threads t ON t.THREAD_ID = l.THREAD_ID " + "WHERE t.PROCESSLIST_ID = %s AND l.OBJECT_SCHEMA = DATABASE() " + "AND l.OBJECT_NAME = %s AND l.LOCK_TYPE = 'RECORD'", + [held.session, OxTask._meta.db_table], + ) + return list(cursor.fetchall()) + + +def check_no_live_node_locked( + workflow: Scenario, project_isolation: str, point: str +) -> None: + if connections[oxpull_db_alias()].vendor != "mysql": + pytest.skip("performance_schema.data_locks is InnoDB's") + outcome = during_the_held_pass(point, lambda: None, locks_held_by_the_pass) + assert_the_pass_was_real_and_held(outcome, workflow, project_isolation) + live_hex = {pk.hex: pk for pk in workflow.live} + root_hex = workflow.ids["root"].hex + locked_live = [] + root_locked = False + other = [] + for index, mode, status, data in outcome.snapshot: + key = (data or "").strip().strip("'") + if index == "PRIMARY" and status == "GRANTED" and key == root_hex: + root_locked = True + elif index == "PRIMARY" and status == "GRANTED" and key in live_hex: + locked_live.append(workflow.name_of(data)) + else: + other.append((index, mode, status, workflow.name_of(data or ""))) + print( # noqa: T201 + "RESULT data_locks", + outcome.describe(), + f"live nodes locked={len(locked_live)} {sorted(locked_live)[:5]} " + f"other task-table locks={other[:8]}", + ) + + # The interlock's own lock is visible, so this really is the pass's + # transaction being read. + assert root_locked, outcome.snapshot + assert not locked_live, ( + f"the cancellation pass holds row locks on {len(locked_live)} live " + f"node(s) {sorted(locked_live)[:5]}: " + outcome.describe() + ) + + +def check_bystander_failure( + workflow: Scenario, project_isolation: str, point: str +) -> None: + worker, row = workflow.bystander_worker, workflow.bystander + outcome = during_the_held_pass(point, lambda: worker.execute(row)) + assert_the_pass_was_real_and_held(outcome, workflow, project_isolation) + print("RESULT bystander", outcome.describe()) # noqa: T201 + + assert outcome.inside, ( + "the final failure of a task in no workflow waited on the cancellation " + "pass: " + outcome.describe() + ) + assert outcome.probe.error is None, repr(outcome.probe.error) + assert workflow.status(row.pk) == OxTask.Status.FAILED + + +# -- the tests --------------------------------------------------------- + + +@pytest.mark.parametrize("point", POINTS) +@pytest.mark.parametrize("history,project_isolation", LIVE_NODE_CELLS) +def test_a_running_nodes_lease_renewal_does_not_wait_on_the_cancellation( + workflow, project_isolation, point +): + """renew_leases() touches only this worker's RUNNING row.""" + check_renewal(workflow, project_isolation, point) + + +@pytest.mark.parametrize("point", POINTS) +@pytest.mark.parametrize("history,project_isolation", LIVE_NODE_CELLS) +def test_a_running_nodes_outcome_write_does_not_wait_on_the_cancellation( + workflow, project_isolation, point +): + """The task returns while the pass is open; its outcome write must land.""" + check_outcome_write(workflow, project_isolation, point) + + +@pytest.mark.parametrize("point", POINTS) +@pytest.mark.parametrize("history,project_isolation", CLAIMABLE_CELLS) +def test_every_claimable_node_can_be_claimed_during_the_cancellation( + workflow, project_isolation, point +): + """No claimable READY node is skipped as locked by a pass not about it.""" + check_claimable(workflow, project_isolation, point) + + +@pytest.mark.parametrize("point", POINTS) +@pytest.mark.parametrize("history,project_isolation", CLAIMABLE_CELLS) +def test_the_cancellation_locks_no_live_node_of_the_workflow( + workflow, project_isolation, point +): + """The docstring's guard, read from InnoDB's own lock table.""" + check_no_live_node_locked(workflow, project_isolation, point) + + +@pytest.mark.parametrize("point", POINTS) +@pytest.mark.parametrize("history,project_isolation", BYSTANDER_CELLS) +def test_an_unrelated_tasks_final_failure_does_not_wait_on_the_cancellation( + workflow, project_isolation, point +): + """A task in no workflow fails its last attempt while the pass is open.""" + check_bystander_failure(workflow, project_isolation, point) diff --git a/tests_oxpull/test_workflows_hazard_nodes_the_reconciler_cancels_emit_no_task.py b/tests_oxpull/test_workflows_hazard_nodes_the_reconciler_cancels_emit_no_task.py new file mode 100644 index 0000000..9dbb0ab --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_nodes_the_reconciler_cancels_emit_no_task.py @@ -0,0 +1,650 @@ +""" +A node the reconciler cancels must say so through oxpull: why it was +cancelled, which upstream task decided it, and in which generation of its +workflow. + +Today _cancel_one_level (workflows.py) moves a doomed cone to DISCARDED +with one queryset UPDATE and records nothing per node. The reconciler logs +a count, and the workflow row stores cancelled_count when it completes. +django-ox maps DISCARDED to TaskResultStatus.FAILED, so the TaskResult a +caller kept from flow.add() reads FAILED with no errors and no attempts, +which is exactly what an operator's discard reads. Nothing tells the two +apart, and nothing says which upstream was to blame. + +The record belongs on oxpull's own node row: a cause, the id of the +upstream task that decided the cancellation, and the workflow generation +it happened in (cancel_cause, cancelled_by_task_id and cancelled_generation +on OxpullWorkflowNode, against OxpullWorkflow.generation). The django-ox +row is left as it is. No task_finished is sent and no error entry is +written: django-ox 1.2.0 sends nothing for an operator's discard either, +and the controls at the bottom pin both facts. + +What the tests require: + +- every node the reconciler cancels carries a cause, the id of a dead + upstream it depends on, and the workflow's current generation. That + holds whether the upstream failed on a worker or an operator discarded + it, and whether the cone is one chain or a fan-out cancelled in one tick. + Any cause counts as recorded except NULL and the empty string, so a + cause stored as a choice whose value is 0 is accepted; +- no task_finished is heard for a node the workflow cancelled. A receiver + is connected for the whole test and must have heard the nodes a worker + did finish, so its silence about the cancelled ones means something; +- a node the workflow did not cancel carries no cause and no upstream: the + dead upstream itself, an independent node that succeeded, and a node an + operator discarded by hand. The last one reads identically through + django.tasks, so the record keyed by task id is the only thing that + separates it from a cancelled node; +- the record is stored, not worked out on read. It still stands, unchanged, + after the failed upstream is retried from the admin and succeeds, and + after a later reconciler tick. That retry revives nothing; +- progress() counts a node cancelled by an upstream that has since + succeeded: none while the upstream is FAILED, one once the admin retry + has made it SUCCESSFUL. The count's name is not fixed; any count + progress() adds that reads that way is accepted. + +A cancelled node that sits further down a chain may name either its dead +parent or the node whose failure started the cancellation; both are +accepted. A LOST upstream is not a case here, because its cone is held +rather than cancelled. + +Known limit: every case runs in a workflow's first generation, because +nothing can start a second one until Workflow.retry() exists. So the +generation check rejects a missing generation and one that differs from +the workflow row, but a record that stamps a constant equal to the first +generation passes here. That a later generation is stamped belongs to the +tests that retry a workflow through Workflow.retry(). + +The module runs with transaction=True, so every enqueue, claim, completion +and reconciler tick commits on its own, anything deferred to commit really +runs, and each record is read as committed. +""" + +from typing import NamedTuple + +import pytest +from django.core.exceptions import FieldDoesNotExist +from django.tasks import task, task_backends +from django.tasks.base import TaskResultStatus +from django.tasks.signals import task_enqueued, task_finished + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowNode +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, flow_worker, order, stored + +pytestmark = pytest.mark.django_db(transaction=True) + +COMPLETE = OxpullWorkflow.Status.COMPLETE + +# Where a cancellation is recorded: on oxpull's node row, against the +# generation stored on the workflow row. +NODE_RECORD_FIELDS = ("cancel_cause", "cancelled_by_task_id", "cancelled_generation") +WORKFLOW_GENERATION_FIELD = "generation" + + +def worker_on(alias): + return OxpullWorker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + + +def reconcile_through_a_worker(worker): + """ + One reconciler tick as production runs it: the task a schedule + enqueues, claimed and executed by a worker. Top priority, so the + worker takes it ahead of any READY node. + """ + result = workflows.reconcile.using(priority=100).enqueue() + assert worker.run_once() is True + row = OxTask.objects.get(pk=result.id) + assert row.status == OxTask.Status.SUCCESSFUL, (row.status, row.errors) + + +def reconcile_until_complete(flow, worker, ticks=3): + for _ in range(ticks): + reconcile_through_a_worker(worker) + if stored(flow).status == COMPLETE: + return + raise AssertionError(f"workflow did not complete: {describe(flow)}") + + +def status_of(task_id): + return OxTask.objects.filter(pk=task_id).values_list("status", flat=True).first() + + +def describe(flow): + return {name: status_of(pk) for name, pk in flow.node_ids().items()} + + +# -- reading what oxpull records ----------------------------------------- + + +class NodeRecord(NamedTuple): + name: str + task_id: str + cause: object + upstream: str | None + generation: object + + +def _has_field(model, name): + try: + model._meta.get_field(name) + except FieldDoesNotExist: + return False + return True + + +def _absent(value): + """NULL and the empty string record nothing. Any other value does, 0 included.""" + return value is None or value == "" + + +def record_from(name, task_id, cause, upstream, generation): + return NodeRecord( + name=name, + task_id=str(task_id), + cause=None if _absent(cause) else cause, + upstream=None if _absent(upstream) else str(upstream), + generation=generation, + ) + + +def cancellations(flow): + """ + What oxpull has committed about cancellation in this workflow: the + workflow's current generation, and each node's record by node name. + An empty cause or upstream reads as None. + """ + missing = [ + f"OxpullWorkflowNode.{name}" + for name in NODE_RECORD_FIELDS + if not _has_field(OxpullWorkflowNode, name) + ] + if not _has_field(OxpullWorkflow, WORKFLOW_GENERATION_FIELD): + missing.append(f"OxpullWorkflow.{WORKFLOW_GENERATION_FIELD}") + assert not missing, ( + "oxpull records nothing about a cancelled node: no cause, no " + "upstream task id and no generation. Missing: " + ", ".join(missing) + ) + generation = OxpullWorkflow.objects.values_list( + WORKFLOW_GENERATION_FIELD, flat=True + ).get(pk=flow.id) + rows = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values_list( + "name", "task_id", *NODE_RECORD_FIELDS + ) + records = {row[0]: record_from(*row) for row in rows} + return generation, records + + +def assert_cancelled(record, generation, decided_by): + """ + decided_by maps the task id of each dead upstream allowed to have + decided this cancellation to its node name. + """ + assert record.cause is not None, ( + f"{record.name} was cancelled, and no cause is recorded: {record}" + ) + assert record.upstream in decided_by, ( + f"{record.name} names upstream {record.upstream!r}, not the dead " + f"upstream that decided it (one of {sorted(decided_by.values())}): {record}" + ) + assert generation is not None and record.generation == generation, ( + f"{record.name} was cancelled in generation {generation!r} and " + f"records {record.generation!r}: {record}" + ) + + +def assert_not_cancelled(record): + assert record.cause is None and record.upstream is None, ( + f"{record.name} was not cancelled by its workflow, and carries a " + f"cancellation record: {record}" + ) + + +# -- task_finished -------------------------------------------------------- + + +@pytest.fixture +def finished_signals(): + """The task ids task_finished is sent for while the test runs.""" + heard: list[str] = [] + + def receiver(sender, task_result, **kwargs): + heard.append(str(task_result.id)) + + task_finished.connect(receiver, dispatch_uid="cancellation-record-finished") + try: + yield heard + finally: + task_finished.disconnect(dispatch_uid="cancellation-record-finished") + + +def assert_no_finished_signal(heard, silent, live): + """ + silent and live map node names to task ids. No task_finished may have + been sent for a silent node. Every live node is one a worker finished, + so the receiver must have heard it; otherwise it vouches for nothing. + """ + deaf = sorted(name for name, task_id in live.items() if task_id not in heard) + if deaf: + pytest.fail( + f"setup: the task_finished receiver heard nothing for {deaf}, " + "which a worker finished, so its silence proves nothing" + ) + sent = sorted(name for name, task_id in silent.items() if task_id in heard) + assert not sent, ( + f"task_finished was sent for {sent}, which no worker finished: the " + "workflow cancelled them, and a cancellation sends no signal" + ) + + +# -- every cancelled node records why --------------------------------------- + + +def _kill_root_on_a_worker(flow, ids): + """Three real attempts on a worker, then FAILED.""" + assert drain(flow_worker()) == 4 # root x3, side + assert status_of(ids["root"]) == OxTask.Status.FAILED + + +def _kill_root_by_operator(flow, ids): + """The django-ox admin's discard, before root ever runs.""" + assert actions.discard(ids["root"]) is True + assert drain(flow_worker()) == 1 # side + assert status_of(ids["root"]) == OxTask.Status.DISCARDED + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="A node the reconciler cancels is moved to DISCARDED, and oxpull " + "records no cause, no upstream task id and no generation for it.", +) +@pytest.mark.parametrize( + ("root_task", "kill"), + [ + (tasks.failing_step.using(priority=10), _kill_root_on_a_worker), + (tasks.step.using(priority=10), _kill_root_by_operator), + ], + ids=["root-failed", "root-discarded"], +) +def test_every_node_the_reconciler_cancels_records_its_cause_upstream_and_generation( + root_task, kill, finished_signals +): + """ + root -> mid -> leaf, plus an independent node `side` that succeeds. + + mid can only have been decided by root. leaf may name root or mid. + root and side were not cancelled by the workflow, so they carry no + record. In the root-discarded case root is an operator's discard, + which must not read as a workflow cancellation. task_finished is + heard for side, and for root when a worker failed it, and never for + mid or leaf. + """ + flow = workflows.create("cone") + flow.add("root", root_task, "root") + flow.add("mid", tasks.step, "mid", depends_on="root") + flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.add("side", tasks.step, "side") + flow.seal() + ids = {name: str(pk) for name, pk in flow.node_ids().items()} + + kill(flow, ids) + reconcile_until_complete(flow, flow_worker()) + + # Preconditions: the cone really was cancelled, and nothing in it ran. + assert stored(flow).status == COMPLETE, describe(flow) + assert status_of(ids["mid"]) == OxTask.Status.DISCARDED, describe(flow) + assert status_of(ids["leaf"]) == OxTask.Status.DISCARDED, describe(flow) + assert status_of(ids["side"]) == OxTask.Status.SUCCESSFUL, describe(flow) + assert "mid" not in order() and "leaf" not in order() + + live = {"side": ids["side"]} + if kill is _kill_root_on_a_worker: + live["root"] = ids["root"] + assert_no_finished_signal( + finished_signals, {"mid": ids["mid"], "leaf": ids["leaf"]}, live + ) + + generation, records = cancellations(flow) + assert_cancelled(records["mid"], generation, {ids["root"]: "root"}) + assert_cancelled( + records["leaf"], generation, {ids["root"]: "root", ids["mid"]: "mid"} + ) + assert_not_cancelled(records["root"]) + assert_not_cancelled(records["side"]) + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="When the reconciler cancels a whole fan-out in one tick, none of " + "the cancelled children records a cause, the upstream that decided it, " + "or the generation.", +) +@pytest.mark.parametrize( + ("alias", "failing", "child"), + [ + ("flows", tasks.failing_step, tasks.step), + ("flows_deferred", tasks.failing_deferred_step, tasks.deferred_step), + ], + ids=["gate", "deferred"], +) +def test_a_whole_fan_out_cancelled_in_one_tick_records_each_child( + alias, failing, child, finished_signals +): + """ + One upstream fails; one reconciler tick cancels every child of it. + task_finished is heard for the failed root and for no child. + """ + width = 12 + flow = workflows.create("fan-out") + flow.add("root", failing, "root") + for index in range(width): + flow.add(f"child-{index}", child, f"child-{index}", depends_on="root") + flow.seal() + ids = {name: str(pk) for name, pk in flow.node_ids().items()} + + worker = worker_on(alias) + assert drain(worker) == 3 # root x3; every child gated + assert status_of(ids["root"]) == OxTask.Status.FAILED + reconcile_through_a_worker(worker) + + children = sorted(name for name in ids if name != "root") + assert all(status_of(ids[name]) == OxTask.Status.DISCARDED for name in children), ( + describe(flow) + ) + assert stored(flow).status == COMPLETE, describe(flow) + + assert_no_finished_signal( + finished_signals, + {name: ids[name] for name in children}, + {"root": ids["root"]}, + ) + + generation, records = cancellations(flow) + unrecorded = [name for name in children if records[name].cause is None] + assert not unrecorded, ( + f"{len(unrecorded)} of {width} cancelled children record no cause: " + f"{[records[name] for name in unrecorded]}" + ) + for name in children: + assert_cancelled(records[name], generation, {ids["root"]: "root"}) + assert_not_cancelled(records["root"]) + + +# -- a cancelled node is told apart from an operator's discard ------------ + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="A cancelled node and a node an operator discarded by hand read " + "the same through django.tasks, and oxpull keeps no record by task id " + "that tells them apart.", +) +def test_a_cancelled_node_is_told_apart_from_an_operators_discard_by_task_id( + finished_signals, +): + """ + The handles a caller keeps from flow.add(), read by their task ids. + + Through django.tasks all three read FAILED, finished, no errors, no + attempts, and they keep reading that way: the record does not go onto + the django-ox row. No task_finished is heard for any of the three. + Only oxpull's record separates the two nodes the workflow cancelled + from the one an operator discarded. + """ + flow = workflows.create("read") + root = flow.add("root", tasks.failing_step, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + leaf = flow.add("leaf", tasks.step, "leaf", depends_on="mid") + by_hand = flow.add("by-hand", tasks.step, "by-hand") + flow.seal() + assert actions.discard(by_hand.id) is True + worker = flow_worker() + assert drain(worker) == 3 # root x3; mid and leaf gated + reconcile_through_a_worker(worker) + assert stored(flow).status == COMPLETE, describe(flow) + + for handle in (mid, leaf, by_hand): + assert status_of(handle.id) == OxTask.Status.DISCARDED + handle.refresh() + via_backend = task_backends["flows"].get_result(str(handle.id)) + for result in (handle, via_backend): + assert ( + result.status, + result.is_finished, + result.errors, + result.attempts, + ) == (TaskResultStatus.FAILED, True, [], 0), (handle.id, result) + + assert_no_finished_signal( + finished_signals, + {"mid": str(mid.id), "leaf": str(leaf.id), "by-hand": str(by_hand.id)}, + {"root": str(root.id)}, + ) + + generation, records = cancellations(flow) + by_task_id = {record.task_id: record for record in records.values()} + decided_by_root = {str(root.id): "root"} + assert_cancelled(by_task_id[str(mid.id)], generation, decided_by_root) + assert_cancelled( + by_task_id[str(leaf.id)], + generation, + {**decided_by_root, str(mid.id): "mid"}, + ) + assert_not_cancelled(by_task_id[str(by_hand.id)]) + + +# -- the record is stored, not inferred ------------------------------------ + + +@task(backend="flows") +def fails_until_allowed(label): + """Fails until the test allows it to succeed, as a fixed bug would.""" + tasks._record(label) + if not tasks.STATE.get(f"{label} may succeed"): + raise ValueError(f"{label} boom") + return label + + +def _root_cancels_mid(): + """root -> mid; root fails for good and a reconciler tick cancels mid.""" + flow = workflows.create("retried") + root = flow.add("root", fails_until_allowed, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + flow.seal() + worker = flow_worker() + assert drain(worker) == 3 # root x3; mid gated + assert status_of(root.id) == OxTask.Status.FAILED + reconcile_through_a_worker(worker) + assert stored(flow).status == COMPLETE, describe(flow) + assert status_of(mid.id) == OxTask.Status.DISCARDED, describe(flow) + return flow, root, mid, worker + + +def _the_admin_retries_root_and_it_succeeds(flow, root, mid, worker): + tasks.STATE["root may succeed"] = True + assert actions.retry(root.id) is True + assert drain(worker) == 1 # root, once more; mid stays DISCARDED + assert status_of(root.id) == OxTask.Status.SUCCESSFUL, describe(flow) + reconcile_through_a_worker(worker) + assert status_of(mid.id) == OxTask.Status.DISCARDED, describe(flow) + assert "mid" not in order() + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="oxpull keeps no record of why a node was cancelled, so nothing " + "survives the failed upstream being retried from the admin and succeeding.", +) +def test_the_record_outlives_the_upstream_being_retried_and_succeeding( + finished_signals, +): + """ + root fails, the reconciler cancels mid, then an operator retries root + from the django-ox admin and it succeeds. + + That retry revives nothing: mid stays DISCARDED and never runs, and no + task_finished is heard for it. The record must still name root, its + cause and the same generation, so a later reading can say mid was + cancelled by an upstream that is now SUCCESSFUL. A record worked out + on read from the upstream's current status would vanish here, and so + would one that the next reconciler tick clears. + """ + flow, root, mid, worker = _root_cancels_mid() + + generation, before = cancellations(flow) + assert_cancelled(before["mid"], generation, {str(root.id): "root"}) + assert_not_cancelled(before["root"]) + + _the_admin_retries_root_and_it_succeeds(flow, root, mid, worker) + + assert_no_finished_signal( + finished_signals, {"mid": str(mid.id)}, {"root": str(root.id)} + ) + generation_after, after = cancellations(flow) + assert generation_after == generation, ( + f"an admin retry moved the workflow from generation {generation!r} " + f"to {generation_after!r}" + ) + assert after["mid"] == before["mid"], ( + "mid's record changed once root was retried and succeeded: " + f"before {before['mid']}, after {after['mid']}" + ) + + +# Counts WorkflowProgress has today. Any other count is one a fix added. +COUNTS_TODAY = frozenset( + {"total", "pending", "blocked", "succeeded", "failed", "unattributed", "finished"} +) + + +def added_progress_counts(flow): + """Every public integer count on progress() that is not one of today's.""" + progress = flow.progress() + counts = {} + for name in dir(progress): + if name.startswith("_") or name in COUNTS_TODAY: + continue + value = getattr(progress, name) + if isinstance(value, int) and not isinstance(value, bool): + counts[name] = value + return progress, counts + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="progress() has no count of nodes cancelled by an upstream that has " + "since succeeded, so a node the admin retry left cancelled does not show " + "in the workflow's counts.", +) +def test_progress_counts_a_node_cancelled_by_an_upstream_that_has_since_succeeded(): + """ + root fails, the reconciler cancels mid, then the admin retries root and + it succeeds. Nothing downstream runs, so mid is left cancelled by an + upstream that is now SUCCESSFUL: work that will not happen unless + someone acts, and the workflow's counts must say so. + + Some count progress() adds reads 0 while root is FAILED and 1 once root + has succeeded. Its name is not fixed. A count of every cancelled node + reads 1 both times and is rejected, and so is a count that forgets the + cancellation once the upstream succeeds, which reads 0 both times. + """ + flow, root, mid, worker = _root_cancels_mid() + progress_before, before = added_progress_counts(flow) + + _the_admin_retries_root_and_it_succeeds(flow, root, mid, worker) + + progress_after, after = added_progress_counts(flow) + counting = sorted( + name for name, value in after.items() if value == 1 and before.get(name) == 0 + ) + assert counting, ( + "progress() has no count that goes from 0 to 1 when the upstream of a " + f"cancelled node succeeds. Added counts before the retry: {before}, " + f"after: {after}. progress() before: {progress_before}, after: " + f"{progress_after}" + ) + + +# -- controls: what holds today, recorded ------------------------------ + + +def _doomed_chain(): + flow = workflows.create("read") + root = flow.add("root", tasks.failing_step, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + leaf = flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.seal() + worker = flow_worker() + assert drain(worker) == 3 + reconcile_through_a_worker(worker) + assert stored(flow).status == COMPLETE, describe(flow) + return flow, root, mid, leaf + + +def test_control_a_worker_written_failure_always_carries_attempts_and_errors(): + """ + The only public tell that exists today. A FAILED written by a worker + has attempts >= 1 and an error per attempt (django-ox increments + attempts and appends worker_ids in the claim), so FAILED with + attempts 0 means "never ran". It does not say why, and an operator's + discard of a READY task reads exactly the same (next control). + """ + _flow, root, mid, _leaf = _doomed_chain() + root.refresh() + mid.refresh() + assert (root.status, len(root.errors), root.attempts) == ( + TaskResultStatus.FAILED, + 3, + 3, + ) + assert (mid.status, mid.is_finished, mid.errors, mid.attempts) == ( + TaskResultStatus.FAILED, + True, + [], + 0, + ) + + +def test_control_django_ox_already_sends_nothing_for_an_operator_discard(): + """ + Pre-existing and documented in django-ox 1.2.0 (docs/monitoring.md, + actions.py): a discard finishes a result without task_finished. Its + public reading is identical to a node the reconciler cancelled, which + is why the tests above ask oxpull for the record rather than + django.tasks. + """ + heard = {"enqueued": set(), "finished": set()} + + def on_enqueued(sender, task_result, **kwargs): + heard["enqueued"].add(str(task_result.id)) + + def on_finished(sender, task_result, **kwargs): + heard["finished"].add(str(task_result.id)) + + task_enqueued.connect(on_enqueued, dispatch_uid="discard-enqueued") + task_finished.connect(on_finished, dispatch_uid="discard-finished") + try: + plain = tasks.default_step.enqueue("plain") + assert actions.discard(plain.id) is True + finally: + task_enqueued.disconnect(dispatch_uid="discard-enqueued") + task_finished.disconnect(dispatch_uid="discard-finished") + assert str(plain.id) in heard["enqueued"] + assert str(plain.id) not in heard["finished"] + plain.refresh() + assert (plain.status, plain.is_finished, plain.errors, plain.attempts) == ( + TaskResultStatus.FAILED, + True, + [], + 0, + ) diff --git a/tests_oxpull/test_workflows_hazard_on_postgresql_a_worker_class_subclass_of.py b/tests_oxpull/test_workflows_hazard_on_postgresql_a_worker_class_subclass_of.py new file mode 100644 index 0000000..602dc95 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_on_postgresql_a_worker_class_subclass_of.py @@ -0,0 +1,499 @@ +""" +A WORKER_CLASS subclass of OxpullWorker that narrows claim_filter_q() alone +must keep its condition on every database, keep the product's own conditions, +and not be refused. + +django-ox promises that a subclass overriding claim_filter_q() and not +claim_filter_sql() "keeps its condition on every database" +(django_ox/worker.py, Worker.claim_filter_q). On PostgreSQL django-ox notices +the empty fragment and claims through SELECT ... FOR UPDATE SKIP LOCKED, +which applies the Q, and logs claim_filter_sql_missing once +(Worker._postgresql_honours_the_claim_filter). Up to 1.2.0 OxpullWorker wrote +a fragment of its own: the dependency gate on a WORKFLOWS backend, and the +rate-limit exclusion once a limit is spent. A subclass inherited that +fragment, so django-ox read it as a subclass that wrote both hooks and took +the single-statement claim. That claim read only the fragment. The gate and +the limit held, and the subclass's own condition was gone. SQLite and MySQL +built the candidate from claim_filter_q() and kept it. + +OxpullWorker now binds its own conditions on the instance, in __init__, and +reads the class's hooks past them. django-ox sees a subclass that wrote +claim_filter_q() alone as exactly that, on every database. + +PinnedWorker below represents a custom pool that leaves the top priority band +to another pool (the same shape as region, tenant or hardware pinning). It +extends the product's condition through super(), as an override must, and +it is wired in through WORKER_CLASS. The worker is built by +django_ox.worker.worker_class(), as ox_worker builds it. A subclass that +replaces the condition without calling super() isn't supported, and nothing +here tests it. + +What a fix has to do to pass: + + * Claim exactly the rows that both the subclass and the product allow. The + pinned worker claims the row inside its band and then nothing, and the + row above the band stays READY with no attempt spent. In a workflow it + runs root, then child, and leaves the node above the band untouched. + With a rate limit spent, the limited row is untouched as well. + * Keep the product's conditions. child outranks root, so a claim that kept + the band but lost the dependency condition would run child first. One + case clears child's run_after so the deferral can't hide that. A claim + that kept the band but lost the rate limit would take the limited row. + * Refuse nothing. This subclass is supported, so each of these fails the + test as a refusal: an Error-level system check in this process, a + non-zero `manage.py check`, an ox_worker that exits at start-up, and + ImproperlyConfigured from worker_class(), from building the worker, or + from add() or seal(). Warnings are allowed. + * The claim path isn't asserted. django-ox's fallback passes, and so would + a single statement that carried the subclass's condition. + +So a fix that refuses the subclass fails here, and so does one that keeps +today's single-statement claim, or one that stops the pinned worker claiming +what it should. + +Every test here passes on SQLite, MySQL and PostgreSQL. They were written +against 1.2.0, where the PostgreSQL cases were expected failures; the markers +came off once the instance-bound claim filter landed. A refusal or a broken +setup goes through pytest.fail, so it reports as a failure and not as a wrong +claim. The "default" parameters are the controls: the same subclass on a +backend that itself declares neither feature. The last test is the same drop +on a rate-limited backend. +""" + +from __future__ import annotations + +import copy +import inspect +import logging +import os +import re +import signal +import subprocess +import sys +import time +from contextlib import contextmanager +from pathlib import Path + +import pytest +from django.conf import settings +from django.core import checks +from django.core.exceptions import ImproperlyConfigured +from django.db import connection +from django.db.models import Q +from django.test import override_settings + +from django_ox.models import OxTask +from django_ox.worker import worker_class +from oxpull import workflows +from oxpull.worker import OxpullWorker + +from . import tasks +from .subprocesses import django_ox_path +from .test_rate_limits import ECHO, captured_sql, open_window, rate_limits, window +from .test_workflows import drain, order + +# Django 6.1 hands every configured alias to the model and field checks +# unless the command pins one, so a system check run here opens "replica" +# to ask SQLite or MySQL whether it supports JSONField. That read leaves +# the alias unwritten, but it has to be declared or the test database +# guard refuses the connection. +pytestmark = pytest.mark.django_db(databases=["default", "replica"]) + +REPO_ROOT = Path(__file__).resolve().parent.parent +SRC = REPO_ROOT / "src" + +BAND = 100 # PinnedWorker claims only priority < BAND +BELOW_BAND = 50 # inside the band, and above priority 0 + + +class PinnedWorker(OxpullWorker): + """ + A user's pool that leaves the top priority band to another pool. + + Extends the product's claim condition through super() and overrides + claim_filter_q() alone, which django-ox documents as keeping the + condition on every database. + """ + + def claim_filter_q(self): + base = super().claim_filter_q() + pinned = Q(priority__lt=100) + return pinned if base is None else base & pinned + + +WORKER_PATH = f"{__name__}.PinnedWorker" + +TASK_ON = { + "default": tasks.default_step, + "flows": tasks.step, + "flows_deferred": tasks.deferred_step, +} + + +@contextmanager +def pinned(*aliases): + """The suite's TASKS, with PinnedWorker as WORKER_CLASS on these aliases.""" + configured = copy.deepcopy(settings.TASKS) + for alias in aliases: + configured[alias]["OPTIONS"]["WORKER_CLASS"] = WORKER_PATH + with override_settings(TASKS=configured): + yield + + +def _is_suite_fixture(message): + """tests_oxpull/settings.py's deliberate oxpull.E007 on "flows_ungated".""" + return message.id == "oxpull.E007" and "'flows_ungated'" in message.msg + + +def build(alias): + """ + The worker ox_worker would start on `alias`, once the system checks it + runs at start-up have passed. + + A refusal fails the test through pytest.fail, not an assertion, so it + never counts as the expected failure: an Error-level check (the suite's + own oxpull.E007 fixture aside), or ImproperlyConfigured from + worker_class() or from building the worker. + """ + errors = [ + f"{message.id}: {message.msg}" + for message in checks.run_checks() + if message.level >= checks.ERROR and not _is_suite_fixture(message) + ] + if errors: + pytest.fail(f"{alias}: Error-level checks refused PinnedWorker: {errors}") + try: + cls = worker_class(alias) + worker = cls(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + except ImproperlyConfigured as exc: + pytest.fail(f"{alias}: refused PinnedWorker: ImproperlyConfigured: {exc}") + if cls is not PinnedWorker: + pytest.fail(f"premise: WORKER_CLASS on {alias!r} built {cls}, not PinnedWorker") + return worker + + +def claim_all(worker, limit=5): + """Claim until the worker finds nothing, without running anything.""" + claimed = [] + while len(claimed) < limit: + row = worker.claim_one() + if row is None: + return claimed + claimed.append(row) + pytest.fail(f"premise: still claiming after {limit} rows") + + +def untouched(task_id): + """READY, with no attempt spent and no lease ever taken.""" + row = OxTask.objects.get(pk=task_id) + return (row.status, row.attempts, row.lease_epoch) == (OxTask.Status.READY, 0, 0) + + +def state_of(task_id): + row = OxTask.objects.get(pk=task_id) + return f"{row.status} attempts={row.attempts} lease_epoch={row.lease_epoch}" + + +def pin_in(statements): + return any(re.search(r"priority[\"`]? < ", sql) for sql in statements) + + +def note(line): + """Which outcome a test saw, shown for passing tests under -rA.""" + sys.stdout.write(line + "\n") + + +# -- in process, through worker_class() ---------------------------------- + + +@pytest.mark.parametrize( + "alias", + [ + "default", + "flows", + "flows_deferred", + ], +) +def test_a_pinned_worker_claims_its_band_and_nothing_else(alias, caplog): + task = TASK_ON[alias] + with pinned(alias): + excluded = task.using(priority=BAND).enqueue("excluded") + allowed = task.using(priority=0).enqueue("allowed") + worker = build(alias) + with ( + caplog.at_level(logging.WARNING, logger="django_ox"), + captured_sql(worker._db_alias) as statements, + ): + rows = claim_all(worker) + fragment = bool(worker.claim_filter_sql()[0]) + label = {excluded.id: "excluded", allowed.id: "allowed"} + claimed = [label.get(str(row.id), row.args) for row in rows] + notice = any( + getattr(record, "event", None) == "claim_filter_sql_missing" + for record in caplog.records + ) + note( + f"OUTCOME {connection.vendor}/{alias}: claimed {claimed}; pin in the SQL: " + f"{pin_in(statements)}; notice: {notice}" + ) + # Claim order is priority DESC, so a claim without the band takes + # "excluded" first. + assert claimed == ["allowed"] and untouched(excluded.id), ( + f"{connection.vendor}/{alias}: PinnedWorker claimed {claimed}, not " + f"['allowed']. Excluded row: {state_of(excluded.id)}. Statements: " + f"{[sql.split()[0] for sql in statements]}; pin in the SQL: " + f"{pin_in(statements)}; django-ox claim_filter_sql_missing notice: {notice}; " + f"claim_filter_sql() fragment non-empty: {fragment}" + ) + + +@pytest.mark.parametrize( + ("alias", "gate_alone"), + [ + pytest.param("flows", False, id="flows"), + pytest.param("flows_deferred", False, id="flows_deferred"), + pytest.param("flows_deferred", True, id="flows_deferred-put-back-to-ready"), + ], +) +def test_a_pinned_worker_runs_its_band_of_a_workflow_in_order(alias, gate_alone): + """ + root (priority 0) -> {vip (priority 100), child (priority 50)}. The + pinned pool runs root, then child, and leaves vip untouched for the pool + that owns the band. + + child outranks root, so a claim that lost the dependency condition runs + child first. The blocked nodes wait, which would hide that, so the + put-back case moves them to READY after the build, the shape a django-ox + retry leaves. Only the condition in the pinned worker's claim holds them + then. + """ + task = TASK_ON[alias] + with pinned(alias): + try: + flow = workflows.create(f"banded-{alias}") + flow.add("root", task.using(priority=0), "root") + flow.add("vip", task.using(priority=BAND), "vip", depends_on="root") + flow.add( + "child", task.using(priority=BELOW_BAND), "child", depends_on="root" + ) + flow.seal() + except ImproperlyConfigured as exc: + pytest.fail(f"{alias}: add() or seal() refused PinnedWorker: {exc}") + ids = flow.node_ids() + if gate_alone: + OxTask.objects.filter(pk__in=[ids["vip"], ids["child"]]).update( + status=OxTask.Status.READY + ) + worker = build(alias) + ran = drain(worker, limit=10) + states = {name: state_of(pk) for name, pk in ids.items()} + note(f"OUTCOME {connection.vendor}/{alias}: ran {order()}; nodes {states}") + assert order() == ["root", "child"] and untouched(ids["vip"]), ( + f"{connection.vendor}/{alias}: PinnedWorker ran {order()} ({ran} tasks), " + f"not ['root', 'child'] with vip untouched; nodes {states}" + ) + + +# -- a real `ox_worker` process ------------------------------------------- + + +def process_env(tmp_path): + """ + A separate process on this session's test database. PinnedWorker lives in + a user module of its own (its source, verbatim) and is WORKER_CLASS on + "default" and "flows". The workflow reconciler is scheduled so the checks + are otherwise silent, and "flows_ungated" stops declaring WORKFLOWS: it is + the suite's deliberate oxpull.E007 fixture and would stop ox_worker at + start-up for a reason that has nothing to do with PinnedWorker. + """ + source = inspect.getsource(PinnedWorker) + (tmp_path / "customer_workers.py").write_text( + "from django.db.models import Q\n" + "from oxpull.worker import OxpullWorker\n\n\n" + source + ) + (tmp_path / "pinned_settings.py").write_text( + "import copy\n" + f"from {settings.SETTINGS_MODULE} import * # noqa: F403\n" + "TASKS = copy.deepcopy(TASKS) # noqa: F405\n" + "WORKER = 'customer_workers.PinnedWorker'\n" + "for alias in ('default', 'flows'):\n" + " TASKS[alias]['OPTIONS']['WORKER_CLASS'] = WORKER\n" + "TASKS['flows_ungated']['OPTIONS']['WORKFLOWS'] = False\n" + "TASKS['default']['OPTIONS']['SCHEDULES']['oxpull-workflow-reconcile'] = " + "{'task': 'oxpull.workflows.reconcile', 'cron': '* * * * *'}\n" + "LOGGING = {\n" + " 'version': 1,\n" + " 'disable_existing_loggers': False,\n" + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},\n" + " 'loggers': {'django_ox': {'handlers': ['console'], 'level': 'DEBUG'}},\n" + "}\n" + ) + name = connection.settings_dict["NAME"] + if connection.vendor == "sqlite": + name = Path(name).resolve() + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "pinned_settings", + "PYTHONPATH": os.pathsep.join( + [str(SRC), django_ox_path(), str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(name), + } + + +def run_python(env, *args): + return subprocess.run( # noqa: S603 + [sys.executable, *args], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def stop(process): + """SIGTERM, as a deploy does; the worker drains and exits.""" + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return process.returncode + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("alias", ["default", "flows"]) +def test_an_ox_worker_process_running_a_pinned_worker_class_stays_in_its_band( + alias, tmp_path +): + env = process_env(tmp_path) + # Prove which oxpull the process imports and which class ox_worker builds. + probe = run_python( + env, + "-c", + "import django; django.setup(); import oxpull; " + "from django_ox.worker import worker_class; " + "from oxpull.worker import OxpullWorker; " + f"cls = worker_class({alias!r}); print(oxpull.__file__); " + "print(cls.__module__ + '.' + cls.__qualname__, issubclass(cls, OxpullWorker))", + ) + lines = probe.stdout.split() + if "ImproperlyConfigured" in probe.stderr: + pytest.fail(f"{alias}: worker_class() refused PinnedWorker: {probe.stderr}") + if not ( + len(lines) == 3 + and lines[0].startswith(str(SRC)) + and lines[1:] == ["customer_workers.PinnedWorker", "True"] + ): + pytest.fail(f"premise: the process imports {lines}: {probe.stderr}") + check = run_python(env, "-m", "django", "check") + check_output = (check.stdout + check.stderr).strip() + if check.returncode != 0: + pytest.fail( + f"{alias}: `manage.py check` refused PinnedWorker (exit " + f"{check.returncode}): {check_output}" + ) + + task = TASK_ON[alias] + excluded = task.using(priority=BAND).enqueue("excluded") + allowed = task.using(priority=0).enqueue("allowed") + label_of = {excluded.id: "excluded", allowed.id: "allowed"} + + log_path = tmp_path / "ox_worker.log" + with log_path.open("w") as log: + worker = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "django", + "ox_worker", + "--backend", + alias, + "--concurrency", + "1", + "--interval", + "0.05", + ], + env=env, + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + # Claim order is priority DESC, so a claim without the band takes + # "excluded" before "allowed": once "allowed" has run, the + # excluded row's fate is already decided. The deadline is only a + # ceiling for a loaded machine; a passing run leaves at once. + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and worker.poll() is None: + if OxTask.objects.get(pk=allowed.id).status == OxTask.Status.SUCCESSFUL: + break + time.sleep(0.1) + exited_on_its_own = worker.poll() is not None + finally: + code = stop(worker) + text = log_path.read_text() + if exited_on_its_own: + pytest.fail( + f"{alias}: ox_worker with WORKER_CLASS customer_workers.PinnedWorker " + f"exited {code} on its own. Log tail: {text[-2000:]}" + ) + claimed = [ + label_of[task_id] + for task_id in re.findall(r"Claimed task id=([0-9a-f-]{36}) ", text) + if task_id in label_of + ] + notice = "but not claim_filter_sql()" in text + note( + f"OUTCOME {connection.vendor}/{alias} process: claimed {claimed}; excluded " + f"{state_of(excluded.id)}; exit {code}; notice in log: {notice}; check: " + f"{check_output!r}" + ) + assert claimed == ["allowed"] and untouched(excluded.id) and code == 0, ( + f"{connection.vendor}: `ox_worker --backend {alias}` with WORKER_CLASS " + f"customer_workers.PinnedWorker claimed {claimed}, not ['allowed']; " + f"excluded row {state_of(excluded.id)}, allowed row {state_of(allowed.id)}; " + f"exit {code}. django-ox claim_filter_sql_missing notice in its log: " + f"{notice}. `manage.py check` in that process: {check_output!r}" + ) + + +# -- scope: the same class on a rate-limited backend (oxpull 1.2.0 code) ---- + + +def test_scope_a_pinned_worker_on_a_rate_limited_backend_keeps_its_band_once_a_limit_is_spent(): # noqa: E501 + with rate_limits( + {"echo": {"limit": 1, "per": 300, "tasks": [ECHO]}}, WORKER_CLASS=WORKER_PATH + ): + open_window("echo") + tasks.plain_echo.enqueue("spend") + worker = build("default") + if not (worker.run_once() and window("echo").used == 1): + pytest.fail("premise: the pinned worker did not spend the echo limit") + excluded = tasks.default_step.using(priority=BAND).enqueue("excluded") + limited = tasks.plain_echo.using(priority=BELOW_BAND).enqueue("limited") + allowed = tasks.default_step.using(priority=0).enqueue("allowed") + with captured_sql(worker._db_alias) as statements: + rows = claim_all(worker) + used = window("echo").used + label = {excluded.id: "excluded", limited.id: "limited", allowed.id: "allowed"} + claimed = [label.get(str(row.id), row.args) for row in rows] + note( + f"OUTCOME {connection.vendor}/default+RATE_LIMITS: claimed {claimed}; " + f"pin in the SQL: {pin_in(statements)}; echo used {used}" + ) + assert ( + claimed == ["allowed"] + and untouched(excluded.id) + and untouched(limited.id) + and used == 1 + ), ( + f"{connection.vendor}/default+RATE_LIMITS: PinnedWorker claimed {claimed} " + f"once the echo limit was spent, not ['allowed']. Excluded row: " + f"{state_of(excluded.id)}; limited row: {state_of(limited.id)}; echo used " + f"{used}. Statements: {[sql.split()[0] for sql in statements]}; pin in the " + f"SQL: {pin_in(statements)}" + ) diff --git a/tests_oxpull/test_workflows_hazard_ox_prune_include_failed_deletes_a_workflow.py b/tests_oxpull/test_workflows_hazard_ox_prune_include_failed_deletes_a_workflow.py new file mode 100644 index 0000000..4891b66 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_ox_prune_include_failed_deletes_a_workflow.py @@ -0,0 +1,430 @@ +""" +ox_prune --include-failed deletes a workflow node an operator +has already retried, and the gate then lets its child run. + +django-ox 1.2.0's ox_prune deletes each batch with +``prunable.filter(pk__in=batch).delete()``, and its comment says the DELETE +re-applies the selection predicate. It does not. OxScheduleTick.task is a +SET_NULL foreign key to OxTask, so QuerySet.delete() cannot fast-delete and +takes Django's Collector path: an unlocked SELECT of the rows matching +status and finished_at, then ``UPDATE SET task_id = NULL``, +then ``DELETE FROM WHERE id IN ()`` with no status +predicate. An actions.retry() that commits between that SELECT and that +DELETE does not save the row. + +For a workflow node that mattered twice. runnable_filter_q() read an absent +upstream as satisfied, so the child became claimable although its parent +never succeeded. And if a worker had already claimed the retried parent, +that execution ran on with no row: its renewal and its outcome write matched +nothing, and the child could run beside it. The django-ox this package pins +re-applies the status predicate in that DELETE, and the claim's dependency +check now blocks on an absent upstream with nothing recorded. + +The window is widened deterministically with connection.execute_wrapper on +ox_prune's own connection. Just before ox_prune's first UPDATE or DELETE +(after the deletion collector has selected its rows), the retry, and where +named a claim or a whole worker execution, runs on other threads with their +own connections through django-ox's public paths. Every statement ox_prune +issues is its real statement, from the real management command. Nothing in +this file measures how often the unwidened window is hit in production. + +Shapes: + open an OPEN workflow with the reconciler run before the + prune, as a scheduled one would be. The reconciler + sweeps SEALED workflows only, so it never cancels + the child here. + sealed-unreconciled a SEALED workflow whose reconciler has not run + (nothing schedules it: oxpull.W003 is a warning, + and create() and seal() do not refuse). + sealed-reconciled a SEALED workflow whose reconciler ran after the + root failed, so the child is already DISCARDED. + Only the lost retry survives this shape; it pins + how far the hazard reaches with a working reconciler. + +Every test asserts the correct behaviour. The in-window tests fail on +django-ox 1.2.0 and pass with the current django-ox implementation. The control +(retry before the prune) passes on both. +""" + +import threading +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command +from django.db import connection, connections +from django.tasks import task +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows + +from . import tasks +from .test_workflows import flow_worker, reconcile +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db(transaction=True) + +SHAPES = ["open", "sealed-unreconciled", "sealed-reconciled"] + +# Past ox_prune's default --older-than of 7d. +PRUNE_AGE = timedelta(days=8) + + +@task(backend="flows") +def parent_that_fails_then_holds(label): + """ + Fails every attempt until the test arms STATE["started"]; after that, + records that it started, signals, and holds until STATE["finish"] is + set, then succeeds. Lets a real worker be mid-execution of the retried + parent while ox_prune deletes its row. + """ + state = tasks.STATE + if "started" not in state: + tasks._record(label) + raise ValueError(f"{label} boom") + tasks._record(f"{label}:started") + state["started"].set() # type: ignore[attr-defined] + if not state["finish"].wait(timeout=30): # type: ignore[attr-defined] + raise RuntimeError("the test never let the parent finish") + tasks._record(f"{label}:finished") + return label + + +# django-ox's ox_prune takes a locking read of each batch before its +# DELETE on PostgreSQL and MySQL. The in-window work here runs inside that +# transaction, so a retry of a selected row waits for ox_prune to commit and +# cannot land inside the window this harness means to widen. +lock_holds_the_window = pytest.mark.xfail( + connection.vendor in ("postgresql", "mysql"), + strict=True, + reason="django-ox's ox_prune locks the rows it selected, so the " + "retry this harness makes inside its transaction waits for the commit and " + "never lands in the window; the window has to move before that read", +) + + +# -- harness ----------------------------------------------------------- + + +def elsewhere(fn): + """Run fn on its own thread and its own connection; return its result.""" + result, error = [], [] + + def run(): + try: + result.append(fn()) + except BaseException as exc: + error.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=30) + assert not thread.is_alive(), "in-window work did not finish" + if error: + raise error[0] + return result[0] + + +def prune_with(in_window): + """ + The real `ox_prune --include-failed`, calling in_window() once, on this + thread, immediately before ox_prune's first UPDATE or DELETE. By then + the deletion collector has run its SELECT and fixed the pk list the + DELETE will use. Returns (stdout, statements ox_prune issued). + """ + statements = [] + fired = [] + + def pause_before_first_write(execute, sql, params, many, context): + head = sql.lstrip().split(None, 1)[0].upper() + statements.append(" ".join(sql.split())[:90]) + if head in ("UPDATE", "DELETE") and not fired: + fired.append(True) + in_window() + return execute(sql, params, many, context) + + out = StringIO() + with connection.execute_wrapper(pause_before_first_write): + call_command("ox_prune", "--include-failed", stdout=out) + assert fired, "ox_prune issued no write; the window was never reached" + return out.getvalue().strip(), statements + + +def prepare( + shape, + *, + root_task=tasks.failing_step, + child_task=None, + worker_factory=flow_worker, +): + """ + root -> child, root run through every attempt to FAILED by a real + worker, the reconciler run or not per shape, and root's finished_at + backdated past the prune cutoff. + + The backdate is the one row written by hand. Production reaches the + same state by the root having failed eight days ago. + """ + if child_task is None: + child_task = tasks.step.using(priority=9) + flow = workflows.create(f"prune-{shape}") + root = flow.add("root", root_task, "root") + child = flow.add("child", child_task, "child", depends_on="root") + if shape != "open": + flow.seal() + worker = worker_factory() + for _ in range(OxTask.objects.get(id=root.id).max_attempts): + worker.run_once() + assert OxTask.objects.get(id=root.id).status == OxTask.Status.FAILED + if shape != "sealed-unreconciled": + reconcile() + OxTask.objects.filter(id=root.id).update(finished_at=timezone.now() - PRUNE_AGE) + connections.close_all() + return flow, root, child + + +def is_node(claimed, node): + return claimed is not None and str(claimed.pk) == str(node.id) + + +def status_of(node): + row = OxTask.objects.filter(id=node.id).first() + return row.status if row is not None else "DELETED" + + +def report(**facts): + statements = facts.pop("statements", []) + print( # noqa: T201 + f"\nS3 vendor={connection.vendor} " + + " ".join(f"{k}={v!r}" for k, v in facts.items()) + + "".join(f"\n ox_prune: {s}" for s in statements) + ) + + +# -- the control ------------------------------------------------------- + + +@pytest.mark.parametrize("shape", SHAPES) +def test_control_a_node_retried_before_ox_prune_survives_and_gates_its_child(shape): + _, root, child = prepare(shape) + + retried = elsewhere(lambda: actions.retry(root.id)) + out = StringIO() + call_command("ox_prune", "--include-failed", stdout=out) + root_after = status_of(root) + claimed = flow_worker().claim_one() + + report( + shape=shape, + retried=retried, + prune=out.getvalue().strip(), + root_after=root_after, + child=status_of(child), + next_claim=claimed.args if claimed else None, + ) + assert retried is True + assert root_after != "DELETED" + assert not is_node(claimed, child) + + +# -- the hazard -------------------------------------------------------- + + +@lock_holds_the_window +@pytest.mark.parametrize("shape", SHAPES) +def test_a_node_retried_inside_ox_prune_keeps_its_row_and_its_child_stays_gated( + shape, +): + _, root, child = prepare(shape) + retried = [] + + out, statements = prune_with( + lambda: retried.append(elsewhere(lambda: actions.retry(root.id))) + ) + root_after = status_of(root) + claimed = flow_worker().claim_one() + + report( + shape=shape, + retried=retried, + prune=out, + root_after=root_after, + child=status_of(child), + next_claim=claimed.args if claimed else None, + statements=statements, + ) + assert retried == [True], "precondition: the retry must land inside the window" + assert root_after == OxTask.Status.READY, ( + "ox_prune deleted a node an operator had already retried" + ) + assert not is_node(claimed, child), ( + "the child was claimed although its upstream never succeeded" + ) + + +@lock_holds_the_window +@pytest.mark.parametrize("shape", SHAPES) +def test_a_node_retried_and_claimed_inside_ox_prune_keeps_its_row_and_gates_its_child( + shape, +): + _, root, child = prepare(shape) + retried, claimed_in_window = [], [] + + def operator_then_worker(): + retried.append(elsewhere(lambda: actions.retry(root.id))) + got = elsewhere(lambda: flow_worker().claim_one()) + claimed_in_window.append(got.args if got else None) + + out, statements = prune_with(operator_then_worker) + root_after = status_of(root) + claimed = flow_worker().claim_one() + + report( + shape=shape, + retried=retried, + claimed_in_window=claimed_in_window, + prune=out, + root_after=root_after, + child=status_of(child), + next_claim=claimed.args if claimed else None, + statements=statements, + ) + assert retried == [True], "precondition: the retry must land inside the window" + assert claimed_in_window == [["root"]], "precondition: a worker holds the root" + assert root_after == OxTask.Status.RUNNING, ( + "ox_prune deleted a node a worker was running after an operator retry" + ) + assert not is_node(claimed, child), ( + "the child was claimed while its upstream was still running" + ) + + +@lock_holds_the_window +@pytest.mark.parametrize("shape", SHAPES) +def test_a_retried_node_executing_across_ox_prune_keeps_its_outcome_and_its_child_waits( + shape, caplog +): + """ + The whole execution, on a real worker: retried and claimed inside the + window, still running when ox_prune commits. The child must not run + until the parent succeeds, and the parent's success must land. + """ + _, root, child = prepare(shape, root_task=parent_that_fails_then_holds) + started, finish = threading.Event(), threading.Event() + tasks.STATE["started"] = started + tasks.STATE["finish"] = finish + + holder = flow_worker() + held, errors, retried = [], [], [] + + def hold(): + try: + held.append(holder.run_once()) + except BaseException as exc: + errors.append(exc) + finally: + connections.close_all() + + holder_thread = threading.Thread(target=hold) + + def operator_then_worker(): + retried.append(elsewhere(lambda: actions.retry(root.id))) + holder_thread.start() + assert started.wait(timeout=30), "the worker never started the retried root" + + try: + out, statements = prune_with(operator_then_worker) + root_while_running = status_of(root) + renewed = holder.renew_leases() + child_ran = flow_worker().run_once() + order_while_running = list(tasks.STATE.get("order", [])) + finally: + finish.set() + if holder_thread.ident is not None: + holder_thread.join(timeout=30) + assert not holder_thread.is_alive(), "the holding worker never finished" + assert not errors, errors + root_after = status_of(root) + lease_lost = [ + record.getMessage() + for record in caplog.records + if getattr(record, "event", None) == "task_lease_lost" + ] + + report( + shape=shape, + retried=retried, + prune=out, + root_while_running=root_while_running, + renewals_that_matched=renewed, + child_ran_while_root_running=child_ran, + order_while_running=order_while_running, + holder_ran=held, + root_after=root_after, + child=status_of(child), + lease_lost_logged=lease_lost, + statements=statements, + ) + assert retried == [True], "precondition: the retry must land inside the window" + assert not child_ran, "the child ran while its retried upstream was still executing" + assert root_after == OxTask.Status.SUCCESSFUL, ( + "the retried root succeeded but its outcome write matched no row" + ) + + +@lock_holds_the_window +@pytest.mark.parametrize("shape", ["open", "sealed-unreconciled"]) +def test_deferral_sweep_keeps_the_child_of_a_node_retried_inside_ox_prune(shape): + """ + The child waits, so the deleted parent alone does not make it claimable. + The reconciler's release backstop runs next, and it must not release the + child of a parent that never succeeded. (sealed-reconciled is left out: + the cancellation pass has already discarded the child, and the lost + retry is covered above.) + """ + _, root, child = prepare( + shape, + root_task=tasks.failing_deferred_step, + child_task=tasks.deferred_step.using(priority=9), + worker_factory=deferring_worker, + ) + assert OxTask.objects.get(id=child.id).status == OxTask.Status.WAITING + retried = [] + + out, statements = prune_with( + lambda: retried.append(elsewhere(lambda: actions.retry(root.id))) + ) + root_after = status_of(root) + child_deferred_after_prune = ( + OxTask.objects.get(id=child.id).status == OxTask.Status.WAITING + ) + reconcile() + child_deferred_after_reconcile = ( + OxTask.objects.get(id=child.id).status == OxTask.Status.WAITING + ) + claimed = deferring_worker().claim_one() + + report( + shape=shape, + retried=retried, + prune=out, + root_after=root_after, + child_deferred_after_prune=child_deferred_after_prune, + child_deferred_after_reconcile=child_deferred_after_reconcile, + next_claim=claimed.args if claimed else None, + statements=statements, + ) + assert retried == [True], "precondition: the retry must land inside the window" + assert root_after == OxTask.Status.READY, ( + "ox_prune deleted a node an operator had already retried" + ) + assert child_deferred_after_reconcile, ( + "the reconciler released a child whose upstream never succeeded" + ) + assert not is_node(claimed, child), ( + "the child was claimed although its upstream never succeeded" + ) diff --git a/tests_oxpull/test_workflows_hazard_postgresql_repeatable_read_concurrent_add.py b/tests_oxpull/test_workflows_hazard_postgresql_repeatable_read_concurrent_add.py new file mode 100644 index 0000000..59c94b5 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_postgresql_repeatable_read_concurrent_add.py @@ -0,0 +1,529 @@ +""" +At PostgreSQL REPEATABLE READ, a no-op write to the workflow +row becomes a serialization failure that nothing retries. + +add(), seal() and both reconciler transactions each open with an UPDATE +of the workflow row. On PostgreSQL even a no-op UPDATE writes a new row +version. A REPEATABLE READ transaction whose snapshot predates another +writer's commit, and which waited on that writer's row lock, gets +SerializationFailure (SQLSTATE 40001) when the writer commits. READ +COMMITTED would re-read the row instead. Nothing in workflows.py reissued +on 40001, so the caller got a raw django.db.OperationalError. + +Now add() (Workflow.add and Batch.add), Workflow.seal() and both +reconcilers' completing transactions run again on a serialization failure +when they own the transaction, and every test here passes. Batch.seal() +still raises for its caller to retry, as 1.2.0 documents and +test_batches.py pins. + +Each test asserts the behaviour the code's own comments promise. +Concurrent add()s all land. seal() waits out an add() in flight and +counts it. A reconciler that loses the completion compare-and-set reads a +rowcount of 0. Each test is parametrized over how a project can set the +isolation level: + +- READ COMMITTED, Django's default. This is the control. +- REPEATABLE READ through DATABASES OPTIONS["isolation_level"]. +- REPEATABLE READ as the server default. On PostgreSQL that is + default_transaction_isolation. On MySQL it is OPTIONS + ["isolation_level"] = None, which leaves InnoDB's own default in place. + +MySQL runs the same tests as a vendor control, because InnoDB answers a +waited-on UPDATE with a current read. SQLite has no isolation level to +set, so it skips. + +Three of the four tests hold the window open deterministically. A monitor +thread sees the contending sessions waiting on a lock before the holder +commits, and every test asserts that it did, so a green control is a +control that actually contended. +""" + +import contextlib +import logging +import threading +import time + +import pytest +from django.db import connections, transaction +from django.tasks.signals import task_enqueued + +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.exceptions import WorkflowSealed +from oxpull.models import OxpullBatch, OxpullBatchMember, OxpullWorkflow + +from . import isolation, tasks +from .racing import THREADS, race_all +from .test_workflows import drain, flow_worker, reconcile, stored + +LEVELS = [ + "read-committed", + "repeatable-read-options", + "repeatable-read-server-default", +] + + +def require_a_database_with_isolation_levels(): + if connections["default"].vendor not in ("postgresql", "mysql"): + pytest.skip("SQLite has no isolation level to configure") + + +def expected_level(level): + return "read committed" if level == "read-committed" else "repeatable read" + + +@contextlib.contextmanager +def new_connections_at(level): + """ + Every connection opened inside the block runs at `level`, set the way + a project sets it. + + Django reads settings_dict["OPTIONS"] when it connects, and each + thread's connection shares that dict, so a thread started inside the + block connects with the level. The test's own connection is already + open at READ COMMITTED and keeps it. Each test also asserts the level + a new transaction on every thread actually ran at, so a setting that + never reached the connection cannot pass as a control. + """ + connection = connections["default"] + options = connection.settings_dict["OPTIONS"] + missing = object() + before = options.get("isolation_level", missing) + altered_database = None + isolation.outlives_the_transaction(level) + if connection.vendor == "postgresql": + from psycopg import IsolationLevel + + if level == "read-committed": + options["isolation_level"] = IsolationLevel.READ_COMMITTED + elif level == "repeatable-read-options": + options["isolation_level"] = IsolationLevel.REPEATABLE_READ + else: + options.pop("isolation_level", None) + altered_database = connection.ops.quote_name( + connection.settings_dict["NAME"] + ) + with connection.cursor() as cursor: + cursor.execute( + f"ALTER DATABASE {altered_database} " + "SET default_transaction_isolation = 'repeatable read'" + ) + # Only a session opened after the ALTER reads the new default, + # and a pooled checkout may be a session opened before it. + isolation.renew_pools() + else: + options["isolation_level"] = { + "read-committed": "read committed", + "repeatable-read-options": "repeatable read", + "repeatable-read-server-default": None, + }[level] + try: + yield + finally: + if altered_database is not None: + with connection.cursor() as cursor: + cursor.execute( + f"ALTER DATABASE {altered_database} " + "RESET default_transaction_isolation" + ) + if before is missing: + options.pop("isolation_level", None) + else: + options["isolation_level"] = before + + +def level_of_a_new_transaction(): + """The isolation level a transaction on this thread's connection gets.""" + connection = connections["default"] + with transaction.atomic(), connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SHOW transaction_isolation") + else: + cursor.execute("SELECT @@SESSION.transaction_isolation") + return str(cursor.fetchone()[0]).lower().replace("-", " ") + + +def lock_waiters(): + """Sessions on this test database waiting on a lock, right now.""" + connection = connections["default"] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute( + "SELECT count(*) FROM pg_stat_activity " + "WHERE wait_event_type = 'Lock' AND datname = current_database()" + ) + else: + cursor.execute( + "SELECT COUNT(DISTINCT w.REQUESTING_ENGINE_TRANSACTION_ID) " + "FROM performance_schema.data_lock_waits w " + "JOIN performance_schema.data_locks l " + "ON l.ENGINE_LOCK_ID = w.REQUESTING_ENGINE_LOCK_ID " + "WHERE l.OBJECT_SCHEMA = DATABASE()" + ) + return int(cursor.fetchone()[0]) + + +def wait_for_lock_waiters(count, timeout=15.0): + """ + Poll until `count` sessions wait on a lock, and return how many did. + + It runs in autocommit on its own thread, so every poll is a new + statement and PostgreSQL's activity snapshot is fresh each time. + """ + deadline = time.monotonic() + timeout + seen = 0 + while time.monotonic() < deadline: + seen = lock_waiters() + if seen >= count: + return seen + time.sleep(0.02) + return seen + + +def describe(errors): + """One line per error: the Django type, the driver's cause, its code.""" + lines = [] + for exc in errors: + cause = exc.__cause__ + code = getattr(cause, "sqlstate", None) or ( + cause.args[0] if cause is not None and cause.args else None + ) + first = str(exc).splitlines()[0] if str(exc) else "" + lines.append( + f"{type(exc).__name__} <- {type(cause).__name__} [{code}]: {first}" + ) + return lines + + +# -- add() ------------------------------------------------------------- + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_concurrent_adds_to_one_open_workflow_all_land(level): + """ + The plain race: eight add()s on one OPEN workflow, released together. + + No add() here runs inside a caller's transaction, so each owns its + transaction and nothing stops it from being reissued. The nodes do not + conflict. They are eight different names under one root, and the only + thing they contend for is the workflow row's no-op UPDATE. + """ + require_a_database_with_isolation_levels() + flow = workflows.create("rr") + flow.add("root", tasks.step, "root") + levels = [] + + def add(index): + def call(): + try: + flow.add( + f"n{index}", + tasks.slow_step, + f"n{index}", + 0.0, + depends_on="root", + ) + finally: + levels.append(level_of_a_new_transaction()) + return f"n{index}" + + return call + + with new_connections_at(level): + _results, errors = race_all([add(index) for index in range(THREADS)]) + + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert len(flow.node_ids()) == THREADS + 1 + assert OxTask.objects.count() == THREADS + 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_adds_that_waited_on_an_add_in_flight_all_land_once_it_commits(level): + """ + The same property, with the window held open. + + One add() runs inside a caller's transaction and holds the workflow + row: a request under ATOMIC_REQUESTS that adds a node and then does + more work. Seven more add()s start while it holds the row. The holder + commits only once all seven are seen waiting on a lock, so the READ + COMMITTED control proves they waited and re-read, and did not simply + run after it. + """ + require_a_database_with_isolation_levels() + flow = workflows.create("rr-held") + flow.add("root", tasks.step, "root") + held = threading.Event() + waited = threading.Event() + observed = {} + levels = [] + others = THREADS - 1 + + def holder(): + with transaction.atomic(): + flow.add("held", tasks.step, "held", depends_on="root") + held.set() + assert waited.wait(timeout=20) + levels.append(level_of_a_new_transaction()) + return "held" + + def add(index): + def call(): + assert held.wait(timeout=10) + try: + flow.add(f"n{index}", tasks.step, f"n{index}", depends_on="root") + finally: + levels.append(level_of_a_new_transaction()) + return f"n{index}" + + return call + + def monitor(): + assert held.wait(timeout=10) + observed["waiters"] = wait_for_lock_waiters(others) + waited.set() + return "monitor" + + with new_connections_at(level): + _results, errors = race_all( + [holder, *[add(index) for index in range(others)], monitor] + ) + + assert observed["waiters"] == others + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert len(flow.node_ids()) == THREADS + 1 + assert OxTask.objects.count() == THREADS + 1 + + +# -- seal() ------------------------------------------------------------ + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_seal_waits_out_an_add_in_flight_and_counts_it(level): + """ + seal()'s comment: "any add() already in flight commits before the + count below". The add() here commits while the workflow is OPEN, so + the one correct outcome is a SEALED workflow that counts two nodes. + WorkflowSealed would be wrong too, because nothing sealed it first. + """ + require_a_database_with_isolation_levels() + flow = workflows.create("rr-seal") + flow.add("root", tasks.step, "root") + held = threading.Event() + waited = threading.Event() + observed = {} + levels = [] + + def holder(): + with transaction.atomic(): + flow.add("held", tasks.step, "held", depends_on="root") + held.set() + assert waited.wait(timeout=20) + return "held" + + def seal(): + assert held.wait(timeout=10) + try: + flow.seal() + except WorkflowSealed: + return "refused" + finally: + levels.append(level_of_a_new_transaction()) + return "sealed" + + def monitor(): + assert held.wait(timeout=10) + observed["waiters"] = wait_for_lock_waiters(1) + waited.set() + return "monitor" + + with new_connections_at(level): + results, errors = race_all([holder, seal, monitor]) + + assert observed["waiters"] == 1 + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert sorted(results) == ["held", "monitor", "sealed"] + row = stored(flow) + assert row.status == OxpullWorkflow.Status.SEALED + assert row.total_count == 2 + + +# -- the reconciler ---------------------------------------------------- + + +class _Records(logging.Handler): + def __init__(self): + super().__init__(level=logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_reconciler_that_loses_the_completion_race_reads_zero_rows(level): + """ + _complete_if_finished's comment: "Racing reconcilers all attempt the + same UPDATE, one matches a SEALED row, and the losers' transactions + change nothing." + + The first reconciler wins the compare-and-set and is held inside its + completing transaction while it enqueues the callback. A task_enqueued + receiver holds it there until the second reconciler is seen waiting + on the workflow row. The loser should return 0 and log nothing. What + it must not do is log workflow_reconcile_failed at ERROR with a + traceback, which is what an operator alerts on. + """ + require_a_database_with_isolation_levels() + flow = workflows.create( + "rr-reconcile", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + flow.add("only", tasks.step, "only") + flow.seal() + assert drain(flow_worker()) == 1 + observed, levels, results, errors, failures = race_the_completion( + level, reconcile, "oxpull.workflows", "workflow_reconcile_failed" + ) + assert observed["waiters"] == 1 + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert failures == [] + assert dict(results) == {"winner": 1, "loser": 0, "monitor": None} + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 1 + ) + + +def race_the_completion(level, reconcile_once, logger_name, failed_event): + """ + Two reconcilers on one finished row: the first is held inside its + completing transaction, after its compare-and-set, until the second is + seen waiting on the row. Returns what the monitor saw, the levels each + thread ran at, the results, the errors, and every `failed_event` record. + """ + cas_held = threading.Event() + waited = threading.Event() + observed = {} + levels = [] + guard = threading.Lock() + + def hold_the_completion(sender, task_result, **kwargs): + if task_result.task.module_path != "tests_oxpull.tasks.record_callback": + return + with guard: + if cas_held.is_set(): + return + cas_held.set() + waited.wait(timeout=20) + + def winner(): + try: + return ("winner", reconcile_once()) + finally: + levels.append(level_of_a_new_transaction()) + + def loser(): + assert cas_held.wait(timeout=10) + try: + return ("loser", reconcile_once()) + finally: + levels.append(level_of_a_new_transaction()) + + def monitor(): + assert cas_held.wait(timeout=10) + observed["waiters"] = wait_for_lock_waiters(1) + waited.set() + return ("monitor", None) + + handler = _Records() + reconciler_logger = logging.getLogger(logger_name) + reconciler_logger.addHandler(handler) + task_enqueued.connect( + hold_the_completion, weak=False, dispatch_uid="hold_the_completion" + ) + try: + with new_connections_at(level): + results, errors = race_all([winner, loser, monitor]) + finally: + task_enqueued.disconnect(dispatch_uid="hold_the_completion") + reconciler_logger.removeHandler(handler) + + failures = [ + describe([record.exc_info[1]]) if record.exc_info else record.getMessage() + for record in handler.records + if getattr(record, "event", None) == failed_event + ] + return observed, levels, results, errors, failures + + +# -- batches, which share add() and the shape of the completion ---------- + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_concurrent_adds_to_one_open_batch_all_land(level): + """ + Batch.add goes through the same reissue as Workflow.add. 1.2.0 landed + one of eight at PostgreSQL REPEATABLE READ. + """ + require_a_database_with_isolation_levels() + batch = batches.create("rr-batch") + levels = [] + + def add(index): + def call(): + try: + batch.add(tasks.plain_echo, f"m{index}") + finally: + levels.append(level_of_a_new_transaction()) + return index + + return call + + with new_connections_at(level): + _results, errors = race_all([add(index) for index in range(THREADS)]) + + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert OxpullBatchMember.objects.filter(batch_id=batch.id).count() == THREADS + assert OxTask.objects.count() == THREADS + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_batch_reconciler_that_loses_the_completion_race_reads_zero_rows( + level, worker +): + """The batch reconciler's compare-and-set, raced the same way.""" + require_a_database_with_isolation_levels() + batch = batches.create( + "rr-batch-reconcile", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + batch.add(tasks.plain_echo, "only") + batch.seal() + assert worker.run_once() is True + observed, levels, results, errors, failures = race_the_completion( + level, batches._reconcile, "oxpull.batches", "batch_reconcile_failed" + ) + assert observed["waiters"] == 1 + assert set(levels) == {expected_level(level)} + assert describe(errors) == [] + assert failures == [] + assert dict(results) == {"winner": 1, "loser": 0, "monitor": None} + assert OxpullBatch.objects.get(pk=batch.id).status == OxpullBatch.Status.COMPLETE + assert ( + OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + == 1 + ) diff --git a/tests_oxpull/test_workflows_hazard_prune_membership.py b/tests_oxpull/test_workflows_hazard_prune_membership.py new file mode 100644 index 0000000..bee364d --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_prune_membership.py @@ -0,0 +1,441 @@ +""" +What oxpull_prune may delete: never batch membership, and a workflow's +structure only once nothing can move it again. + +oxpull 1.2.0's oxpull_prune deleted stale unique locks and nothing else. +The workflows branch first ran the same command line and also deleted the +OxpullBatchMember rows of a batch, plus every OxpullWorkflowNode and +OxpullWorkflowEdge row of a workflow, once the parent had been COMPLETE for +longer than --older-than. Now it never deletes a batch member, and deletes +a workflow's node, with the edges into it, only once no retry can move it. + +What a user is owed: + +(i) Batch membership is kept, whatever the cutoff. Batch.progress() of a + complete batch reads its member task rows as it did on 1.2.0, with + 1.2.0's arithmetic: a member whose task row ox_prune deleted counts + as unattributed, not as succeeded. +(ii) A workflow's nodes and edges are deleted only when the workflow is + COMPLETE and completed before the cutoff. An edge is deleted only + when its downstream task row is SUCCESSFUL, DISCARDED or gone, the + states no retry can move. An edge into a FAILED node stays, because + an operator can still retry that node. + +Everything runs through public paths: a real worker's claim and execution, +the real reconcilers, and the real ox_prune and oxpull_prune commands. The +one shortcut is the clock. Instead of waiting out a cutoff, move_back() +moves timestamps a hundred years into the past: a parent's created_at, +sealed_at and completed_at, its membership rows' created_at, or a task +row's finished_at. It changes no status and adds or deletes no row. Each +prune runs with the default command line and with an explicit +--older-than. The only thing assumed about the default cutoff is that it +is longer than the few seconds a test takes. +""" + +import threading +import uuid +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command +from django.db import connection, connections + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import batches, workflows +from oxpull.models import ( + OxpullBatch, + OxpullBatchMember, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, +) + +from . import tasks +from .test_batches import reconcile as reconcile_batches +from .test_batches import stored as stored_batch +from .test_workflows import drain, flow_worker, order +from .test_workflows import reconcile as reconcile_workflows +from .test_workflows import stored as stored_flow + +pytestmark = pytest.mark.django_db + +LONG_AGO = timedelta(days=36525) # a hundred years + +# Between a workflow that completed a moment ago and one moved back a century. +WORKFLOW_CUTOFF = "30d" + + +def show(*parts): + """Print evidence: these tests exist to be read when red.""" + print(*parts) # noqa: T201 + + +def oxpull_prune(*args): + out = StringIO() + call_command("oxpull_prune", *args, stdout=out) + return out.getvalue() + + +def ox_prune(*args): + out = StringIO() + call_command("ox_prune", *args, stdout=out) + return out.getvalue() + + +def move_back(queryset, *fields, by=LONG_AGO): + """The clock shortcut: move these timestamps back, and nothing else.""" + for row in list(queryset.values("pk", *fields)): + moved = {field: row[field] - by for field in fields if row[field] is not None} + if moved: + queryset.model.objects.filter(pk=row["pk"]).update(**moved) + + +def move_batch_back(batch): + parent = OxpullBatch.objects.filter(pk=batch.id) + assert parent.get().completed_at is not None, "only a batch that completed" + move_back(parent, "created_at", "sealed_at", "completed_at") + move_back(OxpullBatchMember.objects.filter(batch_id=batch.id), "created_at") + + +def move_workflow_back(flow): + move_back( + OxpullWorkflow.objects.filter(pk=flow.id), + "created_at", + "sealed_at", + "completed_at", + ) + move_back(OxpullWorkflowNode.objects.filter(workflow_id=flow.id), "created_at") + move_back(OxpullWorkflowEdge.objects.filter(workflow_id=flow.id), "created_at") + + +def member_rows(batch): + return OxpullBatchMember.objects.filter(batch_id=batch.id).count() + + +def node_rows(flow): + return OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() + + +def edge_rows(flow): + return OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count() + + +def status_of(task_id): + return OxTask.objects.values_list("status", flat=True).get(id=task_id) + + +# -- (i) Batch.progress() after oxpull_prune --------------------------- + + +def _completed_batch(worker): + """Two members that succeed and one that fails, drained and reconciled.""" + batch = batches.create("nightly-invoices") + batch.add(tasks.plain_echo, 1) + batch.add(tasks.plain_echo, 2) + batch.add(tasks.plain_fail) + batch.seal() + ran = 0 + while worker.run_once(): + ran += 1 + assert ran == 5 # two successes, three attempts of the failure + assert reconcile_batches() == 1 + row = stored_batch(batch) + assert row.status == OxpullBatch.Status.COMPLETE + assert row.failed_count == 1 + assert member_rows(batch) == 3 + return batch + + +def test_oxpull_prune_leaves_an_old_batchs_progress_as_1_2_0_did(worker): + """ + 1.2.0's oxpull_prune never touched membership, so progress() of a + COMPLETE batch read the member task rows for as long as they existed. + That has to hold however long ago the batch completed and whatever + cutoff the command is given. + """ + batch = _completed_batch(worker) + before = batch.progress() + assert (before.total, before.pending, before.succeeded, before.failed) == ( + 3, + 0, + 2, + 1, + ) + + move_batch_back(batch) + show(oxpull_prune()) # the operator's cron line, unchanged since 1.2.0 + show(oxpull_prune("--older-than", "0")) # and a cutoff every batch is past + + after = batch.progress() + show(f"before={before} after={after} member rows={member_rows(batch)}") + assert after == before + assert member_rows(batch) == 3 + + +def test_a_failed_member_stays_attributed_under_both_default_prunes(worker): + """ + django-ox's own ox_prune deletes SUCCESSFUL rows past its cutoff. It + does not delete FAILED rows unless --include-failed is passed: they + hold the tracebacks. So an install running both prunes on their + defaults kept progress().failed readable on 1.2.0 indefinitely, and + counted the two pruned successes as unattributed. Both parts of that + arithmetic are the released contract: a pruned member is not reported + as succeeded. + """ + batch = _completed_batch(worker) + move_batch_back(batch) + move_back(OxTask.objects.exclude(finished_at=None), "finished_at") + + show(ox_prune()) + assert OxTask.objects.filter(status=OxTask.Status.SUCCESSFUL).count() == 0 + show(oxpull_prune()) + show(oxpull_prune("--older-than", "0")) + + progress = batch.progress() + show( + f"after both prunes: {progress} unattributed={progress.unattributed} " + f"member rows={member_rows(batch)}" + ) + assert (progress.total, progress.pending, progress.succeeded, progress.failed) == ( + 3, + 0, + 0, + 1, + ) + assert progress.unattributed == 2 # the two SUCCESSFUL rows ox_prune deleted + assert member_rows(batch) == 3 + + +# -- (ii) which workflow rows oxpull_prune may delete ------------------ + +EVERY_EDGE = {("root", "ok"), ("root", "bad"), ("bad", "doomed"), ("root", "gone")} + + +def _mixed_workflow(label): + """ + One sealed workflow whose edges end in every downstream state: + + root -> ok ok runs and ends SUCCESSFUL + root -> bad bad fails three attempts and ends FAILED + bad -> doomed doomed is cancelled by the reconciler: DISCARDED + root -> gone gone ends SUCCESSFUL, then ox_prune deletes its row + + Returns the workflow and {node name: task id}. + """ + flow = workflows.create(label) + results = { + "root": flow.add("root", tasks.step, f"{label}-root"), + "ok": flow.add("ok", tasks.step, f"{label}-ok", depends_on="root"), + "bad": flow.add("bad", tasks.failing_step, f"{label}-bad", depends_on="root"), + "doomed": flow.add("doomed", tasks.step, f"{label}-doomed", depends_on="bad"), + "gone": flow.add("gone", tasks.step, f"{label}-gone", depends_on="root"), + } + flow.seal() + return flow, {name: _pk(result) for name, result in results.items()} + + +def edges_by_name(flow, ids): + names = {task_id: name for name, task_id in ids.items()} + rows = OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).values_list( + "upstream_task_id", "downstream_task_id" + ) + return { + (names.get(_pk(up), str(up)), names.get(_pk(down), str(down))) + for up, down in rows + } + + +def settle(): + """A running install: workers drain and the reconciler ticks, until quiet.""" + worker = flow_worker() + for _ in range(3): + drain(worker) + reconcile_workflows() + + +def test_oxpull_prune_deletes_only_the_workflow_rows_no_retry_can_move(): + """ + Four workflows, one prune: + + - retired: the mixed workflow, completed a hundred years ago. Only + the edges into ok (SUCCESSFUL), doomed (DISCARDED) and gone (row + deleted) may go. The edge into bad (FAILED) stays. + - recent: the same workflow, created, sealed and given its nodes and + edges a hundred years ago like the others, but completed a moment + ago, inside the cutoff. Nothing goes, so the cutoff is read from + when a workflow completed, not from any other timestamp. + - clean: a -> b, both SUCCESSFUL, completed a hundred years ago. + Every node and edge goes, so the prune still reaps. + - unsealed: a -> b, both SUCCESSFUL, created a hundred years ago and + never sealed, so never COMPLETE. Nothing goes. + """ + retired, retired_ids = _mixed_workflow("retired") + recent, recent_ids = _mixed_workflow("recent") + clean = workflows.create("clean") + clean.add("a", tasks.step, "clean-a") + clean.add("b", tasks.step, "clean-b", depends_on="a") + clean.seal() + unsealed = workflows.create("unsealed") + unsealed.add("a", tasks.step, "unsealed-a") + unsealed.add("b", tasks.step, "unsealed-b", depends_on="a") + + settle() + show(f"ran: {order()}") + for ids in (retired_ids, recent_ids): + assert status_of(ids["root"]) == OxTask.Status.SUCCESSFUL + assert status_of(ids["ok"]) == OxTask.Status.SUCCESSFUL + assert status_of(ids["bad"]) == OxTask.Status.FAILED + assert status_of(ids["doomed"]) == OxTask.Status.DISCARDED + assert status_of(ids["gone"]) == OxTask.Status.SUCCESSFUL + assert "retired-doomed" not in order() and "recent-doomed" not in order() + for flow in (retired, recent, clean): + assert stored_flow(flow).status == OxpullWorkflow.Status.COMPLETE + assert stored_flow(unsealed).status == OxpullWorkflow.Status.OPEN + assert OxTask.objects.filter(args=["unsealed-b"]).get().status == ( + OxTask.Status.SUCCESSFUL + ) + + # django-ox's own prune deletes the two `gone` rows, and only those. + gone = [retired_ids["gone"], recent_ids["gone"]] + move_back(OxTask.objects.filter(id__in=gone), "finished_at") + show(ox_prune("--older-than", WORKFLOW_CUTOFF)) + assert not OxTask.objects.filter(id__in=gone).exists() + assert OxTask.objects.filter(args=["retired-bad"]).exists() + assert edges_by_name(retired, retired_ids) == EVERY_EDGE + assert edges_by_name(recent, recent_ids) == EVERY_EDGE + + for flow in (retired, clean, unsealed): + move_workflow_back(flow) + # Everything about recent is as old as retired except its completion. + move_back(OxpullWorkflow.objects.filter(pk=recent.id), "created_at", "sealed_at") + move_back(OxpullWorkflowNode.objects.filter(workflow_id=recent.id), "created_at") + move_back(OxpullWorkflowEdge.objects.filter(workflow_id=recent.id), "created_at") + show(oxpull_prune()) # the operator's cron line + show(oxpull_prune("--older-than", WORKFLOW_CUTOFF)) + + left = edges_by_name(retired, retired_ids) + show( + f"retired: edges left {sorted(left)}, nodes left {node_rows(retired)}; " + f"recent: {edge_rows(recent)} edges, {node_rows(recent)} nodes; " + f"clean: {edge_rows(clean)} edges, {node_rows(clean)} nodes; " + f"unsealed: {edge_rows(unsealed)} edges, {node_rows(unsealed)} nodes" + ) + assert ("root", "bad") in left, ( + "oxpull_prune deleted the edge root -> bad of a workflow that completed " + "a hundred years ago, although bad is FAILED and an operator can still " + "retry it" + ) + assert left == {("root", "bad")}, ( + "only the edge into the FAILED node may stay; the edges into a " + f"SUCCESSFUL, a DISCARDED and a deleted row must go, and {sorted(left)} " + "were left" + ) + retired_nodes = set( + OxpullWorkflowNode.objects.filter(workflow_id=retired.id).values_list( + "name", flat=True + ) + ) + assert retired_nodes == {"bad"}, ( + "only the FAILED node may stay; the nodes of a SUCCESSFUL, a DISCARDED " + f"and a deleted row must go, and {sorted(retired_nodes)} were left" + ) + assert edges_by_name(recent, recent_ids) == EVERY_EDGE, ( + "a workflow that completed inside the cutoff lost edges" + ) + assert node_rows(recent) == 5, "a workflow inside the cutoff lost nodes" + assert (edge_rows(clean), node_rows(clean)) == (0, 0), ( + "a clean workflow that completed a hundred years ago kept its nodes or edges" + ) + assert (edge_rows(unsealed), node_rows(unsealed)) == (1, 2), ( + "a workflow that never completed lost nodes or edges" + ) + + +@pytest.mark.django_db(transaction=True) +def test_the_completion_window_alone_leaves_nothing_for_prune_to_ungate(): + """ + The completion window alone: an operator retry committing inside + workflows._complete_if_finished leaves a READY node on a COMPLETE + workflow, and prune then deletes its edges. + + Expected to PASS. It is here to show what the window does and does + not reach. The completing transaction locks the FAILED and LOST node + rows before it classifies them again, so on PostgreSQL and MySQL the + retry waits for it to commit (asserted below), as it does on SQLite. + The end state is the same either way. The only rows retry() can move + are FAILED and LOST, and a node in either state has been claimed, which + the gate allowed only once every edge into it was satisfied. The node + the window leaves READY is pending, so oxpull_prune deletes nothing of + that workflow, and the node that was really waiting on the failure was + DISCARDED by the reconciler before completion, which is terminal. + """ + flow = workflows.create( + "window", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + root = flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + assert drain(flow_worker()) == 3 + assert OxTask.objects.get(id=root.id).status == OxTask.Status.FAILED + + retried = {} + inside = {} + real_enqueue_callback = workflows._enqueue_callback + + def retry_on_another_connection(): + try: + retried["result"] = actions.retry(root.id) + finally: + connections.close_all() + + def enqueue_callback_with_a_retry_landing(*args, **kwargs): + # Called inside the completing transaction, after the CAS and + # after the counts were re-read under the workflow row lock. + thread = threading.Thread(target=retry_on_another_connection) + thread.start() + thread.join(timeout=3) + inside["retry_committed"] = not thread.is_alive() + inside["thread"] = thread + return real_enqueue_callback(*args, **kwargs) + + workflows._enqueue_callback = enqueue_callback_with_a_retry_landing + try: + assert reconcile_workflows() == 1 + finally: + workflows._enqueue_callback = real_enqueue_callback + inside["thread"].join(timeout=30) + assert retried["result"] is True + show( + f"vendor={connection.vendor} retry committed inside the window: " + f"{inside['retry_committed']}" + ) + if connection.vendor != "sqlite": + # PostgreSQL and MySQL: the completion holds the FAILED root's row, + # so the retry lands just after the commit. SQLite serialises + # writers, so there it lands just after the commit too. + assert inside["retry_committed"] is False + + row = stored_flow(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert OxTask.objects.get(id=root.id).status == OxTask.Status.READY + assert OxTask.objects.get(args=["child"]).status == OxTask.Status.DISCARDED + + move_workflow_back(flow) + show(oxpull_prune()) + show(oxpull_prune("--older-than", WORKFLOW_CUTOFF)) + # root is READY, so the workflow has a pending node and nothing goes. + assert (edge_rows(flow), node_rows(flow)) == (1, 2) + + tasks.STATE.clear() + drain(flow_worker()) + show(f"ran after prune: {order()}") + assert "child" not in order() + + +def _pk(result_or_id): + value = getattr(result_or_id, "id", result_or_id) + return value if isinstance(value, uuid.UUID) else uuid.UUID(str(value)) diff --git a/tests_oxpull/test_workflows_hazard_prune_ungates.py b/tests_oxpull/test_workflows_hazard_prune_ungates.py new file mode 100644 index 0000000..82c162b --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_prune_ungates.py @@ -0,0 +1,500 @@ +""" +Pruning an upstream that never succeeded must not let its downstream run. + +ox_prune deletes finished task rows, and some of them never succeeded: + +(i) The default invocation deletes a DISCARDED row once it is seven days + old. An operator discards a stuck node from the admin, and a week + later the row is gone. +(ii) ox_prune --include-failed deletes FAILED and LOST rows the same way. + +The gate used to read an edge whose upstream row is missing as satisfied, +so after either prune a gating worker ran the child. The reconciler did not +step in first: it cancels cones in SEALED workflows only, and only while +the dead row is still there to find. A blocked node now waits in WAITING, +and leaves it only once every edge into it is marked satisfied by a writer +that saw its upstream succeed. The claim also blocks on an unmarked edge +whose upstream row is missing. + +Every test asserts the promise, not one particular fix: the child of an +upstream that never succeeded never starts. It has no attempt, it was +never RUNNING, and its body never ran. Its status is not one a claim +leads to (RUNNING, SUCCESSFUL, FAILED or LOST); READY, DISCARDED and any +waiting status django-ox may add all pass. Two ends keep that promise, +and most tests accept either one: + +- held: the child waits; +- cancelled: the child is DISCARDED without running. + +Which end is right depends on what was recorded before the row went. A +recorded failure cancels the cone. A missing row with nothing recorded +holds it, because a missing row no longer says whether the upstream +succeeded, and cancelling there would throw away work downstream of a +success. One test pins each end: a failure a worker recorded before the +prune must cancel the child and let the workflow complete, and a discard +nothing recorded must hold the child, not cancel it. + +A control runs alongside them. When the pruned upstream really did +succeed, its child still runs. So a gate that blocks on every missing row +cannot pass this file. + +Each test drives the upstream to its status through real paths: a gating +worker's claim and execution, django_ox.actions, or the real reaper. It +runs the real ox_prune management command and the real reconciler pass, +then lets a real gating worker run everything it can claim. + +One shortcut, used only where a test relies on ox_prune's default 7-day +cutoff: age() moves the upstream's finished_at back eight days with one +UPDATE, standing in for a week passing. No other column is touched. +""" + +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, order, reconcile, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +PAST_THE_DEFAULT_CUTOFF = timedelta(days=8) + +# Only a claim leads to these. A child that never started is in none of +# them: READY while held, DISCARDED once cancelled, or a waiting status. +CLAIMED_STATUSES = ( + OxTask.Status.RUNNING, + OxTask.Status.SUCCESSFUL, + OxTask.Status.FAILED, + OxTask.Status.LOST, +) + + +def ox_prune(*args): + """The real django-ox command, as a cron line would run it.""" + out = StringIO() + call_command("ox_prune", *args, stdout=out) + return out.getvalue() + + +def age(task_id, by=PAST_THE_DEFAULT_CUTOFF): + """Stand-in for time passing: move finished_at back, and nothing else.""" + finished_at = OxTask.objects.values_list("finished_at", flat=True).get(id=task_id) + assert finished_at is not None + OxTask.objects.filter(id=task_id).update(finished_at=finished_at - by) + + +def assert_pruned(task_id, output): + """Precondition: the prune really did delete the upstream row.""" + assert not OxTask.objects.filter(id=task_id).exists(), ( + f"precondition not reached: ox_prune left the upstream row; it said {output!r}" + ) + + +def status_of(task_id): + return OxTask.objects.values_list("status", flat=True).get(id=task_id) + + +def assert_never_started(task_id, label, upstream_status, when): + """ + No worker ever took this row, and its body never ran. + + A claim is the only way into RUNNING. Every claim adds an attempt, + stamps started_at and appends a worker id, and nothing in django-ox + clears any of the three. So they show whether any worker ever took + the row, even after it finished. + """ + row = OxTask.objects.values("status", "attempts", "started_at", "worker_ids").get( + id=task_id + ) + started = ( + row["status"] in CLAIMED_STATUSES + or row["attempts"] != 0 + or row["started_at"] is not None + or bool(row["worker_ids"]) + or label in order() + ) + assert not started, ( + f"{label!r} started {when}, although its upstream settled " + f"{upstream_status} and never succeeded: status={row['status']}, " + f"attempts={row['attempts']}, started_at={row['started_at']}, " + f"worker_ids={row['worker_ids']!r}, execution order {order()}. " + "A gate that reads a missing upstream row as satisfied lets it through." + ) + + +def run_everything_then_assert_never_started(worker, child_id, upstream_status): + """The last step of every test: the worker runs all it can claim.""" + drain(worker) + assert_never_started( + child_id, "child", upstream_status, "after ox_prune deleted its upstream" + ) + + +def exhaust_leases(root_id): + """ + Three workers in turn claim the root with a real lease and die holding + it. expire_lease brings each lease's expiry forward instead of waiting + LOCK_TIMEOUT; the reaper does the rest, requeueing twice and then, with + no attempts left, marking the row LOST. + """ + reaper = flow_worker() + for attempt in (1, 2, 3): + claimed = flow_worker().claim_one() + assert claimed is not None + assert str(claimed.id) == str(root_id) + assert claimed.attempts == attempt + assert actions.expire_lease(root_id) is True + assert reaper.reap() == 1 + assert status_of(root_id) == OxTask.Status.LOST + + +# -- (i) OPEN workflow, DISCARDED upstream, default ox_prune -------------- + + +@pytest.mark.parametrize("closed", ["while-ready", "after-failing"]) +def test_open_workflow_child_of_a_discarded_upstream_never_starts_after_default_prune( + closed, +): + flow = workflows.create(f"open-discarded-{closed}") + upstream = tasks.step if closed == "while-ready" else tasks.failing_step + root = flow.add("root", upstream, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + worker = flow_worker() + if closed == "after-failing": + assert drain(worker) == 3 + assert status_of(root.id) == OxTask.Status.FAILED + + # An operator closes the upstream from the admin. + assert actions.discard(root.id) is True + # The scheduled reconciler ticks. It may hold the child or cancel it. + # Either way the child must not start. + reconcile() + assert stored(flow).status == OxpullWorkflow.Status.OPEN + assert worker.claim_one() is None # control: nothing runs before the prune + assert_never_started(child.id, "child", "DISCARDED", "before the prune") + + age(root.id) + output = ox_prune() # the default invocation: SUCCESSFUL + DISCARDED, 7d + assert_pruned(root.id, output) + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "DISCARDED") + + +# -- (ii) ox_prune --include-failed ---------------------------------------- + + +def test_open_workflow_child_of_a_failed_upstream_never_starts_after_include_failed(): + flow = workflows.create("open-failed") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + worker = flow_worker() + assert drain(worker) == 3 + assert status_of(root.id) == OxTask.Status.FAILED + + reconcile() + assert worker.claim_one() is None # control: nothing runs before the prune + assert_never_started(child.id, "child", "FAILED", "before the prune") + + age(root.id) + output = ox_prune("--include-failed") + assert_pruned(root.id, output) + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "FAILED") + assert order() == ["root", "root", "root"] + + +def test_open_workflow_child_of_a_lost_upstream_never_starts_after_include_failed(): + flow = workflows.create("open-lost") + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + exhaust_leases(root.id) + + worker = flow_worker() + reconcile() + assert worker.claim_one() is None # control: nothing runs before the prune + assert_never_started(child.id, "child", "LOST", "before the prune") + + age(root.id) + output = ox_prune("--include-failed") + assert_pruned(root.id, output) + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "LOST") + assert order() == [] + + +@pytest.mark.parametrize( + ("prune_args", "aged"), + [ + pytest.param( + ["--include-failed"], True, id="reconciler-silent-past-the-default-cutoff" + ), + pytest.param( + ["--include-failed", "--older-than", "0"], + False, + id="prune-runs-before-the-next-reconciler-tick", + ), + ], +) +def test_sealed_workflow_child_of_a_failed_upstream_never_starts_when_prune_comes_first( + prune_args, aged +): + flow = workflows.create("sealed-failed") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + assert drain(worker) == 3 + assert status_of(root.id) == OxTask.Status.FAILED + assert worker.claim_one() is None # control: the gate holds the child + + # No reconciler tick between the failure and the prune. + if aged: + age(root.id) + output = ox_prune(*prune_args) + assert_pruned(root.id, output) + # The reconciler's tick arrives after the prune. + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "FAILED") + assert order() == ["root", "root", "root"] + + +@pytest.mark.parametrize( + ("prune_args", "aged"), + [ + pytest.param( + ["--include-failed"], True, id="reconciler-silent-past-the-default-cutoff" + ), + pytest.param( + ["--include-failed", "--older-than", "0"], + False, + id="prune-runs-before-the-next-reconciler-tick", + ), + ], +) +def test_a_missing_upstream_whose_failure_was_recorded_cancels_its_child( + prune_args, aged +): + """ + The same sequence as the test above, with the other end pinned. + + The worker that ran root's last attempt wrote FAILED, so how root + finished was recorded before ox_prune deleted the row. A missing + upstream with a recorded failure is dead, not unknown: the child is + cancelled without running, and the sealed workflow completes, so + on_complete can fire. A fix that only holds (it blocks on the missing + row and never cancels) leaves the child WAITING and the workflow SEALED + for ever, and fails here. + + Up to three reconciler ticks are allowed after the prune, for a + cancellation and a completion that land on separate ticks. + """ + flow = workflows.create("recorded-failure") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + assert drain(worker) == 3 + assert status_of(root.id) == OxTask.Status.FAILED + assert worker.claim_one() is None # control: the gate holds the child + + # No reconciler tick between the failure and the prune. + if aged: + age(root.id) + output = ox_prune(*prune_args) + assert_pruned(root.id, output) + + for _ in range(3): + reconcile() + drain(worker) + if stored(flow).status == OxpullWorkflow.Status.COMPLETE: + break + + assert_never_started( + child.id, "child", "FAILED", "after ox_prune deleted its upstream" + ) + assert order() == ["root", "root", "root"] + row = stored(flow) + assert (status_of(child.id), row.status) == ( + OxTask.Status.DISCARDED, + OxpullWorkflow.Status.COMPLETE, + ), ( + f"the child is {status_of(child.id)} and the workflow {row.status}, " + "although a worker recorded its upstream FAILED before the prune: the " + "child should be cancelled and the workflow complete" + ) + + +def test_deferred_open_workflow_child_of_a_failed_upstream_is_not_released_by_prune(): + """ + The child waits, so right after the prune it is still out of reach. + The reconciler's release backstop, which is not scoped to sealed + workflows, runs next, and it must not release a child whose upstream + never succeeded. + """ + flow = workflows.create("deferred-open-failed") + root = flow.add("root", tasks.failing_deferred_step, "root") + child = flow.add("child", tasks.deferred_step, "child", depends_on="root") + assert status_of(child.id) == OxTask.Status.WAITING + worker = deferring_worker() + assert drain(worker) == 3 + assert status_of(root.id) == OxTask.Status.FAILED + + reconcile() + assert worker.claim_one() is None # control: nothing runs before the prune + assert_never_started(child.id, "child", "FAILED", "before the prune") + + age(root.id) + output = ox_prune("--include-failed") + assert_pruned(root.id, output) + assert worker.claim_one() is None # nothing runs the child until a tick + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "FAILED") + assert order() == ["root", "root", "root"] + + +# -- a missing row with nothing recorded holds ----------------------------- + + +def test_child_of_an_upstream_pruned_before_its_outcome_was_recorded_is_held(): + """ + An operator discards the upstream, and the prune deletes the row + before any reconciler tick or worker has seen it. Nothing is left + that says how it finished. + + The child must not run, and it must not be cancelled either: it is + held. From a missing row with no record, a discard looks the same as a + success, so a rule that cancels here also cancels the downstream of + an upstream that succeeded. That is lost work. Holding is stuck work + that an operator can see and act on. + + This depends on django-ox 1.2.0's discard recording nothing oxpull can + read. If a later django-ox discard API does record the discard, the + upstream is no longer unrecorded and the right end becomes DISCARDED; + this test then needs changing, not the implementation bending to fit. + """ + flow = workflows.create("unrecorded") + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + + assert actions.discard(root.id) is True + assert worker.claim_one() is None # control: the gate holds the child + + # No reconciler tick between the discard and the prune. + output = ox_prune("--older-than", "0") + assert_pruned(root.id, output) + reconcile() + + run_everything_then_assert_never_started(worker, child.id, "DISCARDED") + assert status_of(child.id) in (OxTask.Status.READY, OxTask.Status.WAITING), ( + f"the child was cancelled (status={status_of(child.id)}) although " + "nothing recorded how its missing upstream finished. It should be " + "held: without a record, cancelling here would also cancel work " + "downstream of an upstream that succeeded." + ) + + +# -- what a user sees ------------------------------------------------------- + + +def test_end_to_end_a_node_downstream_of_a_pruned_failure_never_runs(): + """ + The user's view of (ii) on an OPEN workflow: nodes are added as data + arrives and the workflow is sealed at the end, a reconciler is scheduled, + and ox_prune --include-failed runs from cron with its default cutoff. + """ + flow = workflows.create( + "nightly", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + extract = flow.add("extract", tasks.failing_step, "extract") + load = flow.add("load", tasks.step, "load", depends_on="extract") + worker = flow_worker() + assert drain(worker) == 3 + assert status_of(extract.id) == OxTask.Status.FAILED + reconcile() + + age(extract.id) + output = ox_prune("--include-failed") + assert_pruned(extract.id, output) + drain(worker) + flow.seal() + reconcile() + + row = stored(flow) + progress = flow.progress() + assert "load" not in order(), ( + f"'load' ran after 'extract' FAILED (execution order {order()}). The " + f"workflow then completed as status={row.status}, " + f"failed_count={row.failed_count}, cancelled_count={row.cancelled_count}, " + f"on_complete enqueued={row.callback_task_id is not None}, " + f"on_success enqueued={row.success_task_id is not None}; " + f"progress total={progress.total} succeeded={progress.succeeded} " + f"failed={progress.failed} unattributed={progress.unattributed}" + ) + assert_never_started( + load.id, "load", "FAILED", "after ox_prune deleted its upstream" + ) + + +# -- control: a pruned upstream that succeeded still satisfies its edge ---- + + +@pytest.mark.parametrize("variant", ["open", "sealed", "deferred"]) +def test_control_a_child_whose_pruned_upstream_succeeded_still_runs(variant): + """ + A fan-in: 'first' succeeds and ox_prune deletes it a week later while + 'second' is still waiting. Then 'second' succeeds, and the child must + run. Its missing upstream really did succeed. + + This passes today, and it must keep passing. A fix that blocks on every + missing upstream row, or cancels the cone of one, fails it. So does a + release sweep that never clears a deferral once an upstream is gone. + """ + deferred = variant == "deferred" + step = tasks.deferred_step if deferred else tasks.step + worker = deferring_worker() if deferred else flow_worker() + flow = workflows.create(f"control-{variant}") + # The claim orders by priority first, so 'first' runs alone below. + first = flow.add("first", step.using(priority=10), "first") + flow.add("second", step, "second") + child = flow.add("child", step, "child", depends_on=["first", "second"]) + if variant == "sealed": + flow.seal() + + assert drain(worker, limit=1) == 1 + assert order() == ["first"] + assert status_of(first.id) == OxTask.Status.SUCCESSFUL + reconcile() + + age(first.id) + output = ox_prune() # the default invocation: SUCCESSFUL + DISCARDED, 7d + assert_pruned(first.id, output) + reconcile() + assert order() == ["first"] + + drain(worker) + assert order() == ["first", "second", "child"], ( + f"the child did not run (execution order {order()}, " + f"status={status_of(child.id)}) although both its upstreams " + "succeeded. One of them was pruned after it succeeded, and that " + "must not block or cancel the child." + ) + assert status_of(child.id) == OxTask.Status.SUCCESSFUL diff --git a/tests_oxpull/test_workflows_hazard_reconcile_schedule.py b/tests_oxpull/test_workflows_hazard_reconcile_schedule.py new file mode 100644 index 0000000..7469321 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_reconcile_schedule.py @@ -0,0 +1,277 @@ +""" +The workflow reconciler's schedule requirement. + +1.2.0 hardened the batch reconciler's requirement in three ways the +workflow reconciler did not get: + +(a) batches.reconcile_schedule_aliases() counts a django-ox 1.2.0 stored + schedule (SCHEDULE_SOURCE plus SCHEDULABLE_TASKS exposing the + reconciler), and counts only schedules on a django-ox backend. + backend._reconcile_is_scheduled reads OPTIONS["SCHEDULES"] of every + alias and nothing else. +(b) batches.create() and Batch.seal() refuse when nothing schedules the + reconciler. workflows.create() and Workflow.seal() do not. +(c) oxpull.W002 is emitted once. oxpull.W003 is emitted by every backend + that declares WORKFLOWS. + +Every test asserts the behaviour a user should get. Everything is driven +through public entry points: call_command("check"), +call_command("ox_worker"), django_ox.stored's write API, a real worker's +dispatch_schedules() and run_once(), and workflows.create()/seal(). +""" + +import copy +import io +from datetime import timedelta + +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.core.management import call_command +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, stored + +# Django 6.1 hands every configured alias to the model and field checks +# unless the command pins one, so a system check run here opens "replica" +# to ask SQLite or MySQL whether it supports JSONField. That read leaves +# the alias unwritten, but it has to be declared or the test database +# guard refuses the connection. +pytestmark = pytest.mark.django_db(databases=["default", "replica"]) + +WORKFLOW_RECONCILE = workflows.RECONCILE_TASK_PATH +BATCH_RECONCILE = "oxpull.batches.reconcile" + + +def _names_task(config, path): + """Whether one SCHEDULES or SCHEDULABLE_TASKS entry names `path`.""" + if isinstance(config, str): + return config == path + return isinstance(config, dict) and config.get("task") == path + + +def project_tasks(*, batch_reconciler=True): + """ + settings.TASKS as a project would deploy it, with nothing scheduling the + workflow reconciler in either channel. + + flows_ungated is dropped: it is oxpull.E007 by design, and an Error makes + `manage.py check` exit before the warnings under test are printed. + """ + rebuilt = {} + for alias, params in copy.deepcopy(settings.TASKS).items(): + if alias == "flows_ungated": + continue + options = params.get("OPTIONS", {}) + for key in ("SCHEDULES", "SCHEDULABLE_TASKS"): + entries = options.get(key) + if not isinstance(entries, dict): + continue + unwanted = [WORKFLOW_RECONCILE] + if not batch_reconciler: + unwanted.append(BATCH_RECONCILE) + kept = { + name: config + for name, config in entries.items() + if not any(_names_task(config, path) for path in unwanted) + } + if kept: + options[key] = kept + else: + options.pop(key) + rebuilt[alias] = params + return rebuilt + + +def scheduled_project_tasks(): + """project_tasks() with the workflow reconciler scheduled in settings.""" + rebuilt = project_tasks() + options = rebuilt["flows"].setdefault("OPTIONS", {}) + options.setdefault("SCHEDULES", {})["oxpull-workflow-reconcile"] = { + "task": WORKFLOW_RECONCILE, + "cron": "* * * * *", + } + return rebuilt + + +def run_check(): + """`manage.py check`, as an operator types it; returns what it printed.""" + out, err = io.StringIO(), io.StringIO() + call_command("check", stdout=out, stderr=err) + return out.getvalue() + err.getvalue() + + +# -- (a) the check reads one channel, and reads it on any backend ------- + + +def test_a_stored_schedule_that_runs_the_workflow_reconciler_is_not_reported_missing( + monkeypatch, +): + """ + django-ox 1.2.0 lets a project keep schedules as rows. This project has + no SCHEDULES block anywhere: "default" reads rows, and SCHEDULABLE_TASKS + exposes both reconcilers. A row naming the workflow reconciler is + written through django_ox.stored, a real worker dispatches it, the + reconciler task runs and completes a sealed workflow. + + The configuration demonstrably runs the reconciler. `manage.py check` + must not then tell the operator that nothing schedules it. The batch + reconciler, configured the same way in the same run, is not reported. + """ + from django_ox.stored import create_schedule + + # The registry is process-wide. Start it empty so this test neither + # relies on nor leaves behind another test's keys; the settings channel + # refills it from SCHEDULABLE_TASKS on every lookup. + monkeypatch.setattr("django_ox.registry._registry", {}) + monkeypatch.setattr("django_ox.registry._discovered", True) + + config = project_tasks() + options = config["default"]["OPTIONS"] + options.pop("SCHEDULES") + options["SCHEDULE_SOURCE"] = "django_ox.stored.DatabaseScheduleSource" + options["SCHEDULABLE_TASKS"] = { + "oxpull-batch-reconcile": BATCH_RECONCILE, + "oxpull-workflow-reconcile": WORKFLOW_RECONCILE, + } + assert not any("SCHEDULES" in p.get("OPTIONS", {}) for p in config.values()) + + with override_settings(TASKS=config): + flow = workflows.create("stored-schedule") + flow.add("only", tasks.step, "only") + flow.seal() + assert drain(flow_worker()) == 1 + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + create_schedule( + name="workflow-reconcile", + task_key="oxpull-workflow-reconcile", + trigger="cron", + cron="* * * * *", + # A row fires from its boundary; five minutes back puts the + # latest minute tick after it, whatever second this runs at. + start_time=timezone.now() - timedelta(minutes=5), + ) + dispatcher = Worker( + backend_alias="default", backoff_initial=0, poll_interval=0.01 + ) + assert dispatcher.dispatch_schedules() == 1 + assert OxTask.objects.filter(task_path=WORKFLOW_RECONCILE).count() == 1 + drain(dispatcher) + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE, ( + "premise: the stored row must run the workflow reconciler" + ) + + output = run_check() + + assert "oxpull.W002" not in output, ( + "control: batches count a stored schedule\n" + output + ) + assert "oxpull.W003" not in output, ( + "oxpull.W003 says nothing schedules oxpull.workflows.reconcile, on a " + "project whose stored schedule just ran it\n" + output + ) + + +def test_a_schedule_on_a_backend_no_worker_can_run_does_not_satisfy_the_check(): + """ + The only entry naming the workflow reconciler sits under the OPTIONS of + "immediate", which is Django's ImmediateBackend. Schedules are dispatched + by a django-ox worker for its own backend alias, and no worker can be + built on that one: `ox_worker --backend immediate` refuses. Nothing will + ever tick the entry, so the project has no reconciler, and the check + must say so. + """ + config = project_tasks() + config["immediate"] = { + **config["immediate"], + "OPTIONS": { + "SCHEDULES": { + "oxpull-workflow-reconcile": { + "task": WORKFLOW_RECONCILE, + "cron": "* * * * *", + }, + }, + }, + } + with override_settings(TASKS=config): + with pytest.raises(ImproperlyConfigured, match="not an OxBackend"): + call_command("ox_worker", "--backend", "immediate") + output = run_check() + + assert "oxpull.W003" in output, ( + "a SCHEDULES entry on a backend no worker can dispatch silenced " + "oxpull.W003\n" + output + ) + + +# -- (b) nothing refuses a workflow that nothing will complete ---------- + + +def test_create_refuses_when_nothing_schedules_the_workflow_reconciler(): + """ + batches.create() refuses without a reconciler schedule, on the line that + creates the batch. workflows.create() does not: the check warns, the + warning can be silenced or never read, and the workflow that follows + never completes and never fires a callback. + """ + with override_settings(TASKS=project_tasks()): + assert "oxpull.W003" in run_check(), "premise: the check sees the gap" + with pytest.raises(ImproperlyConfigured, match=r"oxpull\.workflows\.reconcile"): + workflows.create("orphan") + assert not OxpullWorkflow.objects.filter(name="orphan").exists() + + +def test_seal_refuses_when_nothing_schedules_the_workflow_reconciler(): + """ + Settings can change under a workflow that already exists, so Batch.seal() + refuses as well as create(), and a refused seal leaves the batch OPEN. + Workflow.seal() seals regardless. + """ + with override_settings(TASKS=scheduled_project_tasks()): + flow = workflows.create("before-the-schedule-went") + flow.add("only", tasks.step, "only") + with ( + override_settings(TASKS=project_tasks()), + pytest.raises(ImproperlyConfigured, match=r"oxpull\.workflows\.reconcile"), + ): + flow.seal() + assert stored(flow).status == OxpullWorkflow.Status.OPEN + with override_settings(TASKS=scheduled_project_tasks()): + flow.seal() + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + +# -- (c) one condition, one warning -------------------------------------- + + +def test_the_missing_workflow_reconciler_is_reported_once(): + """ + Nothing schedules either reconciler. Both conditions are global: one + schedule anywhere fixes each. W002 is printed once. W003 is printed once + for the backends declaring WORKFLOWS, here "flows" and "flows_deferred". + + That one W003 says what a missing reconciler costs a node that waits: a + release a worker missed is never made. + """ + with override_settings(TASKS=project_tasks(batch_reconciler=False)): + declaring = [ + alias + for alias, params in settings.TASKS.items() + if params.get("OPTIONS", {}).get("WORKFLOWS") is True + ] + output = run_check() + assert sorted(declaring) == ["flows", "flows_deferred"] + assert output.count("oxpull.W002") == 1, "control: W002 is once\n" + output + assert output.count("oxpull.W003") == 1, ( + f"oxpull.W003 printed {output.count('oxpull.W003')} times for one " + "missing schedule\n" + output + ) + assert "waits forever" in output, output diff --git a/tests_oxpull/test_workflows_hazard_retrying_a_failed_node_after_one_reconciler.py b/tests_oxpull/test_workflows_hazard_retrying_a_failed_node_after_one_reconciler.py new file mode 100644 index 0000000..9eb86d4 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_retrying_a_failed_node_after_one_reconciler.py @@ -0,0 +1,395 @@ +""" +Retrying a failed node after one reconciler tick. + +django-ox's supported recovery for a FAILED task is actions.retry, or the +admin's "Retry selected tasks", which calls actions.retry_many. The +workflow reconciler runs once a minute, and its first tick after a node +FAILED discards that node's whole downstream cone. When the cone was the +last pending work, the same tick completes the workflow with +failed_count=1 and fires on_complete. + +An operator who fixes the cause and retries after that tick gets the +upstream rerun and succeeding, and nothing else: the cone is DISCARDED, +DISCARDED is not retryable, the reconciler never looks at a COMPLETE +workflow again, and the stored row keeps the failure count from the +first completion. + +Every test here asserts the state a correct design ends in, whichever fix +is chosen (revive the cone, hold the cone, or anything else): + + * a node whose upstream nodes all end SUCCESSFUL runs, and is not left + DISCARDED; + * a COMPLETE workflow's stored counts agree with its nodes, and + on_success fires once every node succeeded. + +Nothing asserts the intermediate state after the tick, so a design that +never discards the cone passes too. That state is printed in every +failure message, which shows the path was provoked. + +Real paths throughout: nodes fail and succeed inside a real worker's +run_once(), the reconciler is enqueued as the task a SCHEDULES entry +enqueues and is claimed and executed by a worker, and recovery goes +through django-ox's public actions. No task row is written by hand. + +Gap, recorded rather than closed: a fix of the form "wait N minutes before +cancelling a FAILED node's cone" would pass these tests, because a test +cannot let a wall-clock grace elapse, and would still lose the cone of a +retry that arrives after the grace. + +Two controls at the bottom pass today. One runs the same scenario with +the retry landing before any tick: the end-state assertions are +satisfiable by the shipped code when the ordering is favourable. The other +runs the red sequence exactly, retry after the tick included, on a +workflow that was never sealed: the reconciler sweeps SEALED workflows +only, so the same late retry resumes the DAG. Whether an operator's retry +recovers the work turns on whether seal() had been called. +""" + +import time +from datetime import timedelta + +import pytest +from django.tasks import task +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge + +from . import tasks +from .test_workflows import drain, flow_worker, order, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +WAIT = 10 + + +def _transient(label): + """ + Fails until the operator has fixed the cause, then succeeds. The fix + is a flag in the per-test state, the stand-in for a deploy, a + restored credential or a dependency coming back. + """ + tasks._record(label) + if not tasks.STATE.get("cause_fixed"): + raise RuntimeError(f"{label}: transient cause not fixed yet") + return label + + +@task(backend="flows") +def transient_step(label): + return _transient(label) + + +@task(backend="flows_deferred") +def transient_deferred_step(label): + return _transient(label) + + +# backend alias -> (worker factory, the root's task, a plain node task) +BACKENDS = { + "flows": (flow_worker, transient_step, tasks.step), + "flows_deferred": (deferring_worker, transient_deferred_step, tasks.deferred_step), +} + + +def retry_the_node(flow, root_id): + """An operator calls actions.retry on the one FAILED node.""" + assert actions.retry(root_id) is True, "actions.retry did not requeue the root" + + +def retry_the_selection(flow, root_id): + """ + An operator ticks every row of the workflow in the admin and runs + "Retry selected tasks", which calls actions.retry_many on the queryset. + """ + ids = list(flow.node_ids().values()) + changed, skipped = actions.retry_many(OxTask.objects.filter(id__in=ids)) + tasks.STATE["retry_many"] = (changed, skipped) + assert changed == 1, f"retry_many requeued {changed}, skipped {skipped}" + + +RETRIES = {"actions.retry": retry_the_node, "admin.retry_many": retry_the_selection} + + +def build(name, backend, cone_is_last, *, seal=True): + """ + root -> child -> grandchild, both callbacks, sealed unless seal=False. + + With cone_is_last False, an unrelated node scheduled an hour out keeps + the workflow SEALED past the tick, so the cancellation happens without + the completion. + """ + _factory, root_task, step_task = BACKENDS[backend] + flow = workflows.create( + name, + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + root = flow.add("root", root_task, "root") + flow.add("child", step_task, "child", depends_on="root") + flow.add("grandchild", step_task, "grandchild", depends_on="child") + if not cone_is_last: + flow.add( + "later", + step_task.using(run_after=timezone.now() + timedelta(hours=1)), + "later", + ) + if seal: + flow.seal() + return flow, root + + +def reconciler_tick(factory): + """ + One reconciler tick, run the way a SCHEDULES entry runs it: enqueued + as the task, then claimed and executed by a worker. + """ + result = workflows.reconcile.enqueue() + runner = factory() + deadline = time.monotonic() + WAIT + while OxTask.objects.get(id=result.id).status in ( + OxTask.Status.READY, + OxTask.Status.RUNNING, + ): + assert time.monotonic() < deadline, "no worker ran the reconciler" + if not runner.run_once(): + time.sleep(0.01) + row = OxTask.objects.get(id=result.id) + assert row.status == OxTask.Status.SUCCESSFUL, row.errors + return row.return_value + + +def settle(factory, rounds=3): + """Give any recovery a fix might add its chance: ticks and drains.""" + for _ in range(rounds): + reconciler_tick(factory) + drain(factory()) + + +def fail_the_root(flow, root, factory): + """Drain until the root has spent every attempt. Preconditions are hard.""" + drain(factory()) + row = OxTask.objects.get(id=root.id) + assert row.status == OxTask.Status.FAILED, row.status + assert row.attempts == row.max_attempts == 3, (row.attempts, row.max_attempts) + assert order() == ["root"] * 3, order() + + +def describe(flow): + ids = flow.node_ids() + rows = {row.id: row for row in OxTask.objects.filter(id__in=list(ids.values()))} + nodes = {name: str(rows[task_id].status) for name, task_id in sorted(ids.items())} + row = stored(flow) + progress = flow.progress() + return ( + f"nodes={nodes}; workflow={row.status} failed_count={row.failed_count} " + f"cancelled_count={row.cancelled_count} " + f"on_complete_enqueued={row.callback_task_id is not None} " + f"on_success_enqueued={row.success_task_id is not None}; " + f"progress(total={progress.total}, pending={progress.pending}, " + f"succeeded={progress.succeeded}, failed={progress.failed}, " + f"unattributed={progress.unattributed}); " + f"callbacks_ran(complete={tasks.STATE.get('complete', 0)}, " + f"success={tasks.STATE.get('success', 0)})" + ) + + +def cone_problems(flow, names=("child", "grandchild")): + """Nodes left DISCARDED although every upstream they wait on succeeded.""" + ids = flow.node_ids() + by_id = {task_id: name for name, task_id in ids.items()} + statuses = dict( + OxTask.objects.filter(id__in=list(ids.values())).values_list("id", "status") + ) + problems = [] + for name, task_id in sorted(ids.items()): + if statuses.get(task_id) != OxTask.Status.DISCARDED: + continue + upstream = list( + OxpullWorkflowEdge.objects.filter(downstream_task_id=task_id).values_list( + "upstream_task_id", flat=True + ) + ) + if upstream and all( + statuses.get(up) == OxTask.Status.SUCCESSFUL for up in upstream + ): + problems.append( + f"{name!r} is DISCARDED although every upstream it waits on " + f"({sorted(by_id[up] for up in upstream)}) is SUCCESSFUL" + ) + for name in names: + status = statuses.get(ids[name]) + if status != OxTask.Status.SUCCESSFUL: + problems.append(f"{name!r} ended {status}, not SUCCESSFUL") + if name not in order(): + problems.append(f"{name!r} never ran (order={order()})") + return problems + + +def outcome_problems(flow): + """A COMPLETE workflow whose record contradicts its nodes.""" + row = stored(flow) + progress = flow.progress() + ids = list(flow.node_ids().values()) + discarded = OxTask.objects.filter( + id__in=ids, status=OxTask.Status.DISCARDED + ).count() + every_node_succeeded = progress.succeeded == progress.total + problems = [] + if row.status != OxpullWorkflow.Status.COMPLETE: + problems.append(f"workflow is {row.status}, not COMPLETE, after settling") + return problems + if row.failed_count != progress.failed: + problems.append( + f"stored failed_count={row.failed_count} but progress().failed=" + f"{progress.failed}" + ) + if row.cancelled_count != discarded: + problems.append( + f"stored cancelled_count={row.cancelled_count} but {discarded} node(s) " + "are DISCARDED" + ) + if every_node_succeeded and row.success_task_id is None: + problems.append("every node SUCCESSFUL, yet on_success was never enqueued") + if every_node_succeeded and tasks.STATE.get("success", 0) != 1: + problems.append( + f"every node SUCCESSFUL, yet on_success ran " + f"{tasks.STATE.get('success', 0)} time(s)" + ) + return problems + + +def run_retry_after_tick(backend, retry, cone_is_last, name): + factory = BACKENDS[backend][0] + flow, root = build(name, backend, cone_is_last) + fail_the_root(flow, root, factory) + + # The scheduled tick lands before the operator does. + completed = reconciler_tick(factory) + assert OxTask.objects.get(id=root.id).status == OxTask.Status.FAILED + after_tick = describe(flow) + + # The operator fixes the cause and retries through django-ox. + tasks.STATE["cause_fixed"] = True + RETRIES[retry](flow, root.id) + drain(factory()) + assert OxTask.objects.get(id=root.id).status == OxTask.Status.SUCCESSFUL + after_rerun = describe(flow) + + settle(factory) + + ids = flow.node_ids() + child_retryable = actions.retry(ids["child"]) + context = ( + f"\n after the tick (completed {completed}): {after_tick}" + f"\n after retry and rerun: {after_rerun}" + f"\n after 3 more ticks and drains: {describe(flow)}" + f"\n actions.retry(child) now -> {child_retryable}" + f"\n retry_many (changed, skipped) -> {tasks.STATE.get('retry_many')}" + f"\n order={order()}" + ) + return flow, context + + +@pytest.mark.xfail( + strict=True, + reason=( + "retrying a failed node after a reconciler tick reruns only that node; " + "the downstream nodes the tick discarded stay discarded and never run" + ), +) +@pytest.mark.parametrize("cone_is_last", [True, False], ids=["cone_last", "sibling"]) +@pytest.mark.parametrize("retry", sorted(RETRIES)) +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_a_node_retried_after_its_cone_was_cancelled_runs_its_cone( + backend, retry, cone_is_last +): + flow, context = run_retry_after_tick( + backend, retry, cone_is_last, "retry-after-tick" + ) + problems = cone_problems(flow) + assert not problems, "; ".join(problems) + context + + +@pytest.mark.xfail( + strict=True, + reason=( + "a workflow retried after a reconciler tick completed it keeps " + "failed_count from that first completion although no node is failed" + ), +) +@pytest.mark.parametrize("retry", sorted(RETRIES)) +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_a_workflow_recovered_by_retry_keeps_no_stale_outcome(backend, retry): + flow, context = run_retry_after_tick( + backend, retry, cone_is_last=True, name="stale-outcome" + ) + problems = outcome_problems(flow) + assert not problems, "; ".join(problems) + context + + +# -- control: passes today --------------------------------------------- + + +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_control_a_retry_before_any_tick_recovers_the_workflow(backend): + """ + The same scenario with the retry landing before the reconciler's + first tick. Holds on the shipped code, and proves the assertions the + red tests make are satisfiable. + """ + factory = BACKENDS[backend][0] + flow, root = build("control", backend, cone_is_last=True) + fail_the_root(flow, root, factory) + + tasks.STATE["cause_fixed"] = True + retry_the_node(flow, root.id) + drain(factory()) + settle(factory) + + context = f"\n {describe(flow)}\n order={order()}" + problems = cone_problems(flow) + outcome_problems(flow) + assert not problems, "; ".join(problems) + context + assert stored(flow).failed_count == 0 + assert stored(flow).cancelled_count == 0 + + +@pytest.mark.parametrize("backend", sorted(BACKENDS)) +def test_control_an_unsealed_workflow_resumes_after_the_same_late_retry(backend): + """ + The red tests' sequence step for step, the tick before the retry + included, on a workflow nobody has sealed. The reconciler reads only + SEALED workflows, so the tick leaves the cone WAITING, the retry + resumes it, and sealing afterwards completes cleanly with on_success. + """ + factory = BACKENDS[backend][0] + flow, root = build("unsealed", backend, cone_is_last=True, seal=False) + fail_the_root(flow, root, factory) + + reconciler_tick(factory) + after_tick = describe(flow) + ids = flow.node_ids() + assert stored(flow).status == OxpullWorkflow.Status.OPEN, after_tick + for name in ("child", "grandchild"): + assert OxTask.objects.get(id=ids[name]).status == OxTask.Status.WAITING, ( + name, + after_tick, + ) + + tasks.STATE["cause_fixed"] = True + retry_the_node(flow, root.id) + drain(factory()) + assert order() == ["root"] * 4 + ["child", "grandchild"], order() + flow.seal() + settle(factory) + + context = f"\n after the tick: {after_tick}\n end: {describe(flow)}" + problems = cone_problems(flow) + outcome_problems(flow) + assert not problems, "; ".join(problems) + context + assert stored(flow).failed_count == 0, context + assert stored(flow).cancelled_count == 0, context diff --git a/tests_oxpull/test_workflows_hazard_rolling_back_migrate_oxpull_0003_or_downgrade.py b/tests_oxpull/test_workflows_hazard_rolling_back_migrate_oxpull_0003_or_downgrade.py new file mode 100644 index 0000000..005ab44 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_rolling_back_migrate_oxpull_0003_or_downgrade.py @@ -0,0 +1,383 @@ +""" +Rolling Workflows back must not strand the nodes that wait. + +A workflow node whose upstream has not succeeded is a django-ox row in the +WAITING status. Only this version releases it: the OxpullWorker that writes +its upstream's SUCCESSFUL, and the workflow reconciler. oxpull 1.2.0 and the +`django_ox.worker.Worker` never claim, release or reap a WAITING row. + +Two routine ways back off the feature would remove both: + + * `manage.py migrate oxpull 0003_oxpullratewindow`, then running the + worker that remains: oxpull 1.2.0 or the `django_ox.worker.Worker`. The + rewind would drop the workflow tables and leave the task rows WAITING. + * leaving the schema alone and taking WORKFLOWS and the reconcile schedule + out of settings. oxpull.W003 is emitted only by a backend that declares + WORKFLOWS, so with none left it cannot warn. + +Each hazard test asserts an acceptable outcome: the child runs once its +parent has succeeded, or the rollback refuses before it drops a table, or +(for the settings route) a database check reports the waiting nodes. +Migration 0004 refuses to unapply while any workflow row exists, and +oxpull.W007 counts workflow nodes left WAITING with no backend declaring +WORKFLOWS. The controls show the refusal leaves a working install, and that +a rollback before the first workflow goes through. +""" + +from __future__ import annotations + +import io +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import DatabaseError, connection + +import oxpull +from django_ox import actions +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode + +from . import tasks +from ._historical import extract_released_oxpull +from .subprocesses import django_ox_path +from .test_upgrade import _latest_migration, _migrate +from .test_workflows import drain +from .test_workflows_deferred import chain + +REPO_ROOT = Path(__file__).resolve().parent.parent +BEFORE_WORKFLOWS = "0003_oxpullratewindow" +WORKFLOW_TABLES = tuple( + model._meta.db_table + for model in (OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode) +) +# How long the remaining worker gets to run the child once the parent has +# succeeded. A released child is claimed on the next poll (0.1 s here). +GRACE_SECONDS = 5.0 + + +def status_of(label): + return OxTask.objects.filter(args=[label]).values_list("status", flat=True).first() + + +@pytest.fixture +def restore_leaf(): + """Put oxpull back on its newest migration, even when the test fails.""" + latest = _latest_migration() + try: + yield + finally: + _migrate(latest) + + +def roll_back(): + """ + `manage.py migrate oxpull 0003_oxpullratewindow`, the real command. + + Returns what it raised, or None. A rewind that refuses while nodes are + deferred is one of the acceptable fixes, so a refusal goes back to the + test. A database error is not a refusal and propagates. + """ + try: + call_command( + "migrate", "oxpull", BEFORE_WORKFLOWS, interactive=False, verbosity=0 + ) + except DatabaseError: + raise + except Exception as exc: + return exc + return None + + +def assert_refused_before_dropping_anything(refusal): + tables = connection.introspection.table_names() + missing = [table for table in WORKFLOW_TABLES if table not in tables] + assert not missing, f"the rewind raised {refusal!r} after dropping {missing}" + + +def assert_rewound(): + tables = connection.introspection.table_names() + present = [table for table in WORKFLOW_TABLES if table in tables] + assert not present, f"premise: the rewind should drop {present}" + + +def evidence(label="child"): + """What an operator has to go on, read only when an assertion fails.""" + row = OxTask.objects.get(args=[label]) + health = io.StringIO() + try: + call_command("ox_health", "--max-backlog", "0", "--max-age", "1", stdout=health) + health_says = health.getvalue().strip() + except CommandError as exc: + health_says = f"CommandError: {exc}" + parked = OxTask.objects.filter(status=OxTask.Status.WAITING).count() + return ( + f"{label!r} is {row.status} with run_after=" + f"{row.run_after.isoformat() if row.run_after else None}; " + f"WAITING rows: {parked}; " + f"django_ox.actions.retry({label!r}) -> {actions.retry(row.pk)}; " + f"ox_health --max-backlog 0 --max-age 1 -> {health_says!r}" + ) + + +# -- rollback by migration, the `django_ox.worker.Worker` in process ------- + + +@pytest.mark.django_db(transaction=True) +def test_after_migrating_back_to_0003_the_free_worker_runs_the_deferred_child( + restore_leaf, +): + chain("rollback-free") + assert status_of("child") == OxTask.Status.WAITING, "premise: child waiting" + + refusal = roll_back() + if refusal is not None: + assert_refused_before_dropping_anything(refusal) + return + assert_rewound() + + drain(Worker(backend_alias="flows_deferred", backoff_initial=0, poll_interval=0.01)) + + assert status_of("parent") == OxTask.Status.SUCCESSFUL, "premise: parent ran" + assert status_of("child") == OxTask.Status.SUCCESSFUL, evidence() + + +@pytest.mark.django_db(transaction=True) +def test_control_before_the_first_workflow_the_same_rollback_goes_through( + restore_leaf, +): + """ + With no workflow row, nothing can be stranded, so the rewind drops the + workflow tables and a worker without workflows runs the queue. + """ + refusal = roll_back() + assert refusal is None, f"the rewind refused with no workflow present: {refusal!r}" + assert_rewound() + + tasks.default_step.enqueue("plain") + drain(Worker(backoff_initial=0, poll_interval=0.01)) + assert status_of("plain") == OxTask.Status.SUCCESSFUL + + +@pytest.mark.django_db(transaction=True) +def test_control_after_the_refused_rollback_the_workflow_finishes(restore_leaf): + """ + A refused rewind leaves the schema and the rows as they were, so the + workflow's own workers finish it. + """ + chain("rollback-refused-then-run") + refusal = roll_back() + assert refusal is not None, "premise: the rewind refused" + assert_refused_before_dropping_anything(refusal) + + drain(Worker(backend_alias="flows_deferred", backoff_initial=0, poll_interval=0.01)) + assert status_of("parent") == OxTask.Status.SUCCESSFUL + assert status_of("child") == OxTask.Status.WAITING + from oxpull.worker import OxpullWorker + + drain( + OxpullWorker( + backend_alias="flows_deferred", backoff_initial=0, poll_interval=0.01 + ) + ) + workflows._reconcile() + drain( + OxpullWorker( + backend_alias="flows_deferred", backoff_initial=0, poll_interval=0.01 + ) + ) + assert status_of("child") == OxTask.Status.SUCCESSFUL + + +# -- real processes --------------------------------------------------------- + + +@pytest.fixture(scope="module") +def released_oxpull(tmp_path_factory): + """oxpull 1.2.0's source, read from the vendored archive.""" + root = tmp_path_factory.mktemp("oxpull-released") + return extract_released_oxpull(root) + + +def process_env(tmp_path, src): + """ + A separate process on this session's test database, with the settings + an operator has after backing out of Workflows: every backend loses + WORKFLOWS, DEFER_BLOCKED_NODES and the workflow reconciler schedule, + which tests_oxpull.settings carries and oxpull 1.2.0 cannot import. django_ox + logs at DEBUG so the worker's claims are in its log. + """ + (tmp_path / "rollback_settings.py").write_text( + "import copy\n" + f"from {settings.SETTINGS_MODULE} import * # noqa: F403\n" + "TASKS = copy.deepcopy(TASKS)\n" + "for _params in TASKS.values():\n" + " _params.get('OPTIONS', {}).pop('WORKFLOWS', None)\n" + " _params.get('OPTIONS', {}).pop('DEFER_BLOCKED_NODES', None)\n" + " _params.get('OPTIONS', {}).get('SCHEDULES', {}).pop(\n" + " 'oxpull-workflow-reconcile', None\n" + " )\n" + "LOGGING = {\n" + " 'version': 1,\n" + " 'disable_existing_loggers': False,\n" + " 'handlers': {'console': {'class': 'logging.StreamHandler'}},\n" + " 'loggers': {'django_ox': {'handlers': ['console'], 'level': 'DEBUG'}},\n" + "}\n" + ) + name = connection.settings_dict["NAME"] + if connection.vendor == "sqlite": + name = Path(name).resolve() + return { + **os.environ, + "DJANGO_SETTINGS_MODULE": "rollback_settings", + # The current source runs against this process's django-ox. A released + # oxpull runs against the django-ox the interpreter has installed. + "PYTHONPATH": os.pathsep.join( + [str(src)] + + ([django_ox_path()] if src == REPO_ROOT / "src" else []) + + [str(tmp_path), str(REPO_ROOT)] + ), + "OX_TEST_DB_NAME": str(name), + } + + +def run_python(env, *args): + return subprocess.run( # noqa: S603 + [sys.executable, *args], + env=env, + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def assert_imports(env, src, version): + """Prove which oxpull the process runs before believing anything it does.""" + probe = run_python( + env, "-c", "import oxpull; print(oxpull.__version__); print(oxpull.__file__)" + ) + assert probe.returncode == 0, probe.stderr + lines = probe.stdout.split() + assert lines[0] == version, lines + assert lines[1].startswith(str(src)), lines + + +def run_remaining_worker(env, log_path, backend): + """ + `ox_worker --backend ` until the child has succeeded, or for + GRACE_SECONDS after the parent has. Returns (exit code, log text). + """ + parent_done_at = None + with log_path.open("w") as log: + process = subprocess.Popen( # noqa: S603 + [ + sys.executable, + "-m", + "django", + "ox_worker", + "--backend", + backend, + "--interval", + "0.1", + ], + env=env, + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and process.poll() is None: + if status_of("child") == OxTask.Status.SUCCESSFUL: + break + if parent_done_at is None: + if status_of("parent") == OxTask.Status.SUCCESSFUL: + parent_done_at = time.monotonic() + elif time.monotonic() - parent_done_at > GRACE_SECONDS: + break + time.sleep(0.1) + finally: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return process.returncode, log_path.read_text() + + +@pytest.mark.django_db(transaction=True) +def test_after_migrating_back_to_0003_an_oxpull_1_2_0_worker_runs_the_deferred_child( + restore_leaf, released_oxpull, tmp_path +): + env = process_env(tmp_path, released_oxpull) + assert_imports(env, released_oxpull, "1.2.0") + + chain("rollback-1.2.0") + assert status_of("child") == OxTask.Status.WAITING, "premise: child waiting" + + refusal = roll_back() + if refusal is not None: + assert_refused_before_dropping_anything(refusal) + return + assert_rewound() + + check = run_python(env, "-m", "django", "check") + code, log = run_remaining_worker(env, tmp_path / "ox_worker.log", "flows_deferred") + + assert status_of("parent") == OxTask.Status.SUCCESSFUL, ( + f"premise: the 1.2.0 worker ran the parent (exit {code})\n{log[-3000:]}" + ) + assert status_of("child") == OxTask.Status.SUCCESSFUL, ( + f"{evidence()}; oxpull 1.2.0 `manage.py check`: " + f"{(check.stdout + check.stderr).strip()!r}; worker exit {code}; " + f"log tail:\n{log[-1500:]}" + ) + + +# -- rollback by settings, the schema left in place ------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_turning_workflows_off_runs_the_deferred_child_or_a_check_reports_it( + tmp_path, +): + src = REPO_ROOT / "src" + env = process_env(tmp_path, src) + assert_imports(env, src, oxpull.__version__) + + baseline = run_python(env, "-m", "django", "check", "--database", "default") + + # Created while the deployed settings still declare workflows, as in + # production: the workflow is in flight when the deploy that turns it off + # lands. + chain("options-off") + assert status_of("child") == OxTask.Status.WAITING, "premise: child waiting" + + code, log = run_remaining_worker(env, tmp_path / "ox_worker.log", "flows_deferred") + assert status_of("parent") == OxTask.Status.SUCCESSFUL, ( + f"premise: the worker ran the parent (exit {code})\n{log[-3000:]}" + ) + + after = run_python(env, "-m", "django", "check", "--database", "default") + said_before = (baseline.stdout + baseline.stderr).strip() + said_after = (after.stdout + after.stderr).strip() + + child_ran = status_of("child") == OxTask.Status.SUCCESSFUL + reported = said_after != said_before + assert child_ran or reported, ( + f"{evidence()}; `manage.py check --database default` with the child " + f"parked: {said_after!r} (before the workflow existed: {said_before!r}); " + f"worker exit {code}; log tail:\n{log[-1500:]}" + ) diff --git a/tests_oxpull/test_workflows_hazard_seal_lock.py b/tests_oxpull/test_workflows_hazard_seal_lock.py new file mode 100644 index 0000000..1b4d52e --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_seal_lock.py @@ -0,0 +1,416 @@ +""" +Workflow.seal() reads no node row, in or out of a caller's transaction. + +It used to take a row lock on every node, even when it owned its transaction. +oxpull 1.2.0 changed Batch.seal() to count members with a plain COUNT(*) when +seal() owns its transaction, and to keep the locking read only inside a +caller's transaction, where the caller's snapshot can predate a member +(CHANGELOG 1.2.0, batches.py _count_members). Workflow._count_nodes was still +the 1.0.0 shape its docstring called "Batch._count_members, verbatim": +SELECT id ... FOR UPDATE over every node, each primary key pulled into Python, +and every row held until the seal commits. + +Now add() increments the workflow row's node_count in the statement that takes +that row's lock, and seal()'s status write copies node_count into total_count. +An UPDATE reads the row's latest committed version and waits on an add() still +holding it, so the copy is current in and out of a caller's transaction. + +The first three tests pin what seal() does now: its statements read no +node row when it owns its transaction, another connection can lock every +node just before it commits, and on MySQL REPEATABLE READ it does not +stall an add() to another workflow. Before node_count, the first test +pinned a plain COUNT(*). Locking every node violates these contracts. +SQLite has no row locks, so the second and third tests skip there. + +The rest are ported from `oxpull` 1.2.0's batch seal() correctness tests. +They catch a change that goes too far: inside a caller's transaction, +the total must count a node committed after the caller's snapshot, +which a plain COUNT(*) misses on MySQL at REPEATABLE READ. + +The first three drive only public paths: workflows.create(), Workflow.add() +and Workflow.seal(). Their one instrument is a hook on the sealing +connection's commit(), which runs a probe after seal()'s last statement and +before its COMMIT. At a handful of nodes the seal's transaction is open for a +few milliseconds; the hook holds that moment still so another connection +can look at it. At a large node count the locking read itself held it open +for seconds. +""" + +import logging +import threading +import time +from contextlib import contextmanager + +import pytest +from django.db import OperationalError, connections, transaction +from django.test.utils import CaptureQueriesContext + +from oxpull import workflows +from oxpull.exceptions import WorkflowSealed +from oxpull.models import OxpullWorkflow, OxpullWorkflowNode + +from . import isolation, tasks +from .racing import race_all +from .test_batches import a_session_waits_on_this_transaction, session_isolation +from .test_workflows import stored + +pytestmark = pytest.mark.django_db(transaction=True) + +# How long an add() to an unrelated workflow may take while another +# workflow's seal() is open before it counts as waiting on that seal. An +# uncontended add() is a few milliseconds; InnoDB's lock wait is 50 s. +STALL_WINDOW = 2.0 + + +def _require_row_locks(): + if not connections["default"].features.has_select_for_update: + pytest.skip("no row locks on this database, so there is nothing to hold") + + +def node_reads(captured): + table = OxpullWorkflowNode._meta.db_table + return [ + query["sql"] for query in captured.captured_queries if table in query["sql"] + ] + + +@contextmanager +def before_commit(connection, action): + """ + Call action() when the outermost transaction on this connection is + about to commit: every statement has run and every lock is still held. + """ + original = connection.commit + + def commit(): + action() + return original() + + connection.commit = commit + try: + yield + finally: + del connection.commit + + +def test_seal_owning_its_transaction_reads_no_node_row(): + """ + The query shape. seal()'s status UPDATE copies node_count into total_count + and is its only statement on the workflow, so no node row is read, counted + or locked. This test pinned a plain COUNT(*) here until node_count took its + place. + """ + flow = workflows.create("shape") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + connection = connections["default"] + assert not connection.in_atomic_block + + with CaptureQueriesContext(connection) as captured: + flow.seal() + + assert node_reads(captured) == [] + (sealing,) = [ + query["sql"] + for query in captured.captured_queries + if query["sql"].lstrip().upper().startswith("UPDATE") + and OxpullWorkflow._meta.db_table in query["sql"] + ] + assert "node_count" in sealing, sealing + assert stored(flow).total_count == 2 + + +def test_seal_owning_its_transaction_leaves_every_node_lockable_until_commit(): + """ + The same property observed from the lock manager instead of the SQL + text, so a fix that spells the count differently is still held to it. + Just before seal() commits, a second connection asks for every node row + with SKIP LOCKED. It should get all of them. + """ + _require_row_locks() + width = 20 + flow = workflows.create("locks") + for index in range(width): + flow.add(f"n{index}", tasks.step, f"n{index}") + lockable = [] + + def count_lockable_nodes_elsewhere(): + def probe(): + with transaction.atomic(): + return len( + OxpullWorkflowNode.objects.filter(workflow_id=flow.id) + .select_for_update(skip_locked=True) + .values_list("pk", flat=True) + ) + + results, errors = race_all([probe]) + assert errors == [] + lockable.extend(results) + + with before_commit(connections["default"], count_lockable_nodes_elsewhere): + flow.seal() + + assert lockable == [width], ( + f"with seal() about to commit, another connection could lock " + f"{lockable} of {width} node rows" + ) + assert stored(flow).total_count == width + + +def test_sealing_one_workflow_does_not_stall_an_add_to_another_under_repeatable_read(): + """ + Where the lock reaches past the workflow being sealed. Under InnoDB's + REPEATABLE READ a locking read over workflow_id = X takes next-key + locks: the gaps beside X's index records are locked too, and an INSERT + of another workflow's node into either gap waits for the seal to commit. + Django's MySQL backend opens connections at READ COMMITTED, where a + locking read takes no gap locks; a project that sets + OPTIONS["isolation_level"] to "repeatable read", or to None for the + server default, runs at this level, which is the level test_batches' + snapshot tests exercise for the same reason. + + Here the sealing workflow owns the only node records, so the other + workflow's first node lands in one of those gaps. In a table of many + workflows it is whichever workflow's nodes sort next to the one being + sealed, and it waits for as long as that seal's transaction lasts. + + A locking read in seal()'s own transaction can take gap locks. + Rows deleted by earlier tests can bound those locks until the server + purges them, making the regression depend on test order. seal() now + reads no node row when it owns its transaction, so it locks no gap + and that timing no longer matters. + """ + if connections["default"].vendor != "mysql": + pytest.skip( + "gap locks are InnoDB's; PostgreSQL's row locks cover no gap, and " + "SQLite has no row locks" + ) + sealing = workflows.create("sealing") + building = workflows.create("building") + sealing.add("a", tasks.step, "a") + sealing.add("b", tasks.step, "b") + added = threading.Event() + outcome = [] + stalled = [] + + def add_first_node_to_the_other_workflow(): + try: + building.add("first", tasks.step, "first") + outcome.append("added") + except BaseException as exc: + outcome.append(exc) + finally: + added.set() + connections.close_all() + + def while_seal_is_open(): + threading.Thread(target=add_first_node_to_the_other_workflow).start() + stalled.append(not added.wait(timeout=STALL_WINDOW)) + + with ( + session_isolation("REPEATABLE READ"), + before_commit(connections["default"], while_seal_is_open), + ): + sealing.seal() + + assert added.wait(timeout=60) + assert stalled == [False], ( + f"add() to another workflow was still waiting {STALL_WINDOW}s into " + "seal()'s open transaction" + ) + assert outcome == ["added"] + assert stored(sealing).total_count == 2 + assert building.node_ids().keys() == {"first"} + + +# -- guards: the total stays current where a snapshot could lie ---------- + + +def test_seal_inside_a_callers_transaction_reads_no_node_row_either(): + """ + Inside a caller's transaction too. This test pinned a locking read here, + because a plain COUNT could miss a node committed after the caller's + snapshot. The copy of node_count is an UPDATE, which reads the workflow + row's latest committed version, so it needs no read of the nodes (the next + test is the case). + """ + flow = workflows.create("ambient-shape") + flow.add("a", tasks.step, "a") + connection = connections["default"] + + with CaptureQueriesContext(connection) as captured, transaction.atomic(): + flow.seal() + + assert node_reads(captured) == [] + assert stored(flow).total_count == 1 + + +def test_ambient_seal_counts_a_node_committed_after_the_callers_snapshot(): + """ + The case a snapshot could get wrong. Under MySQL REPEATABLE READ the + caller's transaction fixes its read view at its first consistent read, + and an add() that commits after that is invisible to a plain COUNT, + although seal()'s UPDATE waited on nothing: the add() had already + committed. The UPDATE's copy of node_count reads the row's latest + committed version and counts the node. PostgreSQL's READ COMMITTED finds + it either way; the test runs there to show the promise is the same. + """ + if connections["default"].vendor == "sqlite": + pytest.skip( + "a reading SQLite transaction holds its shared lock, so the " + "concurrent add() could not commit under it" + ) + flow = workflows.create("snapshot") + + def add_and_commit(): + flow.add("late", tasks.step, "late") + return "added" + + with session_isolation("REPEATABLE READ"), transaction.atomic(): + # The caller's first consistent read; the read view starts here. + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 0 + results, errors = race_all([add_and_commit]) + assert (results, errors) == (["added"], []) + flow.seal() + + assert stored(flow).total_count == 1 + + +def test_seal_owning_its_transaction_counts_the_node_whose_add_it_waited_for(): + """ + Why the total is current when seal() owns its transaction, even under + MySQL REPEATABLE READ: the status UPDATE is the transaction's first + statement and waits on an add() still holding the workflow row, and it + copies node_count from the version that add() committed. + """ + flow = workflows.create("waited") + added = threading.Event() + + def add_slowly(): + with transaction.atomic(): + flow.add("slow", tasks.step, "slow") + added.set() + # Hold the workflow row across seal()'s arrival. + time.sleep(1.0) + return "added" + + def seal_after_the_add_started(): + assert added.wait(timeout=10) + with session_isolation("REPEATABLE READ"): + flow.seal() + return "sealed" + + results, errors = race_all([add_slowly, seal_after_the_add_started]) + + assert errors == [] + assert sorted(results) == ["added", "sealed"] + assert stored(flow).total_count == 1 + + +def test_an_add_arriving_between_seals_statements_waits_and_is_refused(monkeypatch): + """ + The interleaving a copy without locks might open: an add() committing + after seal()'s status UPDATE returned and before seal() commits. It + cannot happen, because add() claims the workflow row the UPDATE holds + until seal() commits. Proven by starting the add() from inside seal()'s + transaction, after the UPDATE (in its reconciler-schedule check, the step + that follows it), watching it block for a second, and reading its refusal + once seal() has committed. + """ + flow = workflows.create("between") + flow.add("before", tasks.step, "before") + add_finished = threading.Event() + outcome = [] + + def late_add(): + try: + flow.add("late", tasks.step, "late") + outcome.append("added") + except WorkflowSealed: + outcome.append("refused") + finally: + add_finished.set() + connections.close_all() + + original = workflows._require_reconcile_schedule + started = [] + + def check_after_starting_an_add(): + if not started: + started.append(True) + threading.Thread(target=late_add).start() + assert not add_finished.wait(timeout=1.0), "add() did not wait for seal()" + return original() + + monkeypatch.setattr( + workflows, "_require_reconcile_schedule", check_after_starting_an_add + ) + flow.seal() + + assert add_finished.wait(timeout=10) + assert outcome == ["refused"] + assert stored(flow).total_count == 1 + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 1 + + +def test_seal_under_postgresql_repeatable_read_reissues_rather_than_counting_stale( + caplog, +): + """ + PostgreSQL REPEATABLE READ takes the snapshot at seal()'s first + statement, the status UPDATE. When that UPDATE has waited on an add() + holding the workflow row, the row it then finds changed after the + snapshot, and PostgreSQL raises a serialization failure instead of + copying a stale count. seal() owns its transaction here, so it runs it + again on a fresh snapshot, which counts every node. Batch.seal() still + raises for its caller to retry, as test_batches.py pins; this test used + to pin the same for workflows, until the reissue in + test_workflows_hazard_postgresql_repeatable_read_concurrent_add. + """ + if connections["default"].vendor != "postgresql": + pytest.skip("PostgreSQL's REPEATABLE READ takes its snapshot at the UPDATE") + caplog.set_level(logging.INFO, logger="oxpull.workflows") + flow = workflows.create("rr") + flow.add("before", tasks.step, "before") + added = threading.Event() + + def add_slowly(): + with transaction.atomic(): + flow.add("slow", tasks.step, "slow") + added.set() + # The workflow row stays locked until seal()'s UPDATE is waiting + # on it. That UPDATE took the snapshot when it began, so the + # snapshot is older than this commit however late the sealing + # thread got going. + assert a_session_waits_on_this_transaction(), ( + "seal() was never seen waiting on the workflow row add() holds" + ) + return "added" + + def seal_at_repeatable_read(): + assert added.wait(timeout=10) + isolation.outlives_the_transaction("REPEATABLE READ") + with connections["default"].cursor() as cursor: + cursor.execute( + "SET SESSION CHARACTERISTICS AS TRANSACTION ISOLATION LEVEL " + "REPEATABLE READ" + ) + try: + flow.seal() + except OperationalError as exc: + return type(exc.__cause__).__name__ + return "sealed" + + results, errors = race_all([add_slowly, seal_at_repeatable_read]) + + assert errors == [] + assert sorted(results) == ["added", "sealed"] + # The conflict was provoked, and answered by running seal() again. + retries = [ + record + for record in caplog.records + if getattr(record, "event", None) == "workflow_seal_serialization_retry" + ] + assert retries, "seal() never met the serialization failure this test is about" + assert stored(flow).status == OxpullWorkflow.Status.SEALED + assert stored(flow).total_count == 2 diff --git a/tests_oxpull/test_workflows_hazard_the_cancellation_transaction_s_lock_order.py b/tests_oxpull/test_workflows_hazard_the_cancellation_transaction_s_lock_order.py new file mode 100644 index 0000000..e64140a --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_the_cancellation_transaction_s_lock_order.py @@ -0,0 +1,461 @@ +""" +The cancellation transaction's lock order. + +workflows._cancel_one_level takes its row locks in three steps inside one +transaction: the workflow row (the no-op SEALED UPDATE), then every dead +node row of the workflow in primary-key order (SELECT ... FOR UPDATE), then +the READY rows it discards (a second statement, UPDATE ... WHERE pk IN). +The docstring's deadlock guards cover two reconcilers taking the same +locks in the same order. They do not cover a second writer that takes the +same rows in a different order: + + * django_ox.actions.discard_many, the function behind the admin's + "Discard selected tasks", is one UPDATE over the selected ids. Given a + failed workflow's rows it locks a READY child before the FAILED root + and then waits on the root, while the cancel pass holds the root and + waits on that child. + + * ox_prune deletes old DISCARDED rows in batches, and an idle reconciler + tick on a sealed workflow with a dead node locks every one of those + DISCARDED rows with its FOR UPDATE, in its own order. + +Each test asserts the correct behaviour, which is that neither party is +killed by the database's deadlock detector (PostgreSQL 40P01, InnoDB 1213), +so each fails wherever the cycle closes. Every statement on both sides is +the production statement: the real worker fails the root, the real +reconciler entry point (_reconcile, what the scheduled task runs) cancels, +and the operator side is the public actions.discard_many and the real +ox_prune management command. The only interventions are a pause (Django's +public connection.execute_wrapper on the reconciler's connection) so the +operator's statement lands inside the cancel transaction, and in the +prune test a backdated finished_at standing in for eight days passing. + +SQLite is skipped, not passed: it has no row locks, and its writers are +serialised, so no cycle can form. +""" + +from __future__ import annotations + +import io +import logging +import threading +import time +from collections.abc import Callable +from datetime import timedelta +from typing import Any + +import pytest +from django.core.management import call_command +from django.db import DatabaseError, connections, transaction +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowNode +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile, stored + +pytestmark = pytest.mark.django_db(transaction=True) + +TASK_TABLE = OxTask._meta.db_table +WORKFLOW_LOGGER = workflows.__name__ + +# The width of the prune race. +PRUNE_WIDTH = 6000 + +VENDOR = connections[oxpull_db_alias()].vendor + + +@pytest.fixture(autouse=True) +def _row_locks_only(): + if connections[oxpull_db_alias()].vendor == "sqlite": + pytest.skip( + "SQLite has no row locks and serialises writers, so no lock cycle " + "between the cancel pass and another writer can form" + ) + + +# -- harness ----------------------------------------------------------- + + +def deadlock_code(exc: BaseException | None) -> str | int | None: + """40P01 or 1213 if `exc` or anything in its cause chain is a deadlock.""" + seen: set[int] = set() + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + if getattr(exc, "sqlstate", None) == "40P01": + return "40P01" + args = getattr(exc, "args", ()) + if args and args[0] == 1213: + return 1213 + exc = exc.__cause__ or exc.__context__ + return None + + +def describe(exc: BaseException) -> str: + return f"{type(exc).__name__}(deadlock={deadlock_code(exc)!r}): {exc}" + + +def on_own_connection(step: Callable[[], Any], timeout: float = 30) -> Any: + """Run `step` on its own thread and so its own connection, and return its value.""" + out: list[Any] = [] + err: list[BaseException] = [] + + def run() -> None: + try: + out.append(step()) + except BaseException as exc: + err.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(timeout=timeout) + assert not thread.is_alive(), "a probe hung" + if err: + raise err[0] + return out[0] + + +def session_id() -> int: + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SELECT pg_backend_pid()") + else: + cursor.execute("SELECT CONNECTION_ID()") + return int(cursor.fetchone()[0]) + + +def is_waiting_on_a_lock(session: int) -> bool: + """Is that session blocked on a lock now? Read on this thread's connection.""" + connection = connections[oxpull_db_alias()] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute( + "SELECT wait_event_type FROM pg_stat_activity WHERE pid = %s", + [session], + ) + row = cursor.fetchone() + return bool(row) and row[0] == "Lock" + cursor.execute( + "SELECT trx_state FROM information_schema.INNODB_TRX " + "WHERE trx_mysql_thread_id = %s", + [session], + ) + row = cursor.fetchone() + return bool(row) and row[0] == "LOCK WAIT" + + +def row_is_locked(pk: Any) -> bool: + """A NOWAIT probe: is another transaction holding this task row's lock?""" + alias = oxpull_db_alias() + try: + with transaction.atomic(using=alias): + list( + OxTask.objects.using(alias) + .select_for_update(nowait=True) + .filter(pk=pk) + .values_list("pk", flat=True) + ) + except DatabaseError: + return True + return False + + +class AfterTheCancelLock: + """ + Run `step` once, on the reconciler's thread, right after the cancel + pass's SELECT ... FOR UPDATE on the task table has executed: the moment + it holds the workflow row and the dead node rows and has not yet read + or written a READY node. + """ + + def __init__(self, step: Callable[[], None]) -> None: + self.step = step + self.fired = False + + def __call__(self, execute, sql, params, many, context): + result = execute(sql, params, many, context) + if not self.fired and "FOR UPDATE" in sql.upper() and TASK_TABLE in sql: + self.fired = True + self.step() + return result + + +def pause_cancel_passes(monkeypatch, hook: AfterTheCancelLock) -> None: + original = workflows._cancel_one_level + + def cancel_one_level(workflow_pk, db_alias): + with connections[db_alias].execute_wrapper(hook): + return original(workflow_pk, db_alias) + + monkeypatch.setattr(workflows, "_cancel_one_level", cancel_one_level) + + +def reconciler_failures(caplog) -> list[logging.LogRecord]: + """What _reconcile logged and swallowed: its per-workflow exception handler.""" + return [ + record + for record in caplog.records + if record.name == WORKFLOW_LOGGER and record.exc_info + ] + + +def node_statuses(flow) -> dict[str, str]: + ids = flow.node_ids() + by_id = dict(OxTask.objects.filter(id__in=ids.values()).values_list("id", "status")) + return {name: by_id.get(pk, "") for name, pk in ids.items()} + + +# -- (a) an operator's discard_many against the cancel pass --------------- + + +@pytest.mark.xfail( + VENDOR in ("postgresql", "mysql"), + strict=True, + reason=( + "the cancel pass and an operator's discard_many lock the same rows in " + "opposite order, and the database kills one of them as a deadlock" + ), +) +def test_an_operator_discarding_a_failed_workflow_does_not_deadlock_its_cancel_pass( + monkeypatch, caplog +): + """ + A sealed workflow whose root failed, with twelve READY children waiting + on it. The reconciler's cancel pass locks the FAILED root. An operator + who is cleaning up the same failed workflow selects its rows and runs + discard_many (the admin's "Discard selected tasks") in that instant. + Both must finish: the reconciler cancels the children or finds them + already discarded, and the operator's discard of the root lands. + """ + caplog.set_level(logging.INFO, logger=WORKFLOW_LOGGER) + alias = oxpull_db_alias() + + flow = workflows.create("discard-many") + flow.add("root", tasks.failing_step, "root") + root_pk = flow.node_ids()["root"] + added = 0 + # The cycle needs a child that sorts below the root: the discard's + # UPDATE locks it before reaching the root. Twelve children, and more + # only in the rare case that none of the twelve sorts below. + while added < 12 or not any( + pk < root_pk for name, pk in flow.node_ids().items() if name != "root" + ): + flow.add(f"c{added:02d}", tasks.step, f"c{added:02d}", depends_on="root") + added += 1 + flow.seal() + + drain(flow_worker()) + before = node_statuses(flow) + assert before["root"] == OxTask.Status.FAILED, before + assert {s for n, s in before.items() if n != "root"} == {OxTask.Status.READY} + + ids = flow.node_ids() + root_id = ids["root"] + low = sorted(pk for name, pk in ids.items() if name != "root" and pk < root_id) + selection = list(ids.values()) + + operator: dict[str, Any] = {} + operator_ready = threading.Event() + + def discard_many() -> None: + try: + operator["session"] = session_id() + operator_ready.set() + operator["result"] = actions.discard_many(selection) + except BaseException as exc: + operator["error"] = exc + finally: + operator_ready.set() + connections.close_all() + + operator_thread = threading.Thread(target=discard_many) + seen: dict[str, Any] = {} + + def operator_runs_discard_many() -> None: + operator_thread.start() + assert operator_ready.wait(10) + deadline = time.monotonic() + 5 + waiting = False + while time.monotonic() < deadline and operator_thread.is_alive(): + if "session" in operator and on_own_connection( + lambda: is_waiting_on_a_lock(operator["session"]) + ): + waiting = True + break + time.sleep(0.01) + seen["operator_waiting"] = waiting + seen["operator_finished_before_resume"] = not operator_thread.is_alive() + seen["low_children_locked_by_operator"] = on_own_connection( + lambda: sum(row_is_locked(pk) for pk in low) + ) + + hook = AfterTheCancelLock(operator_runs_discard_many) + pause_cancel_passes(monkeypatch, hook) + + reconcile_error: BaseException | None = None + try: + reconcile() + except BaseException as exc: # _reconcile swallows per-workflow errors + reconcile_error = exc + if operator_thread.is_alive() or operator_thread.ident is not None: + operator_thread.join(timeout=60) + assert not operator_thread.is_alive(), "the operator's discard_many never returned" + + assert hook.fired, ( + "the cancel pass never issued its FOR UPDATE, so the operator's " + "discard never overlapped it and this test proved nothing" + ) + + failures = [ + f"reconciler: {record.getMessage()} -> {describe(record.exc_info[1])}" + for record in reconciler_failures(caplog) + ] + if reconcile_error is not None: + failures.append(f"reconciler raised: {describe(reconcile_error)}") + if "error" in operator: + failures.append(f"operator discard_many raised: {describe(operator['error'])}") + after = node_statuses(flow) + assert not failures, ( + f"a lock cycle between the cancel pass and discard_many: {failures}; " + f"db={connections[alias].vendor}, children below root={len(low)} of " + f"{len(ids) - 1}, operator waiting on a lock when the reconciler " + f"resumed={seen.get('operator_waiting')}, low children the operator " + f"held={seen.get('low_children_locked_by_operator')}; after: " + f"root={after['root']}, children=" + f"{sorted({s for n, s in after.items() if n != 'root'})}, " + f"workflow={stored(flow).status}, discard_many returned " + f"{operator.get('result')}" + ) + # Both intents landed. + assert after["root"] == OxTask.Status.DISCARDED, after + assert {s for n, s in after.items() if n != "root"} == {OxTask.Status.DISCARDED} + + +# -- (b) ox_prune against idle reconciler ticks ---------------------------- + + +def test_ox_prune_does_not_deadlock_idle_reconciler_ticks(caplog): + """ + A sealed workflow that cannot complete yet: its root failed, the + reconciler has already cancelled the six thousand children, and one + independent node is still RUNNING. Every reconciler tick on it locks + the root and all six thousand DISCARDED rows and then finds nothing to + do. A week later ox_prune deletes those DISCARDED rows in batches while + the reconciler keeps ticking. Neither may be killed as a deadlock + victim: the reconciler that loses one logs the workflow and skips it + for that tick, and the prune run that loses one dies part-way and + leaves the rest of the rows. + """ + caplog.set_level(logging.INFO, logger=WORKFLOW_LOGGER) + alias = oxpull_db_alias() + + flow = workflows.create("prune") + started = time.monotonic() + # The public add(), inside one caller's transaction so six thousand + # nodes commit once. + with transaction.atomic(using=alias): + flow.add("holder", tasks.step.using(priority=10), "holder") + flow.add("root", tasks.failing_step, "root") + for i in range(PRUNE_WIDTH): + flow.add(f"c{i:05d}", tasks.step, f"c{i:05d}", depends_on="root") + flow.seal() + build_seconds = time.monotonic() - started + + # The holder: a real claim, RUNNING under a live lease, so the + # workflow stays SEALED however many ticks run. + holder = flow_worker().claim_one() + ids = flow.node_ids() + assert holder is not None and holder.id == ids["holder"] + drain(flow_worker(), limit=10) + assert OxTask.objects.get(id=ids["root"]).status == OxTask.Status.FAILED + + reconcile() + nodes = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).values("task_id") + discarded = OxTask.objects.filter(id__in=nodes, status=OxTask.Status.DISCARDED) + assert discarded.count() == PRUNE_WIDTH + assert stored(flow).status == OxpullWorkflow.Status.SEALED + assert not reconciler_failures(caplog), "the setup tick itself failed" + + # Eight days pass. + OxTask.objects.filter( + id__in=nodes, + status__in=[OxTask.Status.DISCARDED, OxTask.Status.FAILED], + ).update(finished_at=timezone.now() - timedelta(days=8)) + + stop = threading.Event() + first_tick = threading.Event() + # Each tick as the span it occupied, not the instant it ended: a tick + # that began before the prune and was still running when it finished is + # the deepest overlap of the two there is, and its end is outside the + # prune's window. + ticks: list[tuple[float, float]] = [] + tick_errors: list[BaseException] = [] + prune_errors: list[BaseException] = [] + prune_out = io.StringIO() + prune_window: dict[str, float] = {} + + def reconciler() -> None: + try: + while not stop.is_set(): + started = time.monotonic() + reconcile() + ticks.append((started, time.monotonic())) + first_tick.set() + except BaseException as exc: + tick_errors.append(exc) + finally: + first_tick.set() + connections.close_all() + + def prune() -> None: + prune_window["start"] = time.monotonic() + try: + call_command("ox_prune", "--batch-size", "500", stdout=prune_out) + except BaseException as exc: + prune_errors.append(exc) + finally: + prune_window["end"] = time.monotonic() + connections.close_all() + + ticking = threading.Thread(target=reconciler) + ticking.start() + assert first_tick.wait(60) + pruning = threading.Thread(target=prune) + pruning.start() + pruning.join(timeout=300) + stop.set() + ticking.join(timeout=120) + assert not pruning.is_alive() and not ticking.is_alive() + + ticks_during_prune = sum( + started <= prune_window["end"] and ended >= prune_window["start"] + for started, ended in ticks + ) + assert ticks_during_prune, ( + f"no reconciler tick overlapped the prune run: {len(ticks)} tick(s), " + f"the prune ran for {prune_window['end'] - prune_window['start']:.2f}s" + ) + + failures = [ + f"reconciler: {record.getMessage()} -> {describe(record.exc_info[1])}" + for record in reconciler_failures(caplog) + ] + failures += [f"reconciler loop raised: {describe(e)}" for e in tick_errors] + failures += [f"ox_prune raised: {describe(e)}" for e in prune_errors] + remaining = discarded.count() + assert not failures, ( + f"{len(failures)} failure(s) while ox_prune raced idle reconciler ticks " + f"on {connections[alias].vendor}: {failures[:5]}; ticks during the prune=" + f"{ticks_during_prune}, prune took " + f"{prune_window['end'] - prune_window['start']:.2f}s, DISCARDED rows left=" + f"{remaining}, prune said {prune_out.getvalue().strip()!r}; " + f"(fixture built in {build_seconds:.1f}s)" + ) + assert remaining == 0, prune_out.getvalue() diff --git a/tests_oxpull/test_workflows_hazard_the_reconciler_s_per_workflow_except_exception.py b/tests_oxpull/test_workflows_hazard_the_reconciler_s_per_workflow_except_exception.py new file mode 100644 index 0000000..bb31930 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_the_reconciler_s_per_workflow_except_exception.py @@ -0,0 +1,941 @@ +""" +The reconcilers' per-item `except Exception` swallows TaskTimeout. + +django-ox raises TaskTimeout inside a sync task when its TASK_TIMEOUT +expires, and records that attempt as failed. It documents one exception: +a task that swallows the TaskTimeout "has its attempt recorded as whatever +it goes on to do", and one still running TASK_TIMEOUT_GRACE later is +recorded failed by the backstop, which then recycles the worker. + +`workflows.reconcile` and `batches.reconcile` are ordinary tasks on the +default queue, so whichever worker claims one applies its backend's +TASK_TIMEOUT. Both sweeps wrap each item in `except Exception`, and +TaskTimeout subclasses TimeoutError. A sweep past its deadline therefore +catches its own timeout, logs "could not be reconciled", and carries on. + +The timeout does not always arrive bare. A delivery that lands inside a +database driver comes out of it as the driver's error, chained from the +TaskTimeout (django-ox's worker._timeout_in follows that chain for +exactly this reason). The connection it leaves behind can be unusable, +and the swallowed sweep goes on using it. + +The fix lets the timeout out wherever it sits in that chain. It also +stops a tick between items when the next item would probably not finish +in time, and visits items in a new order each tick, so a tick that ends +early cannot leave the same items behind every time. Tests 8 and 9 switch +the stop off so the timeout still lands mid-sweep; the tests at the end +of the file cover the stop and the order. + +Every test here asserts what django-ox promises for a timed-out attempt. +Each worker test built on a controlled delay first runs a plain busy +task through an identically configured worker and requires django-ox to +record that attempt as a timeout without recycling. A failing result +therefore cannot come from the timeout injection not working. +""" + +import logging +import math +import random +import sys +import threading +import time +from contextlib import suppress +from datetime import timedelta +from inspect import iscoroutinefunction + +import pytest +from django.db import Error, OperationalError, connection, connections +from django.tasks import task +from django.utils import timezone + +from django_ox import actions +from django_ox import worker as worker_module +from django_ox.exceptions import TaskTimeout +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import batches, workflows +from oxpull.models import OxpullBatch, OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker + +TIMEOUT_PATH = "django_ox.exceptions.TaskTimeout" +VENDOR = connection.vendor + + +def _busy(seconds): + """Pure Python, so an injected TaskTimeout lands inside it.""" + end = time.monotonic() + seconds + while time.monotonic() < end: + pass + + +@task +def busy(seconds): + _busy(seconds) + return "done" + + +def default_worker(**kwargs): + return Worker(backoff_initial=0, poll_interval=0.01, **kwargs) + + +def _chain(exc): + """The exceptions Python's traceback would print, outermost first.""" + found, seen = [], set() + while exc is not None and id(exc) not in seen: + found.append(exc) + seen.add(id(exc)) + if exc.__cause__ is not None: + exc = exc.__cause__ + elif exc.__suppress_context__: + exc = None + else: + exc = exc.__context__ + return found + + +def _timeout_in_chain(exc): + return any(isinstance(link, TaskTimeout) for link in _chain(exc)) + + +def _chain_names(exc): + return "<-".join(type(link).__name__ for link in _chain(exc)) + + +def _swallowed_timeouts(records, event): + return [ + r + for r in records + if getattr(r, "event", None) == event + and r.exc_info + and _timeout_in_chain(r.exc_info[1]) + ] + + +def _errors(row): + return [e["exception_class_path"] for e in row.errors] + + +def _reset_connection(): + """ + What django-ox's Worker._discard_connections does after a timeout, so + the test can read the row back. It only matters when the attempt left + this thread's connection unusable, which is part of what is under test. + """ + for conn in connections.all(initialized_only=True): + conn.in_atomic_block = False + conn.savepoint_ids = [] + conn.atomic_blocks = [] + conn.needs_rollback = False + conn.closed_in_transaction = False + with suppress(Error): + conn.close() + + +class _EndIfWedged: + """ + Turns a wedged attempt into a failure of this test instead of a hang. + + Seen once while probing this hazard on MySQL: after a swallowed + delivery, the sweep blocked for ten minutes in PyMySQL's rollback() + reading an OK packet that never came, its server session idle. After + `after` seconds this ends the database's other sessions from the + server side. It never fires on an attempt that finishes. + """ + + def __init__(self, after): + self.after = after + self.fired = False + self._done = threading.Event() + self._thread = threading.Thread(target=self._watch, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc_info): + self._done.set() + self._thread.join(60) + return False + + def _watch(self): + if self._done.wait(self.after): + return + self.fired = True + conn = connections["default"] + try: + with conn.cursor() as cursor: + if conn.vendor == "mysql": + cursor.execute( + "SELECT id FROM information_schema.processlist " + "WHERE db = DATABASE() AND id <> CONNECTION_ID()" + ) + for (pid,) in cursor.fetchall(): + with suppress(Error): + cursor.execute(f"KILL {int(pid)}") + elif conn.vendor == "postgresql": + cursor.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE datname = current_database() " + "AND pid <> pg_backend_pid()" + ) + finally: + connections.close_all() + + +def watching_monitoring_tools(): + """ + Registered sys.monitoring tool ids with events enabled, by id. A + reserved id with no events runs no callback, so it watches nothing. + """ + return { + tool_id: name + for tool_id in range(6) + if (name := sys.monitoring.get_tool(tool_id)) is not None + and sys.monitoring.get_events(tool_id) + } + + +@pytest.fixture +def interruptible_attempts(monkeypatch): + """ + Run each attempt on a thread the worker will raise TaskTimeout inside. + + django-ox leaves a watched thread alone on purpose. An exception raised + between two bytecodes can land inside a tracing tool's own callback, so + the worker declines to inject one and the grace backstop becomes the + whole enforcement. Coverage measurement is such a tool. The sqlite job + runs `pytest -q --cov`, so without this every test below would lose the + interruption it is about and read red on its own control. + + The trace hook is per thread, so it comes off for the length of the + attempt and goes back afterwards, and the attempt then runs unwatched + for real. A sys.monitoring tool cannot be taken off one thread, so a + session measured that way stubs the worker's probe instead. Every other + thread, and the rest of the test, is measured as usual. + + django-ox carries the same fixture, for the same reason, over the same + private names. The `_call_task` monkeypatch runs on every session, so a + rename there raises and these tests say so. The `_active_tracer` one is + reached only on a sys.monitoring session, so a rename of that name is + caught only there, not under a sys.settrace session. + """ + original = Worker._call_task + + def call_task(self, task, db_task, task_result, timeout): + if iscoroutinefunction(task.func): + return original(self, task, db_task, task_result, timeout) + installed = sys.gettrace() + sys.settrace(None) + try: + return original(self, task, db_task, task_result, timeout) + finally: + sys.settrace(installed) + + monkeypatch.setattr(Worker, "_call_task", call_task) + if watching_monitoring_tools(): + monkeypatch.setattr(worker_module, "_active_tracer", lambda: None) + + +def require_timeouts_enforced(make_worker, timeout, grace): + """ + The control. A plain task that would run past timeout + grace, on a + worker built exactly like the one under test, is stopped by + TaskTimeout: recorded with that error, requeued, and no recycle. + """ + seconds = timeout + min(grace, 1.0) + 0.7 + result = busy.enqueue(seconds) + worker = make_worker(task_timeout=timeout, task_timeout_grace=grace) + started = time.monotonic() + assert worker.run_once() + elapsed = time.monotonic() - started + row = OxTask.objects.get(id=result.id) + control = ( + f"control: status={row.status} errors={_errors(row)} " + f"recycling={worker.recycling} elapsed={elapsed:.2f}s" + ) + print(control) # noqa: T201 + assert _errors(row)[-1:] == [TIMEOUT_PATH], control + assert row.status == OxTask.Status.READY, control + assert not worker.recycling, control + assert elapsed < seconds, control + # Out of the way, so the next claim takes the reconciler's row. + assert actions.discard(result.id) + + +def sealed_finished_workflows(count): + flows = [] + for i in range(count): + flow = workflows.create(f"finished-{i}") + flow.add("n", tasks.step, f"n{i}") + flow.seal() + flows.append(flow) + assert drain(flow_worker()) == count + return flows + + +def workflow_statuses(flows): + return [OxpullWorkflow.objects.get(pk=f.id).status for f in flows] + + +# -- the sweep itself -------------------------------------------------- + + +@pytest.mark.django_db +def test_the_workflow_sweep_lets_a_task_timeout_out(monkeypatch): + flow = workflows.create("unit") + flow.add("n", tasks.step, "n") + flow.seal() + + def timed_out(workflow_pk, db_alias): + raise TaskTimeout + + monkeypatch.setattr(workflows, "_cancel_unreachable", timed_out) + with pytest.raises(TaskTimeout): + workflows._reconcile() + + +@pytest.mark.django_db +def test_the_batch_sweep_lets_a_task_timeout_out(monkeypatch): + batch = batches.create("unit") + batch.add(tasks.plain_echo, 1) + batch.seal() + + def timed_out(batch_pk, db_alias): + raise TaskTimeout + + monkeypatch.setattr(batches, "_complete_if_finished", timed_out) + with pytest.raises(TaskTimeout): + batches._reconcile() + + +def _lost_connection_raised_by_a_timeout(*args): + """ + The shape a delivery inside PyMySQL's socket read takes, as measured: + PyMySQL catches OSError around the read, TaskTimeout is a + TimeoutError, so the driver closes the socket and raises + OperationalError(2013) while handling it; Django re-raises that from + the driver's exception. On MySQL, 266 of 300 unplaced deliveries + reached the sweep this way and 6 arrived bare. + """ + try: + raise TaskTimeout + except TaskTimeout: + try: + raise OSError(2013, "Lost connection to MySQL server during query") + except OSError as driver_error: + raise OperationalError(*driver_error.args) from driver_error + + +@pytest.mark.django_db +@pytest.mark.parametrize( + ("module", "item_step"), + [(workflows, "_cancel_unreachable"), (batches, "_complete_if_finished")], + ids=["workflows", "batches"], +) +def test_a_driver_error_raised_by_a_timeout_is_let_out_too( + monkeypatch, module, item_step +): + if module is workflows: + flow = workflows.create("unit") + flow.add("n", tasks.step, "n") + flow.seal() + else: + batch = batches.create("unit") + batch.add(tasks.plain_echo, 1) + batch.seal() + monkeypatch.setattr(module, item_step, _lost_connection_raised_by_a_timeout) + with pytest.raises((OperationalError, TaskTimeout)) as raised: + module._reconcile() + assert _timeout_in_chain(raised.value), _chain_names(raised.value) + + +# -- (a) a timed-out sweep recorded SUCCESSFUL ------------------------- + + +@pytest.mark.django_db(transaction=True) +def test_a_workflow_reconcile_past_its_timeout_is_recorded_as_timed_out( + monkeypatch, caplog, interruptible_attempts +): + flows = sealed_finished_workflows(2) + timeout, grace = 0.3, 30 + require_timeouts_enforced(flow_worker, timeout, grace) + + original = workflows._cancel_unreachable + calls = [] + + def slow_first(workflow_pk, db_alias): + calls.append(workflow_pk) + if len(calls) == 1: + _busy(1.5) + return original(workflow_pk, db_alias) + + monkeypatch.setattr(workflows, "_cancel_unreachable", slow_first) + # The release backstop runs first in the same attempt, and its reads come + # out of the same 0.3 s. It is not what this test is about, so it is + # switched off: the sweep then reaches its first workflow one read after + # the attempt starts, as the batch sweep below does, and the timeout + # lands in the delay and not before it. + monkeypatch.setattr(workflows, "_release_backstop", lambda db_alias: 0) + result = workflows.reconcile.enqueue() + worker = flow_worker(task_timeout=timeout, task_timeout_grace=grace) + with caplog.at_level(logging.ERROR, logger="oxpull.workflows"): + assert worker.run_once() + row = OxTask.objects.get(id=result.id) + swallowed = _swallowed_timeouts(caplog.records, "workflow_reconcile_failed") + seen = ( + f"status={row.status} return_value={row.return_value!r} " + f"errors={_errors(row)} workflows={workflow_statuses(flows)} " + f"swallowed_timeouts={len(swallowed)} recycling={worker.recycling}" + ) + print(seen) # noqa: T201 + assert calls, "the sweep never reached a sealed workflow; nothing was tested" + assert row.status != OxTask.Status.SUCCESSFUL, seen + assert _errors(row)[-1:] == [TIMEOUT_PATH], seen + + +@pytest.mark.django_db(transaction=True) +def test_a_batch_reconcile_past_its_timeout_is_recorded_as_timed_out( + monkeypatch, caplog, interruptible_attempts +): + made = [] + for i in range(2): + batch = batches.create(f"finished-{i}") + batch.add(tasks.plain_echo, i) + batch.seal() + made.append(batch) + assert drain(default_worker()) == 2 + timeout, grace = 0.3, 30 + require_timeouts_enforced(default_worker, timeout, grace) + + original = batches._complete_if_finished + calls = [] + + def slow_first(batch_pk, db_alias): + calls.append(batch_pk) + if len(calls) == 1: + _busy(1.5) + return original(batch_pk, db_alias) + + monkeypatch.setattr(batches, "_complete_if_finished", slow_first) + result = batches.reconcile.enqueue() + worker = default_worker(task_timeout=timeout, task_timeout_grace=grace) + with caplog.at_level(logging.ERROR, logger="oxpull.batches"): + assert worker.run_once() + row = OxTask.objects.get(id=result.id) + swallowed = _swallowed_timeouts(caplog.records, "batch_reconcile_failed") + seen = ( + f"status={row.status} return_value={row.return_value!r} " + f"errors={_errors(row)} " + f"batches={[OxpullBatch.objects.get(pk=b.id).status for b in made]} " + f"swallowed_timeouts={len(swallowed)} recycling={worker.recycling}" + ) + print(seen) # noqa: T201 + assert calls, "the sweep never reached a sealed batch; nothing was tested" + assert row.status != OxTask.Status.SUCCESSFUL, seen + assert _errors(row)[-1:] == [TIMEOUT_PATH], seen + + +# -- (b) a timed-out sweep that outlives the grace recycles the worker -- + + +@pytest.mark.django_db(transaction=True) +def test_a_workflow_reconcile_past_its_timeout_does_not_recycle_the_worker( + monkeypatch, interruptible_attempts +): + flows = sealed_finished_workflows(4) + # The two outcomes are seconds apart on any machine. A sweep that lets + # the timeout out ends when it is delivered, 0.3 s in, with the whole + # grace still ahead of it. One that swallows it has at least three more + # workflows to visit at 1.5 s each, a delay that a slow machine can only + # lengthen, so it is still running when the grace ends 3 s later. + timeout, grace = 0.3, 3.0 + require_timeouts_enforced(flow_worker, timeout, grace) + + original = workflows._cancel_unreachable + + def slow_each(workflow_pk, db_alias): + _busy(1.5) + return original(workflow_pk, db_alias) + + monkeypatch.setattr(workflows, "_cancel_unreachable", slow_each) + result = workflows.reconcile.enqueue() + worker = flow_worker(task_timeout=timeout, task_timeout_grace=grace) + started = time.monotonic() + assert worker.run_once() + elapsed = time.monotonic() - started + row = OxTask.objects.get(id=result.id) + seen = ( + f"status={row.status} epoch={row.lease_epoch} errors={_errors(row)} " + f"recycling={worker.recycling} run_once={elapsed:.2f}s " + f"workflows={workflow_statuses(flows)}" + ) + print(seen) # noqa: T201 + assert not worker.recycling, seen + assert _errors(row)[-1:] == [TIMEOUT_PATH], seen + + +# -- a real sweep that is simply longer than its timeout ----------------- + + +def without_the_cooperative_stop(monkeypatch): + """ + Switch off the sweep's stop between items, so the timeout lands inside + the sweep and the test exercises the backstop: a TaskTimeout let out + of the per-item except, wherever it was delivered. The stop itself is + covered at the end of this file. + + raising=False, so a tree without the stop reads red for the test's own + reason. If the stop ever reads its deadline from somewhere else, this + patch no longer reaches it, and the sweep test below fails on a + SUCCESSFUL row instead of passing quietly. + """ + monkeypatch.setattr(batches, "remaining", lambda: None, raising=False) + + +def dead_chains(count, prefix, depth=4): + """ + Sealed workflows whose root an operator discarded, each with a chain + of `depth` nodes behind it. One reconciler pass cancels the chain a + level per transaction (depth + 1 cancel passes) and then completes + the workflow. The shape the hazard calls a long sweep. + """ + flows = [] + for i in range(count): + flow = workflows.create(f"{prefix}-{i}") + root = flow.add("n0", tasks.step, f"{prefix}{i}-0") + for level in range(1, depth + 1): + flow.add( + f"n{level}", + tasks.step, + f"{prefix}{i}-{level}", + depends_on=f"n{level - 1}", + ) + flow.seal() + assert actions.discard(root.id) + flows.append(flow) + return flows + + +@pytest.mark.django_db(transaction=True) +def test_a_real_sweep_longer_than_its_timeout_is_recorded_as_timed_out( + monkeypatch, caplog, interruptible_attempts +): + timeout, grace = 0.15, 30 + # Calibrate on this machine and database, so the sweep under test is + # roughly twelve timeouts long wherever it runs. + dead_chains(10, "calibrate") + started = time.monotonic() + assert workflows._reconcile() == 10 + per_workflow = (time.monotonic() - started) / 10 + count = min(max(math.ceil(12 * timeout / per_workflow), 20), 2000) + flows = dead_chains(count, "sweep") + require_timeouts_enforced(flow_worker, timeout, grace) + without_the_cooperative_stop(monkeypatch) + + result = workflows.reconcile.enqueue() + worker = flow_worker(task_timeout=timeout, task_timeout_grace=grace) + raised = None + started = time.monotonic() + with ( + caplog.at_level(logging.ERROR, logger="oxpull.workflows"), + _EndIfWedged(timeout + grace + 3) as rescue, + ): + try: + worker.run_once() + except Exception as exc: + raised = exc + elapsed = time.monotonic() - started + _reset_connection() + row = OxTask.objects.get(id=result.id) + swallowed = _swallowed_timeouts(caplog.records, "workflow_reconcile_failed") + statuses = workflow_statuses(flows) + seen = ( + f"workflows={count} per_workflow={per_workflow * 1000:.2f}ms " + f"run_once={elapsed:.2f}s raised={raised!r} wedged={rescue.fired} " + f"status={row.status} return_value={row.return_value!r} " + f"errors={_errors(row)} swallowed_timeouts={len(swallowed)} " + f"swallowed_chains={sorted({_chain_names(r.exc_info[1]) for r in swallowed})} " + f"complete={statuses.count(OxpullWorkflow.Status.COMPLETE)} " + f"sealed={statuses.count(OxpullWorkflow.Status.SEALED)}" + ) + print(seen) # noqa: T201 + delivered = bool(swallowed) or TIMEOUT_PATH in _errors(row) + assert delivered, f"the sweep finished inside its timeout; nothing tested. {seen}" + assert raised is None, seen + assert row.status != OxTask.Status.SUCCESSFUL, seen + assert _errors(row)[-1:] == [TIMEOUT_PATH], seen + + +# -- (c) a timeout that lands in the driver ------------------------------ + + +@pytest.mark.django_db(transaction=True) +def test_a_timeout_landing_anywhere_in_a_sweep_leaves_the_attempt_an_outcome( + monkeypatch, caplog, interruptible_attempts +): + """ + No placement: the watchdog's own delivery, at a TASK_TIMEOUT drawn at + random inside a sweep of real sealed workflows. The only patch + switches off the stop between workflows, which would otherwise end + most trials before the delivery that this test is about. + + Each workflow holds an operator-discarded node (so every tick opens a + cancel transaction on it) and a node deferred a month (so it never + completes), which makes the sweep repeatable. The correct outcome of + every trial is the one django-ox promises: run_once returns, and the + attempt is recorded, never left RUNNING for the reaper, and never + recycles the worker. Stops at the first trial that breaks it. + """ + if connection.vendor == "sqlite": + pytest.skip( + "0 of 350 unplaced deliveries on SQLite left the connection " + "unusable while this hazard was probed; sqlite3 runs its protocol " + "in C, where psycopg and PyMySQL run theirs in Python." + ) + far = timezone.now() + timedelta(days=30) + for i in range(30): + flow = workflows.create(f"repeatable-{i}") + root = flow.add("a", tasks.step, f"a{i}") + flow.add("b", tasks.step.using(run_after=far), f"b{i}") + flow.seal() + assert actions.discard(root.id) + samples = [] + for _ in range(5): + started = time.monotonic() + assert workflows._reconcile() == 0 + samples.append(time.monotonic() - started) + sweep = sorted(samples)[2] + + without_the_cooperative_stop(monkeypatch) + grace = 3.0 + rng = random.Random(20260913) # noqa: S311 + delivered = 0 + broken = [] + for trial in range(40): + for stale in OxTask.objects.filter( + task_path="oxpull.workflows.reconcile", status=OxTask.Status.READY + ).values_list("id", flat=True): + actions.discard(stale) + caplog.clear() + timeout = rng.uniform(0.05, 0.95) * sweep + result = workflows.reconcile.enqueue() + worker = flow_worker(task_timeout=timeout, task_timeout_grace=grace) + raised = None + with ( + caplog.at_level(logging.ERROR, logger="oxpull.workflows"), + _EndIfWedged(timeout + grace + 3) as rescue, + ): + try: + worker.run_once() + except Exception as exc: + raised = exc + _reset_connection() + row = OxTask.objects.get(id=result.id) + not_reconciled = [ + r + for r in caplog.records + if getattr(r, "event", None) == "workflow_reconcile_failed" + ] + if _swallowed_timeouts( + not_reconciled, "workflow_reconcile_failed" + ) or TIMEOUT_PATH in _errors(row): + delivered += 1 + if ( + raised is not None + or row.status == OxTask.Status.RUNNING + or worker.recycling + or rescue.fired + ): + chains = sorted( + {_chain_names(r.exc_info[1]) for r in not_reconciled if r.exc_info} + ) + broken.append( + f"trial {trial}: timeout {timeout * 1000:.1f}ms into a " + f"{sweep * 1000:.0f}ms sweep on {connection.vendor}; run_once " + f"raised {raised!r}; reconcile row {row.status} " + f"errors={_errors(row)}; recycling={worker.recycling} " + f"wedged={rescue.fired}; {len(not_reconciled)} of 30 workflows " + f"logged 'could not be reconciled', chains {chains}" + ) + break + print( # noqa: T201 + f"trials run={trial + 1} delivered={delivered} sweep={sweep * 1000:.0f}ms " + f"broken={broken}" + ) + assert not broken, broken[0] if broken else "" + assert delivered, "no trial delivered a TaskTimeout; nothing was tested" + + +# -- the stop between items, and a new order every tick ------------------ + +ITEM_STEPS = pytest.mark.parametrize( + ("module", "item_step"), + [(workflows, "_cancel_unreachable"), (batches, "_complete_if_finished")], + ids=["workflows", "batches"], +) + + +def finished_units(module, count): + """ + The ids of `count` sealed workflows or batches whose tasks have all + run, oldest first, so each needs only its completion. + """ + if module is workflows: + return [flow.id for flow in sealed_finished_workflows(count)] + ids = [] + for i in range(count): + batch = batches.create(f"finished-{i}") + batch.add(tasks.plain_echo, i) + batch.seal() + ids.append(batch.id) + assert drain(default_worker()) == count + return ids + + +def unit_statuses(module, ids): + model = OxpullWorkflow if module is workflows else OxpullBatch + rows = {str(pk): status for pk, status in model.objects.values_list("pk", "status")} + return [rows[str(pk)] for pk in ids] + + +def complete_status(module): + model = OxpullWorkflow if module is workflows else OxpullBatch + return model.Status.COMPLETE + + +class SteppedClock: + """ + Stands in for the `time` module inside batches and workflows, with a + monotonic() that moves only when the test moves it, so a tick timed on + it spends the same time on any machine. Every other name is the real + module's, which keeps the retry pauses in both modules sleeping. + """ + + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def advance(self, seconds): + self.now += seconds + + def __getattr__(self, name): + return getattr(time, name) + + +def time_a_tick_on(monkeypatch, clock, budget): + """ + Put every clock a reconciler tick reads under `clock`, with `budget` + seconds until the task timeout, or no timeout when it is None. + + A tick reads two: remaining() for what the timeout leaves, and + time.monotonic() for how long a step took. The sweep reads both through + batches, for workflows as well. The release backstop, which a workflow + tick runs first, reads both through workflows. All four names go to the + one clock, because a tick timed partly on the real one would again + depend on the machine. + + remaining is set with raising=False, so a tree without the stop reads + red for the test's own reason. + """ + deadline = None if budget is None else clock.monotonic() + budget + + def remaining(): + return None if deadline is None else deadline - clock.monotonic() + + for reader in (batches, workflows): + monkeypatch.setattr(reader, "time", clock) + monkeypatch.setattr(reader, "remaining", remaining, raising=False) + + +@pytest.mark.django_db +@ITEM_STEPS +def test_an_item_that_times_out_every_tick_does_not_hold_back_the_rest( + monkeypatch, caplog, module, item_step +): + """ + A tick now ends at the item it timed out on. Visited oldest first, the + items after one that always overruns would never be reached. Here the + oldest raises TaskTimeout on every visit, which is that item without a + real deadline per tick, and every other item must still complete + within a bounded number of ticks, none of them logged as failing. + """ + ids = finished_units(module, 6) + broken, healthy = str(ids[0]), ids[1:] + original = getattr(module, item_step) + visits = [] + + def times_out_on_the_oldest(pk, db_alias): + visits[-1] += 1 + if str(pk) == broken: + raise TaskTimeout + return original(pk, db_alias) + + monkeypatch.setattr(module, item_step, times_out_on_the_oldest) + complete = complete_status(module) + kind = "workflow" if module is workflows else "batch" + ticks = timed_out = 0 + done_before = [] + with caplog.at_level(logging.ERROR, logger=module.__name__): + while ticks < 60: + done = unit_statuses(module, healthy).count(complete) + if done == len(healthy): + break + done_before.append(done) + visits.append(0) + ticks += 1 + try: + module._reconcile() + except TaskTimeout: + timed_out += 1 + failed = [ + r.getMessage() + for r in caplog.records + if getattr(r, "event", None) == f"{kind}_reconcile_failed" + ] + statuses = unit_statuses(module, ids) + seen = ( + f"ticks={ticks} timed_out={timed_out} done_before_each={done_before} " + f"visited_each={visits} failed={failed} statuses={statuses}" + ) + print(seen) # noqa: T201 + assert not failed, seen + assert timed_out, f"no tick let the TaskTimeout out. {seen}" + assert statuses[1:] == [complete] * len(healthy), seen + assert statuses[0] != complete, seen + + +@pytest.mark.django_db +@ITEM_STEPS +def test_a_tick_short_of_time_stops_between_items_and_the_next_tick_finishes( + monkeypatch, caplog, module, item_step +): + """ + The stop, on a clock this test moves (SteppedClock): each item takes + 0.1 s of it and the tick has 1 s. The tick must stop between items with + items left, return what it completed, and log what it did not visit. A + tick with no deadline then completes the rest. + """ + count = 20 + ids = finished_units(module, count) + original = getattr(module, item_step) + clock = SteppedClock() + + def slow(pk, db_alias): + clock.advance(0.1) + return original(pk, db_alias) + + monkeypatch.setattr(module, item_step, slow) + kind = "workflow" if module is workflows else "batch" + time_a_tick_on(monkeypatch, clock, budget=1.0) + with caplog.at_level(logging.WARNING, logger=module.__name__): + completed = module._reconcile() + stops = [ + r + for r in caplog.records + if getattr(r, "event", None) == f"{kind}_reconcile_stopped" + ] + complete = complete_status(module) + done = unit_statuses(module, ids).count(complete) + seen = ( + f"completed={completed} done={done} clock={clock.now:.2f}s " + f"stops={[(r.visited_count, r.unvisited_count) for r in stops]} " + f"left={[round(r.remaining_seconds, 3) for r in stops]}" + ) + print(seen) # noqa: T201 + assert len(stops) == 1, seen + assert stops[0].remaining_seconds > 0, f"stopped after the deadline. {seen}" + assert 1 <= completed < count, seen + assert done == completed == stops[0].visited_count, seen + assert stops[0].unvisited_count == count - completed, seen + + time_a_tick_on(monkeypatch, clock, budget=None) + assert module._reconcile() == count - completed + assert unit_statuses(module, ids).count(complete) == count + + +@pytest.mark.django_db(transaction=True) +def test_a_real_sweep_longer_than_its_timeout_stops_in_time_over_several_ticks( + caplog, interruptible_attempts +): + """ + The sweep test 8 runs, with the stop left on and nothing patched: + about six timeouts of work, one reconcile task per tick through a + worker with that timeout, until every workflow is complete. + + A tick may end either way the fix allows: stopped between workflows + and SUCCESSFUL, or recorded as a timeout when one workflow overran the + margin. It may not swallow a timeout, raise out of run_once, leave the + row RUNNING, or recycle the worker. The first tick must be cut short, + and at least one tick must end with the stop. + """ + timeout, grace = 0.4, 30 + dead_chains(10, "calibrate") + started = time.monotonic() + assert workflows._reconcile() == 10 + per_workflow = (time.monotonic() - started) / 10 + count = min(max(math.ceil(6 * timeout / per_workflow), 20), 2000) + chains = dead_chains(count, "ticks") + require_timeouts_enforced(flow_worker, timeout, grace) + + outcomes, broken = [], [] + for tick in range(60): + statuses = workflow_statuses(chains) + if statuses.count(OxpullWorkflow.Status.COMPLETE) == count: + break + for stale in OxTask.objects.filter( + task_path="oxpull.workflows.reconcile", status=OxTask.Status.READY + ).values_list("id", flat=True): + actions.discard(stale) + caplog.clear() + result = workflows.reconcile.enqueue() + worker = flow_worker(task_timeout=timeout, task_timeout_grace=grace) + raised = None + with ( + caplog.at_level(logging.WARNING, logger="oxpull.workflows"), + _EndIfWedged(timeout + grace + 3) as rescue, + ): + try: + worker.run_once() + except Exception as exc: + raised = exc + _reset_connection() + row = OxTask.objects.get(id=result.id) + swallowed = _swallowed_timeouts(caplog.records, "workflow_reconcile_failed") + stopped = any( + getattr(r, "event", None) == "workflow_reconcile_stopped" + for r in caplog.records + ) + if stopped: + outcomes.append("stopped") + elif TIMEOUT_PATH in _errors(row): + outcomes.append("timed out") + else: + outcomes.append(str(row.status)) + if ( + raised is not None + or row.status == OxTask.Status.RUNNING + or worker.recycling + or rescue.fired + or swallowed + ): + broken.append( + f"tick {tick}: run_once raised {raised!r}; row {row.status} " + f"errors={_errors(row)}; recycling={worker.recycling} " + f"wedged={rescue.fired}; swallowed={len(swallowed)}" + ) + break + statuses = workflow_statuses(chains) + seen = ( + f"workflows={count} per_workflow={per_workflow * 1000:.2f}ms " + f"ticks={len(outcomes)} outcomes={outcomes} broken={broken} " + f"complete={statuses.count(OxpullWorkflow.Status.COMPLETE)}" + ) + print(seen) # noqa: T201 + assert not broken, seen + assert outcomes[:1] in (["stopped"], ["timed out"]), seen + assert statuses.count(OxpullWorkflow.Status.COMPLETE) == count, seen + assert "stopped" in outcomes, seen diff --git a/tests_oxpull/test_workflows_hazard_the_reconciler_s_release_sweep_reads_every.py b/tests_oxpull/test_workflows_hazard_the_reconciler_s_release_sweep_reads_every.py new file mode 100644 index 0000000..c7578d0 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_the_reconciler_s_release_sweep_reads_every.py @@ -0,0 +1,445 @@ +""" +The reconciler's release backstop must not read the READY backlog on every +tick. + +The backstop runs first in every reconciler pass, whatever the backends +declare. Development builds swept `status = READY AND run_after = ` +across the whole task table. No django-ox index contains run_after, so the +database reached those rows only through status (or the primary key), and an +install with few or no blocked nodes never filled the LIMIT: the sweep read +every READY row, or on MySQL at some shapes every row in the table, to +release nothing. The backstop now reads the unsatisfied edges and the +WAITING nodes of OPEN and SEALED workflows, which no READY backlog enlarges. + +Every test here builds an idle tick through real paths: + + * a workflow in flight, built with `create()`, `add()` and `seal()`, whose + root a real `OxpullWorker.claim_one()` holds RUNNING under a lease, so the + reconciler has a SEALED workflow to look at and nothing to release; + * a backlog of ordinary READY tasks written by django-ox's public + `django_ox.bulk.enqueue_many`, the bulk form of `task.enqueue()`; + * one real `workflows._reconcile()` pass, the function the scheduled + `oxpull.workflows.reconcile` task runs. + +The tick is measured server-side, never by wall clock: + + * PostgreSQL: heap tuples read from the task table during the tick + (`seq_tup_read + idx_tup_fetch` from `pg_stat_user_tables`, parallel + workers included), with the tick in autocommit as a worker runs it. + * MySQL: the session's `Handler_read_*` counters around the tick, less the + cost of reading them. + * SQLite: virtual-machine instructions executed during the tick, from the + connection's progress handler. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import time +from dataclasses import dataclass + +import pytest +from django.db import connection +from django.db.models import Exists, OuterRef +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox.bulk import enqueue_many +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.worker import OxpullWorker +from oxpull.workflows import BACKSTOP_PAGE + +from . import tasks +from .test_workflows import reconcile, stored +from .test_workflows_cost import rows_examined + +pytestmark = pytest.mark.django_db(transaction=True) + +# Backlog sizes for the growth test. Overridable so a smoke run can use small +# ones. +SIZES = tuple( + int(n) for n in os.environ.get("OX_SWEEP_SIZES", "20000,200000").split(",") +) +# The MySQL whole-table shape: (SUCCESSFUL history rows, READY backlog rows). +MYSQL_SHAPE = tuple( + int(n) for n in os.environ.get("OX_SWEEP_MYSQL_SHAPE", "70000,20000").split(",") +) +WIDTH = 10 +ENQUEUE_CHUNK = 10_000 + +row_counting = pytest.mark.skipif( + connection.vendor not in ("mysql", "postgresql"), + reason="the bound is in rows; SQLite's counter is VM instructions, and the " + "growth test covers SQLite", +) + + +@dataclass +class TickCost: + reads: int + unit: str + ms: float + completed: int + + +# -- configuration ----------------------------------------------------- + + +@contextlib.contextmanager +def configured(backend: str): + """ + Yield (node task, worker alias) for the workflow backend under test: + "flows", or "flows_deferred", which carries the removed + DEFER_BLOCKED_NODES option and must cost the same. + """ + if backend == "flows": + yield tasks.step, "flows" + elif backend == "flows_deferred": + yield tasks.deferred_step, "flows_deferred" + else: # pragma: no cover + raise AssertionError(backend) + + +def in_flight_workflow(task, alias: str): + """ + A SEALED workflow whose root a real worker holds RUNNING, with WIDTH + children waiting on it. The reconciler has something to look at and + nothing it may release or complete. + """ + flow = workflows.create(f"in-flight-{alias}") + flow.add("root", task, "root") + for i in range(WIDTH): + flow.add(f"c{i:02d}", task, f"c{i:02d}", depends_on="root") + flow.seal() + worker = OxpullWorker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + claimed = worker.claim_one() + assert claimed is not None, "the worker claimed nothing" + assert str(claimed.id) == str(flow.node_ids()["root"]), "the worker took a child" + return flow + + +def enqueue_backlog(count: int, start: int = 0) -> None: + """`count` ordinary READY tasks, through django-ox's public bulk enqueue.""" + done = 0 + while done < count: + n = min(ENQUEUE_CHUNK, count - done) + enqueue_many(tasks.plain_echo, [((start + done + i,), {}) for i in range(n)]) + done += n + + +def refresh_statistics() -> None: + """ + Fresh planner statistics, so a plan is not an artefact of stale ones. + + PostgreSQL also gets VACUUM. History rows were READY before they were + finished, and until autovacuum reaches them their dead READY versions + stay in the status indexes and inflate what an index scan visits. The + figure after VACUUM is the conservative one. + + SQLite is left as Django leaves it: nothing runs ANALYZE there, so a + production SQLite install plans without sqlite_stat1. + """ + tables = [OxTask._meta.db_table, "oxpull_oxpullworkflowedge"] + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + for table in tables: + cursor.execute(f'VACUUM (ANALYZE) "{table}"') + elif connection.vendor == "mysql": + cursor.execute("ANALYZE TABLE " + ", ".join(f"`{t}`" for t in tables)) + cursor.fetchall() + + +# -- measurement ------------------------------------------------------- + + +def _pg_task_rows_read() -> int: + """ + Heap tuples read from the task table so far, parallel workers included. + + Not pg_stat_xact_user_tables: that counts the leader process only, and + at 200,000 READY rows the sweep plans as a parallel Seq Scan, so it + reported about a third of the rows scanned. The cumulative view includes + what each parallel worker reports at exit. Forcing a flush makes this + backend's own pending counts visible first; nothing else connects to the + test database. + """ + with connection.cursor() as cursor: + cursor.execute("SELECT pg_stat_force_next_flush()") + with connection.cursor() as cursor: + cursor.execute("SELECT pg_stat_clear_snapshot()") + cursor.execute( + "SELECT COALESCE(seq_tup_read, 0) + COALESCE(idx_tup_fetch, 0) " + "FROM pg_stat_user_tables WHERE relname = %s", + [OxTask._meta.db_table], + ) + row = cursor.fetchone() + return int(row[0]) if row else 0 + + +def _mysql_handler_reads() -> int: + with connection.cursor() as cursor: + cursor.execute("SHOW SESSION STATUS LIKE 'Handler_read%'") + return sum(int(value) for _name, value in cursor.fetchall()) + + +def measure(fn) -> tuple[int, str, float, object]: + """Run fn() once and return (server-side reads, unit, ms, fn's result).""" + vendor = connection.vendor + connection.ensure_connection() + if vendor == "postgresql": + before = _pg_task_rows_read() + started = time.perf_counter() + result = fn() + ms = (time.perf_counter() - started) * 1000 + after = _pg_task_rows_read() + return after - before, "task-table heap tuples read", ms, result + if vendor == "mysql": + first = _mysql_handler_reads() + second = _mysql_handler_reads() + overhead = second - first + started = time.perf_counter() + result = fn() + ms = (time.perf_counter() - started) * 1000 + after = _mysql_handler_reads() + return after - second - overhead, "Handler_read_* rows", ms, result + if vendor == "sqlite": + raw = connection.connection + step = 100 + calls = [0] + + def tick_handler() -> int: + calls[0] += 1 + return 0 + + raw.set_progress_handler(tick_handler, step) + try: + started = time.perf_counter() + result = fn() + ms = (time.perf_counter() - started) * 1000 + finally: + raw.set_progress_handler(None, step) + return calls[0] * step, "SQLite VM instructions", ms, result + pytest.skip(f"no server-side read counter for {vendor}") # pragma: no cover + + +def measure_tick() -> TickCost: + reads, unit, ms, completed = measure(reconcile) + return TickCost(reads=reads, unit=unit, ms=ms, completed=int(completed)) + + +def sweep_queryset(): + """ + The task-table read behind the backstop's release page: the waiting rows + among a page of open nodes of the open workflows, by primary key. The page + of nodes is read first, as _release_page reads it. + """ + live = OxpullWorkflow.objects.filter( + status__in=(OxpullWorkflow.Status.OPEN, OxpullWorkflow.Status.SEALED) + ) + page = list( + OxpullWorkflowNode.objects.filter( + workflow_id__in=live.values("pk"), succeeded=False + ) + .order_by("pk") + .values_list("task_id", flat=True)[:BACKSTOP_PAGE] + ) + return ( + OxTask.objects.filter(status=OxTask.Status.WAITING, pk__in=page) + .exclude( + Exists( + OxpullWorkflowEdge.objects.filter( + downstream_task_id=OuterRef("pk"), satisfied=False + ) + ) + ) + .order_by("pk") + .values_list("pk", flat=True) + ) + + +def sweep_evidence() -> str: + """The sweep statement's plan and its own read cost, for the log.""" + qs = sweep_queryset() + vendor = connection.vendor + lines = [] + if vendor == "postgresql": + plan = qs.explain(analyze=True, format="JSON") + if isinstance(plan, str): + plan = json.loads(plan) + root = plan[0]["Plan"] + + scanned = [0] + + def walk(node, depth=0): + label = node["Node Type"] + if "Index Name" in node: + label += f" on {node['Index Name']}" + if "Relation Name" in node: + label += f" [{node['Relation Name']}]" + loops = int(node.get("Actual Loops", 1)) + removed = int(node.get("Rows Removed by Filter", 0)) + int( + node.get("Rows Removed by Index Recheck", 0) + ) + if "Relation Name" in node: + scanned[0] += (int(node.get("Actual Rows", 0)) + removed) * loops + lines.append( + f" {' ' * depth}{label}: rows={node.get('Actual Rows')} " + f"removed_by_filter={removed} loops={loops}" + ) + for child in node.get("Plans", []): + walk(child, depth + 1) + + walk(root) + # rows_examined sums rows each node emits, so a scan that filters + # everything out counts as 0; the second figure adds what the scan + # nodes threw away. + lines.insert( + 0, + f" sweep EXPLAIN ANALYZE: rows x loops = {rows_examined(qs)}; " + f"relation rows scanned incl. removed by filter = {scanned[0]}", + ) + elif vendor == "mysql": + reads, _unit, ms, found = measure(lambda: list(qs)) + lines.append( + f" sweep alone: Handler_read_* = {reads}, returned {len(found)}, " + f"{ms:.1f} ms" + ) + for line in qs.explain(analyze=True).splitlines(): + lines.append(f" {line}") + else: + reads, _unit, ms, found = measure(lambda: list(qs)) + lines.append( + f" sweep alone: VM instructions = {reads}, returned {len(found)}, " + f"{ms:.1f} ms" + ) + for line in qs.explain().splitlines(): + lines.append(f" {line}") + with CaptureQueriesContext(connection) as captured: + reconcile() + first = captured.captured_queries[0]["sql"] if captured.captured_queries else "" + lines.append( + f" tick issued {len(captured.captured_queries)} statement(s); first: " + f"{first[:400]}" + ) + return "\n".join(lines) + + +def assert_idle(flow, waiting: int, ready: int, cost: TickCost) -> None: + """The tick under test had nothing to release, cancel or complete.""" + assert cost.completed == 0, "the tick completed a workflow; it was not idle" + assert stored(flow).status == OxpullWorkflow.Status.SEALED + assert OxTask.objects.filter(status=OxTask.Status.WAITING).count() == waiting, ( + "the tick released a waiting node; it was not idle" + ) + assert OxTask.objects.filter(status=OxTask.Status.READY).count() == ready + + +# -- the tests --------------------------------------------------------- + + +@pytest.mark.parametrize("backend", ["flows", "flows_deferred"]) +def test_an_idle_reconciler_tick_costs_the_same_whatever_the_ready_backlog(backend): + """ + An idle tick's cost must not track the READY backlog. + + With nothing to release, the correct cost is O(waiting nodes), bounded + by BACKSTOP_PAGE. Growing the ordinary READY backlog tenfold, from + SIZES[0] to SIZES[1], must leave it roughly where it was. + """ + small, large = SIZES + assert large > small + with configured(backend) as (task, alias): + flow = in_flight_workflow(task, alias) + waiting = WIDTH + assert OxTask.objects.filter(status=OxTask.Status.WAITING).count() == waiting, ( + "precondition: the in-flight workflow's children are not waiting" + ) + + started = time.perf_counter() + enqueue_backlog(small) + built_small = time.perf_counter() - started + refresh_statistics() + at_small = measure_tick() + assert_idle(flow, waiting, small, at_small) + + started = time.perf_counter() + enqueue_backlog(large - small, start=small) + built_large = time.perf_counter() - started + refresh_statistics() + at_large = measure_tick() + assert_idle(flow, waiting, large, at_large) + + evidence = sweep_evidence() + + print( # noqa: T201 + f"\n [s1 sweep] {connection.vendor} backend={backend} " + f"waiting_rows={waiting} unit={at_small.unit}\n" + f" READY={small:>7} (built {built_small:.1f}s): " + f"tick reads={at_small.reads:>8} in {at_small.ms:.1f} ms\n" + f" READY={large:>7} (built {built_large:.1f}s): " + f"tick reads={at_large.reads:>8} in {at_large.ms:.1f} ms\n" + f"{evidence}" + ) + + assert at_large.reads < 2 * at_small.reads + BACKSTOP_PAGE, ( + f"an idle reconciler tick read {at_small.reads} ({at_small.unit}) over " + f"{small} READY rows and {at_large.reads} over {large}, with {waiting} " + "waiting row(s) and nothing to release: its cost tracks the READY backlog" + ) + + +@row_counting +def test_an_idle_reconciler_tick_over_a_mostly_finished_table_reads_few_rows(): + """ + A table that is mostly history, with a READY backlog and nothing to + release: an idle tick over MYSQL_SHAPE must stay under 10,000 row reads. + + The old sweep's `ORDER BY pk LIMIT 1000` could plan on MySQL as a walk of + the primary key that never reached its limit, reading SUCCESSFUL history + as well as the READY backlog. The log records whether the reads exceed + the READY row count. + """ + history, backlog = MYSQL_SHAPE + with configured("flows") as (task, alias): + flow = in_flight_workflow(task, alias) + + started = time.perf_counter() + enqueue_backlog(history) + # Stands in for workers having run them: django-ox's own worker writes + # exactly this status on each row it finishes. + now = timezone.now() + finished = OxTask.objects.filter( + status=OxTask.Status.READY, task_path=tasks.plain_echo.module_path + ).update( + status=OxTask.Status.SUCCESSFUL, + attempts=1, + started_at=now, + last_attempted_at=now, + finished_at=now, + ) + assert finished == history + enqueue_backlog(backlog, start=history) + built = time.perf_counter() - started + refresh_statistics() + + cost = measure_tick() + assert_idle(flow, WIDTH, backlog, cost) + evidence = sweep_evidence() + + total = OxTask.objects.count() + print( # noqa: T201 + f"\n [s1 sweep] whole-table shape: rows={total} READY={backlog} " + f"SUCCESSFUL={history} (built {built:.1f}s)\n" + f" tick reads={cost.reads} ({cost.unit}) in {cost.ms:.1f} ms; " + f"reads beyond the READY row count: {cost.reads - backlog}\n" + f"{evidence}" + ) + + assert cost.reads < 10_000, ( + f"an idle reconciler tick read {cost.reads} rows ({cost.unit}) over a table " + f"of {total} rows with {backlog} READY and nothing to release" + ) diff --git a/tests_oxpull/test_workflows_hazard_the_stored_cancelled_count_counts_every.py b/tests_oxpull/test_workflows_hazard_the_stored_cancelled_count_counts_every.py new file mode 100644 index 0000000..9e2454e --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_the_stored_cancelled_count_counts_every.py @@ -0,0 +1,301 @@ +""" +The stored cancelled count is the number of nodes the workflow's own +cancellation changed, and nothing else. + +One definition, shared with +tests_oxpull/test_workflows.py::test_an_operator_discarding_a_node_cancels_its_cone. +OxpullWorkflow.cancelled_count is the reconciler's own row count: the +READY or WAITING node rows the workflow's cancellation moved to DISCARDED +because an +upstream of theirs settled without succeeding, added up over every +reconciler tick and stored when the workflow completes. It records what +the workflow did, so: + +- a node a person discarded (actions.discard, discard_many, the admin's + "Discard selected tasks") is never counted, whether or not it has + edges; it shows only as unattributed; +- a node the workflow cancelled still counts after ox_prune deletes its + task row; +- a workflow that cancelled nothing stores 0. + +Today the completing transaction stores the DISCARDED rows it can still +see instead. That counts people's discards and drops pruned +cancellations. + +Wrong implementations these tests reject: + +- counting DISCARDED rows at completion, which is today's code: every + test marked as an expected failure; +- counting DISCARDED nodes that have an upstream, or a dead upstream: + the operator-discards test, where a person also discards a dependent + node, and the pruning test; +- keeping one tick's count instead of the total (the tick that + completes, or the last tick that cancelled anything): the pruning + test, and the unmarked test whose cancellations land in two ticks. + +Adding each tick's row count onto the workflow row passes, and so does +counting oxpull node rows that record a cancellation. +""" + +import io +import logging +from datetime import timedelta + +import pytest +from django.core.management import call_command +from django.utils import timezone + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import order, stored + +pytestmark = pytest.mark.django_db + +COMPLETE = OxpullWorkflow.Status.COMPLETE +SEALED = OxpullWorkflow.Status.SEALED +DISCARDED = OxTask.Status.DISCARDED + + +def worker_on(alias): + return OxpullWorker(backend_alias=alias, backoff_initial=0, poll_interval=0.01) + + +def reconcile_through_a_worker(worker): + """ + One reconciler tick the way production runs it: the task a schedule + enqueues, claimed and executed by a worker. Top priority so the worker + takes it ahead of any READY node. + """ + result = workflows.reconcile.using(priority=100).enqueue() + assert worker.run_once() is True + row = OxTask.objects.get(pk=result.id) + assert row.status == OxTask.Status.SUCCESSFUL, (row.status, row.errors) + + +def reconcile_until_complete(worker, flow): + for _ in range(3): + reconcile_through_a_worker(worker) + if stored(flow).status == COMPLETE: + break + + +def status_of(ids, name): + return OxTask.objects.filter(pk=ids[name]).values_list("status", flat=True).first() + + +def logged_cancellations(caplog, flow): + """What the reconciler reported cancelling in this workflow, for messages.""" + return sum( + record.cancelled_count + for record in caplog.records + if getattr(record, "event", None) == "workflow_nodes_cancelled" + and record.workflow_id == str(flow.id) + ) + + +def describe(flow, caplog): + row = stored(flow) + ids = flow.node_ids() + statuses = {name: status_of(ids, name) for name in ids} + return ( + f"status={row.status} failed_count={row.failed_count} " + f"cancelled_count={row.cancelled_count} " + f"reconciler_logged={logged_cancellations(caplog, flow)} nodes={statuses}" + ) + + +@pytest.mark.xfail( + strict=True, + reason=( + "The stored cancelled count includes nodes a person discarded; it should " + "count only the nodes the workflow itself cancelled" + ), +) +@pytest.mark.parametrize( + ("alias", "step"), + [("flows", tasks.step), ("flows_deferred", tasks.deferred_step)], + ids=["gate", "deferred"], +) +def test_operator_discards_are_not_counted_as_nodes_the_workflow_cancelled( + caplog, alias, step +): + """ + a -> b, a -> c, plus an independent leaf. A person discards the leaf, + the dependent node b, and then a. The workflow cancels c, and only c, + so the stored count is 1. + + b is the case that separates the definition from "DISCARDED nodes + with an upstream": b has an upstream, and a dead one, and it is still + a person's discard. + """ + caplog.set_level(logging.INFO, logger="oxpull.workflows") + flow = workflows.create("operator-discards") + flow.add("a", step, "a") + flow.add("b", step, "b", depends_on="a") + flow.add("c", step, "c", depends_on="a") + flow.add("leaf", step, "leaf") + flow.seal() + ids = flow.node_ids() + + assert actions.discard(ids["leaf"]) is True + assert actions.discard(ids["b"]) is True + assert actions.discard(ids["a"]) is True + assert status_of(ids, "c") == OxTask.Status.READY + + worker = worker_on(alias) + reconcile_until_complete(worker, flow) + + row = stored(flow) + assert row.status == COMPLETE, describe(flow, caplog) + assert order() == [] # nothing ran + assert status_of(ids, "c") == DISCARDED # the workflow's one cancellation + assert row.failed_count == 0 + assert flow.progress().unattributed == 4 # all four stay visible there + assert row.cancelled_count == 1, ( + "cancelled_count should count only c, the node the workflow cancelled; " + "leaf, b and a are a person's discards. " + describe(flow, caplog) + ) + + +@pytest.mark.xfail( + strict=True, + reason=( + "A workflow that cancelled nothing stores a non-zero cancelled count when " + "a person discarded one of its nodes" + ), +) +def test_a_workflow_that_cancelled_nothing_stores_no_cancellations(caplog): + """ + No edges anywhere, so there is nothing for the workflow to cancel. A + person closes one node through discard_many, which is what the + django-ox admin's "Discard selected tasks" action calls; the other runs. + """ + caplog.set_level(logging.INFO, logger="oxpull.workflows") + flow = workflows.create("nothing-cancelled") + flow.add("work", tasks.step, "work") + flow.add("leaf", tasks.step, "leaf") + flow.seal() + ids = flow.node_ids() + + assert actions.discard_many([ids["leaf"]]) == (1, 0) + worker = worker_on("flows") + assert worker.run_once() is True + assert order() == ["work"] + reconcile_until_complete(worker, flow) + + row = stored(flow) + assert row.status == COMPLETE, describe(flow, caplog) + assert row.failed_count == 0 + assert flow.progress().unattributed == 1 + assert row.cancelled_count == 0, ( + "the workflow cancelled nothing; a person discarded one node. " + + describe(flow, caplog) + ) + + +@pytest.mark.xfail( + strict=True, + reason=( + "The stored cancelled count drops a node the workflow cancelled once " + "pruning deletes that node's task row before the workflow completes" + ), +) +def test_a_cancelled_node_pruned_before_the_workflow_completes_still_counts(caplog): + """ + root fails permanently and the workflow cancels child in one tick. An + independent sibling keeps the workflow SEALED. The default ox_prune + deletes child's DISCARDED row once it is older than 7 days. A later + tick completes the workflow and cancels nothing. + + Shortcut: child's finished_at is backdated 8 days. In production the + same state is a sibling that stays pending for longer than ox_prune's + --older-than after the cancellation (a backlog, a stopped fleet, a + long-deferred node) while ox_prune runs on its usual schedule. + """ + caplog.set_level(logging.INFO, logger="oxpull.workflows") + flow = workflows.create("pruned-cancellation") + flow.add("root", tasks.failing_step.using(priority=10), "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.add("sibling", tasks.step, "sibling") + flow.seal() + ids = flow.node_ids() + + worker = worker_on("flows") + for _ in range(3): + assert worker.run_once() is True + assert order() == ["root", "root", "root"] + assert status_of(ids, "root") == OxTask.Status.FAILED + + reconcile_through_a_worker(worker) + assert status_of(ids, "child") == DISCARDED # the workflow cancelled it + assert stored(flow).status == SEALED # sibling is still READY + + OxTask.objects.filter(pk=ids["child"]).update( + finished_at=timezone.now() - timedelta(days=8) + ) + call_command("ox_prune", stdout=io.StringIO()) + assert not OxTask.objects.filter(pk=ids["child"]).exists() + + assert worker.run_once() is True + assert order()[-1] == "sibling" + reconcile_until_complete(worker, flow) + + row = stored(flow) + assert row.status == COMPLETE, describe(flow, caplog) + assert row.failed_count == 1 + assert row.cancelled_count == 1, ( + "the workflow cancelled child; ox_prune deleting the row afterwards " + "does not undo that. " + describe(flow, caplog) + ) + + +def test_cancellations_in_separate_ticks_add_up(caplog): + """ + Control: passes today and must keep passing. Two branches fail at + different times. The workflow cancels two nodes in one tick and one + in a later tick, then completes, and the stored count is the total of + 3, not either tick's count. + """ + caplog.set_level(logging.INFO, logger="oxpull.workflows") + flow = workflows.create("two-ticks") + flow.add("first", tasks.failing_step.using(priority=10), "first") + flow.add("first-child", tasks.step, "first-child", depends_on="first") + flow.add( + "first-grandchild", tasks.step, "first-grandchild", depends_on="first-child" + ) + flow.add("second", tasks.failing_step.using(priority=5), "second") + flow.add("second-child", tasks.step, "second-child", depends_on="second") + flow.seal() + ids = flow.node_ids() + + worker = worker_on("flows") + for _ in range(3): + assert worker.run_once() is True + assert order() == ["first"] * 3 + assert status_of(ids, "first") == OxTask.Status.FAILED + + reconcile_through_a_worker(worker) + assert status_of(ids, "first-child") == DISCARDED, describe(flow, caplog) + assert status_of(ids, "first-grandchild") == DISCARDED, describe(flow, caplog) + assert status_of(ids, "second-child") == OxTask.Status.WAITING + assert stored(flow).status == SEALED # second is still READY + + for _ in range(3): + assert worker.run_once() is True + assert order()[3:] == ["second"] * 3 + assert status_of(ids, "second") == OxTask.Status.FAILED + + reconcile_until_complete(worker, flow) + row = stored(flow) + assert row.status == COMPLETE, describe(flow, caplog) + assert status_of(ids, "second-child") == DISCARDED + assert row.failed_count == 2 + assert row.cancelled_count == 3, ( + "two cancelled in the first tick and one in a later tick. " + + describe(flow, caplog) + ) diff --git a/tests_oxpull/test_workflows_hazard_under_use_tz_false_on_sqlite_or_mysql_every.py b/tests_oxpull/test_workflows_hazard_under_use_tz_false_on_sqlite_or_mysql_every.py new file mode 100644 index 0000000..32eea30 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_under_use_tz_false_on_sqlite_or_mysql_every.py @@ -0,0 +1,579 @@ +""" +Workflows under USE_TZ=False on SQLite and MySQL. + +Django's SQLite and MySQL backends refuse to bind an aware datetime to a +DateTimeField when USE_TZ is off, filter values included, with a ValueError +raised in Python before any SQL is sent. PostgreSQL accepts it. Development +builds of workflows bound an aware run_after sentinel on every reconciler +tick, on every release and on every add() of a blocked node, and each of +those raised. A blocked node now waits in django-ox's WAITING status +instead, and its release binds the process clock, which is naive under +USE_TZ=False. + +USE_TZ=False is a configuration django-ox supports (django_ox.W001 exists +for it, CI runs a naive leg). `docs/features/rate-limiting.md` documents +rate limits under it. Every test here passes on all three databases. + +Every test asserts the behaviour a user is owed. Two layers: + + * In-process, under override_settings(USE_TZ=False), through the public + API and real OxpullWorker.run_once() claims. + * A fresh interpreter configured with USE_TZ=False from the start + (TIME_ZONE="UTC", the zone django_ox.W001's own hint recommends), + migrated, and driven by the real `ox_worker` management command for + each backend, with the workflow reconciler on a real django-ox + schedule. This is the path a user runs, and it rules out + override_settings as the cause. +""" + +import json +import os +import subprocess +import sys +import traceback +from datetime import timedelta +from pathlib import Path + +import pytest +from django.db import connections +from django.tasks.signals import task_finished +from django.test import override_settings +from django.utils import timezone + +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, order, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def naive(): + return override_settings(USE_TZ=False) + + +def _raised(fn): + """ + Call fn; return (value, None), or (None, the raise path) if it raised. + + The raise path is every frame in oxpull or django_ox plus the frame + that raised, so a failure names the line that bound the value. + """ + try: + return fn(), None + except Exception as exc: + frames = traceback.extract_tb(exc.__traceback__) + kept = [ + frame + for index, frame in enumerate(frames) + if "/oxpull/" in frame.filename + or "/django_ox/" in frame.filename + or index == len(frames) - 1 + ] + path = "".join(traceback.format_list(kept)) + return None, f"{path}{type(exc).__name__}: {exc}" + + +# -- (a) the reconciler ------------------------------------------------ + + +def test_a_scheduled_reconcile_task_completes_a_sealed_workflow(): + """ + The reconciler as a schedule runs it: the oxpull.workflows.reconcile + task enqueued and executed by a django-ox worker. No backend here + defers anything; only the "flows" backend is involved. + """ + with naive(): + flow = workflows.create( + "naive-chain", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + assert drain(flow_worker()) == 2 + assert order() == ["root", "child"] + + tick = workflows.reconcile.enqueue() + drain(Worker(backoff_initial=0, poll_interval=0.01), limit=10) + + row = OxTask.objects.get(pk=tick.id) + assert row.status == OxTask.Status.SUCCESSFUL, ( + f"the reconcile task ended {row.status} after {row.attempts} " + f"attempt(s); last error: {row.errors[-1:]}" + ) + workflow = stored(flow) + assert workflow.status == OxpullWorkflow.Status.COMPLETE + assert workflow.callback_task_id is not None + assert workflow.success_task_id is not None + + +def test_one_reconciler_pass_cancels_a_doomed_cone_and_completes_the_workflow(): + """ + The pass itself, called as the scheduled task calls it, on a workflow + that needs both of the reconciler's jobs: cancel the child of a FAILED + root, then complete the workflow. + """ + with naive(): + flow = workflows.create( + "naive-doomed", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert OxTask.objects.get(args=["root"]).status == OxTask.Status.FAILED + + completed, error = _raised(workflows._reconcile) + assert error is None, f"_reconcile() raised:\n{error}" + assert completed == 1 + assert OxTask.objects.get(pk=child.id).status == OxTask.Status.DISCARDED + workflow = stored(flow) + assert workflow.status == OxpullWorkflow.Status.COMPLETE + assert workflow.cancelled_count == 1 + assert workflow.callback_task_id is not None + + +# -- (b) add() of a node that waits ------------------------------------ + + +def test_add_holds_a_blocked_node_on_the_backend_with_the_leftover_option(): + with naive(): + flow = workflows.create("naive-deferred") + flow.add("parent", tasks.deferred_step, "parent") + child, error = _raised( + lambda: flow.add("child", tasks.deferred_step, "child", depends_on="parent") + ) + assert error is None, f"add() of a blocked node raised:\n{error}" + row = OxTask.objects.get(pk=child.id) + assert (row.status, row.run_after) == (OxTask.Status.WAITING, None), ( + "the child is blocked, so it should wait with no run_after of its own" + ) + flow.seal() + assert drain(deferring_worker()) == 2 + assert order() == ["parent", "child"] + + +# -- (c) the eager release after a SUCCESSFUL write ------------------------ + + +def _finish_upstream_and_report(root_id): + """ + Run one claim on the deferring worker. Return (run_once's value, its + raise path or None, the root's stored status, whether task_finished + was sent for the root). + """ + finished = [] + + def receiver(sender, task_result, **kwargs): + finished.append(str(task_result.id)) + + task_finished.connect(receiver) + try: + ran, error = _raised(deferring_worker().run_once) + finally: + task_finished.disconnect(receiver) + status = OxTask.objects.get(pk=root_id).status + return ran, error, status, str(root_id) in finished + + +def test_a_deferring_worker_finishing_an_upstream_of_another_backends_node(): + """ + Root on the deferring backend, child on the non-deferring "flows" + backend, so add() binds no sentinel and succeeds. The root's success + then reaches release_dependents. + """ + with naive(): + flow = workflows.create("naive-mixed") + root = flow.add("root", tasks.deferred_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + + ran, error, status, signalled = _finish_upstream_and_report(root.id) + assert error is None, ( + f"run_once() raised after the root was written {status} " + f"(task_finished sent: {signalled}):\n{error}" + ) + assert ran is True + assert status == OxTask.Status.SUCCESSFUL + assert signalled, "task_finished was not sent for a task that succeeded" + assert drain(flow_worker()) == 1 + assert order() == ["root", "child"] + + +def test_a_deferring_worker_finishing_an_upstream_of_a_scheduled_node(): + """ + The same release on one deferring backend: the child carries the + caller's own run_after, so add() leaves it alone (a caller's run_after + is never overwritten) and binds nothing. Not a cross-backend corner. + """ + with naive(): + flow = workflows.create("naive-scheduled") + root = flow.add("root", tasks.deferred_step, "root") + flow.add( + "child", + tasks.deferred_step.using(run_after=timezone.now() - timedelta(seconds=1)), + "child", + depends_on="root", + ) + flow.seal() + + ran, error, status, signalled = _finish_upstream_and_report(root.id) + assert error is None, ( + f"run_once() raised after the root was written {status} " + f"(task_finished sent: {signalled}):\n{error}" + ) + assert ran is True + assert status == OxTask.Status.SUCCESSFUL + assert signalled, "task_finished was not sent for a task that succeeded" + assert drain(deferring_worker()) == 1 + assert order() == ["root", "child"] + + +# -- a real USE_TZ=False project, driven by ox_worker ---------------------- + +DRIVER = r""" +import collections +import json +import os +import signal +import subprocess +import sys +import time +import traceback +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +mode, cfg_path = sys.argv[1], sys.argv[2] +cfg = json.loads(Path(cfg_path).read_text()) + +if "mysql" in cfg["database"]["ENGINE"]: + try: + import MySQLdb # noqa: F401 + except ModuleNotFoundError: + import pymysql + + pymysql.install_as_MySQLdb() + +import django +from django.conf import settings + +GATED = {"WORKER_CLASS": "oxpull.worker.OxpullWorker", "WORKFLOWS": True} +settings.configure( + SECRET_KEY="naive-tz", + USE_TZ=False, + TIME_ZONE="UTC", + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + INSTALLED_APPS=["django_ox", "oxpull"], + DATABASES={"default": cfg["database"]}, + TASKS={ + "default": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["default"], + "OPTIONS": { + "SCHEDULES": { + "oxpull-workflow-reconcile": { + "task": "oxpull.workflows.reconcile", + "every": 1, + }, + "oxpull-batch-reconcile": { + "task": "oxpull.batches.reconcile", + "cron": "* * * * *", + }, + }, + }, + }, + "flows": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["flowq"], + "OPTIONS": dict(GATED), + }, + "flows_deferred": { + "BACKEND": "oxpull.backend.OxpullBackend", + "QUEUES": ["deferq"], + "OPTIONS": dict(GATED, DEFER_BLOCKED_NODES=True), + }, + }, + LOGGING={ + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "plain": {"format": "%(levelname)s %(name)s %(message)s"}, + }, + "handlers": { + "stderr": {"class": "logging.StreamHandler", "formatter": "plain"}, + }, + "loggers": { + "django_ox": {"handlers": ["stderr"], "level": "INFO"}, + "oxpull": {"handlers": ["stderr"], "level": "INFO"}, + }, + }, +) + +if mode == "manage": + django.setup() + from django.core.management import execute_from_command_line + + execute_from_command_line(["manage.py", *sys.argv[3:]]) + sys.exit(0) + +name = cfg["database"]["NAME"] +engine = cfg["database"]["ENGINE"] +django.setup() +from django.db import connection + +if "sqlite" not in engine: + extra = "" + if "mysql" in engine: + extra = " CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + quote = connection.ops.quote_name + with connection._nodb_cursor() as cursor: + cursor.execute(f"DROP DATABASE IF EXISTS {quote(name)}") + cursor.execute(f"CREATE DATABASE {quote(name)}{extra}") + +result = {} +procs = [] +try: + import oxpull + from django.core import checks + from django.core.management import call_command + from django_ox.models import OxTask + + result["oxpull_file"] = oxpull.__file__ + result["use_tz"] = settings.USE_TZ + result["vendor"] = connection.vendor + result["check_issues"] = [f"{m.id}: {m.msg}" for m in checks.run_checks()] + call_command("migrate", verbosity=0) + + import naive_tz_tasks as t + from oxpull import workflows + from oxpull.models import OxpullWorkflow + + chain = workflows.create( + "chain", + on_complete=t.record, on_complete_args=["complete"], + on_success=t.record, on_success_args=["success"], + ) + chain.add("root", t.step, "chain-root") + chain.add("child", t.step, "chain-child", depends_on="root") + chain.seal() + + mixed = workflows.create("mixed") + mixed.add("root", t.deferred_step, "mixed-root") + mixed.add("child", t.step, "mixed-child", depends_on="root") + mixed.seal() + + for alias in ("default", "flows", "flows_deferred"): + log = open(HERE / f"worker-{alias}.log", "w") + proc = subprocess.Popen( + [sys.executable, __file__, "manage", cfg_path, "ox_worker", + "--backend", alias, "--interval", "0.2"], + stdout=log, stderr=subprocess.STDOUT, + ) + procs.append((alias, proc, log)) + + def statuses(): + return { + str(pk): status + for pk, status in OxpullWorkflow.objects.values_list("pk", "status") + } + + deadline = time.monotonic() + cfg["seconds"] + started = time.monotonic() + while time.monotonic() < deadline: + now = statuses() + wanted = OxpullWorkflow.Status.COMPLETE + if all(now.get(str(f.id)) == wanted for f in (chain, mixed)): + break + time.sleep(0.25) + result["waited_seconds"] = round(time.monotonic() - started, 1) +finally: + for alias, proc, log in procs: + proc.send_signal(signal.SIGTERM) + for alias, proc, log in procs: + try: + proc.wait(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + log.close() + result.setdefault("exit_codes", {})[alias] = proc.returncode + +try: + def wf(flow): + row = OxpullWorkflow.objects.get(pk=flow.id) + return { + "status": row.status, + "callback_task_id": row.callback_task_id and str(row.callback_task_id), + "success_task_id": row.success_task_id and str(row.success_task_id), + } + + result["chain"] = wf(chain) + result["mixed"] = wf(mixed) + result["nodes"] = { + args[0]: status + for args, status in OxTask.objects.filter( + queue_name__in=["flowq", "deferq"] + ).values_list("args", "status") + } + ticks = list( + OxTask.objects.filter(task_path="oxpull.workflows.reconcile") + .order_by("enqueued_at") + .values_list("status", "errors") + ) + result["reconcile_ticks"] = dict(collections.Counter(s for s, _ in ticks)) + errored = [e for _, e in ticks if e] + result["reconcile_last_error"] = str(errored[-1][-1])[-700:] if errored else None + logs = {} + for alias, proc, log in procs: + text = (HERE / f"worker-{alias}.log").read_text() + lines = text.splitlines() + unhandled = [x for x in lines if "Unhandled error executing task" in x] + logs[alias] = { + "unhandled_error": len(unhandled), + "unhandled_lines": unhandled[:2], + "value_error_lines": [x for x in lines if x.startswith("ValueError")][:3], + "succeeded_lines": [x for x in lines if " succeeded in " in x][:6], + } + result["worker_logs"] = logs + + blocked = workflows.create("blocked-add") + blocked.add("root", t.deferred_step, "blocked-root") + try: + blocked.add("child", t.deferred_step, "blocked-child", depends_on="root") + result["blocked_add"] = "ok" + except Exception as exc: + result["blocked_add"] = f"{type(exc).__name__}: {exc}" + print("NAIVE-TZ-RESULT " + json.dumps(result), flush=True) +finally: + if "sqlite" not in engine: + connection.close() + with connection._nodb_cursor() as cursor: + cursor.execute(f"DROP DATABASE IF EXISTS {connection.ops.quote_name(name)}") +""" + +DRIVER_TASKS = """ +from django.tasks import task + + +@task(backend="flows", queue_name="flowq") +def step(label): + return label + + +@task(backend="flows_deferred", queue_name="deferq") +def deferred_step(label): + return label + + +@task(queue_name="default") +def record(label): + return label +""" + + +@pytest.fixture(scope="module") +def real_project(django_db_setup, tmp_path_factory): + here = tmp_path_factory.mktemp("naive_tz_project") + source = connections["default"].settings_dict + database = { + key: source[key] + for key in ("ENGINE", "USER", "PASSWORD", "HOST", "PORT", "OPTIONS") + if key in source + } + if connections["default"].vendor == "sqlite": + database["NAME"] = str(here / "naive.sqlite3") + database["OPTIONS"] = {"timeout": 20} + else: + database["NAME"] = f"{source['NAME']}_naive" + # "seconds" is a ceiling on three ox_worker interpreters starting cold and + # finishing both workflows. The driver stops waiting the moment both are + # COMPLETE, so a passing run never waits it out. + (here / "config.json").write_text( + json.dumps({"database": database, "seconds": 120}) + ) + (here / "driver.py").write_text(DRIVER) + (here / "naive_tz_tasks.py").write_text(DRIVER_TASKS) + env = {k: v for k, v in os.environ.items() if k != "DJANGO_SETTINGS_MODULE"} + env["PYTHONPATH"] = os.pathsep.join( + filter(None, [str(REPO_ROOT / "src"), env.get("PYTHONPATH")]) + ) + proc = subprocess.run( # noqa: S603 + [sys.executable, str(here / "driver.py"), "drive", str(here / "config.json")], + capture_output=True, + text=True, + env=env, + cwd=here, + # Past the driver's own worst case, so a failing run still reports: + # the wait above, then up to a minute for each worker to stop. + timeout=420, + ) + marker = "NAIVE-TZ-RESULT " + found = [line for line in proc.stdout.splitlines() if line.startswith(marker)] + assert found, ( + f"driver exited {proc.returncode} with no result\n" + f"stdout:\n{proc.stdout[-4000:]}\nstderr:\n{proc.stderr[-4000:]}" + ) + return json.loads(found[-1][len(marker) :]) + + +def _assert_supported(result): + """Preconditions: this is the worktree's oxpull, in a configuration + that raises no error and no workflow or timezone warning.""" + print(json.dumps(result, indent=2)) # noqa: T201 + assert result["use_tz"] is False + assert result["oxpull_file"].startswith(str(REPO_ROOT / "src")), result[ + "oxpull_file" + ] + blocking = [ + issue + for issue in result["check_issues"] + if ".E" in issue.split(":")[0] + or issue.startswith(("django_ox.W001", "oxpull.W002", "oxpull.W003")) + ] + assert not blocking, blocking + + +def test_ox_worker_completes_sealed_workflows_in_a_use_tz_false_project(real_project): + _assert_supported(real_project) + nodes = real_project["nodes"] + assert nodes.get("chain-root") == nodes.get("chain-child") == "SUCCESSFUL", nodes + assert real_project["chain"]["status"] == "COMPLETE", ( + f"after {real_project['waited_seconds']}s of ox_worker with the reconciler " + f"scheduled every second: chain={real_project['chain']} " + f"mixed={real_project['mixed']} reconcile ticks=" + f"{real_project['reconcile_ticks']} last tick error: " + f"{real_project['reconcile_last_error']}" + ) + assert real_project["chain"]["callback_task_id"] + assert real_project["chain"]["success_task_id"] + assert real_project["mixed"]["status"] == "COMPLETE", real_project["mixed"] + + +def test_ox_worker_finishing_a_deferring_upstream_reports_it(real_project): + _assert_supported(real_project) + assert real_project["nodes"].get("mixed-root") == "SUCCESSFUL", real_project[ + "nodes" + ] + deferq = real_project["worker_logs"]["flows_deferred"] + assert deferq["unhandled_error"] == 0, deferq + assert deferq["succeeded_lines"], deferq + + +def test_add_of_a_blocked_node_in_a_use_tz_false_project(real_project): + _assert_supported(real_project) + assert real_project["blocked_add"] == "ok", real_project["blocked_add"] diff --git a/tests_oxpull/test_workflows_hazard_unique_nodes.py b/tests_oxpull/test_workflows_hazard_unique_nodes.py new file mode 100644 index 0000000..2c76e22 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_unique_nodes.py @@ -0,0 +1,628 @@ +""" +A unique task as a workflow node. + +A unique task deduplicates while a task with the same key is READY or +RUNNING: enqueueing it again returns the pending task. Workflow.add() +enqueues through the backend, so a unique node that add() accepted would +be a row that an application call or another workflow already holds, or +a row that a later enqueue of the same call is deduplicated onto. Every +holder's edges gate that one row, and every holder's reconciler can +discard it. `oxpull` 1.2.0 refuses a unique task as a batch or workflow +callback (batches._reject_unique). The tests below also require node +handling to prevent the hazards described in their docstrings. + +What is required: add() refuses a unique task as a node. It decides with +the same metadata the enqueue itself uses (the @unique decorator, or +UNIQUE_TASKS on the backend the task enqueues through), whether or not +anything holds the key, and it refuses before anything happens: no call to +the backend's enqueue(), no task_enqueued signal, no task, lock, node or +edge row, and no change to the workflow row. add_node() checks all of it. + +Every test then drives a real worker (claim_one, execute, run_once), the +reconciler's own body and public APIs, and asserts, separately, the +contract each party was given: + + * a workflow runs a node only after every upstream it declared has + succeeded, and a workflow in which nothing failed runs every node and + completes with on_success; + * an ordinary enqueue that was accepted runs, unless somebody discards it. + +Each test collects every problem before it asserts, so a failure shows the +accepted node and the damage that followed it together. + +Two fixes that look right fail here. One refuses a unique node only when +its key is already held: in the node-first test the key is free when add() +runs, so that fix accepts the node, and the application's later enqueue of +the same call is discarded with it. The other enqueues first and refuses +afterwards: its rollback leaves no rows, but add_node() sees the enqueue() +call and the task_enqueued signal. + +The "not-unique" parameter runs the same shapes with an ordinary task, +which add() must accept. It passed before the refusal existed, which +showed that the assertions can be satisfied and that uniqueness was the +cause of every failure next to it. +""" + +import uuid +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any +from unittest import mock + +import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.tasks import task +from django.tasks.exceptions import TaskException +from django.tasks.signals import task_enqueued +from django.test import override_settings + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.backend import OxpullBackend +from oxpull.models import ( + OxpullUniqueLock, + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, +) +from oxpull.unique import unique +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, order, reconcile, stored + +pytestmark = pytest.mark.django_db + +NOT_UNIQUE = "not-unique" + +# What a refusal may raise. The message must also say the task is unique, +# so an unrelated ValueError or configuration error does not count. +REFUSALS = (ImproperlyConfigured, TaskException, ValueError) + + +# -- tasks ------------------------------------------------------------- +# +# Module-level, as django.tasks requires, and resolved by the worker from +# this module's dotted path. + + +@unique +@task(backend="flows") +def unique_step(label): + return tasks._record(label) + + +@unique +@task(backend="flows_deferred") +def unique_deferred_step(label): + return tasks._record(label) + + +# Unique only through OPTIONS["UNIQUE_TASKS"], which the `shape` fixture +# sets for its "UNIQUE_TASKS" parameter. +@task(backend="flows") +def listed_step(label): + return tasks._record(label) + + +@task(backend="flows_deferred") +def listed_deferred_step(label): + return tasks._record(label) + + +# alias -> (ordinary node, failing node, @unique task, UNIQUE_TASKS task) +BACKENDS = { + "flows": (tasks.step, tasks.failing_step, unique_step, listed_step), + "flows_deferred": ( + tasks.deferred_step, + tasks.failing_deferred_step, + unique_deferred_step, + listed_deferred_step, + ), +} + + +def unique_tasks_on(listed): + """settings.TASKS with UNIQUE_TASKS set per alias, from {alias: [task]}.""" + configured = dict(settings.TASKS) + for alias, listed_tasks in listed.items(): + config = settings.TASKS[alias] + configured[alias] = { + **config, + "OPTIONS": { + **config["OPTIONS"], + "UNIQUE_TASKS": [each.module_path for each in listed_tasks], + }, + } + return configured + + +@dataclass(frozen=True) +class Shape: + alias: str + marking: str + step: Any + failing: Any + # The task each test is about: unique, except in the control. + shared: Any + + def worker(self): + return OxpullWorker( + backend_alias=self.alias, backoff_initial=0, poll_interval=0.01 + ) + + +@pytest.fixture(params=["flows", "flows_deferred"]) +def backend_alias(request): + return request.param + + +@pytest.fixture(params=["decorator", "UNIQUE_TASKS", NOT_UNIQUE]) +def shape(request, backend_alias): + step, failing, decorated, listed = BACKENDS[backend_alias] + marking = request.param + if marking != "UNIQUE_TASKS": + shared = decorated if marking == "decorator" else step + yield Shape(backend_alias, marking, step, failing, shared) + return + # Each listed task is listed on its own alias only, so a refusal that + # reads UNIQUE_TASKS from one fixed backend fails on the other alias. + listed_on_own_alias = { + alias: [listed_task] for alias, (*_, listed_task) in BACKENDS.items() + } + with override_settings(TASKS=unique_tasks_on(listed_on_own_alias)): + yield Shape(backend_alias, marking, step, failing, listed) + + +# -- helpers ----------------------------------------------------------- + + +@contextmanager +def enqueues_of(node_task): + """ + Collect, as descriptions, every enqueue of `node_task` while the block + runs: calls to the backend's enqueue() and task_enqueued signals. + """ + seen = [] + path = node_task.module_path + original = OxpullBackend.enqueue + + def spy(backend, enqueued_task, args, kwargs): + if enqueued_task.module_path == path: + seen.append(f"a call to the {backend.alias!r} backend's enqueue()") + return original(backend, enqueued_task, args, kwargs) + + def receiver(sender, task_result, **kwargs): + if task_result.task.module_path == path: + seen.append(f"a task_enqueued signal for row {str(task_result.id)[:8]}") + + task_enqueued.connect(receiver, weak=False) + try: + with mock.patch.object(OxpullBackend, "enqueue", spy): + yield seen + finally: + task_enqueued.disconnect(receiver) + + +def footprint(flow): + """Everything a refused add() must leave exactly as it found it.""" + return { + "task rows": OxTask.objects.count(), + "unique lock rows": OxpullUniqueLock.objects.count(), + "node rows": OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count(), + "edge rows": OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count(), + "workflow row": OxpullWorkflow.objects.filter(pk=flow.id).values().get(), + } + + +def add_node(problems, shape, flow, name, node_task, *args, **kwargs): + """ + flow.add() of the task the test is about. Returns its result, or None + when add() refused it. + + An ordinary task must be accepted. A unique task must be refused, and + refused before anything happened; anything else is appended to + `problems`. + """ + if shape.marking == NOT_UNIQUE: + result = flow.add(name, node_task, *args, **kwargs) + assert flow.node_ids()[name] == uuid.UUID(str(result.id)) + return result + + where = f"{flow.name}.add({name!r}, {node_task.module_path})" + before = footprint(flow) + with enqueues_of(node_task) as enqueued: + try: + result = flow.add(name, node_task, *args, **kwargs) + except REFUSALS as exc: + refusal = exc + else: + refusal = None + if refusal is None: + problems.append( + f"{where} accepted a unique task as a node, as row " + f"{str(result.id)[:8]}; add() must refuse it" + ) + return result + if "unique" not in str(refusal).lower(): + problems.append( + f"{where} raised {type(refusal).__name__} without saying the task " + f"is unique: {refusal}" + ) + if enqueued: + problems.append( + f"{where} refused only after {' and '.join(enqueued)}; it must " + "refuse before enqueueing anything" + ) + after = footprint(flow) + changed = [ + f"{label} {before[label]} became {after[label]}" + for label in before + if after[label] != before[label] + ] + if changed: + problems.append(f"{where} refused, but changed: " + "; ".join(changed)) + return None + + +def settle(worker, passes=4): + """Drain and reconcile until nothing moves: a fleet plus the schedule.""" + for _ in range(passes): + drain(worker) + reconcile() + + +def _row(task_id): + return OxTask.objects.filter(pk=task_id).first() + + +def _state(task_id): + row = _row(task_id) + if row is None: + return "absent" + return f"{row.status} attempts={row.attempts}" + + +def describe(*flows): + """Every workflow row and node, with a row id prefix to show sharing.""" + lines = [] + for flow in flows: + row = stored(flow) + lines.append( + f"{row.name}: status={row.status} failed_count={row.failed_count} " + f"cancelled_count={row.cancelled_count} " + f"on_success={'enqueued' if row.success_task_id else 'not enqueued'}" + ) + for name, task_id in flow.node_ids().items(): + lines.append(f" {name} -> row {str(task_id)[:8]} {_state(task_id)}") + lines.append(f"execution order: {order()}") + return "\n".join(lines) + + +def ran_past_an_unsucceeded_upstream(flow): + """ + Edges of `flow` whose downstream row has been claimed although its + upstream row exists and has not succeeded: this workflow's own ordering + contract, broken. + """ + broken = [] + edges = OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).values_list( + "upstream_task_id", "downstream_task_id" + ) + for upstream_id, downstream_id in edges: + up, down = _row(upstream_id), _row(downstream_id) + if down is None or down.attempts == 0 or up is None: + continue + if up.status != OxTask.Status.SUCCESSFUL: + broken.append( + f"{flow.name}: row {str(downstream_id)[:8]} was claimed " + f"({down.status}) while its upstream is {up.status}" + ) + return broken + + +def not_run_clean(flow): + """ + What is wrong with `flow`, which has no failing node of its own: every + node should have succeeded and the workflow completed with on_success. + """ + row = stored(flow) + problems = [] + if row.status != OxpullWorkflow.Status.COMPLETE: + problems.append(f"{row.name} is {row.status}, not COMPLETE") + else: + if row.failed_count or row.cancelled_count: + problems.append( + f"{row.name} completed with failed_count={row.failed_count} " + f"cancelled_count={row.cancelled_count}" + ) + if row.success_task_id is None: + problems.append(f"{row.name}'s on_success was not enqueued") + for name, task_id in flow.node_ids().items(): + node = _row(task_id) + if node is None or node.status != OxTask.Status.SUCCESSFUL: + problems.append(f"{row.name}.{name} is {_state(task_id)}") + return problems + + +def not_run_for_the_application(queued): + """ + What is wrong with a call the application enqueued on its own: it + should have run, on a row that belongs to no workflow. + """ + problems = [] + holders = OxpullWorkflowNode.objects.filter(task_id=queued.id).values_list( + "workflow_id", "name" + ) + for workflow_id, name in holders: + workflow_name = OxpullWorkflow.objects.get(pk=workflow_id).name + problems.append( + f"the application's own enqueue, row {queued.id[:8]}, is also " + f"node {workflow_name}.{name}" + ) + own = _row(queued.id) + if own is None or own.status != OxTask.Status.SUCCESSFUL: + problems.append( + f"the application's own enqueue, row {queued.id[:8]}, is " + f"{_state(queued.id)}: it never ran" + ) + return problems + + +def report(problems, *flows, extra=()): + return "\n".join([*problems, "", *extra, describe(*flows)]) + + +# -- controls ---------------------------------------------------------- + + +def test_each_parameter_marks_the_task_as_it_says(shape): + """ + A control, and it passes today. It pins that the unique parameters + really deduplicate and the not-unique one really does not, and what + 1.2.0 already refuses: a unique task as a workflow callback. + """ + first = shape.shared.enqueue("key") + second = shape.shared.enqueue("key") + if shape.marking == NOT_UNIQUE: + assert first.id != second.id + return + assert first.id == second.id + with pytest.raises(ImproperlyConfigured, match="unique"): + workflows.create("callback", on_complete=shape.shared, on_complete_args=["k"]) + + +def test_add_accepts_a_task_that_is_unique_only_on_another_backend(): + """ + A control, and it passes today. UNIQUE_TASKS belongs to one backend, and + so does the enqueue's deduplication. listed_step is listed on "flows" + only, so through "flows_deferred" it is an ordinary task: its enqueue + writes a new row every time, and add() must accept it there. A refusal + that asks whether any backend lists the task refuses it, and fails here. + """ + with override_settings(TASKS=unique_tasks_on({"flows": [listed_step]})): + elsewhere = listed_step.using(backend="flows_deferred") + # Preconditions: unique on its own backend, ordinary on the other. + assert listed_step.enqueue("key").id == listed_step.enqueue("key").id + assert elsewhere.enqueue("key").id != elsewhere.enqueue("key").id + + flow = workflows.create("elsewhere") + first = flow.add("first", elsewhere, "node") + second = flow.add("second", elsewhere, "node", depends_on="first") + + assert first.id != second.id + assert flow.node_ids() == { + "first": uuid.UUID(first.id), + "second": uuid.UUID(second.id), + } + + +# -- two workflows share one unique node -------------------------------- + + +def test_a_failure_in_one_workflow_does_not_cancel_a_node_of_another(shape): + """ + doomed: bad (fails) -> shared. healthy: shared -> after. + + Both add the same unique call. Today healthy's `shared` is doomed's row, + held by doomed's edge from `bad`. When `bad` fails, doomed's + cancellation discards that row, and healthy's reconciler then cancels + `after` because its upstream is DISCARDED. Nothing in healthy failed, + yet none of its nodes runs. It completes with failed_count=0 and + on_success withheld, and nothing says which workflow did it. + """ + problems = [] + doomed = workflows.create("doomed") + doomed.add("bad", shape.failing, "bad") + add_node( + problems, shape, doomed, "shared", shape.shared, "shared", depends_on="bad" + ) + doomed.seal() + + healthy = workflows.create( + "healthy", on_success=tasks.record_callback, on_success_args=["healthy"] + ) + shared = add_node(problems, shape, healthy, "shared", shape.shared, "shared") + healthy.add("after", shape.step, "after", depends_on="shared" if shared else None) + healthy.seal() + + settle(shape.worker()) + + problems += not_run_clean(healthy) + ran_past_an_unsucceeded_upstream(doomed) + assert not problems, report(problems, doomed, healthy) + + +def test_an_unsealed_workflows_failure_does_not_hang_another_workflow(shape): + """ + stalled (never sealed): bad (fails) -> shared. waiting: shared -> after. + + The workflow that owns the failure is OPEN, because the process building + it died before seal(). Its nodes run anyway. The reconciler sweeps + SEALED workflows only, so nothing cancels the shared row. waiting's own + sweep finds no dead node among its own nodes, because `bad` is not + one of them. Today the shared row stays READY and gated forever, waiting + stays SEALED forever, and no callback fires. + """ + problems = [] + stalled = workflows.create("stalled") + stalled.add("bad", shape.failing, "bad") + add_node( + problems, shape, stalled, "shared", shape.shared, "shared", depends_on="bad" + ) + + waiting = workflows.create( + "waiting", on_success=tasks.record_callback, on_success_args=["waiting"] + ) + shared = add_node(problems, shape, waiting, "shared", shape.shared, "shared") + waiting.add("after", shape.step, "after", depends_on="shared" if shared else None) + waiting.seal() + + settle(shape.worker()) + + problems += not_run_clean(waiting) + ran_past_an_unsucceeded_upstream(stalled) + progress = waiting.progress() + assert not problems, report( + problems, + stalled, + waiting, + extra=[ + ( + f"waiting.progress(): total={progress.total} " + f"pending={progress.pending} blocked={progress.blocked}" + ) + ], + ) + + +# -- a unique node and the application's own enqueue of the same call --- + + +def test_a_workflow_does_not_cancel_a_task_the_application_enqueued(shape): + """ + The application comes first. It enqueues the unique call on its own, + say from a view. A workflow then adds the same call behind an upstream + that fails. + + Today add() returns the application's pending row. The workflow's edge + gates it, so it waits on `bad`'s retries (under DEFER_BLOCKED_NODES + add() also writes DEFERRED_UNTIL onto it). When `bad` fails, the + reconciler discards it. The task the application enqueued, which was + never part of any workflow, never runs. + """ + problems = [] + queued = shape.shared.enqueue("shared") + + flow = workflows.create("adopter") + flow.add("bad", shape.failing, "bad") + add_node(problems, shape, flow, "shared", shape.shared, "shared", depends_on="bad") + flow.seal() + + settle(shape.worker()) + + problems += ran_past_an_unsucceeded_upstream(flow) + problems += not_run_for_the_application(queued) + assert not problems, report( + problems, flow, extra=[f"application enqueue: row {queued.id[:8]}"] + ) + + +def test_an_application_enqueue_made_after_a_unique_node_was_added_runs(shape): + """ + The node comes first. A workflow adds a unique call behind an upstream + that fails, while nothing holds that call's key. Later the application + enqueues the same call on its own. + + Today add() takes the free key for the node. The application's enqueue + is deduplicated onto the node's row and returns the node's id, so the + application's call waits behind `bad`, a node of a workflow it never + joined. When `bad` fails, the reconciler discards that row with 0 + attempts, and the application's call never runs. + + A refusal that checks whether the key is already held accepts this + node, because the key is free when add() runs, and fails here twice: at + add(), and when the application's call is discarded. + """ + problems = [] + flow = workflows.create("first") + flow.add("bad", shape.failing, "bad") + add_node(problems, shape, flow, "shared", shape.shared, "shared", depends_on="bad") + flow.seal() + + queued = shape.shared.enqueue("shared") + + settle(shape.worker()) + + problems += ran_past_an_unsucceeded_upstream(flow) + problems += not_run_for_the_application(queued) + assert not problems, report( + problems, flow, extra=[f"application enqueue: row {queued.id[:8]}"] + ) + + +def test_a_node_does_not_adopt_an_execution_that_began_before_its_upstream( + shape, +): + """ + extract -> transform (unique) -> load, where a worker has already + claimed the application's own `transform` call when the workflow is + built. + + Today add() returns that RUNNING row as the `transform` node. Its + execution began before `extract` existed and finishes SUCCESSFUL, which + satisfies load's edge, so `load` runs before `extract`. The workflow + then completes clean, with on_success enqueued. + """ + queued = shape.shared.enqueue("transform") + worker = shape.worker() + in_flight = worker.claim_one() + # Precondition, true under any fix: no workflow exists yet. + assert in_flight is not None + assert uuid.UUID(str(in_flight.id)) == uuid.UUID(queued.id) + + problems = [] + flow = workflows.create( + "late", on_success=tasks.record_callback, on_success_args=["late"] + ) + flow.add("extract", shape.step, "extract") + transform = add_node( + problems, + shape, + flow, + "transform", + shape.shared, + "transform", + depends_on="extract", + ) + flow.add( + "load", + shape.step.using(priority=10), + "load", + depends_on="transform" if transform else "extract", + ) + flow.seal() + + # The rest of what run_once() does with the row it claimed. + worker.execute(in_flight, inline=True) + nodes = flow.node_ids() + extract = _row(nodes["extract"]) + if transform is not None: + node = _row(nodes["transform"]) + if node.attempts and extract.status != OxTask.Status.SUCCESSFUL: + problems.append( + f"late.transform is {node.status} (attempts={node.attempts}) " + f"while its upstream extract is {extract.status} " + f"(attempts={extract.attempts})" + ) + + settle(worker) + + ran = order() + if "load" not in ran or "extract" not in ran: + problems.append(f"load or extract never ran: {ran}") + elif ran.index("load") < ran.index("extract"): + problems.append(f"load ran before extract: {ran}") + problems += not_run_clean(flow) + assert not problems, report( + problems, flow, extra=[f"application enqueue: row {queued.id[:8]}"] + ) diff --git a/tests_oxpull/test_workflows_hazard_unique_nodes_let_two_workflows_build_a_dependency.py b/tests_oxpull/test_workflows_hazard_unique_nodes_let_two_workflows_build_a_dependency.py new file mode 100644 index 0000000..d494bd3 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_unique_nodes_let_two_workflows_build_a_dependency.py @@ -0,0 +1,253 @@ +""" +Unique nodes let two workflows build a dependency cycle. + +Workflow.add() documents that a cycle is unconstructible because every +edge points backwards in insertion order. That holds per workflow. The +gate, though, reads the global edge table keyed by task id, and a unique +task deduplicates to whichever row already holds its key. So: + + W1: a = unique_echo(1) -> T1 + b = unique_add(1, 2) after a -> T2, edge T1 -> T2 + W2: x = unique_add(1, 2) -> dedups to T2 + y = unique_echo(1) after x -> dedups to T1, edge T2 -> T1 + +T1 and T2 now each wait on the other. Neither is dead, so the reconciler +cancels nothing, and neither workflow ever completes. + +Every test asserts the correct behaviour and accepts either fix: add() +refuses unique nodes (every one, as it now does, or only the one that +would close the cycle), or the workflows run to completion anyway. +A refusal counts only when it says the task is unique. Both outcomes +are checked through the public API, the real claim path and the real +reconciler pass. Accepting the nodes without preventing the cycle fails. +""" + +import threading +import time + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.db import connections +from django.tasks.exceptions import TaskException + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, flow_worker, reconcile, stored +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +# A refusal at add() is a correct fix, so it is caught. Nothing broader +# is: a database error must still fail the test. +REFUSALS = (TaskException, ImproperlyConfigured, ValueError) + + +def edge_cycle(): + """A cycle in the global edge table, as a list of task ids, or None.""" + graph = {} + for up, down in OxpullWorkflowEdge.objects.values_list( + "upstream_task_id", "downstream_task_id" + ): + graph.setdefault(up, set()).add(down) + state, path = {}, [] + + def visit(node): + state[node] = "open" + path.append(node) + for nxt in graph.get(node, ()): + if state.get(nxt) == "open": + return [*path[path.index(nxt) :], nxt] + if nxt not in state: + found = visit(nxt) + if found: + return found + path.pop() + state[node] = "done" + return None + + for node in list(graph): + if node not in state: + found = visit(node) + if found: + return found + return None + + +def settle(worker, rounds=5): + """ + What a production install does while it waits: claim and run + whatever the gate lets through, reap, and tick the reconciler. + """ + for _ in range(rounds): + drain(worker) + worker.reap() + reconcile() + + +def describe(flows): + lines = [f"edge cycle: {edge_cycle()}"] + for flow in flows: + row = stored(flow) + lines.append(f"{flow.name}: {row.status} {flow.progress()}") + rows = OxTask.objects.values_list("task_path", "args", "status", "run_after") + lines.append(f"task rows: {sorted(rows, key=str)}") + return "\n".join(lines) + + +def assert_every_workflow_completed(flows): + detail = describe(flows) + assert edge_cycle() is None, ( + "the global edge table holds a cycle, which add() documents as " + f"unconstructible\n{detail}" + ) + for flow in flows: + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE, ( + f"{flow.name} never completes: its nodes wait on each other " + f"across workflows and none of them is dead\n{detail}" + ) + assert row.callback_task_id is not None, ( + f"{flow.name} completed without enqueueing on_complete\n{detail}" + ) + + +def refused_for_uniqueness(refusals): + """Every refusal add() raised must say the task is unique.""" + others = [exc for exc in refusals if "unique" not in str(exc).lower()] + assert not others, f"add() refused a node for another reason: {others!r}" + + +def try_add(flow, refusals, name, node_task, *args, depends_on=None): + """flow.add(), with a refusal recorded rather than raised. None if refused.""" + try: + return flow.add(name, node_task, *args, depends_on=depends_on) + except REFUSALS as exc: + refusals.append(exc) + return None + + +def build_opposite_order(unique_a, unique_b): + """The hazard's own sequence. Returns (w1, w2, refusals).""" + refusals = [] + w1 = workflows.create( + "w1", on_complete=tasks.record_callback, on_complete_args=["w1"] + ) + a = try_add(w1, refusals, "a", unique_a, 1) + b = try_add(w1, refusals, "b", unique_b, 1, 2, depends_on="a" if a else None) + w2 = workflows.create( + "w2", on_complete=tasks.record_callback, on_complete_args=["w2"] + ) + x = try_add(w2, refusals, "x", unique_b, 1, 2) + y = try_add(w2, refusals, "y", unique_a, 1, depends_on="x" if x else None) + if None not in (a, b, x, y): + # Not a precondition of the fix, but proof the run provoked the + # hazard rather than passing because dedup never happened. + tasks.STATE["dedup"] = (str(x.id) == str(b.id), str(y.id) == str(a.id)) + w1.seal() + w2.seal() + refused_for_uniqueness(refusals) + return w1, w2, refusals + + +def test_two_workflows_sharing_unique_nodes_in_opposite_order_both_complete(): + w1, w2, _refusals = build_opposite_order( + tasks.unique_echo.using(backend="flows"), + tasks.unique_add.using(backend="flows"), + ) + worker = flow_worker() + if OxpullWorkflowNode.objects.exists(): + # add() accepted a node, so a worker on the gated backend must be + # able to start on one. + first = worker.claim_one() + assert first is not None, ( + "a worker on the gated backend can claim nothing: every node of " + f"both workflows waits on another\ndedup={tasks.STATE.get('dedup')}\n" + f"{describe([w1, w2])}" + ) + worker.execute(first, inline=True) + settle(worker) + assert_every_workflow_completed([w1, w2]) + + +def test_a_ring_of_three_workflows_sharing_unique_nodes_all_complete(): + """ + The same hazard one workflow wider, so a fix that only looks for the + reverse of the new edge does not pass. + + W1: echo(1) -> echo(2) + W2: echo(2) -> echo(3) + W3: echo(3) -> echo(1) + """ + node = tasks.unique_echo.using(backend="flows") + flows, refusals = [], [] + for i in range(3): + head, tail = i + 1, (i + 1) % 3 + 1 + flow = workflows.create( + f"ring-{i}", on_complete=tasks.record_callback, on_complete_args=[i] + ) + if try_add(flow, refusals, "head", node, head): + try_add(flow, refusals, "tail", node, tail, depends_on="head") + flow.seal() + flows.append(flow) + refused_for_uniqueness(refusals) + settle(flow_worker()) + assert_every_workflow_completed(flows) + + +def test_the_same_cycle_on_the_deferring_backend_both_complete(): + """ + DEFER_BLOCKED_NODES takes a different path through add(): + _defer_if_blocked runs on the id the enqueue returned. On a dedup hit + that is another workflow's committed, visible row, and today it + defers T1, which had been claimable, so the claim never even sees + either node and the release sweep never clears either. + """ + w1, w2, _refusals = build_opposite_order( + tasks.unique_echo.using(backend="flows_deferred"), + tasks.unique_add.using(backend="flows_deferred"), + ) + settle(deferring_worker()) + assert_every_workflow_completed([w1, w2]) + + +@pytest.mark.django_db(transaction=True) +def test_a_running_worker_completes_both_workflows_sharing_unique_nodes(): + """ + The real run() loop in its own thread: its own connection, its own + reap on the first pass, the gated claim. The reconciler ticks beside + it, as the schedule would on another worker. + """ + w1, w2, _refusals = build_opposite_order( + tasks.unique_echo.using(backend="flows"), + tasks.unique_add.using(backend="flows"), + ) + worker = OxpullWorker(backend_alias="flows", backoff_initial=0, poll_interval=0.01) + + def run(): + try: + worker.run() + finally: + connections.close_all() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + try: + # A ceiling: the loop leaves as soon as both workflows are COMPLETE. + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + reconcile() + if all( + stored(flow).status == OxpullWorkflow.Status.COMPLETE + for flow in (w1, w2) + ): + break + time.sleep(0.1) + finally: + worker.request_stop() + thread.join(timeout=30) + assert not thread.is_alive() + assert_every_workflow_completed([w1, w2]) diff --git a/tests_oxpull/test_workflows_hazard_unknown_status.py b/tests_oxpull/test_workflows_hazard_unknown_status.py new file mode 100644 index 0000000..fc2933e --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_unknown_status.py @@ -0,0 +1,305 @@ +""" +A node status the workflow reconciler does not know. + +Workflow completion and progress() must not decide finished-ness by +treating every status outside PENDING_STATUSES (READY, RUNNING) as +finished. `oxpull` 1.2.0's batches use explicit SETTLED_STATUSES, so a +status the package has never seen holds a batch open +(tests_oxpull/test_batches.py, +test_a_status_the_reconciler_does_not_know_holds_the_batch_open). +The tests here require the same conservative handling for workflows. + +How a row gets a status this package does not know, in production. With +the current django-ox task statuses; no supported code path writes an unknown one: every +django-ox ref carries the same six statuses, and the admin cannot edit +status. The sequences that do produce it are: + +- a rolling upgrade to a later django-ox release that adds a status. + The new processes write it while a process still on this version runs + the scheduled reconciler (any worker in the fleet can pick that task up); +- a rollback after such an upgrade, which leaves the new rows behind, + because migrations do not rewrite statuses; +- an out-of-band write (SQL, a custom operator tool). + +None of those can be driven from this version's code, so the one shortcut +in this file is the UPDATE that writes the unknown value. Everything around +it uses the real paths: nodes run, fail and succeed under a real +OxpullWorker, the LOST row comes from the real reaper after +actions.expire_lease, and the DISCARDED row from actions.discard. +""" + +import threading +import time + +import pytest +from django.db import connections + +from django_ox import actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflow + +from . import tasks +from .test_workflows import drain, flow_worker, order, reconcile, stored + +pytestmark = pytest.mark.django_db + +# Not a status in django-ox 1.2.0. Ten characters at most, the status +# column's max_length, so PostgreSQL and strict-mode MySQL store it exactly +# as SQLite does. The same value the batches test uses. +UNKNOWN = "ZOMBIE" + +# What a sealed one-node workflow must do at the next reconciler pass, for +# each status its node can be in. A status missing from this table is a +# decision nobody has taken yet, and test 1 fails on it. +COMPLETES_THE_WORKFLOW = { + OxTask.Status.READY: False, + OxTask.Status.RUNNING: False, + OxTask.Status.SUCCESSFUL: True, + OxTask.Status.FAILED: True, + OxTask.Status.LOST: True, + OxTask.Status.DISCARDED: True, + OxTask.Status.WAITING: False, + UNKNOWN: False, +} + + +def callback_flow(name): + """An OPEN workflow carrying both callbacks, so a completion is visible.""" + return workflows.create( + name, + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + + +def write_unknown_status(task_id, from_status): + """ + The shortcut: the write a process on a later release would make. + + Compare-and-set on the status the row is in, the shape django-ox's own + actions use, so it cannot land on a row the test did not expect. + """ + return OxTask.objects.filter(pk=task_id, status=from_status).update(status=UNKNOWN) + + +def put_node_in(status, task_id): + """Drive a one-node workflow's node into `status` through the real path.""" + worker = flow_worker() + if status == OxTask.Status.READY: + pass + elif status == OxTask.Status.RUNNING: + claimed = worker.claim_one() + assert claimed is not None and str(claimed.pk) == str(task_id) + elif status == OxTask.Status.SUCCESSFUL: + assert drain(worker) == 1 + elif status == OxTask.Status.FAILED: + assert drain(worker) == 3 # MAX_ATTEMPTS on "flows" + elif status == OxTask.Status.LOST: + # Two failed attempts, then a third claimed and never finished: the + # reaper's attempts-exhausted branch, with a real 1.2.0 lease. + assert worker.run_once() is True + assert worker.run_once() is True + claimed = worker.claim_one() + assert claimed is not None and str(claimed.pk) == str(task_id) + assert claimed.attempts == claimed.max_attempts + assert actions.expire_lease(task_id) is True + assert worker.reap() == 1 + elif status == OxTask.Status.DISCARDED: + assert actions.discard(task_id) is True + elif status == OxTask.Status.WAITING: + # A one-node workflow's node has no upstream, so add() enqueues it + # READY. This write gives it the status add() gives a node that + # waits for one. + assert ( + OxTask.objects.filter(pk=task_id, status=OxTask.Status.READY).update( + status=OxTask.Status.WAITING + ) + == 1 + ) + else: + assert write_unknown_status(task_id, OxTask.Status.READY) == 1 + assert OxTask.objects.get(pk=task_id).status == status + + +def callback_rows(): + return OxTask.objects.filter(task_path="tests_oxpull.tasks.record_callback").count() + + +# -- the contract with django-ox --------------------------------------- + + +def test_workflow_status_sets_cover_every_django_ox_status(): + """ + The workflow reconciler classifies a node status three ways: pending + (holds completion), SUCCESSFUL (satisfies the gate), dead (cancels the + downstream cone). The three sets must be disjoint and cover every status + django-ox defines, and the table above must name every status. A new + django-ox status must fail this test until the sets and table account + for it. + + Completion reads SETTLED_STATUSES, shared with batches: a status + outside it is pending. SETTLED_STATUSES must be exactly the gate's + SUCCESSFUL plus the cancellation's dead statuses, and together with + PENDING_STATUSES it must cover the enum. This is the batches contract + (test_the_reconciler_knows_every_django_ox_status). Set coverage alone + cannot detect completion treating every status outside + PENDING_STATUSES as finished; the tests below catch that behaviour. + """ + pending = set(workflows.PENDING_STATUSES) + settled = set(workflows.SETTLED_STATUSES) + succeeded = {OxTask.Status.SUCCESSFUL} + dead = set(workflows.DEAD_STATUSES) + assert pending & succeeded == set() + assert pending & dead == set() + assert succeeded & dead == set() + assert pending | succeeded | dead == set(OxTask.Status) + assert pending & settled == set() + assert pending | settled == set(OxTask.Status) + assert settled == succeeded | dead + assert set(workflows.UNATTRIBUTED_STATUSES) <= dead + assert set(OxTask.Status) <= set(COMPLETES_THE_WORKFLOW) + assert UNKNOWN not in set(OxTask.Status) + + +@pytest.mark.parametrize("status", list(COMPLETES_THE_WORKFLOW), ids=str) +def test_every_node_status_has_one_completion_reading(status): + """ + Read the same contract through reconcile() rather than the module's + constants: seal a one-node workflow, drive its node into each status, + then run one reconciler pass. The six known django-ox statuses guard + against over-correction: LOST and DISCARDED must still complete. + The unknown status must hold the workflow open. + """ + flow = workflows.create(f"contract-{status.lower()}") + task = ( + tasks.failing_step + if status in (OxTask.Status.FAILED, OxTask.Status.LOST) + else tasks.step + ) + node = flow.add("only", task, "only") + flow.seal() + put_node_in(status, node.id) + + completes = COMPLETES_THE_WORKFLOW[status] + observed = (reconcile(), stored(flow).status) + expected = ( + 1 if completes else 0, + OxpullWorkflow.Status.COMPLETE if completes else OxpullWorkflow.Status.SEALED, + ) + assert observed == expected + + +# -- the hazard ---------------------------------------------------------- + + +def test_a_sealed_workflow_whose_node_status_is_unknown_stays_sealed(): + """ + A node whose row carries a status this package cannot read + must not count as finished, and must not be attributed to on_success. + It stays pending, and the workflow stays SEALED, until a release that + knows the status reads it. Every observable is collected before the one + comparison, so a failure shows what the reconciler did. + """ + flow = callback_flow("unknown-status") + node = flow.add("only", tasks.step, "only") + flow.seal() + put_node_in(UNKNOWN, node.id) + + progress = flow.progress() + completed = reconcile() + row = stored(flow) + observed = { + "progress (total, pending, finished)": ( + progress.total, + progress.pending, + progress.finished, + ), + "reconcile() completed": completed, + "workflow status": row.status, + "on_complete enqueued": row.callback_task_id is not None, + "on_success enqueued": row.success_task_id is not None, + "callback task rows": callback_rows(), + } + assert observed == { + "progress (total, pending, finished)": (1, 1, 0), + "reconcile() completed": 0, + "workflow status": OxpullWorkflow.Status.SEALED, + "on_complete enqueued": False, + "on_success enqueued": False, + "callback task rows": 0, + } + + +@pytest.mark.django_db(transaction=True) +def test_an_unknown_status_over_a_running_leaf_does_not_complete_the_workflow(): + """ + The mid-upgrade shape, with a real worker thread holding a real lease. + + root -> leaf. root succeeds under a real OxpullWorker. A second worker, + in its own thread, claims leaf and starts executing it. While it runs, + a process on a later release writes a status this one does not know + onto the RUNNING row. The holder's finish write is then refused + (WRITABLE_STATUSES is RUNNING and LOST), so the row keeps the unknown + status although the task body ran. No SUCCESSFUL was ever recorded for + leaf, so on_success ("the run was clean") must not fire, and the + workflow must not complete while leaf's status is unreadable. + """ + flow = callback_flow("unknown-under-lease") + flow.add("root", tasks.step, "root") + leaf = flow.add("leaf", tasks.slow_step, "leaf", seconds=1.5, depends_on="root") + flow.seal() + + assert flow_worker().run_once() is True + assert order() == ["root"] + + outcome = {} + + def run_leaf(): + try: + outcome["ran"] = flow_worker().run_once() + except Exception as exc: # pragma: no cover - reported below + outcome["error"] = exc + finally: + connections.close_all() + + holder = threading.Thread(target=run_leaf) + holder.start() + deadline = time.monotonic() + 10 + while not OxTask.objects.filter(pk=leaf.id, status=OxTask.Status.RUNNING).exists(): + assert time.monotonic() < deadline, "leaf was never claimed" + time.sleep(0.01) + assert write_unknown_status(leaf.id, OxTask.Status.RUNNING) == 1 + holder.join(timeout=30) + + # Preconditions: the holder ran the body and its finish write was refused. + assert not holder.is_alive() + assert "error" not in outcome + assert order() == ["root", "leaf"] + assert OxTask.objects.get(pk=leaf.id).status == UNKNOWN + + progress = flow.progress() + completed = reconcile() + row = stored(flow) + observed = { + "progress (total, pending, succeeded)": ( + progress.total, + progress.pending, + progress.succeeded, + ), + "reconcile() completed": completed, + "workflow status": row.status, + "on_complete enqueued": row.callback_task_id is not None, + "on_success enqueued": row.success_task_id is not None, + "callback task rows": callback_rows(), + } + assert observed == { + "progress (total, pending, succeeded)": (2, 1, 1), + "reconcile() completed": 0, + "workflow status": OxpullWorkflow.Status.SEALED, + "on_complete enqueued": False, + "on_success enqueued": False, + "callback task rows": 0, + } diff --git a/tests_oxpull/test_workflows_hazard_with_defer_blocked_nodes_releasing_a_node.py b/tests_oxpull/test_workflows_hazard_with_defer_blocked_nodes_releasing_a_node.py new file mode 100644 index 0000000..a6f1ae6 --- /dev/null +++ b/tests_oxpull/test_workflows_hazard_with_defer_blocked_nodes_releasing_a_node.py @@ -0,0 +1,234 @@ +""" +A released workflow node must not read as backlog for the whole time it +was blocked. + +django-ox measures a waiting task's age from the moment it became +eligible: Coalesce(run_after, enqueued_at) in stats.oldest_ready_age, +the django_ox_oldest_ready_age_seconds gauge, and ox_health --max-age. +Its own re-eligibility writer, the retry path, writes run_after=now+delay +so the clock restarts. A blocked node used to carry a far-future run_after, +and its release wrote run_after=NULL, so its age fell back to enqueued_at: +the moment add() ran, before the upstream chain executed. + +A blocked node now waits in django-ox's WAITING status, which no backlog +reading counts, and the release writes run_after as the later of now and +the time the caller asked for, through the worker's release and through +the reconciler's backstop alike. + +Every test asserts that a node that just became claimable reads as just +eligible. The first test is the control, django-ox's own retry. +""" + +import re +import time +from datetime import timedelta +from io import StringIO + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import DatabaseError +from django.db.models.functions import Coalesce +from django.tasks import task +from django.utils import timezone + +from django_ox import metrics, stats +from django_ox.models import OxTask +from oxpull import workflows + +from . import tasks +from .test_workflows_deferred import deferring_worker + +pytestmark = pytest.mark.django_db + +# How long the upstream chain took, standing in for a long extract. +BLOCKED_FOR = timedelta(hours=2) +# How far apart the database server's clock and the worker's may be. +CLOCK_SKEW = timedelta(seconds=1) +# An ox_health threshold an operator pages on (docs/monitoring.md's cron +# example uses 600); far above any real claim latency in these tests. +MAX_AGE_SECONDS = 300 + + +@task(backend="flows_deferred") +def slow_deferred_step(label, seconds): + time.sleep(seconds) + return tasks._record(label) + + +def order(): + return list(tasks.STATE.get("order", [])) + + +def eligible_since(task_id): + """The instant django-ox's age readings measure this row from.""" + return ( + OxTask.objects.filter(pk=task_id) + .annotate(since=Coalesce("run_after", "enqueued_at")) + .values_list("since", flat=True) + .get() + ) + + +def gauge(queue="default"): + """django_ox_oldest_ready_age_seconds for a queue, as a scrape renders it.""" + match = re.search( + rf'^django_ox_oldest_ready_age_seconds\{{queue="{queue}"\}} (\S+)$', + metrics.render_prometheus(), + re.MULTILINE, + ) + return None if match is None else float(match.group(1)) + + +def health(max_age): + """None when ox_health --max-age exits 0, else the reason it gave.""" + try: + call_command("ox_health", "--max-age", str(max_age), stdout=StringIO()) + except CommandError as exc: + return str(exc) + return None + + +def backlog_problems(max_age=MAX_AGE_SECONDS): + """Every age reading an operator alerts on, and which ones fire.""" + age = stats.oldest_ready_age() + scraped = gauge() + reason = health(max_age) + problems = [] + if age is None or age.total_seconds() > max_age: + problems.append(f"stats.oldest_ready_age() = {age}") + if scraped is None or scraped > max_age: + problems.append(f"django_ox_oldest_ready_age_seconds = {scraped}") + if reason is not None: + problems.append(f"ox_health --max-age {max_age} failed: {reason}") + return problems + + +def released_problems(child_id, upstream): + """backlog_problems, plus the eligibility instant against the upstream.""" + problems = [] + since = eligible_since(child_id) + if since < upstream.finished_at: + problems.append( + f"child eligible since {since.isoformat()}, " + f"{upstream.finished_at - since} before its only upstream " + f"finished at {upstream.finished_at.isoformat()}" + ) + return problems + backlog_problems() + + +def backdate_every_row(by=BLOCKED_FOR): + OxTask.objects.update(enqueued_at=timezone.now() - by) + + +def deferred_chain(name): + flow = workflows.create(name) + root = flow.add("root", tasks.deferred_step, "root") + child = flow.add("child", tasks.deferred_step, "child", depends_on="root") + flow.seal() + assert OxTask.objects.get(pk=child.id).status == OxTask.Status.WAITING + return root, child + + +# -- control ----------------------------------------------------------- + + +def test_control_a_retry_restarts_the_age_clock(worker): + """ + django-ox's own contract, through its own writer: a task that failed + and went back to READY is measured from its retry, not its enqueue. + Passes today; it is what the released node should match. + """ + result = tasks.plain_flaky.enqueue(2) + backdate_every_row() + assert worker.run_once() is True + row = OxTask.objects.get(pk=result.id) + assert row.status == OxTask.Status.READY and row.run_after is not None + # last_attempted_at is written by the claim that ran the failed attempt, + # on the database's clock; run_after by the retry, on the worker's. A + # second either way is still two hours clear of the backdated enqueue. + assert eligible_since(row.pk) >= row.last_attempted_at - CLOCK_SKEW + assert backlog_problems() == [] + + +# -- the hazard -------------------------------------------------------- + + +def test_a_node_released_by_its_upstreams_worker_reads_as_just_eligible(): + """ + The eager path: OxpullWorker._write_outcome writes the root + SUCCESSFUL and releases the child in the same pass. The child became + claimable at that moment. + """ + root, child = deferred_chain("eager") + backdate_every_row() + assert deferring_worker().run_once() is True + root_row = OxTask.objects.get(pk=root.id) + child_row = OxTask.objects.get(pk=child.id) + assert root_row.status == OxTask.Status.SUCCESSFUL + assert child_row.status == OxTask.Status.READY, "the release did not run" + assert stats.ready_count() == 1 + + problems = released_problems(child.id, root_row) + assert problems == [], "\n".join(problems) + + +def test_a_node_released_by_the_reconciler_backstop_reads_as_just_eligible( + monkeypatch, worker +): + """ + The backstop path. The worker's release fails after the outcome + landed (OxpullWorker._write_outcome logs a DatabaseError and returns + the outcome, as it does for a lock wait timeout or a dropped + connection), so the child waits until the scheduled reconciler task + runs on a real worker and releases it. + """ + root, child = deferred_chain("backstop") + backdate_every_row() + + def release_fails(task_ids, db_alias): + raise DatabaseError("simulated: the release statement failed") + + with monkeypatch.context() as patch: + patch.setattr("oxpull.worker.release_downstream", release_fails) + assert deferring_worker().run_once() is True + root_row = OxTask.objects.get(pk=root.id) + assert root_row.status == OxTask.Status.SUCCESSFUL + assert OxTask.objects.get(pk=child.id).status == OxTask.Status.WAITING + + reconciler = workflows.reconcile.enqueue() + assert worker.run_once() is True + assert OxTask.objects.get(pk=reconciler.id).status == OxTask.Status.SUCCESSFUL + child_row = OxTask.objects.get(pk=child.id) + assert child_row.status == OxTask.Status.READY, "the backstop did not run" + assert stats.ready_count() == 1 + + problems = released_problems(child.id, root_row) + assert problems == [], "\n".join(problems) + + +def test_every_stage_of_a_real_time_chain_reads_as_just_eligible(): + """ + No row written by hand: extract and transform really take 1.5 s + each, and the threshold is 1 s. Each release should read as + milliseconds old. Today the age grows by one stage's run time per + stage, so the page fires at every stage and gets louder down the DAG. + """ + flow = workflows.create("etl") + flow.add("extract", slow_deferred_step, "extract", 1.5) + transform = flow.add( + "transform", slow_deferred_step, "transform", 1.5, depends_on="extract" + ) + load = flow.add("load", tasks.deferred_step, "load", depends_on="transform") + flow.seal() + + readings = {} + for released in (transform, load): + assert deferring_worker().run_once() is True + row = OxTask.objects.get(pk=released.id) + assert row.status == OxTask.Status.READY + readings[row.args[0]] = backlog_problems(max_age=1) + assert order() == ["extract", "transform"] + assert readings == {"transform": [], "load": []}, "\n".join( + f"{stage}: {problems}" for stage, problems in readings.items() + ) diff --git a/tests_oxpull/test_workflows_one_key_statistics.py b/tests_oxpull/test_workflows_one_key_statistics.py new file mode 100644 index 0000000..223ef44 --- /dev/null +++ b/tests_oxpull/test_workflows_one_key_statistics.py @@ -0,0 +1,821 @@ +""" +Statistics that show one key for a whole index do not turn the edge reads +into scans. + +One wide fan-in alone in the edge table gives oxpull_wf_edge_down_idx the +same (downstream_task_id, satisfied) key on every row, and one wide fan-out +does the same to oxpull_wf_edge_up_idx. ANALYZE then writes a statistics line +whose numbers are all the row count. From 3.45 to 3.49 SQLite marks such an +index when a connection loads the line, and does not probe a marked index +with =, IN or IS. 3.45 applies that under INDEXED BY too, where the only plan +left is a read of the whole named index. The mark stays on the connection +until its schema is read again, whatever statistics are gathered later. + +Six reads look an edge up by one of those two columns with no workflow in +hand, and run for every claim, every node or every reconciler page: the +claim's dependency check, the same check unhinted in Workflow.progress(), the +release's read of the edges into the nodes it was given, the outcome's read of +the edges out of the node that succeeded, the read that marks the edges into a +chunk of nodes, and the record repair's read of the edges out of a chunk of +them. Each has a test here that builds the state on the test's own connection, +keeps that connection, grows the unrelated workflow tenfold, and asserts the +read's work stays what it was and its plan seeks the index. Two more tests ask +a marked connection what the reads decide, and the last ones read the SQL each +database is sent. + +A read given a chunk of ids sends one range an id. Being allowed to probe the +index is not the same as being planned to: a read of columns the index does +not carry looks dearer as a branch an id than as a read of the table, and a +planner that takes the table tests every row against every range. So those +three reads name their index, and two of the tests below are given a chunk +long enough to show it. + +**The state is built, never assumed.** On a SQLite that marks indexes, +each test first shows a plain equality on the column planned as a whole +read, which is the mark at work, and shows it again after the table grew. +A test whose own rows give the index a second key, where that equality +has another index to take, shows the mark to an equality that names the +index instead. A newer SQLite cannot be put into that state, so there +the same tests assert a seek and bounded work and say nothing about the +mark. Compatibility coverage includes SQLite 3.45 and 3.46. + +Work is SQLite VM instructions, counted by the hazard suite's Meter. Plans +are read with the statistics suite's parser. +""" + +from __future__ import annotations + +import re +import sqlite3 +import uuid + +import pytest +from django.db import DEFAULT_DB_ALIAS, connection, transaction +from django.db.models import Exists, OuterRef, Subquery, Value +from django.db.models.functions import Coalesce +from django.db.utils import ConnectionHandler +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox.models import OxTask +from oxpull import worker as oxpull_worker +from oxpull import workflows +from oxpull.models import OxpullWorkflowEdge +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import flow_worker +from .test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost import ( + Meter, + Reading, + claim_cost, + fan_in, + mark_maps_running, +) +from .test_workflows_hazard_unknown_status import UNKNOWN +from .test_workflows_statistics import EDGE, gather_statistics, whole_reads + +pytestmark = pytest.mark.django_db(transaction=True) + +sqlite_only = pytest.mark.skipif( + connection.vendor != "sqlite", + reason="the index mark, the VM instruction count and the plan text are SQLite's", +) + +# The releases that do not probe a marked index with =, IN or IS, and the ones +# among them that hold to it under INDEXED BY. +MARKS_INDEXES = (3, 45, 0) <= sqlite3.sqlite_version_info < (3, 50, 0) +MARKS_UNDER_A_HINT = (3, 45, 0) <= sqlite3.sqlite_version_info < (3, 46, 0) + +# Every index with more than about a hundred rows can be marked, so the narrow +# workflow is already wide enough, and the wide one is ten times it. +NARROW = 400 +WIDE = 4000 + +# Ids in one list read. Long enough that a planner counting one-key statistics +# makes more of a branch an id than of a read of the whole table. +LISTED = 100 + +DOWN = ("oxpull_wf_edge_down_idx", "downstream_task_id") +UP = ("oxpull_wf_edge_up_idx", "upstream_task_id") + + +# -- the state --------------------------------------------------------- + + +def fan_out(name: str, width: int): + """One root and `width` nodes that each wait for it, through Workflow.add().""" + flow = workflows.create(name) + with transaction.atomic(): + flow.add("root", tasks.step, f"{name}-root") + for i in range(width): + flow.add( + f"leaf-{i:05d}", tasks.step, f"{name}-leaf-{i:05d}", depends_on="root" + ) + flow.seal() + return flow + + +def statistics_line(index: str) -> str: + with connection.cursor() as cursor: + cursor.execute("SELECT stat FROM sqlite_stat1 WHERE idx = %s", [index]) + return cursor.fetchone()[0] + + +def plain_equality_plan(column: str) -> str: + """How this connection plans `column = ?` on the edge table, with no hint.""" + with connection.cursor() as cursor: + # S608: a column name from this file, in a statement that is only explained. + cursor.execute( + f'EXPLAIN QUERY PLAN SELECT "id" FROM "{EDGE}" WHERE "{column}" = %s', # noqa: S608 + [uuid.uuid4().hex], + ) + return " | ".join(row[3] for row in cursor.fetchall()) + + +def assert_the_mark_is_in_force(index: str, column: str) -> None: + """ + On a SQLite that marks indexes, show this connection's mark on `index`: a + plain equality on its leading column reads a table or an index whole. A + SQLite that has no mark has nothing to show, and the caller goes on to + assert the seek and the bounded work there. + """ + if not MARKS_INDEXES: + return + plan = plain_equality_plan(column) + assert plan.startswith("SCAN "), ( + f"SQLite {sqlite3.sqlite_version} should hold a mark on {index} after " + f"statistics {statistics_line(index)!r}, and plans a plain equality on " + f"{column} as: {plan}" + ) + + +def named_index_equality_plan(index: str, column: str) -> str: + """How this connection plans `column = ?` on the edge table for a read + that names `index`, which is how the list reads meet the mark.""" + with connection.cursor() as cursor: + # S608: an index and a column name from this file, in a statement that + # is only explained. + cursor.execute( + f'EXPLAIN QUERY PLAN SELECT "id" FROM "{EDGE}" ' # noqa: S608 + f'INDEXED BY "{index}" WHERE "{column}" = %s', + [uuid.uuid4().hex], + ) + return " | ".join(row[3] for row in cursor.fetchall()) + + +def assert_the_mark_is_in_force_under_a_hint(index: str, column: str) -> None: + """ + Show the mark to a read that names `index`: a plain equality there is + planned as a read of the whole named index. From 3.46 on a named index is + exempt, so there is nothing to show and the caller goes on to assert the + seek and the bounded work. + + For a read whose own rows give the index a second key, where the plain + equality assert_the_mark_is_in_force sends has another index to take. + """ + if not MARKS_UNDER_A_HINT: + return + plan = named_index_equality_plan(index, column) + assert plan.startswith("SCAN "), ( + f"SQLite {sqlite3.sqlite_version} should hold a mark on {index} after " + f"statistics {statistics_line(index)!r}, and plans an equality on " + f"{column} that names it as: {plan}" + ) + + +class KeptConnection: + """The test's connection, and a check that it is still the one it was.""" + + def __init__(self) -> None: + connection.ensure_connection() + self.raw = connection.connection + + def check(self) -> None: + assert connection.connection is self.raw, ( + "the connection was replaced, and the mark went with the old one" + ) + + +def edge_plans(captured, column: str, *, gate: bool = False) -> list[tuple[list, str]]: + """ + The whole reads and the plan of every captured statement that looks an + edge up by `column`: in the dependency check's subquery, which Django + aliases V0, or in a statement of its own on the edge table, with or + without the index its FROM item names. + """ + # S608: a pattern that captured SQL is matched against, and never sent anywhere. + if gate: + wanted = re.compile(rf'FROM "{EDGE}" "?V0"?.* WHERE .*V0"?\."{column}"') + else: + wanted = re.compile( + rf'^SELECT .* FROM "{EDGE}"( INDEXED BY "\w+")? WHERE ' # noqa: S608 + rf'.*"{EDGE}"\."{column}"' + ) + return [ + whole_reads(query["sql"]) + for query in captured.captured_queries + if wanted.search(query["sql"]) + ] + + +def assert_a_bounded_seek( + read: str, + plans: list[tuple[list, str]], + readings: list[Reading], + index: str, + column: str, + *, + hinted: bool, +) -> None: + """ + Every plan seeks `index` on `column` and reads no edge table whole, and the + work beside the wide workflow is what it was beside the narrow one. Where + this SQLite marks the index for this kind of read, the seek is the range: + the equality is not allowed. Everything wrong is reported together, so a + failure shows the plan and what the plan cost. + """ + assert plans, "no statement read the edge table by that column" + ranged = MARKS_UNDER_A_HINT if hinted else MARKS_INDEXES + how = rf"\({column}>\? AND {column}<\?\)" if ranged else rf"\({column}[=>]" + seek = re.compile(rf"SEARCH \w+ USING (COVERING )?INDEX {index} {how}") + wrong = [] + for reads, plan in plans: + whole = [read for read in reads if read.table == EDGE] + if whole or not seek.search(plan): + expected = "a range seek" if ranged else "a seek" + wrong.append( + f"expected {expected} of {index}, whole reads {whole}:\n{plan}" + ) + narrow, wide = readings + print( # noqa: T201 + f"\n [one key, SQLite {sqlite3.sqlite_version}] {read}: {narrow.work} VM " + f"instructions beside {NARROW} edges, {wide.work} beside {WIDE}", + end="", + ) + if wide.work > 2 * narrow.work: + wrong.append( + f"{read} cost {narrow.work} VM instructions beside a {NARROW}-wide " + f"workflow and {wide.work} beside a {WIDE}-wide one " + f"({wide.work / max(narrow.work, 1):.1f}x)" + ) + assert wrong == [], f"SQLite {sqlite3.sqlite_version}:\n" + "\n".join(wrong) + + +def small_flow(name: str, *, priority: int = 0): + """ + `first`, at `priority`, and a `second` that waits for it: the workflow the + reads are about. + """ + flow = workflows.create(name) + flow.add("first", tasks.step.using(priority=priority), f"{name}-first") + second = flow.add("second", tasks.step, f"{name}-second", depends_on="first") + return flow, second + + +def nodes_no_worker_recorded(name: str, count: int): + """ + A workflow of `count` nodes that succeeded with no record written, each + with a node below it. What a writer that died between the task row and the + record leaves, and what the record repair is given. + """ + flow = workflows.create(name) + with transaction.atomic(): + for i in range(count): + flow.add(f"up-{i:03d}", tasks.step, f"{name}-up-{i:03d}") + flow.add( + f"down-{i:03d}", + tasks.step, + f"{name}-down-{i:03d}", + depends_on=f"up-{i:03d}", + ) + flow.seal() + ids = flow.node_ids() + OxTask.objects.filter(pk__in=[ids[f"up-{i:03d}"] for i in range(count)]).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + return flow + + +# -- the six reads, beside a workflow that grows ----------------------- + + +@sqlite_only +def test_a_claim_seeks_the_edge_index_under_one_key_statistics(): + """ + The claim's dependency check, hinted, once per candidate row. The claims + take plain tasks enqueued first, so every candidate is outside the fan-in. + """ + meter = Meter() + worker = flow_worker() + for i in range(30): + tasks.step.enqueue(f"unrelated-{i:02d}") + kept = KeptConnection() + readings, plans = [], [] + for width in (NARROW, WIDE): + fan_in(f"fan-in-{width}", width, tasks.step) + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + readings.append(claim_cost(meter, worker, label_prefix="unrelated")) + with CaptureQueriesContext(connection) as captured: + assert worker.claim_one() is not None + plans += edge_plans(captured, DOWN[1], gate=True) + kept.check() + assert_a_bounded_seek("a claim", plans, readings, *DOWN, hinted=True) + + +@sqlite_only +def test_progress_seeks_the_edge_index_under_one_key_statistics(): + """ + The same check with no hint, for each READY node of the workflow asked + about. That workflow is added once the first statistics are gathered: an + edge of its own in them would be a second key. + """ + meter = Meter() + flow = None + kept = KeptConnection() + readings, plans = [], [] + for width in (NARROW, WIDE): + fan_in(f"fan-in-{width}", width, tasks.step) + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + flow = flow or small_flow("asked-about")[0] + with meter.measure() as reading, CaptureQueriesContext(connection) as captured: + progress = flow.progress() + assert (progress.total, progress.pending, progress.blocked) == (2, 2, 1) + readings.append(reading) + plans += edge_plans(captured, DOWN[1], gate=True) + kept.check() + assert_a_bounded_seek( + "progress() of a two-node workflow", plans, readings, *DOWN, hinted=False + ) + + +@sqlite_only +def test_a_release_seeks_the_edge_index_under_one_key_statistics(): + """ + release_downstream's read of the edges into the nodes it was given. The + node still waits for its upstream, so the read is the whole of the call. + Its workflow is added once the first statistics are gathered: an edge of + its own in them would be a second key. + """ + meter = Meter() + alias = oxpull_db_alias() + second = None + kept = KeptConnection() + readings, plans = [], [] + for width in (NARROW, WIDE): + fan_in(f"fan-in-{width}", width, tasks.step) + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + second = second or small_flow("released")[1] + with meter.measure() as reading, CaptureQueriesContext(connection) as captured: + assert workflows.release_downstream([second.id], alias) == 0 + readings.append(reading) + plans += edge_plans(captured, DOWN[1]) + kept.check() + assert_a_bounded_seek( + "the release of one waiting node", plans, readings, *DOWN, hinted=False + ) + + +@sqlite_only +def test_an_outcome_seeks_the_edge_index_under_one_key_statistics(monkeypatch): + """ + record_node_outcome's read of the edges out of the node that succeeded, + inside the worker's own outcome transaction. A fan-out is what gives the + upstream index one key. The node that runs outranks the fan-out's root. + """ + meter = Meter() + worker = flow_worker() + outcomes: list[Reading] = [] + real_outcome = oxpull_worker.record_node_outcome + + def metered_outcome(*args): + with meter.measure() as reading: + downstream = real_outcome(*args) + outcomes.append(reading) + return downstream + + monkeypatch.setattr(oxpull_worker, "record_node_outcome", metered_outcome) + kept = KeptConnection() + plans = [] + for width in (NARROW, WIDE): + fan_out(f"fan-out-{width}", width) + gather_statistics() + assert_the_mark_is_in_force(*UP) + _, second = small_flow(f"ran-beside-{width}", priority=10) + with CaptureQueriesContext(connection) as captured: + assert worker.run_once() is True + assert tasks.STATE["order"][-1] == f"ran-beside-{width}-first" + assert OxTask.objects.get(pk=second.id).status == OxTask.Status.READY + plans += edge_plans(captured, UP[1]) + kept.check() + assert len(outcomes) == 2 + assert_a_bounded_seek( + "recording one node's success", plans, outcomes, *UP, hinted=False + ) + + +@sqlite_only +def test_the_read_that_marks_edges_seeks_the_edge_index_under_one_key_statistics(): + """ + _satisfy_edges_into's read of the edges into the nodes it was given, a + reconciler page of them at a time. It selects columns the index does not + carry, which is where a planner weighing a range an id against a read of + the table takes the table. The ids name no edge, as a page of nodes + already released does, so the read is the whole of the call. + """ + meter = Meter() + alias = oxpull_db_alias() + given = sorted(uuid.uuid4() for _ in range(LISTED)) + kept = KeptConnection() + readings, plans = [], [] + for width in (NARROW, WIDE): + fan_in(f"fan-in-{width}", width, tasks.step) + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + with meter.measure() as reading, CaptureQueriesContext(connection) as captured: + assert workflows._satisfy_edges_into(given, alias) == 0 + readings.append(reading) + plans += edge_plans(captured, DOWN[1]) + kept.check() + assert_a_bounded_seek( + f"the read that marks the edges into {LISTED} nodes", + plans, + readings, + *DOWN, + hinted=True, + ) + + +@sqlite_only +def test_a_record_repair_seeks_the_edge_index_under_one_key_statistics(): + """ + _repair_records_once's read of the edges out of the nodes whose outcome no + worker recorded, on the other column and with the same shape. A fan-out is + what gives the upstream index one key. Each pass repairs a workflow of its + own, because a repair marks the edges it found and a second pass over the + same workflow would find none. Both are added once the first statistics + are gathered: an edge of their own in them would be a second key. + + A hundred nodes that succeeded are a hundred upstream keys, so the mark is + shown to a read that names the index rather than to a plain equality, + which by the second pass has another index to take. + """ + meter = Meter() + alias = oxpull_db_alias() + waiting: list = [] + kept = KeptConnection() + readings, plans = [], [] + for width in (NARROW, WIDE): + fan_out(f"fan-out-{width}", width) + gather_statistics() + assert_the_mark_is_in_force_under_a_hint(*UP) + waiting = waiting or [ + nodes_no_worker_recorded(f"repaired-{i}", LISTED) for i in range(2) + ] + flow = waiting.pop(0) + with meter.measure() as reading, CaptureQueriesContext(connection) as captured: + downstream = workflows._repair_records_once(flow.id, alias) + assert len(downstream) == LISTED + readings.append(reading) + plans += edge_plans(captured, UP[1]) + kept.check() + assert_a_bounded_seek( + f"the repair of {LISTED} nodes' records", + plans, + readings, + *UP, + hinted=True, + ) + + +# -- what the reads decide, on a marked connection --------------------- + + +def chain(name: str): + """`up` and a `down` that waits for it. Returns the two task ids.""" + flow = workflows.create(name) + flow.add("up", tasks.step, f"{name}-up") + down = flow.add("down", tasks.step, f"{name}-down", depends_on="up") + return flow.node_ids()["up"], uuid.UUID(str(down.id)) + + +@sqlite_only +def test_the_gate_fails_closed_under_one_key_statistics(): + """ + The range beside the equality decides nothing the equality did not. On a + connection whose edge index is marked, a READY node behind an open edge + runs only when its upstream row is there and SUCCESSFUL, and a satisfied + edge lets its node run with the upstream row gone. Asked three ways: the + claim's condition, the unhinted check progress() uses, and real claims. + """ + wide = fan_in("fan-in", NARROW, tasks.step) + assert mark_maps_running(wide) == NARROW + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + kept = KeptConnection() + + done_up, after_done = chain("open-edge-upstream-successful") + OxTask.objects.filter(pk=done_up).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + running_up, after_running = chain("open-edge-upstream-running") + OxTask.objects.filter(pk=running_up).update(status=OxTask.Status.RUNNING) + gone_up, after_gone = chain("open-edge-upstream-gone") + OxTask.objects.filter(pk=gone_up).delete() + unknown_up, after_unknown = chain("open-edge-upstream-unknown") + OxTask.objects.filter(pk=unknown_up).update(status=UNKNOWN) + marked_up, after_marked = chain("satisfied-edge-upstream-gone") + OxpullWorkflowEdge.objects.filter(upstream_task_id=marked_up).update(satisfied=True) + OxTask.objects.filter(pk=marked_up).delete() + + downs = [after_done, after_running, after_gone, after_unknown, after_marked] + assert OxTask.objects.filter(pk__in=downs, status=OxTask.Status.WAITING).update( + status=OxTask.Status.READY + ) == len(downs) + runnable = {after_done, after_marked} + + ready = OxTask.objects.filter(pk__in=downs, status=OxTask.Status.READY) + hinted = set( + ready.filter(workflows.runnable_filter_q()).values_list("pk", flat=True) + ) + unhinted = set( + ready.filter(~Exists(workflows._blocking_edges(OuterRef("pk")))).values_list( + "pk", flat=True + ) + ) + assert hinted == runnable + assert unhinted == runnable + + worker = flow_worker() + claimed = set() + while (task := worker.claim_one()) is not None: + claimed.add(task.pk) + assert claimed & set(downs) == runnable + kept.check() + assert_the_mark_is_in_force(*DOWN) + + +@sqlite_only +def test_the_list_reads_take_exactly_the_ids_they_were_given_under_one_key_statistics(): + """ + The list reads hold a range per id, and a range must not take in an id + that sorts between two of the list. Seven nodes wait behind an upstream + that succeeded with its edge never marked, as a missed mark leaves them. + Every other one is named, among ids of nothing at all, first to the read + that marks edges and then to the release. The named ones are marked and + released. The ones that sort between them, a node behind an upstream that + has not run, and the wide fan-in's own node are left as they were. + """ + alias = oxpull_db_alias() + wide = fan_in("fan-in", NARROW, tasks.step) + gather_statistics() + assert_the_mark_is_in_force(*DOWN) + kept = KeptConnection() + + missed = sorted((chain(f"missed-mark-{i}") for i in range(7)), key=lambda c: c[1]) + OxTask.objects.filter(pk__in=[up for up, _ in missed]).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + waiting = [down for _, down in missed] + listed, between = waiting[::2], waiting[1::2] + _, held = chain("upstream-not-run") + reduce_id = wide.node_ids()["reduce"] + given = [*listed, held, *(uuid.uuid4() for _ in range(40))] + + in_the_list = set( + OxpullWorkflowEdge.objects.filter( + downstream_task_id__probe_in=given + ).values_list("downstream_task_id", flat=True) + ) + assert in_the_list == {*listed, held} + + assert workflows._satisfy_edges_into(given, alias) == len(listed) + marked = OxpullWorkflowEdge.objects.filter(satisfied=True) + assert set(marked.values_list("downstream_task_id", flat=True)) == set(listed) + + assert workflows.release_downstream(given, alias) == len(listed) + status = dict(OxTask.objects.values_list("pk", "status")) + assert {status[pk] for pk in listed} == {OxTask.Status.READY} + assert {status[pk] for pk in (*between, held, reduce_id)} == {OxTask.Status.WAITING} + kept.check() + assert_the_mark_is_in_force(*DOWN) + + +# -- what each database is sent ---------------------------------------- + + +def _gate(edges) -> str: + """A claim-shaped query over the task table with `edges` as its check.""" + return OxTask.objects.filter(~workflows._EdgeProbe(edges)).query + + +def _exact_edges(downstream): + """workflows._blocking_edges, written with Django's own `exact`.""" + upstream_status = Subquery( + OxTask.objects.filter(pk=OuterRef("upstream_task_id")).values("status")[:1] + ) + return ( + OxpullWorkflowEdge.objects.filter( + downstream_task_id=downstream, satisfied=False + ) + .alias(upstream_status=Coalesce(upstream_status, Value(""))) + .exclude(upstream_status=OxTask.Status.SUCCESSFUL) + ) + + +def compiles_for(vendor: str): + """ + A connection that compiles SQL as `vendor` does: the test's own where that + is its vendor, and otherwise a PostgreSQL one that never connects. MySQL's + compiler asks the server whether it is MariaDB before it binds a UUID, so + MySQL's SQL is compared on a MySQL run alone. + """ + if connection.vendor == vendor: + return connection + if vendor == "mysql": + pytest.skip("compiling a UUID for MySQL asks a server which MySQL it is") + engine = f"django.db.backends.{vendor}" + handler = ConnectionHandler( + {DEFAULT_DB_ALIAS: {"ENGINE": engine, "NAME": "unused"}} + ) + return handler[DEFAULT_DB_ALIAS] + + +@pytest.mark.parametrize("vendor", ["postgresql", "mysql"]) +def test_the_other_databases_are_sent_what_exact_and_in_send_them(vendor): + """ + The ranges, the index a read names and the plus that puts a term beyond it + are SQLite's alone. MySQL's claim is planned from FORCE INDEX and + NO_SEMIJOIN over this exact text and PostgreSQL's from its column + statistics, so the three list reads are compiled here as the source builds + them, with the claim's check, and compared byte for byte with the same + querysets under Django's own lookups. + """ + database = compiles_for(vendor) + ids = [uuid.UUID(int=n) for n in (3, 1, 2)] + edges = OxpullWorkflowEdge.objects + down, up = DOWN[0], UP[0] + pairs = [ + ( + edges.filter(downstream_task_id__probe=ids[0], satisfied=False), + edges.filter(downstream_task_id=ids[0], satisfied=False), + ), + ( + edges.filter(upstream_task_id__probe=ids[0], satisfied=False), + edges.filter(upstream_task_id=ids[0], satisfied=False), + ), + ( + workflows._reading_index( + edges.filter( + downstream_task_id__probe_in=ids, satisfied__aside=False + ).values_list("pk", "upstream_task_id", "downstream_task_id"), + down, + ), + edges.filter(downstream_task_id__in=ids, satisfied=False).values_list( + "pk", "upstream_task_id", "downstream_task_id" + ), + ), + ( + workflows._reading_index( + edges.filter( + downstream_task_id__probe_in=ids, satisfied__aside=False + ).values_list("downstream_task_id", flat=True), + down, + ), + edges.filter(downstream_task_id__in=ids, satisfied=False).values_list( + "downstream_task_id", flat=True + ), + ), + ( + workflows._reading_index( + edges.filter( + upstream_task_id__probe_in=ids, satisfied__aside=False + ).values_list("pk", "downstream_task_id"), + up, + ), + edges.filter(upstream_task_id__in=ids, satisfied=False).values_list( + "pk", "downstream_task_id" + ), + ), + ] + for probe, builtin in pairs: + ours = probe.query.get_compiler(connection=database).as_sql() + theirs = builtin.query.get_compiler(connection=database).as_sql() + assert ours == theirs + assert "BETWEEN" not in ours[0] + assert "INDEXED BY" not in ours[0] + assert " +" not in ours[0] + + ours = _gate(workflows._blocking_edges(OuterRef("pk"))) + theirs = _gate(_exact_edges(OuterRef("pk"))) + ours = ours.get_compiler(connection=database).as_sql() + assert ours == theirs.get_compiler(connection=database).as_sql() + assert "BETWEEN" not in ours[0] + if vendor == "mysql": + assert "FORCE INDEX (`oxpull_wf_edge_down_idx`)" in ours[0] + assert "EXISTS(SELECT /*+ NO_SEMIJOIN() */ " in ours[0] + + +@sqlite_only +def test_sqlite_is_sent_the_equality_and_a_range_that_holds_the_same_key(): + # Django quotes the subquery's alias from 6.1 on and did not before. + outer = r'\("django_ox_oxtask"\."id"\)' + column = r'"?V0"?\."downstream_task_id"' + both = rf"\({column} = {outer} AND {column} BETWEEN {outer} AND {outer}\)" + hinted = str(_gate(workflows._blocking_edges(OuterRef("pk")))) + unhinted = str( + OxTask.objects.filter(Exists(workflows._blocking_edges(OuterRef("pk")))).query + ) + assert re.search(both, hinted), hinted + assert re.search(both, unhinted), unhinted + assert ' INDEXED BY "oxpull_wf_edge_down_idx" WHERE ' in hinted + assert "INDEXED BY" not in unhinted + + one = uuid.UUID(int=7) + sql, params = ( + OxpullWorkflowEdge.objects.filter(upstream_task_id__probe=one) + .values("pk") + .query.sql_with_params() + ) + column = f'"{EDGE}"."upstream_task_id"' + assert f"({column} = %s AND {column} BETWEEN %s AND %s)" in sql + assert params == (one.hex,) * 3 + + +@sqlite_only +def test_sqlite_is_sent_one_range_for_each_id_in_a_list(): + ids = [uuid.UUID(int=n) for n in (3, 1, 2, 3)] + sql, params = ( + OxpullWorkflowEdge.objects.filter(downstream_task_id__probe_in=[*ids, None]) + .values("pk") + .query.sql_with_params() + ) + column = f'"{EDGE}"."downstream_task_id"' + one = f"{column} BETWEEN %s AND %s" + assert f"WHERE ({one} OR ({one} OR {one}))" in sql + assert " IN " not in sql + assert params == tuple(uuid.UUID(int=n).hex for n in (3, 3, 1, 1, 2, 2)) + + +@sqlite_only +def test_sqlite_is_sent_the_index_a_list_read_takes_and_a_plus_beside_it(): + """ + A list read names its index in its FROM item, and its term on the index's + second column carries the plus that puts it beyond the index. Django spells + `satisfied=False` as an equality from 6.1 on and negated the column before, + and SQLite is sent the one spelling whichever it is. + """ + ids = [uuid.UUID(int=n) for n in (1, 2)] + sql, params = workflows._reading_index( + OxpullWorkflowEdge.objects.filter( + downstream_task_id__probe_in=ids, satisfied__aside=False + ).values_list("downstream_task_id", flat=True), + workflows.EDGE_DOWNSTREAM_INDEX, + ).query.sql_with_params() + assert f'FROM "{EDGE}" INDEXED BY "{DOWN[0]}" WHERE ' in sql + assert f'AND +"{EDGE}"."satisfied" = %s)' in sql + assert params[-1] is False + + out = workflows._reading_index( + OxpullWorkflowEdge.objects.filter( + upstream_task_id__probe_in=ids, satisfied__aside=False + ).values_list("pk", "downstream_task_id"), + workflows.EDGE_UPSTREAM_INDEX, + ) + assert f'FROM "{EDGE}" INDEXED BY "{UP[0]}" WHERE ' in str(out.query) + + nothing = OxpullWorkflowEdge.objects.filter(downstream_task_id__probe_in=[]) + assert list(nothing) == [] + by_subquery = OxpullWorkflowEdge.objects.filter( + downstream_task_id__probe_in=OxTask.objects.values("pk") + ) + assert " IN (SELECT " in str(by_subquery.query) + + +@sqlite_only +def test_a_full_chunk_of_ids_is_one_statement_sqlite_accepts(): + """ + A flat OR of a full chunk is deeper than SQLite's expression limit and + binds more parameters than an old default build allows. The ranges nest + as a tree, and a chunk is sized for two parameters an id. + """ + alias = oxpull_db_alias() + _, held = chain("held") + ids = [uuid.uuid4() for _ in range(workflows.ID_CHUNK_SIZE - 1)] + found = OxpullWorkflowEdge.objects.filter(downstream_task_id__probe_in=[*ids, held]) + assert [edge.downstream_task_id for edge in found] == [held] + + connection.ensure_connection() + raw = connection.connection + limit = sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER + previous = raw.setlimit(limit, 999) + try: + assert workflows._id_chunk_size(alias) == (999 - 8) // 2 + assert workflows.release_downstream([*ids, held], alias) == 0 + assert connection.connection is raw and raw.getlimit(limit) == 999 + finally: + raw.setlimit(limit, previous) + assert workflows._id_chunk_size(alias) == workflows.ID_CHUNK_SIZE diff --git a/tests_oxpull/test_workflows_outcome_reconnect.py b/tests_oxpull/test_workflows_outcome_reconnect.py new file mode 100644 index 0000000..476e2e1 --- /dev/null +++ b/tests_oxpull/test_workflows_outcome_reconnect.py @@ -0,0 +1,1055 @@ +""" +A workflow node's outcome, written while the server ends the worker's session. + +django-ox writes an attempt's outcome once more on a new connection when the +first write finds its connection gone (Worker._write_outcome_reconnecting). +OxpullWorker writes a node's SUCCESSFUL or FAILED in a transaction of its own, +with the node's record and the marks on the edges out of it. Before 1.5.0 it +failed that transaction closed on every database error, a lost connection's +included, so django-ox never made its second write: the row stayed RUNNING, +the reaper took it back once its lease ran out, and the node's body ran again. + +Every test here has the server end the worker's session for real, with +pg_terminate_backend or KILL from a second connection, at one point of the +outcome: + +- before: after the worker's pre-write check, which drops only a connection a + failed statement flagged, and before the node transaction's first statement; +- inside: after the fenced write of the task row, inside the transaction; +- committed: after the transaction committed, before the worker hears back; +- before-release: after the commit came back, before the downstream release. + +The worker is a real OxpullWorker claiming and running the node, on PostgreSQL +and on MySQL. SQLite has no server session to end. +""" + +import logging +import threading +import time +import warnings +from contextlib import contextmanager, suppress + +import pytest +from django.db import ( + DatabaseError, + InterfaceError, + OperationalError, + connection, + connections, + transaction, +) +from django.tasks import task + +from django_ox import actions +from django_ox.compat import task_finished +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.models import OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.routing import oxpull_db_alias +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_workflows import drain, flow_worker +from .test_workflows_hazard_a_node_that_builds_workflow_structure_is import ( + reconciler_tick, +) +from .test_workflows_outcomes import node_lock_read +from .test_workflows_waiting import events + +pytestmark = pytest.mark.django_db(transaction=True) + +READY = OxTask.Status.READY +RUNNING = OxTask.Status.RUNNING +WAITING = OxTask.Status.WAITING +SUCCESSFUL = OxTask.Status.SUCCESSFUL +FAILED = OxTask.Status.FAILED + +TASK_TABLE = OxTask._meta.db_table + +needs_a_server = pytest.mark.skipif( + connection.vendor not in ("postgresql", "mysql"), + reason="has the server end a session, with pg_terminate_backend or KILL", +) + +# Bodies run per label, the points armed by the next body, the points armed +# now, and what each hook saw. Module state, because the worker resolves the +# task by its dotted path, and the pool thread in one test shares it. +RUNS: dict[str, int] = {} +PLAN: dict[str, int] = {} +ARMED: dict[str, int] = {} +SEEN: dict[str, list] = {} + + +@pytest.fixture(autouse=True) +def _reset(transactional_db): + for state in (RUNS, PLAN, ARMED, SEEN): + state.clear() + yield + for state in (RUNS, PLAN, ARMED, SEEN): + state.clear() + # Before the database is flushed, whatever a test left of its connection. + forget_the_connection() + + +def forget_the_connection(): + """ + Drop this thread's connection for "default", whatever state a broken + transaction left it in. Django leaves in_atomic_block set when an atomic + block cannot create its savepoint on a dead connection, and close() then + keeps the dead driver connection. connect() resets the rest. + """ + conn = connections["default"] + driver = conn.connection + with suppress(Exception): + if driver is not None: + driver.close() + # A pooled connection goes back to its pool even closed, which throws it + # away and opens another. One it never gets back is gone from it for good, + # and a pool of four is empty after four tests. + pool = getattr(driver, "_pool", None) + if pool is not None: + with suppress(Exception): + pool.putconn(driver) + conn.connection = None + conn.in_atomic_block = False + conn.closed_in_transaction = False + + +# -- ending a session ------------------------------------------------------------ + + +def raw_session_id(conn): + """The server session of `conn`'s driver connection, asked past Django.""" + conn.ensure_connection() + cursor = conn.connection.cursor() + try: + cursor.execute( + "SELECT pg_backend_pid()" + if conn.vendor == "postgresql" + else "SELECT CONNECTION_ID()" + ) + return int(cursor.fetchone()[0]) + finally: + with suppress(Exception): + cursor.close() + + +def end_session(session): + """ + Have the server end `session` from a connection of another thread, and + return once it has gone. + """ + failed = [] + + def run(): + try: + other = connections["default"] + with other.cursor() as cursor: + if other.vendor == "postgresql": + cursor.execute("SELECT pg_terminate_backend(%s, 10000)", [session]) + if not cursor.fetchone()[0]: + raise AssertionError(f"backend {session} did not exit") + return + cursor.execute(f"KILL {int(session)}") + deadline = time.monotonic() + 10 + while True: + cursor.execute( + "SELECT COUNT(*) FROM information_schema.processlist " + "WHERE id = %s", + [session], + ) + if not cursor.fetchone()[0]: + return + if time.monotonic() > deadline: + raise AssertionError(f"connection {session} did not end") + time.sleep(0.02) + except BaseException as exc: + failed.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run, name="reconnect-test-other") + thread.start() + thread.join(30) + if failed: + raise failed[0] + + +def end_own_session(): + """End this thread's session for "default", and return its id.""" + session = raw_session_id(connections["default"]) + end_session(session) + return session + + +def fire(point): + """End this thread's session if `point` is armed, and disarm it once.""" + left = ARMED.get(point, 0) + if not left: + return False + ARMED[point] = left - 1 + SEEN.setdefault(point, []).append(end_own_session()) + return True + + +def settle_connection(): + """ + Drop this thread's connection if a worker pass left it unusable, as the + next request or task of a real process would, so the test reads the rows. + """ + connections["default"].close_if_unusable_or_obsolete() + + +def from_another_connection(read): + box = {} + + def run(): + try: + box["value"] = read() + except BaseException as exc: + box["error"] = exc + finally: + connections.close_all() + + thread = threading.Thread(target=run, name="reconnect-test-reader") + thread.start() + thread.join(30) + if "error" in box: + raise box["error"] + return box["value"] + + +# -- the node, the worker and the hooks -------------------------------------------- + + +@task(backend="flows") +def node_body(label, *, fail=False): + """Counts its runs, uses its connection, arms the planned points.""" + RUNS[label] = RUNS.get(label, 0) + 1 + with connections["default"].cursor() as cursor: + cursor.execute("SELECT 1") + ARMED.update(PLAN) + PLAN.clear() + if fail: + raise ValueError(f"{label} failed") + return label + + +@task(backend="flows") +def offline_body(label): + """ + Query, then have the server end this session while the body works on + without the database, and return with no further query. Nothing on this + thread sees the error. + """ + RUNS[label] = RUNS.get(label, 0) + 1 + SEEN.setdefault("offline", []).append(end_own_session()) + return label + + +@task(backend="flows") +def grafting_body(workflow_id, label): + """Discovers its work, adds a child that depends on itself, then returns.""" + RUNS[label] = RUNS.get(label, 0) + 1 + flow = workflows.get(workflow_id) + flow.add(f"{label}-child", tasks.step, f"{label}-child", depends_on=label) + ARMED.update(PLAN) + PLAN.clear() + return label + + +class HookedWorker(OxpullWorker): + """OxpullWorker, with the test's hooks between the steps of an outcome.""" + + def _discard_unusable_connections(self): + super()._discard_unusable_connections() + fire("before") + + def _release_after_outcome(self, db_task, downstream): + fire("before-release") + super()._release_after_outcome(db_task, downstream) + + +def hooked_worker(**kwargs): + return HookedWorker( + backend_alias="flows", backoff_initial=0, poll_interval=0.01, **kwargs + ) + + +def is_outcome_write(sql, params): + return ( + sql.lstrip().startswith(f"UPDATE {connection.ops.quote_name(TASK_TABLE)}") + and isinstance(params, (list, tuple)) + and any(str(value) in (SUCCESSFUL, FAILED) for value in params) + ) + + +@contextmanager +def outcome_hooks(): + """ + Log this thread's outcome writes of the task row; end the session after one + inside a transaction when "inside" is armed; and when "committed" is armed, + let the next commit after an outcome write land on the server, end the + session, and have the commit raise the error the dead session gives, as a + reply lost after the commit does. + """ + log = [] + since_armed = [] + conn = connections["default"] + + def wrapper(execute, sql, params, many, context): + write = is_outcome_write(sql, params) + if write: + log.append(sql) + if ARMED.get("committed"): + since_armed.append(sql) + result = execute(sql, params, many, context) + if write and conn.in_atomic_block: + fire("inside") + return result + + real_commit = conn._commit + + def commit_then_lose_the_reply(): + result = real_commit() + if since_armed and fire("committed"): + since_armed.clear() + cursor = conn.connection.cursor() + with conn.wrap_database_errors: + cursor.execute("SELECT 1") + return result + + conn._commit = commit_then_lose_the_reply + try: + with conn.execute_wrapper(wrapper): + yield log + finally: + del conn._commit + + +@contextmanager +def finished_signals(): + sent = [] + + def receiver(sender, task_result, **kwargs): + sent.append(str(task_result.id)) + + task_finished.connect(receiver) + try: + yield sent + finally: + task_finished.disconnect(receiver) + + +@contextmanager +def between_the_tries(monkeypatch): + """ + What another connection sees when django-ox asks whether the row already + holds the write, which it does only on its second try. + """ + seen = [] + real = Worker._outcome_already_written + + def observed(self, db_task, status, fields): + pk = db_task.pk + seen.append( + from_another_connection( + lambda: { + "row": OxTask.objects.values_list("status", "lease_epoch").get( + pk=pk + ), + "record": OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch" + ).get(task_id=pk), + "marks": sorted( + OxpullWorkflowEdge.objects.filter( + upstream_task_id=pk + ).values_list("satisfied", flat=True) + ), + } + ) + ) + return real(self, db_task, status, fields) + + monkeypatch.setattr(Worker, "_outcome_already_written", observed) + yield seen + + +def logs(caplog): + caplog.set_level(logging.INFO) + caplog.set_level(logging.INFO, logger="django_ox") + caplog.set_level(logging.INFO, logger="oxpull") + + +def row(result): + return OxTask.objects.get(pk=result.id) + + +def record_of(result): + return OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch", "succeeded" + ).get(task_id=result.id) + + +def marks_into(result): + return sorted( + OxpullWorkflowEdge.objects.filter(downstream_task_id=result.id).values_list( + "satisfied", flat=True + ) + ) + + +def a_chain(name, *, fail=False): + """parent -> child, OPEN; the parent's failure is final on its first attempt.""" + flow = workflows.create(name) + parent = flow.add("parent", node_body, "parent", fail=fail) + child = flow.add("child", tasks.step, "child", depends_on="parent") + if fail: + OxTask.objects.filter(pk=parent.id).update(max_attempts=1) + return flow, parent, child + + +def reap_what_the_lease_left(): + """ + What production does once the lease runs out: expire it and run a reaper. + Returns whether the reaper took a row back. + """ + return any( + actions.expire_lease(pk) + for pk in OxTask.objects.filter(status=RUNNING).values_list("pk", flat=True) + ) and bool(flow_worker().reap()) + + +# -- the reproducers --------------------------------------------------------------- + +POINTS = ["before", "inside", "committed"] +STATUSES = ["SUCCESSFUL", "FAILED"] + + +@needs_a_server +@pytest.mark.parametrize("status", STATUSES) +@pytest.mark.parametrize("point", POINTS) +def test_a_node_outcome_whose_connection_the_server_ends_is_written_once_on_another( + point, status, monkeypatch, caplog +): + logs(caplog) + fail = status == "FAILED" + _, parent, child = a_chain(f"reconnect-{point}-{status}", fail=fail) + PLAN[point] = 1 + with ( + between_the_tries(monkeypatch) as between, + finished_signals() as finished, + outcome_hooks() as writes, + ): + worker = hooked_worker() + claimed = worker.claim_one() + epoch = claimed.lease_epoch + worker.execute(claimed) + settle_connection() + + written = SUCCESSFUL if not fail else FAILED + first = { + "row": (row(parent).status, row(parent).lease_epoch, row(parent).locked_by), + "record": record_of(parent), + "marks": marks_into(child), + "child": row(child).status, + "finished": list(finished), + "writes": len(writes), + "reconnected": [ + getattr(r, "already_written", None) + for r in events(caplog, "task_outcome_reconnected") + ], + "unrecorded": len(events(caplog, "workflow_outcome_unrecorded")) + + len(events(caplog, "task_outcome_unrecorded")), + } + expected = { + "row": (written, epoch, None), + "record": (written, epoch, not fail), + "marks": [not fail], + # Before and inside, the second try's transaction commits and its + # release runs. After a commit whose reply was lost, the second try + # finds the row written and releases nothing: the reconciler does. + "child": READY if (not fail and point != "committed") else WAITING, + "finished": [str(parent.id)], + # Before, the first try fails ahead of its write; inside, it wrote and + # rolled back; committed, the second try writes nothing. + "writes": {"before": 1, "inside": 2, "committed": 1}[point], + "reconnected": [point == "committed"], + "unrecorded": 0, + } + problems = [ + f"{key}: {first[key]!r}, expected {expected[key]!r}" + for key in expected + if first[key] != expected[key] + ] + if SEEN.get(point, []) == []: + problems.append(f"the session was never ended at {point}") + if point == "committed": + # The first transaction committed whole before its reply was lost. + whole = {"row": (written, epoch), "record": (written, epoch)} + else: + # Nothing of the first transaction is left: no partial record or mark. + whole = {"row": (RUNNING, epoch), "record": (None, None)} + if between != [{**whole, "marks": [point == "committed" and not fail]}]: + problems.append( + f"between the tries another connection saw {between}, expected one " + f"look at {whole} with its marks" + ) + + # Production, once the lease would have run out, and the reconciler tick + # every workflow deployment schedules. + reaped = reap_what_the_lease_left() + reconciler_tick() + drain(flow_worker()) + if reaped: + problems.append("the reaper took the row back") + if RUNS.get("parent") != 1: + problems.append(f"the parent's body ran {RUNS.get('parent')} time(s)") + if row(parent).status != written: + problems.append(f"the parent ended {row(parent).status}") + if not fail and row(child).status != SUCCESSFUL: + problems.append(f"the child is {row(child).status} after the reconciler") + assert not problems, "\n".join( + [f"disconnect {point}, {status}", *problems, f"first pass: {first}"] + ) + + +@needs_a_server +def test_worker_pool_body_returns_without_query_after_session_ends(caplog): + """ + The node's session ends while its body works on without the database + on the worker's pool thread, and the body returns with no further query. + """ + logs(caplog) + flow = workflows.create("pool-offline") + node = flow.add("only", offline_body, "only") + flow.seal() + with finished_signals() as finished: + OxpullWorker( + backend_alias="flows", + concurrency=1, + poll_interval=0.05, + lock_timeout=2, + max_tasks=1, + backoff_initial=0, + ).run() + first = (row(node).status, row(node).attempts, RUNS.get("only")) + reaped = reap_what_the_lease_left() + drain(flow_worker()) + assert SEEN.get("offline"), "the session was never ended" + assert (first, reaped) == ((SUCCESSFUL, 1, 1), False) + assert RUNS.get("only") == 1 + assert finished == [str(node.id)] + assert len(events(caplog, "task_outcome_reconnected")) == 1 + assert not events(caplog, "workflow_outcome_unrecorded") + + +@needs_a_server +def test_a_node_that_grafts_onto_its_own_workflow_keeps_its_child_and_runs_once( + caplog, +): + """ + The shape the hazard test calls not re-executable: a node that adds a child + depending on itself. Its outcome write finds the connection gone. Before the + fix the reaper ran it again, the second add() raised, and the node ended + FAILED with the child it had committed discarded. + """ + logs(caplog) + flow = workflows.create( + "graft-reconnect", + on_success=tasks.record_callback, + on_success_args=["graft-success"], + ) + root = flow.add("root", grafting_body, str(flow.id), "root") + PLAN["before"] = 1 + assert hooked_worker().run_once() is True + settle_connection() + after_first = row(root).status + reap_what_the_lease_left() + flow.seal() + drain(flow_worker()) + reconciler_tick() + drain(flow_worker()) + reconciler_tick() + + nodes = flow.node_ids() + stored = OxpullWorkflow.objects.get(pk=flow.id) + child_runs = tasks.STATE.get("order", []).count("root-child") + problems = [] + if not SEEN.get("before"): + problems.append("the session was never ended") + if after_first != SUCCESSFUL: + problems.append(f"root was {after_first} after the worker's pass") + if RUNS.get("root") != 1: + problems.append(f"root's body ran {RUNS.get('root')} time(s)") + if row(root).status != SUCCESSFUL: + problems.append(f"root ended {row(root).status}") + if set(nodes) != {"root", "root-child"}: + problems.append(f"nodes {sorted(nodes)}") + elif OxTask.objects.get(pk=nodes["root-child"]).status != SUCCESSFUL: + problems.append( + f"root-child is {OxTask.objects.get(pk=nodes['root-child']).status}" + ) + if child_runs != 1: + problems.append(f"root-child ran {child_runs} time(s)") + if ( + stored.status != OxpullWorkflow.Status.COMPLETE + or stored.failed_count + or stored.cancelled_count + or stored.success_task_id is None + ): + problems.append( + f"workflow {stored.status}, failed_count={stored.failed_count}, " + f"cancelled_count={stored.cancelled_count}, on_success " + f"{'withheld' if stored.success_task_id is None else 'enqueued'}" + ) + assert not problems, "\n".join(problems) + + +@needs_a_server +def test_a_second_try_that_loses_its_connection_too_leaves_the_row_to_the_reaper( + caplog, +): + logs(caplog) + _, parent, child = a_chain("reconnect-exhausted") + PLAN["inside"] = 2 + with finished_signals() as finished, outcome_hooks() as writes: + worker = hooked_worker() + claimed = worker.claim_one() + epoch = claimed.lease_epoch + worker.execute(claimed) + settle_connection() + + assert len(SEEN.get("inside", [])) == 2 + assert len(writes) == 2, "the second try was not made" + assert (row(parent).status, row(parent).lease_epoch) == (RUNNING, epoch) + assert claimed.status == RUNNING, "the instance kept a status that never committed" + assert record_of(parent) == (None, None, False) + assert marks_into(child) == [False] + assert finished == [] + assert len(events(caplog, "task_outcome_unrecorded")) == 1 + assert not events(caplog, "task_outcome_reconnected") + assert not events(caplog, "task_succeeded") + # The reaper takes it back once the lease runs out. + assert actions.expire_lease(parent.id) is True + assert flow_worker().reap() == 1 + assert row(parent).status == READY + + +# -- Downstream release after a lost reply ---------------------------------------- + + +@needs_a_server +@pytest.mark.parametrize("point", ["committed", "before-release"]) +def test_the_reconciler_releases_what_a_committed_outcome_held(point, caplog): + """ + The outcome committed, and the worker's release never ran: after a reply + lost with the commit, django-ox's second try finds the row written and runs + no release; when the session ends after the commit came back, the release + fails on it and is logged. The edges are marked in the committed + transaction, and the reconciler, which every workflow deployment has to + schedule, releases the child, from a worker that is not the one that wrote + the outcome. The parent does not run again. + """ + logs(caplog) + flow, parent, child = a_chain(f"release-{point}") + flow.add("grandchild", tasks.step, "grandchild", depends_on="child") + PLAN[point] = 1 + with finished_signals() as finished, outcome_hooks(): + assert hooked_worker().run_once() is True + settle_connection() + assert SEEN.get(point), "the session was never ended" + assert row(parent).status == SUCCESSFUL + assert marks_into(child) == [True] + assert row(child).status == WAITING, "the release was not skipped" + assert finished == [str(parent.id)] + + reconciler_tick() + assert row(child).status == READY + drain(flow_worker()) + assert row(child).status == SUCCESSFUL + assert OxTask.objects.get(pk=flow.node_ids()["grandchild"]).status == SUCCESSFUL + assert RUNS.get("parent") == 1 + assert row(parent).attempts == 1 + + +# -- what must not change ----------------------------------------------------------- + + +def simulated(kind): + def make(): + if kind == "OperationalError": + return OperationalError("simulated: not a lost connection") + if kind == "InterfaceError": + return InterfaceError("simulated: not a lost connection") + return DatabaseError("simulated: not a lost connection") + + return make + + +@pytest.mark.parametrize( + "kind", ["OperationalError", "InterfaceError", "DatabaseError"] +) +def test_an_error_on_a_connection_that_still_answers_fails_closed(kind, caplog): + """ + Only a lost connection goes to django-ox's second try. The same classes of + error on a connection that answers fail closed after one try, as before. + """ + logs(caplog) + _, parent, child = a_chain(f"not-lost-{kind}") + raised = [] + + def wrapper(execute, sql, params, many, context): + if not raised and node_lock_read(sql, params): + raised.append(sql) + raise simulated(kind)() + return execute(sql, params, many, context) + + with ( + finished_signals() as finished, + outcome_hooks() as writes, + connection.execute_wrapper(wrapper), + ): + assert flow_worker().run_once() is True + + assert len(raised) == 1 + assert len(writes) == 1 + assert row(parent).status == RUNNING + assert record_of(parent) == (None, None, False) + assert marks_into(child) == [False] + assert finished == [] + assert len(events(caplog, "workflow_outcome_unrecorded")) == 1 + assert not events(caplog, "task_outcome_reconnected") + + +def test_an_interface_error_in_the_release_after_the_commit_is_logged(caplog): + """ + The release after a node's committed success logs its failure and leaves + the nodes to the reconciler. Django's InterfaceError is not a + DatabaseError: raised by a release statement on a connection that still + answers, it escaped execute(), and no task_finished was sent for a row + that was SUCCESSFUL. + """ + logs(caplog) + _, parent, child = a_chain("release-interface-error") + releasing = [] + raised = [] + real = HookedWorker._release_after_outcome + + def marked(self, db_task, downstream): + releasing.append(db_task.pk) + try: + real(self, db_task, downstream) + finally: + releasing.clear() + + def wrapper(execute, sql, params, many, context): + if releasing and not raised: + raised.append(sql) + raise simulated("InterfaceError")() + return execute(sql, params, many, context) + + escaped = [] + with ( + pytest.MonkeyPatch.context() as patch, + finished_signals() as finished, + connection.execute_wrapper(wrapper), + ): + patch.setattr(HookedWorker, "_release_after_outcome", marked) + worker = hooked_worker() + claimed = worker.claim_one() + epoch = claimed.lease_epoch + try: + worker.execute(claimed) + except InterfaceError as exc: + escaped.append(repr(exc)) + + first = { + "escaped": escaped, + "release statements failed": len(raised), + "row": row(parent).status, + "record": record_of(parent), + "marks": marks_into(child), + "child": row(child).status, + "finished": list(finished), + "release failed": len(events(caplog, "workflow_release_failed")), + } + assert first == { + "escaped": [], + "release statements failed": 1, + "row": SUCCESSFUL, + "record": (SUCCESSFUL, epoch, True), + "marks": [True], + "child": WAITING, + "finished": [str(parent.id)], + "release failed": 1, + } + + reconciler_tick() + assert row(child).status == READY + drain(flow_worker()) + assert row(child).status == SUCCESSFUL + assert RUNS.get("parent") == 1 + assert row(parent).attempts == 1 + + +@needs_a_server +@pytest.mark.parametrize("caller", ["atomic", "autocommit-off"]) +def test_inside_a_callers_transaction_a_lost_connection_reaches_the_caller( + caller, caplog +): + """ + A caller's transaction is theirs, whatever ended it: the error reaches them, + and nothing writes a second time. + """ + logs(caplog) + _, parent, _child = a_chain(f"caller-lost-{caller}") + worker = hooked_worker() + claimed = worker.claim_one() + PLAN["before"] = 1 + with outcome_hooks() as writes, pytest.raises((OperationalError, InterfaceError)): + if caller == "atomic": + with transaction.atomic(): + worker.execute(claimed, inline=True) + else: + connection.set_autocommit(False) + worker.execute(claimed, inline=True) + forget_the_connection() + assert SEEN.get("before"), "the session was never ended" + assert len(writes) <= 1 + assert row(parent).status == RUNNING + assert not events(caplog, "task_outcome_reconnected") + + +@needs_a_server +def test_a_second_try_is_fenced_when_another_worker_took_the_row(caplog): + """ + The lease ran out while the connection was going, and another worker holds + the row at a later epoch. The second try matches nothing, and nothing is + recorded, sent or released for an outcome this execution no longer owns. + """ + logs(caplog) + _, parent, child = a_chain("reconnect-fenced") + worker = hooked_worker() + claimed = worker.claim_one() + other = {} + + def take_over_then_end(): + def take(): + thief = flow_worker() + assert actions.expire_lease(parent.id) is True + assert thief.reap() == 1 + taken = thief.claim_one() + return (taken.pk, taken.lease_epoch, thief.worker_id) + + other["claim"] = from_another_connection(take) + end_own_session() + + real = HookedWorker._discard_unusable_connections + + def hook(self): + real(self) + if RUNS.get("parent") and "claim" not in other: + take_over_then_end() + + with ( + pytest.MonkeyPatch.context() as patch, + finished_signals() as finished, + outcome_hooks() as writes, + ): + patch.setattr(HookedWorker, "_discard_unusable_connections", hook) + worker.execute(claimed) + settle_connection() + + pk, epoch, thief = other["claim"] + assert str(pk) == str(parent.id) + assert epoch > claimed.lease_epoch + # The first try fails ahead of its write; the second is fenced out. + assert len(writes) == 1 + stored = row(parent) + assert (stored.status, stored.lease_epoch, stored.locked_by) == ( + RUNNING, + epoch, + thief, + ) + assert record_of(parent) == (None, None, False) + assert marks_into(child) == [False] + assert finished == [] + assert len(events(caplog, "task_lease_lost")) == 1 + assert not events(caplog, "task_outcome_reconnected") + assert not events(caplog, "workflow_outcome_unrecorded") + + +@needs_a_server +def test_a_stuck_thread_handover_on_a_lost_connection_leaves_the_row_as_it_was(): + """ + The watchdog's record of a stuck attempt calls the outcome write directly, + with no second try. A lost connection there is raised, which the watchdog + logs as task_stuck_unrecorded before it recycles the worker, and the row and + the instance are left at the epoch the execution holds. + """ + _, parent, child = a_chain("handover-lost") + OxTask.objects.filter(pk=parent.id).update(max_attempts=1) + worker = flow_worker() + claimed = worker.claim_one() + epoch = claimed.lease_epoch + end_own_session() + with pytest.raises((OperationalError, InterfaceError)): + worker._handle_failure(claimed, RuntimeError("stuck"), 0, release=True) + forget_the_connection() + assert claimed.lease_epoch == epoch + assert claimed.status == RUNNING + assert (row(parent).status, row(parent).lease_epoch) == (RUNNING, epoch) + assert record_of(parent) == (None, None, False) + assert marks_into(child) == [False] + + +# -- MySQL session bounds and the driver's liveness check ------------------------- + +needs_mysql = pytest.mark.skipif( + connection.vendor != "mysql", reason="MySQL's lock wait bound is a session value" +) + + +def restoring(value): + def match(sql, params): + return sql.startswith("SET SESSION innodb_lock_wait_timeout") and list( + params or [] + )[:1] == [value] + + return match + + +@contextmanager +def statements_matching(match, error=None): + seen = [] + + def wrapper(execute, sql, params, many, context): + if match(sql, params): + seen.append(sql) + if error is not None: + raise error() + return execute(sql, params, many, context) + + with connection.execute_wrapper(wrapper): + yield seen + + +class TheBlocksOwnError(Exception): + pass + + +@needs_mysql +@pytest.mark.parametrize("restore_error", ["InterfaceError", "OperationalError"]) +def test_on_mysql_a_bound_that_cannot_be_put_back_never_replaces_the_blocks_error( + restore_error, caplog +): + logs(caplog) + with connection.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 37") + error = InterfaceError if restore_error == "InterfaceError" else OperationalError + with ( + statements_matching(restoring(37), lambda: error("restore failed")) as seen, + pytest.raises(TheBlocksOwnError), + workflows.session_timeouts(oxpull_db_alias(), lock_wait=5), + ): + raise TheBlocksOwnError + assert len(seen) == 1 + assert connection.connection is None, "the session was not closed" + assert len(events(caplog, "workflow_session_bounds_not_restored")) == 1 + + +@needs_mysql +@pytest.mark.parametrize("where", ["outside a transaction", "inside a transaction"]) +def test_on_mysql_nothing_is_put_back_on_a_session_the_server_ended(where, caplog): + """ + The block's error is the lost connection's own, and it is what leaves the + block: no statement goes to put back bounds that went with the session. + Outside a transaction the dead connection is closed; inside one Django has + already dropped it and opened another, which never carried the bounds. + """ + logs(caplog) + with connection.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 37") + with ( + statements_matching(restoring(37)) as restores, + pytest.raises(OperationalError), + workflows.session_timeouts(oxpull_db_alias(), lock_wait=5), + ): + if where == "inside a transaction": + with transaction.atomic(), connection.cursor() as cursor: + cursor.execute("SELECT 1") + end_own_session() + cursor.execute("SELECT 1") + else: + end_own_session() + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + assert restores == [] + assert not events(caplog, "workflow_session_bounds_not_restored") + if where == "outside a transaction": + assert connection.connection is None, "the dead connection was left open" + connection.close() + + +@needs_mysql +@pytest.mark.parametrize("restore_error", ["OperationalError", "InterfaceError"]) +def test_on_mysql_a_bound_that_cannot_be_put_back_after_a_clean_block_is_reported( + restore_error, caplog +): + """ + With no error leaving the block, a failure to put the bounds back is never + silent: it is logged and the session closed, and one that is not a database + error is raised as well. + """ + logs(caplog) + with connection.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 37") + error = InterfaceError if restore_error == "InterfaceError" else OperationalError + raised = [] + with statements_matching(restoring(37), lambda: error("restore failed")): + try: + with workflows.session_timeouts(oxpull_db_alias(), lock_wait=5): + pass + except InterfaceError as exc: + raised.append(exc) + assert connection.connection is None + assert len(events(caplog, "workflow_session_bounds_not_restored")) == 1 + assert len(raised) == (1 if restore_error == "InterfaceError" else 0) + + +@needs_mysql +def test_on_mysql_the_drivers_liveness_check_does_not_reconnect(): + """ + django-ox's check that a connection is lost is Django's is_usable(), which + on MySQL is the driver's ping() with its default arguments. A ping that + reconnected would answer for a session the server ended, and the outcome + would never be written a second time. The last part shows these assertions + tell a reconnecting ping apart. + """ + driver = connection.Database + name = f"{driver.__name__} {getattr(driver, 'VERSION_STRING', '')}".strip() + connection.ensure_connection() + raw = connection.connection + ended = end_own_session() + assert connection.is_usable() is False, f"{name}: is_usable() answered" + assert connection.connection is raw + assert not raw.open, f"{name}: ping() opened a new session" + with pytest.raises(driver.Error): + raw.ping() + assert not raw.open + assert flow_worker()._outcome_connection_lost() is True + with suppress(Exception): + connection.close() + + connection.ensure_connection() + raw = connection.connection + ended = end_own_session() + with warnings.catch_warnings(): + # PyMySQL deprecates the argument; this is the one call that wants it. + warnings.simplefilter("ignore", DeprecationWarning) + raw.ping(reconnect=True) + assert raw.open + assert raw_session_id(connection) != ended + connection.close() diff --git a/tests_oxpull/test_workflows_outcomes.py b/tests_oxpull/test_workflows_outcomes.py new file mode 100644 index 0000000..232bb22 --- /dev/null +++ b/tests_oxpull/test_workflows_outcomes.py @@ -0,0 +1,1997 @@ +""" +What a workflow node's outcome writes, and the release that follows it. + +OxpullWorker writes a node's SUCCESSFUL or FAILED, the node's record and, on +success, the marks on the edges out of it, in one transaction. It releases the +nodes those edges held once that transaction has committed. The claim says +which rows are nodes, so a task that is not one pays for nothing more. + +These tests pin that transaction and what it does when it loses a lock +conflict, the table that decides whether a record accepts a write, the flag +each claim path carries, and the bounds, second try and skip queue of the +release. +""" + +import logging +import re +import statistics +import threading +import time +import uuid +from contextlib import contextmanager + +import pytest +from django.db import ( + OperationalError, + connection, + connections, + transaction, +) +from django.db.models import Q +from django.utils import timezone + +import oxpull.worker as oxpull_worker +from django_ox.compat import task_finished +from django_ox.models import OxTask +from django_ox.worker import Worker +from oxpull import workflows +from oxpull.models import OxpullWorkflowEdge, OxpullWorkflowNode, OxpullWorkflowScan +from oxpull.routing import oxpull_db_alias +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_batches import session_isolation +from .test_upgrade import _latest_migration, _migrate +from .test_workflows import drain, flow_worker, reconcile +from .test_workflows_hazard_mixed_fleet_migration import ( + before_workflows_migration, # noqa: F401 - a fixture +) +from .test_workflows_waiting import ( + SimulatedDriverError, + chain, + children_ready_to_release, + conflicting_release_many, + edge_marks, + events, + lock_conflict, + not_a_lock_conflict, + status_of, + statuses, +) + +pytestmark = pytest.mark.django_db + +READY = OxTask.Status.READY +RUNNING = OxTask.Status.RUNNING +WAITING = OxTask.Status.WAITING +SUCCESSFUL = OxTask.Status.SUCCESSFUL +FAILED = OxTask.Status.FAILED +LOST = OxTask.Status.LOST +DISCARDED = OxTask.Status.DISCARDED + +TASK_TABLE = OxTask._meta.db_table +NODE_TABLE = OxpullWorkflowNode._meta.db_table +EDGE_TABLE = OxpullWorkflowEdge._meta.db_table + +# The attribute a claimed row carries, spelled out so this module imports on a +# package without it and each test fails on its own. +NODE_FLAG = "ox_wf_node" + + +def quoted(name): + return connection.ops.quote_name(name) + + +def record_of(result): + return OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch", "succeeded" + ).get(task_id=result.id) + + +def epoch_of(result): + return OxTask.objects.get(pk=result.id).lease_epoch + + +def node_lock_read(sql, params): + """The outcome's read of its node row: the first statement after the task row.""" + node, column, alias = (re.escape(quoted(name)) for name in (NODE_TABLE, "id", "pk")) + # S608: a pattern matched against SQL, not a query. + return re.match(rf"\s*SELECT {node}\.{column}(?: AS {alias})? FROM {node} ", sql) # noqa: S608 + + +def updates_of(table): + def match(sql, params): + return sql.lstrip().startswith(f"UPDATE {quoted(table)}") + + return match + + +@contextmanager +def statements(match=None, error=None, times=None): + """ + Every statement this thread's connection runs in the block, as (sql, + params), and the ones `match` accepts. With `error`, each accepted + statement raises error() instead of running, the first `times` of them, or + all of them when `times` is None. + """ + log, seen = [], [] + left = [times] + + def wrapper(execute, sql, params, many, context): + log.append((sql, list(params) if isinstance(params, (list, tuple)) else params)) + if match is not None and match(sql, params): + seen.append(sql) + if error is not None and (left[0] is None or left[0] > 0): + if left[0] is not None: + left[0] -= 1 + raise error() + return execute(sql, params, many, context) + + with connection.execute_wrapper(wrapper): + yield log, seen + + +@contextmanager +def finished_signals(): + sent = [] + + def receiver(sender, task_result, **kwargs): + sent.append(str(task_result.id)) + + task_finished.connect(receiver) + try: + yield sent + finally: + task_finished.disconnect(receiver) + + +def outcome_writes(log, status): + """The worker's outcome UPDATEs of the task table that write `status`.""" + task = quoted(TASK_TABLE) + return [ + sql + for sql, params in log + if sql.lstrip().startswith(f"UPDATE {task}") + and isinstance(params, list) + and str(status) in [str(value) for value in params] + ] + + +# -- the transition table ----------------------------------------------------- + +# (record outcome, record epoch), (written status, written epoch), accepted. +TRANSITIONS = [ + pytest.param((None, None), (SUCCESSFUL, 1), True, id="none-any"), + pytest.param((None, None), (FAILED, 0), True, id="none-epoch-0"), + pytest.param((None, 4), (SUCCESSFUL, 4), True, id="cleared-equal"), + pytest.param((None, 4), (FAILED, 5), True, id="cleared-greater"), + pytest.param((None, 4), (SUCCESSFUL, 3), False, id="cleared-lesser"), + pytest.param((FAILED, 3), (SUCCESSFUL, 4), True, id="failed-greater"), + pytest.param((SUCCESSFUL, 3), (FAILED, 4), True, id="successful-greater"), + pytest.param((DISCARDED, 3), (LOST, 4), True, id="discarded-greater-lost"), + pytest.param((SUCCESSFUL, 3), (FAILED, 2), False, id="successful-lesser"), + pytest.param((LOST, 3), (SUCCESSFUL, 2), False, id="lost-lesser"), + pytest.param((LOST, 3), (SUCCESSFUL, 3), True, id="lost-equal-successful"), + pytest.param((LOST, 3), (FAILED, 3), True, id="lost-equal-failed"), + pytest.param((LOST, 3), (DISCARDED, 3), True, id="lost-equal-discarded"), + pytest.param((LOST, 3), (LOST, 3), False, id="lost-equal-lost"), + pytest.param((FAILED, 3), (DISCARDED, 3), False, id="failed-equal-discarded"), + pytest.param((FAILED, 3), (LOST, 3), False, id="failed-equal-lost"), + pytest.param((SUCCESSFUL, 3), (FAILED, 3), False, id="successful-equal-failed"), + pytest.param((SUCCESSFUL, 3), (SUCCESSFUL, 3), False, id="successful-equal-same"), + pytest.param((SUCCESSFUL, 3), (LOST, 3), False, id="successful-equal-lost"), + pytest.param((DISCARDED, 3), (SUCCESSFUL, 3), False, id="discarded-equal-success"), + # A settled outcome with no epoch is not a state any writer leaves. It is + # corrupt, and it accepts anything, so the next observation repairs it. + pytest.param((SUCCESSFUL, None), (FAILED, 0), True, id="corrupt-no-epoch"), +] + + +def a_node(name): + flow = workflows.create(name) + result = flow.add("only", tasks.step, "only") + return OxpullWorkflowNode.objects.get(task_id=result.id) + + +def set_record(node, record, succeeded): + OxpullWorkflowNode.objects.filter(pk=node.pk).update( + outcome=record[0], outcome_epoch=record[1], succeeded=succeeded + ) + + +def stored_record(node): + return OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch", "succeeded" + ).get(pk=node.pk) + + +@pytest.mark.parametrize(("record", "write", "accepted"), TRANSITIONS) +def test_a_record_takes_a_write_exactly_where_the_transition_table_says( + record, write, accepted +): + node = a_node("transitions") + set_record(node, record, succeeded=record[0] == SUCCESSFUL) + + took = workflows.record_outcome(node.pk, write[0], write[1], oxpull_db_alias()) + + assert took is accepted + if accepted: + assert stored_record(node) == (write[0], write[1], write[0] == SUCCESSFUL) + else: + assert stored_record(node) == (record[0], record[1], record[0] == SUCCESSFUL) + + +@pytest.mark.parametrize(("record", "write", "accepted"), TRANSITIONS) +def test_succeeded_changes_only_when_the_record_takes_the_write( + record, write, accepted +): + """ + succeeded starts out saying the opposite of the record's outcome, which no + writer leaves behind. A refused write must leave it that way, and a write + the record takes must set it from the written status. + """ + node = a_node("succeeded") + marker = record[0] != SUCCESSFUL + set_record(node, record, succeeded=marker) + + workflows.record_outcome(node.pk, write[0], write[1], oxpull_db_alias()) + + succeeded = stored_record(node)[2] + assert succeeded is ((write[0] == SUCCESSFUL) if accepted else marker) + + +def test_a_record_refuses_a_status_that_is_not_settled(): + node = a_node("unsettled") + for status in (READY, RUNNING, WAITING): + with pytest.raises(ValueError, match="not a settled status"): + workflows.record_outcome(node.pk, status, 1, oxpull_db_alias()) + assert stored_record(node) == (None, None, False) + + +# -- what the outcome writes ---------------------------------------------------- + + +def test_a_node_that_succeeds_records_it_and_marks_the_edges_out_of_it(): + _, parent, child = chain("succeeds") + assert flow_worker().run_once() is True + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert edge_marks(child) == [True] + assert status_of(child) == READY + assert record_of(child) == (None, None, False) + + +def test_a_node_that_fails_records_the_failure_and_marks_nothing(): + flow = workflows.create("fails") + root = flow.add("root", tasks.failing_step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + worker = flow_worker() + assert worker.run_once() is True + assert status_of(root) == READY, "the first attempt should be retried" + assert record_of(root) == (None, None, False), "a retry write recorded something" + drain(worker) + assert status_of(root) == FAILED + assert record_of(root) == (FAILED, epoch_of(root), False) + assert edge_marks(child) == [False] + assert status_of(child) == WAITING + + +def test_a_worker_on_a_backend_without_workflows_records_and_releases(): + """ + "limited" declares no WORKFLOWS, and it shares the "default" queue with + "flows". Any backend on the database declaring WORKFLOWS is enough. + """ + _, parent, child = chain("limited-peer") + worker = OxpullWorker( + backend_alias="limited", backoff_initial=0, poll_interval=0.01 + ) + assert worker._workflows is False + assert worker.run_once() is True + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert edge_marks(child) == [True] + assert status_of(child) == READY + + +@pytest.mark.django_db(transaction=True) +def test_a_worker_started_before_the_workflow_migration_records_once_it_is_applied( + before_workflows_migration, # noqa: F811 + monkeypatch, +): + """ + "limited" declares no WORKFLOWS, so it records only because "flows" does. + Before 0004_workflows its claim cannot read the node table, and it runs a + plain task as it did before. Once the migration is applied it records. + """ + monkeypatch.setattr(oxpull_worker, "NODE_TABLE_RECHECK", 0.0, raising=False) + worker = OxpullWorker( + backend_alias="limited", backoff_initial=0, poll_interval=0.01 + ) + loose = tasks.step.enqueue("loose") + assert worker.run_once() is True + assert status_of(loose) == SUCCESSFUL + + _migrate(_latest_migration()) + _, parent, child = chain("after-the-migration") + assert worker.run_once() is True + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert status_of(child) == READY + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("failing", ["node record", "edge marks"]) +def test_a_failure_part_way_through_a_node_outcome_leaves_the_task_row_running( + failing, caplog +): + """ + The outcome, the record and the marks are one transaction. A statement of + the record or the marks that fails takes the task row's write back with it, + and the worker reports no outcome: the row stays RUNNING for the reaper. + """ + _, parent, child = chain(f"part-way-{failing.replace(' ', '-')}") + table = NODE_TABLE if failing == "node record" else EDGE_TABLE + with ( + caplog.at_level(logging.WARNING, logger="oxpull"), + finished_signals() as finished, + statements(updates_of(table), error=not_a_lock_conflict) as (_, seen), + ): + assert flow_worker().run_once() is True + + assert status_of(parent) == RUNNING, ( + f"the task row's outcome committed although its {failing} failed" + ) + assert len(seen) == 1, "an error that is not a lock conflict was tried again" + assert record_of(parent) == (None, None, False) + assert edge_marks(child) == [False] + assert status_of(child) == WAITING + assert finished == [] + assert len(events(caplog, "workflow_outcome_unrecorded")) == 1 + + +@pytest.mark.django_db(transaction=True) +def test_a_node_outcome_that_loses_a_lock_conflict_runs_its_whole_transaction_again( + caplog, +): + _, parent, child = chain("conflict-once") + with ( + caplog.at_level(logging.INFO, logger="oxpull"), + finished_signals() as finished, + statements(node_lock_read, error=lock_conflict, times=1) as (log, seen), + ): + assert flow_worker().run_once() is True + + assert len(seen) == 2 + assert len(outcome_writes(log, SUCCESSFUL)) == 2, "the fence was not written again" + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert edge_marks(child) == [True] + assert status_of(child) == READY + assert finished == [str(parent.id)] + assert len(events(caplog, "workflow_outcome_retry")) == 1 + assert not events(caplog, "workflow_outcome_unrecorded") + + +@pytest.mark.django_db(transaction=True) +def test_a_stuck_thread_handover_that_loses_a_lock_conflict_keeps_its_epoch(): + """ + The handover writes FAILED at the next epoch. The try that lost had already + copied that epoch onto the instance, and the second try must fence on the + epoch the execution holds, not on the one the lost try wrote. + """ + _, parent, child = chain("handover") + OxTask.objects.filter(pk=parent.id).update(max_attempts=1) + worker = flow_worker() + claimed = worker.claim_one() + assert str(claimed.pk) == str(parent.id) + epoch = claimed.lease_epoch + + with statements(node_lock_read, error=lock_conflict, times=1) as (_, seen): + landed = worker._handle_failure(claimed, RuntimeError("stuck"), 0, release=True) + + assert landed is True + assert len(seen) == 2 + row = OxTask.objects.get(pk=parent.id) + assert (row.status, row.lease_epoch) == (FAILED, epoch + 1) + assert claimed.lease_epoch == epoch + 1 + assert record_of(parent) == (FAILED, epoch + 1, False) + assert edge_marks(child) == [False] + + +@pytest.mark.django_db(transaction=True) +def test_a_node_outcome_that_keeps_losing_lock_conflicts_fails_closed(caplog): + _, parent, child = chain("conflict-always") + worker = flow_worker() + claimed = worker.claim_one() + epoch = claimed.lease_epoch + with ( + caplog.at_level(logging.INFO, logger="oxpull"), + finished_signals() as finished, + statements(node_lock_read, error=lock_conflict) as (_, seen), + ): + worker.execute(claimed) + + assert len(seen) == oxpull_worker.OUTCOME_ATTEMPTS + row = OxTask.objects.get(pk=parent.id) + assert (row.status, row.lease_epoch) == (RUNNING, epoch) + assert claimed.status == RUNNING, "the instance kept a status that never committed" + assert finished == [] + assert len(events(caplog, "workflow_outcome_retry")) == len(seen) - 1 + assert len(events(caplog, "workflow_outcome_unrecorded")) == 1 + assert record_of(parent) == (None, None, False) + assert status_of(child) == WAITING + + +@pytest.mark.django_db(transaction=True) +def test_a_node_outcome_stops_trying_once_the_lease_is_nearly_spent(caplog): + """ + A try holds the task row, which stalls the lease's renewal. With a lease of + 0.6 s renewed every 0.2 s, a try may start only in the first 0.2 s. + """ + _, parent, _child = chain("lease-spent") + worker = flow_worker(lock_timeout=0.6, renew_interval=0.2) + + def slow_conflict(): + time.sleep(0.3) + return lock_conflict() + + with ( + caplog.at_level(logging.INFO, logger="oxpull"), + statements(node_lock_read, error=slow_conflict) as (_, seen), + ): + assert worker.run_once() is True + + assert len(seen) == 1 + assert status_of(parent) == RUNNING + assert len(events(caplog, "workflow_outcome_unrecorded")) == 1 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("caller", ["atomic", "autocommit-off"]) +def test_a_node_outcome_inside_a_callers_transaction_propagates_a_lock_conflict( + caller, +): + """ + Whether the call owns its transaction is read from the connection: an open + atomic block is the caller's, and so is a transaction the caller started + by turning autocommit off. + """ + _, parent, _child = chain(f"caller-{caller}") + worker = flow_worker() + claimed = worker.claim_one() + with ( + statements(node_lock_read, error=lock_conflict) as (_, seen), + pytest.raises(OperationalError), + ): + if caller == "atomic": + with transaction.atomic(): + worker.execute(claimed) + else: + connection.set_autocommit(False) + try: + worker.execute(claimed) + finally: + connection.rollback() + connection.set_autocommit(True) + assert len(seen) == 1 + assert status_of(parent) == RUNNING + + +def recording_release_many(monkeypatch, seen): + """Record every release_many call, and what another connection sees then.""" + real = workflows._waiting.release_many + + def release_many(rows, *, using): + rows = list(rows) + + def look(): + try: + return ( + [status_of(OxTask.objects.get(pk=pk)) for pk, _ in rows], + OxpullWorkflowEdge.objects.filter( + downstream_task_id__in=[pk for pk, _ in rows] + ).values_list("satisfied", flat=True)[::1], + ) + finally: + connections.close_all() + + box = {} + thread = threading.Thread(target=lambda: box.setdefault("seen", look())) + thread.start() + thread.join(30) + seen.append(box.get("seen")) + return real(rows, using=using) + + monkeypatch.setattr(workflows._waiting, "release_many", release_many) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("caller", ["atomic", "autocommit-off"]) +def test_inside_a_callers_transaction_the_release_waits_for_its_commit( + caller, monkeypatch +): + """ + A worker's pass inside a caller's transaction writes the outcome, the record + and the marks there, and releases nothing yet. The release runs once the + caller commits, and from another connection it then sees the child's edge + marked. + """ + _, parent, child = chain(f"release-waits-for-the-commit-{caller}") + worker = flow_worker() + claimed = worker.claim_one() + seen = [] + recording_release_many(monkeypatch, seen) + + if caller == "atomic": + with transaction.atomic(): + worker.execute(claimed) + inside = (status_of(parent), status_of(child), edge_marks(child), seen[:]) + else: + connection.set_autocommit(False) + try: + worker.execute(claimed) + inside = (status_of(parent), status_of(child), edge_marks(child), seen[:]) + connection.commit() + finally: + connection.set_autocommit(True) + + assert inside == (SUCCESSFUL, WAITING, [True], []) + assert seen == [([WAITING], [True])] + assert status_of(child) == READY + + +@pytest.mark.django_db(transaction=True) +def test_a_caller_that_rolls_back_releases_nothing(monkeypatch): + _, parent, child = chain("caller-rolls-back") + worker = flow_worker() + claimed = worker.claim_one() + seen = [] + recording_release_many(monkeypatch, seen) + + with pytest.raises(RuntimeError), transaction.atomic(): + worker.execute(claimed) + raise RuntimeError("the caller's own work failed") + + assert seen == [] + assert (status_of(parent), status_of(child), edge_marks(child)) == ( + RUNNING, + WAITING, + [False], + ) + + +@pytest.mark.django_db(transaction=True) +def test_a_release_failure_after_the_callers_commit_cannot_reach_its_transaction( + monkeypatch, caplog +): + """ + Before, the release ran last inside a caller's transaction, and a release + error that ended that transaction had to propagate, because it took the + outcome with it. The release now runs after the caller's commit, in + transactions of its own. The same error there ends only the release's own + transaction: the outcome and the marks stay committed, task_finished was + sent, and the child goes to its workflow's skip queue for the reconciler. + """ + flow, parent, child = chain("release-error-after-the-commit") + worker = flow_worker() + claimed = worker.claim_one() + + def ends_the_transaction(rows, *, using): + with connections[using].cursor() as cursor: + cursor.execute("ROLLBACK") + raise lock_conflict() + + monkeypatch.setattr(workflows._waiting, "release_many", ends_the_transaction) + with ( + caplog.at_level(logging.WARNING, logger="oxpull"), + finished_signals() as sent, + transaction.atomic(), + ): + worker.execute(claimed) + + assert len(sent) == 1 + assert (status_of(parent), status_of(child), edge_marks(child)) == ( + SUCCESSFUL, + WAITING, + [True], + ) + assert len(events(caplog, "workflow_release_deferred")) == 1 + assert OxpullWorkflowScan.objects.get(workflow_id=flow.id).skipped == [ + str(child.id) + ] + + +@pytest.mark.commit_callbacks_wait +def test_a_node_outcome_in_a_callers_transaction_registers_its_release_on_commit( + django_capture_on_commit_callbacks, +): + """ + The release is a commit callback on the database the record was written to. + Inside the block nothing is released. When the block runs the callbacks, + the child is released. A success with nothing downstream, a final failure + and a task that is not a node register none. + + This test runs inside one transaction and carries commit_callbacks_wait, so + tests_oxpull/conftest.py leaves the worker's callbacks to this block. + """ + _, parent, child = chain("on-commit") + with django_capture_on_commit_callbacks( + using=oxpull_db_alias(), execute=True + ) as callbacks: + assert flow_worker().run_once() is True + inside = (status_of(parent), status_of(child), edge_marks(child)) + assert inside == (SUCCESSFUL, WAITING, [True]) + assert len(callbacks) == 1 + assert status_of(child) == READY + + flow = workflows.create("on-commit-failure") + bad = flow.add("bad", tasks.failing_step, "bad") + flow.add("after-bad", tasks.step, "after-bad", depends_on="bad") + plain = tasks.plain_echo.enqueue("not a node") + with django_capture_on_commit_callbacks(using=oxpull_db_alias()) as callbacks: + # The child, bad's three attempts, and the plain task on a queue the + # flows backend shares. + assert drain(flow_worker()) == 5 + assert (status_of(child), status_of(bad)) == (SUCCESSFUL, FAILED) + assert OxTask.objects.get(pk=plain.id).status == SUCCESSFUL + assert callbacks == [] + + +@pytest.mark.django_db(transaction=True) +def test_the_release_after_a_node_outcome_sees_the_outcome_committed(monkeypatch): + _, parent, child = chain("after-commit") + seen = {} + real = oxpull_worker.release_downstream + + def look_from_another_connection(task_ids, db_alias): + def look(): + try: + seen["parent"] = status_of(parent) + seen["marks"] = edge_marks(child) + finally: + connections.close_all() + + reader = threading.Thread(target=look) + reader.start() + reader.join(30) + seen["ids"] = sorted(str(task_id) for task_id in task_ids) + return real(task_ids, db_alias) + + monkeypatch.setattr( + oxpull_worker, "release_downstream", look_from_another_connection, raising=False + ) + assert flow_worker().run_once() is True + assert seen == {"parent": SUCCESSFUL, "marks": [True], "ids": [str(child.id)]} + assert status_of(child) == READY + + +@pytest.mark.django_db(transaction=True) +def test_a_node_outcome_waits_on_a_held_node_row_for_its_bound_then_tries_again( + monkeypatch, caplog +): + """ + Another transaction holds the upstream's node row, as an add() naming it + does. The outcome waits for the lock no longer than its bound, then runs + its transaction again, and lands once the holder commits. + """ + if connection.vendor not in ("postgresql", "mysql"): + pytest.skip("SQLite has no row locks") + monkeypatch.setattr(oxpull_worker, "OUTCOME_LOCK_TIMEOUT", 1.0, raising=False) + _, parent, child = chain("held-node") + held = threading.Event() + box = {} + + def hold(): + try: + with transaction.atomic(): + list( + OxpullWorkflowNode.objects.select_for_update() + .filter(task_id=parent.id) + .values_list("pk", flat=True) + ) + held.set() + time.sleep(3.5) + except BaseException as exc: + box["error"] = exc + finally: + held.set() + connections.close_all() + + holder = threading.Thread(target=hold) + holder.start() + assert held.wait(30) + started = time.monotonic() + with caplog.at_level(logging.INFO, logger="oxpull"): + assert flow_worker().run_once() is True + elapsed = time.monotonic() - started + holder.join(30) + + assert box == {} + assert elapsed >= 1.0, elapsed + assert len(events(caplog, "workflow_outcome_retry")) >= 1 + assert not events(caplog, "workflow_outcome_unrecorded") + assert status_of(parent) == SUCCESSFUL + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert status_of(child) == READY + + +@contextmanager +def probing_at(match, probe): + """Run `probe(cursor)` on this connection before each statement `match` accepts.""" + found = [] + inside = [False] + + def wrapper(execute, sql, params, many, context): + if match(sql, params) and not inside[0]: + inside[0] = True + try: + with connection.cursor() as cursor: + found.append(probe(cursor)) + finally: + inside[0] = False + return execute(sql, params, many, context) + + with connection.execute_wrapper(wrapper): + yield found + + +def mysql_lock_wait(cursor): + cursor.execute("SELECT @@SESSION.innodb_lock_wait_timeout") + return int(cursor.fetchone()[0]) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("outcome", ["lands", "fails closed"]) +def test_on_mysql_a_node_outcome_bounds_its_lock_waits_and_puts_the_session_back( + outcome, +): + if connection.vendor != "mysql": + pytest.skip("MySQL's lock wait bound is a session variable") + _, parent, _child = chain(f"mysql-bounds-{outcome.replace(' ', '-')}") + with connection.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 37") + error = lock_conflict if outcome == "fails closed" else None + try: + with ( + probing_at(node_lock_read, mysql_lock_wait) as during, + statements(node_lock_read, error=error), + ): + assert flow_worker().run_once() is True + with connection.cursor() as cursor: + after = mysql_lock_wait(cursor) + finally: + connection.close() + expected_tries = oxpull_worker.OUTCOME_ATTEMPTS if error else 1 + assert during == [5] * expected_tries + assert after == 37 + assert status_of(parent) == (RUNNING if error else SUCCESSFUL) + + +@pytest.mark.django_db(transaction=True) +def test_on_mysql_a_session_that_cannot_be_put_back_is_closed(caplog): + if connection.vendor != "mysql": + pytest.skip("MySQL's lock wait bound is a session variable") + _, parent, child = chain("mysql-restore-fails") + with connection.cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 37") + closed = [] + real_close = connection.close + + def close(): + closed.append(True) + real_close() + + def restoring_37(sql, params): + return sql.startswith("SET SESSION innodb_lock_wait_timeout") and list( + params or [] + )[:1] == [37] + + try: + connection.close = close + with ( + caplog.at_level(logging.WARNING, logger="oxpull"), + statements(restoring_37, error=not_a_lock_conflict, times=1) as (_, seen), + ): + assert flow_worker().run_once() is True + finally: + del connection.close + connection.close() + assert len(seen) == 1 + assert closed, "the connection was not closed" + assert len(events(caplog, "workflow_session_bounds_not_restored")) == 1 + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert status_of(child) == READY + + +@pytest.mark.django_db(transaction=True) +def test_on_postgresql_each_try_of_a_node_outcome_bounds_its_lock_waits(): + if connection.vendor != "postgresql": + pytest.skip("PostgreSQL bounds lock waits per transaction") + _, parent, _child = chain("pg-bounds") + + def lock_timeout(cursor): + cursor.execute("SHOW lock_timeout") + return cursor.fetchone()[0] + + with ( + probing_at(node_lock_read, lock_timeout) as during, + statements(node_lock_read, error=lock_conflict, times=1), + ): + assert flow_worker().run_once() is True + with connection.cursor() as cursor: + after = lock_timeout(cursor) + assert during == ["5s", "5s"] + assert after == "0" + assert status_of(parent) == SUCCESSFUL + + +# -- the flag the claim carries --------------------------------------------------- + +# OxpullWorker copies django-ox's PostgreSQL claim statement and the method that +# formats it, to add the node flag. One distribution now ships both, so a +# digest of the copied text would only say the text changed, not whether the +# copy still behaves like the original. These tests hold the behaviour: the +# row a claim leaves, what the claimed object carries, the fence on the lease, +# and the outcome a failure or a success writes must be the same whichever of +# the two workers took the task. On PostgreSQL the claim these workers take +# is the single statement; claim_path() names the path each database runs. + +WORKERS = { + "free": lambda: Worker(backoff_initial=0, poll_interval=0.01), + "oxpull": flow_worker, +} + + +def test_the_node_flag_name_is_the_one_the_claim_sets(): + assert workflows.NODE_FLAG == NODE_FLAG + + +def stored(task_id): + return OxTask.objects.get(pk=task_id) + + +def shape(row): + """What a claim leaves on a row, with the values that differ per row left out.""" + return { + "status": row.status, + "attempts": row.attempts, + "lease_epoch": row.lease_epoch, + "worker_ids": len(row.worker_ids), + "locked_by": bool(row.locked_by), + "locked_at": row.locked_at is not None, + "started_at": row.started_at is not None, + "last_attempted_at": row.last_attempted_at is not None, + "finished_at": row.finished_at is not None, + "lease_seconds": round((row.lease_expires_at - row.locked_at).total_seconds()), + } + + +@pytest.mark.parametrize("kind", sorted(WORKERS)) +def test_a_claim_leaves_a_running_row_with_a_lease_and_a_new_epoch(kind): + queued = tasks.plain_echo.enqueue("claimed") + assert stored(queued.id).lease_epoch == 0 + claimed = WORKERS[kind]().claim_one() + assert str(claimed.pk) == str(queued.id) + assert shape(stored(queued.id)) == { + "status": RUNNING, + "attempts": 1, + "lease_epoch": 1, + "worker_ids": 1, + "locked_by": True, + "locked_at": True, + "started_at": True, + "last_attempted_at": True, + "finished_at": False, + "lease_seconds": shape(stored(queued.id))["lease_seconds"], + } + assert shape(stored(queued.id))["lease_seconds"] > 0 + + +def test_both_workers_leave_the_same_row_behind(): + shapes = {} + for kind in sorted(WORKERS): + queued = tasks.plain_echo.enqueue(kind) + WORKERS[kind]().claim_one() + shapes[kind] = shape(stored(queued.id)) + assert shapes["free"] == shapes["oxpull"] + + +@pytest.mark.parametrize("kind", sorted(WORKERS)) +def test_the_claimed_object_is_the_row_as_the_database_now_holds_it(kind): + queued = tasks.plain_echo.enqueue( + "reloaded", + ) + claimed = WORKERS[kind]().claim_one() + fresh = stored(queued.id) + differing = { + field.attname: (getattr(claimed, field.attname), getattr(fresh, field.attname)) + for field in OxTask._meta.concrete_fields + if getattr(claimed, field.attname) != getattr(fresh, field.attname) + } + assert differing == {} + + +@pytest.mark.parametrize("kind", sorted(WORKERS)) +def test_a_worker_reaped_off_its_task_cannot_write_the_outcome(kind): + queued = tasks.plain_echo.enqueue("fenced") + worker = WORKERS[kind]() + claimed = worker.claim_one() + # What the reaper and a second worker do between this worker's claim and + # its finish: the epoch moves on and another holder has the row. + OxTask.objects.filter(pk=claimed.pk).update( + lease_epoch=claimed.lease_epoch + 1, locked_by="the-new-holder" + ) + worker.execute(claimed) + row = stored(queued.id) + assert (row.status, row.return_value, row.locked_by, row.finished_at) == ( + RUNNING, + None, + "the-new-holder", + None, + ) + assert row.lease_epoch == claimed.lease_epoch + 1 + + +def outcome(row): + return { + "status": row.status, + "attempts": row.attempts, + "errors": len(row.errors), + "finished": row.finished_at is not None, + "locked_by": row.locked_by, + "lease_expires_at": row.lease_expires_at, + } + + +def test_a_success_writes_the_same_outcome_from_either_worker(): + written = {} + for kind in sorted(WORKERS): + queued = tasks.plain_echo.enqueue(f"value-{kind}") + assert WORKERS[kind]().run_once() is True + row = stored(queued.id) + assert row.return_value == f"value-{kind}" + written[kind] = outcome(row) + assert written["free"] == written["oxpull"] + assert written["free"] == { + "status": SUCCESSFUL, + "attempts": 1, + "errors": 0, + "finished": True, + "locked_by": None, + "lease_expires_at": None, + } + + +def test_a_failure_retries_and_then_fails_the_same_from_either_worker(): + history = {} + for kind in sorted(WORKERS): + queued = tasks.plain_fail.enqueue() + worker = WORKERS[kind]() + seen = [] + while worker.run_once(): + row = outcome(stored(queued.id)) + seen.append( + (row["status"], row["attempts"], row["errors"], row["finished"]) + ) + history[kind] = seen + assert history["free"] == history["oxpull"] + assert history["free"] == [ + (READY, 1, 1, False), + (READY, 2, 2, False), + (FAILED, 3, 3, True), + ] + + +class NarrowedWorker(OxpullWorker): + """A subclass that narrows claim_filter_q() alone, and so has no SQL fragment.""" + + def claim_filter_q(self): + condition = super().claim_filter_q() + narrowed = Q(queue_name="default") + return narrowed if condition is None else condition & narrowed + + def claim_filter_sql(self): + return "", {} + + +CLAIMERS = { + "workflow-backend": flow_worker, + "claim-filter-q-only": lambda: NarrowedWorker( + backend_alias="flows", backoff_initial=0, poll_interval=0.01 + ), + "backend-without-workflows": lambda: OxpullWorker( + backend_alias="limited", backoff_initial=0, poll_interval=0.01 + ), +} + + +def claim_path(worker): + """Which django-ox claim path this worker takes on this database.""" + if connection.vendor == "sqlite": + return "compare-and-set" + if connection.vendor == "postgresql" and ( + worker.claim_filter_sql()[0] or worker.claim_filter_q() is None + ): + return "single-statement" + return "skip-locked" + + +@pytest.mark.parametrize("claimer", sorted(CLAIMERS)) +def test_every_claim_path_flags_whether_the_claimed_task_is_a_node(claimer): + plain = tasks.plain_echo.enqueue("plain") + flow = workflows.create(f"flag-{claimer}") + node = flow.add("node", tasks.step, "node") + worker = CLAIMERS[claimer]() + path = claim_path(worker) + + with statements() as (log, _): + first = worker.claim_one() + second = worker.claim_one() + + flags = {str(task.pk): getattr(task, NODE_FLAG) for task in (first, second)} + assert flags == {str(plain.id): False, str(node.id): True} + assert None not in flags.values() + flagged = [sql for sql, _ in log if NODE_FLAG in sql] + assert len(flagged) == 2, flagged + for sql in flagged: + if path == "single-statement": + assert sql.lstrip().startswith("UPDATE") and "RETURNING *" in sql + elif path == "skip-locked": + assert "SKIP LOCKED" in sql + else: + assert sql.lstrip().startswith("SELECT") and "SKIP LOCKED" not in sql + + +@pytest.mark.django_db(transaction=True) +def test_a_task_the_claim_flagged_as_no_node_writes_its_outcome_alone(): + plain = tasks.step.enqueue("plain") + with statements() as (log, _): + assert flow_worker().run_once() is True + assert status_of(plain) == SUCCESSFUL + node_reads = [sql for sql, _ in log if NODE_TABLE in sql and NODE_FLAG not in sql] + assert node_reads == [] + bounds = [sql for sql, _ in log if "set_config" in sql or "SET SESSION" in sql] + assert bounds == [] + + +@pytest.mark.django_db(transaction=True) +def test_a_row_fetched_outside_the_claim_is_looked_up(): + plain = tasks.step.enqueue("plain") + _, parent, child = chain("fetched") + worker = flow_worker() + for expected in (plain, parent): + claimed = worker.claim_one() + assert str(claimed.pk) == str(expected.id) + fetched = OxTask.objects.get(pk=claimed.pk) + assert not hasattr(fetched, NODE_FLAG) + worker.execute(fetched) + assert status_of(plain) == SUCCESSFUL + assert not OxpullWorkflowNode.objects.filter(task_id=plain.id).exists() + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert edge_marks(child) == [True] + assert status_of(child) == READY + + +def data_locks_on(table): + with connection.cursor() as cursor: + cursor.execute( + "SELECT COUNT(*) FROM performance_schema.data_locks " + "WHERE OBJECT_SCHEMA = DATABASE() AND OBJECT_NAME = %s", + [table], + ) + return int(cursor.fetchone()[0]) + + +def lock_rows_on(table): + """Lock rows on `table` in this database, as (index, type, mode, locked data).""" + with connection.cursor() as cursor: + cursor.execute( + "SELECT INDEX_NAME, LOCK_TYPE, LOCK_MODE, LOCK_DATA " + "FROM performance_schema.data_locks " + "WHERE OBJECT_SCHEMA = DATABASE() AND OBJECT_NAME = %s", + [table], + ) + return sorted( + tuple("" if value is None else str(value) for value in row) + for row in cursor.fetchall() + ) + + +def holds_the_gap_only(lock_row): + """True when a lock row covers the gap before a record and not the record.""" + return "GAP" in lock_row[2].split(",") + + +def holds_record_and_gap(lock_row): + """True when a lock row is a next-key lock, over a record and the gap before it.""" + modes = lock_row[2].split(",") + return "GAP" not in modes and "REC_NOT_GAP" not in modes + + +def entry_of(lock_row): + """The index entry a lock row names, as (index, locked data).""" + return lock_row[0], lock_row[3] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["READ COMMITTED", "REPEATABLE READ"]) +def test_on_mysql_a_held_claim_that_flags_a_node_locks_no_node_row(level): + if connection.vendor != "mysql": + pytest.skip("performance_schema.data_locks is MySQL's") + _, parent, _child = chain(f"held-claim-{level.replace(' ', '-')}") + with session_isolation(level): + worker = flow_worker() + with transaction.atomic(): + unflagged = ( + Worker._ready_queryset(worker) + .select_for_update(skip_locked=True) + .first() + ) + assert str(unflagged.pk) == str(parent.id) + unflagged_task_locks = lock_rows_on(TASK_TABLE) + with transaction.atomic(): + candidate = ( + worker._ready_queryset().select_for_update(skip_locked=True).first() + ) + assert str(candidate.pk) == str(parent.id) + assert getattr(candidate, NODE_FLAG) is True + task_locks = lock_rows_on(TASK_TABLE) + node_locks = data_locks_on(NODE_TABLE) + assert task_locks, "the claim held no lock at all, so nothing was measured" + # Compare the locks that decide whether another session waits. A second + # claim skips a locked record and a worker's own UPDATE waits on one, so a + # record the flag locked and django-ox's read did not would cost another + # worker its row. A locking read also locks index records purge has not + # removed yet, so the later read can hold fewer of them than the earlier. + held = [row for row in task_locks if not holds_the_gap_only(row)] + unflagged_held = [ + row for row in unflagged_task_locks if not holds_the_gap_only(row) + ] + assert set(held) <= set(unflagged_held), (held, unflagged_held) + # The gaps themselves are not the same gaps between the two reads: purge + # removes a delete-marked entry the read had locked and hands its gap to + # the successor, so a gap-only lock appears that neither read took. It + # widens nothing, because the claim already holds that record and the gap + # before it, which is what this asserts. + next_key = {entry_of(row) for row in held if holds_record_and_gap(row)} + inherited = [row for row in task_locks if holds_the_gap_only(row)] + assert all(entry_of(row) in next_key for row in inherited), (inherited, held) + assert [row for row in task_locks if row[2].startswith("S")] == [] + assert node_locks == 0 + + +def mysql_only_read_view(): + if connection.vendor != "mysql": + pytest.skip("MySQL answers a locking read's subqueries from a read view") + + +@pytest.mark.django_db(transaction=True) +def test_on_mysql_a_node_added_after_the_claims_read_view_opened_records_its_outcome(): + """ + MySQL answers the claim's subqueries from a read view that can open before + the row the claim returns was committed. In a real claim the gate on a row + the scan passes first opens it. Here a plain read inside the claim's + transaction opens it at REPEATABLE READ, and add() commits a root and its + child from another connection before the locking read runs. + """ + mysql_only_read_view() + flow = workflows.create("view-before-node") + added = {} + + def add_from_another_connection(): + try: + handle = workflows.get(flow.id) + added["root"] = handle.add("root", tasks.step, "root") + added["kid"] = handle.add("kid", tasks.step, "kid", depends_on="root") + finally: + connections.close_all() + + def open_the_view_then_add(execute, sql, params, many, context): + if "SKIP LOCKED" in sql and not added: + with connection.cursor() as cursor: + # S608: a table name from the model's _meta. + cursor.execute(f"SELECT COUNT(*) FROM {quoted(NODE_TABLE)}") # noqa: S608 + adder = threading.Thread(target=add_from_another_connection) + adder.start() + adder.join(30) + return execute(sql, params, many, context) + + with session_isolation("REPEATABLE READ"): + worker = flow_worker() + with connection.execute_wrapper(open_the_view_then_add): + claimed = worker.claim_one() + root, kid = added["root"], added["kid"] + assert str(claimed.pk) == str(root.id) + worker.execute(claimed) + + assert status_of(root) == SUCCESSFUL + assert record_of(root) == (SUCCESSFUL, epoch_of(root), True) + assert edge_marks(kid) == [True] + assert status_of(kid) == READY + assert getattr(claimed, NODE_FLAG) is None + + +# Rows the gate refuses ahead of every root in claim order. The claim probes the +# edge index for each one, so their number is how long a scan runs, and a root is +# claimed from outside the scan's read view only if its COMMIT lands inside that +# time. Measured on MySQL 8.4 held to two CPUs and to half of one, 800 rows make +# a scan 6 to 17 ms against a COMMIT of 2 to 10 ms, and 19 rounds in 20 are hits. +REFUSED_AHEAD = 800 +# Rows added after a COMMIT that landed behind the scan it was aimed at. +REFUSED_GROWTH = 800 +# Roots claimed from outside their claim's read view before the adding stops, and +# the rounds it may take to get them. +HITS_WANTED = 5 +MAX_ROUNDS = 200 +# How many of the latest scans, and of the latest COMMITs, a median is taken over. +MEDIAN_OVER = 9 +# No thread waits on the other for longer than RENDEZVOUS_SECONDS, and no round +# starts after LOOP_SECONDS. +RENDEZVOUS_SECONDS = 30 +LOOP_SECONDS = 300 + + +@contextmanager +def every_connection_at(level): + """ + DATABASES OPTIONS["isolation_level"] for every connection opened in the + block, in any thread. MySQL only. + """ + alias = oxpull_db_alias() + targets = { + id(options): options + for options in ( + connections.settings[alias].setdefault("OPTIONS", {}), + connections[alias].settings_dict.setdefault("OPTIONS", {}), + ) + } + missing = object() + saved = { + key: options.get("isolation_level", missing) for key, options in targets.items() + } + for options in targets.values(): + options["isolation_level"] = level.lower() + connections[alias].close() + try: + yield + finally: + for key, options in targets.items(): + if saved[key] is missing: + options.pop("isolation_level", None) + else: + options["isolation_level"] = saved[key] + connections[alias].close() + + +def refused_rows_ahead(count): + """ + `count` READY plain tasks the claim's gate refuses, each held by an + unsatisfied edge from a FAILED node, enqueued before anything else. + """ + blocker = workflows.create("refused-ahead") + upstream = blocker.add("upstream", tasks.step, "upstream") + OxTask.objects.filter(pk=upstream.id).update( + status=FAILED, finished_at=timezone.now() + ) + with transaction.atomic(): + refused = [tasks.plain_echo.enqueue(f"refused-{i}").id for i in range(count)] + now = timezone.now() + OxpullWorkflowEdge.objects.bulk_create( + [ + OxpullWorkflowEdge( + workflow_id=blocker.id, + upstream_task_id=upstream.id, + downstream_task_id=task_id, + satisfied=False, + created_at=now, + ) + for task_id in refused + ], + batch_size=500, + ) + + +class Aim: + """ + How far into the claimer's scan a root's COMMIT is aimed, and what each + round's verdict does to the next round. + + hit: the claim could not flag the root. It committed after the view of the + scan that claimed it opened, and before that scan reached it. + early: the signalled scan claimed the root and flagged it exactly, so the + COMMIT was in before the scan's view opened. The next COMMIT waits longer. + late: a later scan claimed the root and flagged it exactly, so the COMMIT + landed behind the signalled scan and between two views. The caller adds + refused rows, which makes every scan after it longer. + """ + + # Fractions of a scan: where a COMMIT is aimed, how much later each early + # round puts the next one, and the latest it is ever aimed. The last is + # under 1, so however many rounds were early the wait ends inside the scan. + START = 0.4 + STEP = 0.15 + CEILING = 0.9 + + def __init__(self): + self.later = 0.0 + self.rounds = [] + + @property + def hits(self): + return self.rounds.count("hit") + + def delay(self, scan, commit): + """ + Seconds to wait after the claimer's signal before starting the COMMIT. + + A COMMIT that takes `commit` seconds is aimed to land START of the way + into a scan of `scan` seconds. What the early rounds added comes on top + and does not shrink with `commit`: a COMMIT slower than the scan leaves + nothing to wait for, and an early round must still move the next one. + """ + return max(0.0, self.START * scan - commit) + self.later * scan + + def judge(self, *, flag, claimed_by, signalled): + if flag is None: + verdict = "hit" + elif claimed_by == signalled: + verdict = "early" + self.later = min(self.later + self.STEP, self.CEILING - self.START) + else: + verdict = "late" + self.rounds.append(verdict) + return verdict + + +def test_an_early_round_makes_the_next_commit_wait_longer_and_a_late_one_does_not(): + aim = Aim() + scan, commit = 0.010, 0.002 + first = aim.delay(scan, commit) + assert first == pytest.approx(Aim.START * scan - commit) + assert aim.delay(scan, 2 * scan) == 0.0 + + assert aim.judge(flag=True, claimed_by=7, signalled=7) == "early" + second = aim.delay(scan, commit) + assert second == pytest.approx(first + Aim.STEP * scan) + assert aim.delay(scan, 2 * scan) == pytest.approx(Aim.STEP * scan) + + assert aim.judge(flag=True, claimed_by=9, signalled=8) == "late" + assert aim.judge(flag=None, claimed_by=9, signalled=9) == "hit" + assert aim.judge(flag=None, claimed_by=11, signalled=10) == "hit" + assert aim.delay(scan, commit) == second + assert aim.rounds == ["early", "late", "hit", "hit"] + assert aim.hits == 2 + + for _ in range(10): + aim.judge(flag=True, claimed_by=1, signalled=1) + assert aim.delay(scan, 0.0) == pytest.approx(Aim.CEILING * scan) + assert aim.delay(scan, 0.0) < scan + + +class RootsAddedMidScan: + """ + One thread claims and runs whatever is runnable. Another adds roots to + `flow`, each with one child, and commits each pair while a claim's scan is + under way. + + A root is claimed from outside a read view only if it commits while the + scan that claims it runs, so the claimer must never be behind the adder. One + pair is in flight at a time: a round starts once the pair before it has + finished and a claim has come back empty. add(root) and add(child) run in + one transaction, and SKIP LOCKED passes over both rows until it commits. The + claimer signals as it is about to send its next locking read, and the adder + commits Aim.delay() after the signal. The claim's flag and the scan that + claimed the root make the round's verdict, and the verdict aims the next + round (Aim). + + Each of the first `forced_early` rounds holds the signalled scan back until + the COMMIT is in. The scan's view then has the root, so the flag is exact + and the verdict is "early". + + Every wait ends by RENDEZVOUS_SECONDS, or as soon as the other thread has + stopped. A wait on the other thread that runs out is an error. A wait on the + product, for a committed root to be claimed and for the pair to finish, ends + the rounds and leaves the verdict to the caller's assertions. + """ + + def __init__(self, flow, *, hits_wanted=HITS_WANTED, forced_early=0): + self.flow = flow + self.hits_wanted = hits_wanted + self.forced_early = forced_early + self.alias = oxpull_db_alias() + self.aim = Aim() + self.added, self.levels, self.errors, self.log = [], [], [], [] + self.claimed, self.finished = {}, set() + self.scan_seconds, self.commit_seconds = [], [] + self.scans, self.signalled, self.grown = 0, None, 0 + self.stalled = None + self.armed, self.fired, self.hold, self.committed = ( + threading.Event() for _ in range(4) + ) + self.idle, self.park, self.parked, self.stop = ( + threading.Event() for _ in range(4) + ) + + def run(self): + """Both threads to their end. Raises what either of them raised.""" + threads = [ + threading.Thread(target=target, name=target.__name__, daemon=True) + for target in (self.adder, self.claimer) + ] + for thread in threads: + thread.start() + # The last round starts by LOOP_SECONDS and has six waits in it. + deadline = time.monotonic() + LOOP_SECONDS + 6 * RENDEZVOUS_SECONDS + for thread in threads: + thread.join(max(0.0, deadline - time.monotonic())) + self.stop.set() + if self.errors: + raise AssertionError(self.report()) from self.errors[0][1] + alive = [thread.name for thread in threads if thread.is_alive()] + assert alive == [], self.report() + + def report(self): + def ms(seconds): + recent = seconds[-MEDIAN_OVER:] + return f"{1000 * statistics.median(recent):.1f}" if recent else "-" + + return ( + f"rounds={''.join(verdict[0] for verdict in self.aim.rounds) or '-'}" + f" hits={self.aim.hits}/{self.hits_wanted} later={self.aim.later:.2f}" + f" refused_rows_grown={self.grown} scans={self.scans}" + f" scan_ms={ms(self.scan_seconds)} commit_ms={ms(self.commit_seconds)}" + f" stalled={self.stalled}" + f" errors={[(who, repr(exc)) for who, exc in self.errors]}" + f" last_rounds={self.log[-10:]}" + ) + + def until(self, condition): + """ + Whether `condition`, an Event or a callable, came true before the + deadline and before either thread stopped. + """ + event = condition if isinstance(condition, threading.Event) else None + holds = condition if event is None else event.is_set + deadline = time.monotonic() + RENDEZVOUS_SECONDS + while not holds(): + if self.stop.is_set() or time.monotonic() > deadline: + return holds() + if event is None: + time.sleep(0.001) + else: + event.wait(0.05) + return True + + def rendezvous(self, condition, what): + if not self.until(condition): + why = ( + "the other thread stopped" + if self.stop.is_set() + else f"waited {RENDEZVOUS_SECONDS} s" + ) + raise AssertionError(f"no {what}: {why}") + + def on_scan(self, execute, sql, params, many, context): + if "SKIP LOCKED" not in sql: + return execute(sql, params, many, context) + self.scans += 1 + if self.armed.is_set(): + self.armed.clear() + self.signalled = self.scans + self.fired.set() + if self.hold.is_set(): + self.rendezvous(self.committed, "COMMIT for the held scan") + started = time.monotonic() + try: + return execute(sql, params, many, context) + finally: + self.scan_seconds.append(time.monotonic() - started) + + def claimer(self): + worker = flow_worker() + try: + with connections[self.alias].cursor() as cursor: + cursor.execute("SELECT @@SESSION.transaction_isolation") + self.levels.append(cursor.fetchone()[0]) + with connections[self.alias].execute_wrapper(self.on_scan): + while not self.stop.is_set(): + if self.park.is_set(): + self.parked.set() + self.rendezvous( + lambda: not self.park.is_set(), "end to the adder's park" + ) + self.parked.clear() + continue + task = worker.claim_one() + if task is None: + self.idle.set() + time.sleep(0.002) + continue + self.claimed[str(task.pk)] = (getattr(task, NODE_FLAG), self.scans) + worker.execute(task) + self.finished.add(str(task.pk)) + except BaseException as exc: + self.errors.append(("claimer", exc)) + self.stop.set() + finally: + connections.close_all() + + def adder(self): + deadline = time.monotonic() + LOOP_SECONDS + try: + handle = workflows.get(self.flow.id) + self.rendezvous( + lambda: len(self.scan_seconds) >= MEDIAN_OVER, "scans to time" + ) + for number in range(MAX_ROUNDS): + if self.aim.hits >= self.hits_wanted and number >= self.forced_early: + break + if time.monotonic() > deadline: + raise AssertionError(f"no round may start after {LOOP_SECONDS} s") + if not self.round(handle, number, hold=number < self.forced_early): + break + except BaseException as exc: + self.errors.append(("adder", exc)) + finally: + self.stop.set() + connections.close_all() + + def grow(self): + """ + REFUSED_GROWTH more refused rows, added while the claimer stands still. + + Only the gate holds a refused row back, and the gate reads its edge from + the scan's view. A scan under way when the rows commit would find one of + them READY with no edge in its view, and claim it. + """ + self.park.set() + try: + self.rendezvous(self.parked, "claimer standing still") + refused_rows_ahead(REFUSED_GROWTH) + finally: + self.park.clear() + self.grown += REFUSED_GROWTH + + def round(self, handle, number, *, hold): + """One pair, added, committed and finished. False where it did not finish.""" + self.idle.clear() + self.rendezvous(self.idle, "claim that came back empty") + # Until one COMMIT has been timed there is nothing to aim with, and the + # first one starts at the signal. + delay = ( + 0.0 + if hold or not self.commit_seconds + else self.aim.delay( + statistics.median(self.scan_seconds[-MEDIAN_OVER:]), + statistics.median(self.commit_seconds[-MEDIAN_OVER:]), + ) + ) + self.committed.clear() + if hold: + self.hold.set() + else: + self.hold.clear() + landed = [] + with transaction.atomic(using=self.alias): + # Registered ahead of add()'s own callbacks, so it runs as soon as + # COMMIT returns and times nothing but the COMMIT. + transaction.on_commit( + lambda: landed.append(time.monotonic()), using=self.alias + ) + root = handle.add(f"root{number}", tasks.step, f"root{number}") + kid = handle.add( + f"kid{number}", tasks.step, f"kid{number}", depends_on=f"root{number}" + ) + self.fired.clear() + self.armed.set() + self.rendezvous(self.fired, "scan sent by the claimer") + time.sleep(delay) + started = time.monotonic() + self.commit_seconds.append(landed[0] - started) + self.committed.set() + pair = (str(root.id), str(kid.id)) + self.added.append(pair) + if not self.until(lambda: pair[0] in self.claimed): + self.stalled = f"round {number}: the root was not claimed" + return False + flag, claimed_by = self.claimed[pair[0]] + verdict = self.aim.judge( + flag=flag, claimed_by=claimed_by, signalled=self.signalled + ) + self.log.append( + f"{number}:{verdict} flag={flag}" + f" scans_behind={claimed_by - self.signalled}" + f" delay_ms={1000 * delay:.1f}" + f" commit_ms={1000 * self.commit_seconds[-1]:.1f}" + ) + if not self.until(lambda: set(pair) <= self.finished): + self.stalled = f"round {number}: the pair did not finish" + return False + if verdict == "late": + self.grow() + seen = len(self.scan_seconds) + self.rendezvous( + lambda: len(self.scan_seconds) >= seen + MEDIAN_OVER, + "scans of the new length", + ) + return True + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["READ COMMITTED", "REPEATABLE READ"]) +def test_on_mysql_roots_added_while_claims_scan_past_refused_rows_record_their_outcomes( + level, +): + """ + The view opening as it does in production. Every claim scans past + REFUSED_AHEAD rows its gate refuses, and the gate on the first one opens the + view. Another thread adds roots, each with one child, and commits each pair + part-way through a scan (RootsAddedMidScan), until HITS_WANTED roots have + been claimed from outside their claim's view. Every root must still record + its outcome and release its child. + """ + mysql_only_read_view() + with every_connection_at(level): + refused_rows_ahead(REFUSED_AHEAD) + flow = workflows.create("added-while-claims-scan") + loop = RootsAddedMidScan(flow) + loop.run() + + added, report = loop.added, loop.report() + assert loop.levels == [level.replace(" ", "-")] + roots = [root for root, _ in added] + records = { + str(task_id): outcome + for task_id, outcome in OxpullWorkflowNode.objects.filter( + workflow_id=flow.id + ).values_list("task_id", "outcome") + } + status = { + str(pk): value + for pk, value in OxTask.objects.filter( + pk__in=[task_id for pair in added for task_id in pair] + ).values_list("pk", "status") + } + unrecorded = [ + root for root in roots if status[root] == SUCCESSFUL and records[root] is None + ] + stranded = [ + kid + for root, kid in added + if status[root] == SUCCESSFUL and status[kid] == WAITING + ] + assert unrecorded == [], report + assert stranded == [], report + assert set(status.values()) == {SUCCESSFUL}, report + # The scans were as long as the refused rows make them only if none of those + # rows was claimed. + assert set(loop.claimed) == {task_id for pair in added for task_id in pair}, report + unseen = [root for root in roots if loop.claimed[root][0] is None] + assert len(unseen) >= HITS_WANTED, ( + f"{len(unseen)} roots were claimed from outside their claim's read view, " + f"and {HITS_WANTED} are wanted, so too little was provoked: {report}" + ) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", ["READ COMMITTED", "REPEATABLE READ"]) +def test_on_mysql_a_root_committed_before_its_claims_scan_is_sent_reads_as_early(level): + """ + What separates a hit from a miss. A root that is in its claim's view is + flagged exactly, so a flag of None is not what every claim carries, and the + round it ends makes the next COMMIT wait longer. + """ + mysql_only_read_view() + with every_connection_at(level): + refused_rows_ahead(50) + flow = workflows.create("committed-before-the-scan") + loop = RootsAddedMidScan(flow, hits_wanted=0, forced_early=2) + loop.run() + + report = loop.report() + assert loop.aim.rounds == ["early", "early"], report + assert [loop.claimed[root][0] for root, _ in loop.added] == [True, True], report + assert loop.aim.later == pytest.approx(2 * Aim.STEP), report + assert loop.stalled is None, report + + +# -- the release ---------------------------------------------------------------- + + +def skip_queue(workflow_id): + row = OxpullWorkflowScan.objects.filter(workflow_id=workflow_id).first() + return [] if row is None else list(row.skipped) + + +def workflow_of(task_id): + return OxpullWorkflowNode.objects.get(task_id=task_id).workflow_id + + +@pytest.mark.django_db(transaction=True) +def test_an_owned_release_decides_and_moves_each_chunk_inside_one_transaction( + monkeypatch, +): + monkeypatch.setattr(workflows, "ID_CHUNK_SIZE", 1) + ids = children_ready_to_release("one-transaction", 2) + inside = [] + + def wrapper(execute, sql, params, many, context): + if quoted(TASK_TABLE) in sql or quoted(EDGE_TABLE) in sql: + inside.append((connection.in_atomic_block, sql)) + return execute(sql, params, many, context) + + with connection.execute_wrapper(wrapper): + assert workflows.release_downstream(ids, oxpull_db_alias()) == 2 + assert inside + assert [sql for atomic, sql in inside if not atomic] == [] + assert statuses(ids) == [READY, READY] + + +@pytest.mark.django_db(transaction=True) +def test_an_owned_release_bounds_its_lock_waits_and_statements(): + if connection.vendor == "sqlite": + pytest.skip("SQLite's only bound is the connection's busy timeout") + ids = children_ready_to_release("release-bounds", 1) + if connection.vendor == "postgresql": + + def probe(cursor): + cursor.execute("SHOW lock_timeout") + lock = cursor.fetchone()[0] + cursor.execute("SHOW statement_timeout") + return lock, cursor.fetchone()[0] + + expected, reset = ("1s", "2s"), ("0", "0") + else: + with connection.cursor() as cursor: + cursor.execute( + "SET SESSION innodb_lock_wait_timeout = 37, max_execution_time = 0" + ) + + def probe(cursor): + cursor.execute( + "SELECT @@SESSION.innodb_lock_wait_timeout, " + "@@SESSION.max_execution_time" + ) + lock, statement = cursor.fetchone() + return int(lock), int(statement) + + expected, reset = (1, 2000), (37, 0) + try: + with probing_at(updates_of(TASK_TABLE), probe) as during: + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + with connection.cursor() as cursor: + after = probe(cursor) + finally: + connection.close() + assert during == [expected] + assert after == reset + + +@pytest.mark.django_db(transaction=True) +def test_a_release_inside_a_callers_transaction_sets_no_bounds(): + ids = children_ready_to_release("caller-no-bounds", 1) + with statements() as (log, _), transaction.atomic(): + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + assert [sql for sql, _ in log if "set_config" in sql or "SET SESSION" in sql] == [] + + +@pytest.mark.django_db(transaction=True) +def test_a_release_behind_a_held_row_gives_up_at_its_bound_and_queues_the_row(caplog): + """ + Another transaction holds a waiting node's task row. The release waits for + it no longer than its bound, twice, then leaves the row in its workflow's + skip queue. The reconciler releases it from there once the row is free. + """ + if connection.vendor not in ("postgresql", "mysql"): + pytest.skip("SQLite has no row locks") + ids = children_ready_to_release("held-row", 1) + held, done = threading.Event(), threading.Event() + box = {} + + def hold(): + try: + with transaction.atomic(): + list( + OxTask.objects.select_for_update() + .filter(pk=ids[0]) + .values_list("pk", flat=True) + ) + held.set() + done.wait(15) + except BaseException as exc: + box["error"] = exc + finally: + held.set() + connections.close_all() + + holder = threading.Thread(target=hold) + holder.start() + assert held.wait(30) + started = time.monotonic() + with caplog.at_level(logging.WARNING, logger="oxpull"): + released = workflows.release_downstream(ids, oxpull_db_alias()) + elapsed = time.monotonic() - started + done.set() + holder.join(30) + + assert box == {} + assert elapsed < 8, f"the release waited {elapsed:.1f}s behind the held row" + assert released == 0 + assert len(events(caplog, "workflow_release_deferred")) == 1 + workflow_id = workflow_of(ids[0]) + assert skip_queue(workflow_id) == [str(ids[0])] + reconcile() + assert statuses(ids) == [READY] + assert skip_queue(workflow_id) == [] + + +@pytest.mark.django_db(transaction=True) +def test_a_release_chunk_that_loses_twice_leaves_its_rows_in_the_skip_queue( + monkeypatch, caplog +): + monkeypatch.setattr(workflows, "ID_CHUNK_SIZE", 1) + ids = children_ready_to_release("lost-twice-queued", 2) + conflicting_release_many(monkeypatch, {ids[0]}, conflicts=2) + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + assert statuses(ids) == [WAITING, READY] + assert len(events(caplog, "workflow_release_deferred")) == 1 + assert skip_queue(workflow_of(ids[0])) == [str(ids[0])] + + +def conflict_with_code(code): + """An OperationalError carrying one engine's code for a lost lock conflict.""" + if connection.vendor == "mysql": + return OperationalError(code, "simulated") + cause = SimulatedDriverError("simulated") + if connection.vendor == "postgresql": + cause.sqlstate = code + else: + cause.sqlite_errorcode = code + error = OperationalError("simulated") + error.__cause__ = cause + return error + + +CONFLICT_CODES = { + "postgresql": ["40001", "40P01", "55P03", "57014"], + "mysql": [1205, 1213, 3024, 1969], + "sqlite": [5, 5 | (2 << 8)], +} + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("code", CONFLICT_CODES[connection.vendor], ids=str) +def test_every_lock_conflict_and_timeout_is_decided_again_once(monkeypatch, code): + ids = children_ready_to_release(f"code-{code}", 1) + calls = conflicting_release_many( + monkeypatch, set(ids), conflicts=1, error=lambda: conflict_with_code(code) + ) + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + assert calls == [set(ids), set(ids)] + assert statuses(ids) == [READY] + + +def nodes_without_rows(workflow, count): + """`count` node rows on `workflow` whose task ids name no task row.""" + now = timezone.now() + task_ids = [uuid.uuid4() for _ in range(count)] + OxpullWorkflowNode.objects.bulk_create( + [ + OxpullWorkflowNode( + workflow_id=workflow.id, + name=f"n{index}", + name_key=workflows.name_key(f"n{index}"), + task_id=task_id, + created_at=now, + ) + for index, task_id in enumerate(task_ids) + ] + ) + return task_ids + + +def test_the_skip_queue_keeps_each_node_once_oldest_first_up_to_its_limit(caplog): + flow = workflows.create("queue-limit") + limit = workflows.SKIP_QUEUE_LIMIT + task_ids = nodes_without_rows(flow, limit + 50) + alias = oxpull_db_alias() + first = sorted(task_ids[:10]) + + workflows.queue_skipped(first, alias) + workflows.queue_skipped([*first, uuid.uuid4()], alias) + assert skip_queue(flow.id) == [str(task_id) for task_id in first] + + with caplog.at_level(logging.WARNING, logger="oxpull"): + workflows.queue_skipped(task_ids, alias) + queued = skip_queue(flow.id) + assert len(queued) == limit + assert len(set(queued)) == limit + assert queued[:10] == [str(task_id) for task_id in first] + (overflow,) = events(caplog, "workflow_release_skip_overflow") + assert overflow.dropped_count == 50 + + +@pytest.mark.django_db(transaction=True) +def test_the_reconciler_retries_the_skip_queue_before_its_pass_over_waiting_nodes( + monkeypatch, +): + ids = children_ready_to_release("queue-first", 3) + alias = oxpull_db_alias() + workflows.queue_skipped(ids[1:], alias) + calls = [] + real = workflows.release_downstream + + def recording(task_ids, db_alias): + calls.append(sorted(uuid.UUID(str(task_id)) for task_id in task_ids)) + return real(task_ids, db_alias) + + monkeypatch.setattr(workflows, "release_downstream", recording) + reconcile() + assert calls[0] == ids[1:] + assert statuses(ids) == [READY, READY, READY] + assert skip_queue(workflow_of(ids[0])) == [] + + +@pytest.mark.django_db(transaction=True) +def test_a_worker_marks_edges_only_inside_its_outcome_transaction(): + """ + On a worker's path, every edge mark is an UPDATE inside the outcome's own + transaction, after the task row's outcome and before that transaction's + commit. After the commit the worker writes no edge row: the release moves + task rows only. So the mark is covered by the outcome's second try after a + lock conflict (the next test), and no mark on this path runs outside it. + """ + flow = workflows.create("marks-inside-the-outcome") + parent = flow.add("parent", tasks.step, "parent") + children = [ + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="parent") for i in range(3) + ] + marker = "-- commit --" + real_commit = connection.commit + + def commit(): + log.append((marker, None)) + return real_commit() + + connection.commit = commit + try: + with statements() as (log, _): + assert flow_worker().run_once() is True + finally: + del connection.commit + + sql = [text for text, _ in log] + edge_updates = [ + at for at, text in enumerate(sql) if updates_of(EDGE_TABLE)(text, None) + ] + (outcome,) = [ + at for at, entry in enumerate(log) if outcome_writes([entry], SUCCESSFUL) + ] + commit_after = next( + at for at, text in enumerate(sql) if text == marker and at > outcome + ) + assert edge_updates, sql + assert outcome < edge_updates[0] and edge_updates[-1] < commit_after, sql + assert all(edge_marks(child) == [True] for child in children) + assert record_of(parent)[0] == SUCCESSFUL + + +@pytest.mark.django_db(transaction=True) +def test_a_node_outcome_whose_edge_mark_loses_a_lock_conflict_runs_again(caplog): + """ + The mark itself loses the conflict, as a serialization failure at PostgreSQL + REPEATABLE READ would make it. Simulated: no writer in this package updates + an edge row while an outcome holds it, because every other writer marks an + edge only once it sees the upstream SUCCESSFUL, which this outcome's commit + is the first to show. + """ + _, parent, child = chain("mark-conflict-once") + with ( + caplog.at_level(logging.INFO, logger="oxpull"), + statements(updates_of(EDGE_TABLE), error=lock_conflict, times=1) as (log, seen), + ): + assert flow_worker().run_once() is True + + assert len(seen) == 2 + assert len(outcome_writes(log, SUCCESSFUL)) == 2, "the fence was not written again" + assert record_of(parent) == (SUCCESSFUL, epoch_of(parent), True) + assert edge_marks(child) == [True] + assert status_of(child) == READY + assert len(events(caplog, "workflow_outcome_retry")) == 1 diff --git a/tests_oxpull/test_workflows_reconciler.py b/tests_oxpull/test_workflows_reconciler.py new file mode 100644 index 0000000..ab80f14 --- /dev/null +++ b/tests_oxpull/test_workflows_reconciler.py @@ -0,0 +1,1699 @@ +""" +The workflow reconciler: its backstop, its completion and its cancellation. + +The backstop pages through each open workflow in three phases: the edges not +marked satisfied, the nodes whose record may be behind their task row, and the +waiting nodes to release. Its place survives between ticks, it stops at the +high-water marks its pass began with, and it visits workflows round-robin. A +row another transaction holds is queued and passed, and retried on every tick. + +Completion records outcomes before it completes, locks the task rows of +FAILED and LOST nodes before it classifies them again, and writes what it +counted under the workflow's generation once. + +The cancellation decides only on dead upstream rows it holds locked, or on +missing rows whose record is dead, and it records what it discarded. Its +discard and a release of the same row are each pinned to the epoch their writer +read. + +Every write of several node or edge rows locks them in key order first. Those +tests read the statements each path sends. + +Shortcuts, each named where it is taken: + +- settled_by_hand() writes FAILED onto a task row with one UPDATE. It stands in + for any writer that records nothing on the node, such as a django-ox worker + of another version. +- Several tests shrink BACKSTOP_PAGE, VISIT_PAGES or BACKSTOP_BUDGET so a pass + spans visits with a handful of rows. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from datetime import datetime, timedelta +from io import StringIO +from typing import Any + +import pytest +from django.core.management import call_command +from django.db import connection, connections, transaction +from django.tasks import task +from django.utils import timezone + +from django_ox import _waiting, actions +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import ( + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, + OxpullWorkflowRound, + OxpullWorkflowScan, +) +from oxpull.routing import oxpull_db_alias + +from . import isolation, tasks +from .test_workflows import drain, flow_worker, free_worker, order, reconcile, stored +from .test_workflows_callers import DEADLOCK_CODE, conflict + +pytestmark = pytest.mark.django_db + +READY = OxTask.Status.READY +WAITING = OxTask.Status.WAITING +SUCCESSFUL = OxTask.Status.SUCCESSFUL +FAILED = OxTask.Status.FAILED +DISCARDED = OxTask.Status.DISCARDED +EDGES = OxpullWorkflowScan.Phase.EDGES +NODES = OxpullWorkflowScan.Phase.NODES +RELEASE = OxpullWorkflowScan.Phase.RELEASE + +TASK_TABLE = OxTask._meta.db_table.upper() +WORKFLOW_TABLE = OxpullWorkflow._meta.db_table.upper() + +needs_row_locks = pytest.mark.skipif( + connection.vendor == "sqlite", + reason="SQLite has no row locks, and its writers serialise", +) + + +@task(backend="flows") +def flagged_step(label): + """Fails while its label is flagged, and succeeds once the flag is cleared.""" + tasks._record(label) + if tasks.STATE.get(f"fail:{label}"): + raise ValueError(f"{label} fails while flagged") + return label + + +# -- helpers ----------------------------------------------------------------- + + +def backstop() -> int: + return workflows._release_backstop(oxpull_db_alias()) + + +def status(task_id) -> str: + return OxTask.objects.values_list("status", flat=True).get(pk=task_id) + + +def record_of(task_id) -> tuple[str | None, int | None]: + return OxpullWorkflowNode.objects.values_list("outcome", "outcome_epoch").get( + task_id=task_id + ) + + +def recorded(results) -> int: + return OxpullWorkflowNode.objects.filter( + task_id__in=[result.id for result in results], outcome__isnull=False + ).count() + + +def edge_marks(result) -> list[bool]: + return sorted( + OxpullWorkflowEdge.objects.filter(downstream_task_id=result.id).values_list( + "satisfied", flat=True + ) + ) + + +def scan_phase(flow) -> str | None: + return ( + OxpullWorkflowScan.objects.filter(workflow_id=flow.id) + .values_list("phase", flat=True) + .first() + ) + + +def skip_queue(flow) -> list[str]: + row = OxpullWorkflowScan.objects.filter(workflow_id=flow.id).first() + return [] if row is None else list(row.skipped) + + +def settled_by_hand(results, to=FAILED) -> None: + """Shortcut: settle rows the way a writer that records nothing does.""" + OxTask.objects.filter(pk__in=[result.id for result in results]).update( + status=to, finished_at=timezone.now() + ) + + +def open_roots(name: str, count: int): + """An OPEN workflow of `count` nodes with no dependencies.""" + flow = workflows.create(name) + return flow, [flow.add(f"n{i:02d}", tasks.step, f"n{i:02d}") for i in range(count)] + + +def on_another_connection(step: Callable[[], Any]) -> Any: + out: list[Any] = [] + err: list[BaseException] = [] + + def run() -> None: + try: + out.append(step()) + except BaseException as exc: + err.append(exc) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(60) + assert not thread.is_alive(), "the step on another connection never finished" + if err: + raise err[0] + return out[0] + + +@contextmanager +def holding_node_row(node_pk: int) -> Iterator[None]: + """Another transaction holds this node row's lock for the block.""" + held = threading.Event() + done = threading.Event() + errors: list[BaseException] = [] + + def hold() -> None: + try: + with transaction.atomic(): + list( + OxpullWorkflowNode.objects.select_for_update() + .filter(pk=node_pk) + .values_list("pk", flat=True) + ) + held.set() + done.wait(120) + except BaseException as exc: + errors.append(exc) + held.set() + finally: + connections.close_all() + + thread = threading.Thread(target=hold) + thread.start() + assert held.wait(30) + assert not errors, errors + try: + yield + finally: + done.set() + thread.join(60) + + +# -- the backstop -------------------------------------------------------------- + + +def test_the_backstop_records_an_outcome_no_worker_recorded(): + """ + A django-ox worker writes the outcome and nothing else. The backstop + records it. + """ + _, (root,) = open_roots("unrecorded", 1) + assert free_worker().run_once() is True + row = OxTask.objects.get(pk=root.id) + assert row.status == SUCCESSFUL + assert record_of(root.id) == (None, None) + + backstop() + assert record_of(root.id) == (SUCCESSFUL, row.lease_epoch) + assert OxpullWorkflowNode.objects.get(task_id=root.id).succeeded is True + + +def test_an_edge_whose_pruned_upstream_recorded_its_success_is_satisfied_and_released(): + """ + root ran on a django-ox worker, so nothing marked its edge. Its success was + recorded, and then ox_prune deleted its row. The recorded success is proof + enough: the backstop marks the edge and releases the child. + + Shortcut: the record is written with record_outcome(), the write the + backstop's node phase and completion make, so the edge is still unmarked + when the row goes. + """ + flow = workflows.create("recorded-then-pruned") + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + assert free_worker().run_once() is True + epoch = OxTask.objects.get(pk=root.id).lease_epoch + node = OxpullWorkflowNode.objects.get(task_id=root.id) + assert workflows.record_outcome(node.pk, SUCCESSFUL, epoch, oxpull_db_alias()) + call_command("ox_prune", "--older-than", "0", stdout=StringIO()) + assert not OxTask.objects.filter(pk=root.id).exists() + assert (edge_marks(child), status(child.id)) == ([False], WAITING) + + backstop() + assert edge_marks(child) == [True] + assert status(child.id) == READY + assert flow_worker().run_once() is True + assert order() == ["root", "child"] + + +def test_an_edge_whose_upstream_was_pruned_before_anything_recorded_it_holds(): + """The control: a pruned row with no record proves nothing, so the child waits.""" + flow = workflows.create("pruned-unrecorded") + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + assert free_worker().run_once() is True + call_command("ox_prune", "--older-than", "0", stdout=StringIO()) + assert not OxTask.objects.filter(pk=root.id).exists() + + for _ in range(3): + backstop() + assert record_of(root.id) == (None, None) + assert (edge_marks(child), status(child.id)) == ([False], WAITING) + assert flow_worker().claim_one() is None + + +def test_a_pass_longer_than_a_visit_goes_on_from_where_it_stopped(monkeypatch): + """ + Five nodes that never succeed, two to a page, one page a visit. Each tick + records the next page. A visit that began its pass again from the start + would read the same first page for ever. + """ + monkeypatch.setattr(workflows, "BACKSTOP_PAGE", 2) + monkeypatch.setattr(workflows, "VISIT_PAGES", 1, raising=False) + flow, roots = open_roots("longer-than-a-visit", 5) + settled_by_hand(roots) + + seen = [] + for _ in range(4): + backstop() + seen.append((recorded(roots), scan_phase(flow))) + assert seen == [(0, NODES), (2, NODES), (4, NODES), (5, RELEASE)] + + +def test_nodes_added_after_a_pass_began_wait_for_the_next_pass(monkeypatch): + """ + The pass records its high-water marks when it begins and stops there, so a + workflow that keeps growing cannot keep one pass going for ever. + """ + monkeypatch.setattr(workflows, "BACKSTOP_PAGE", 2) + monkeypatch.setattr(workflows, "VISIT_PAGES", 1, raising=False) + flow, roots = open_roots("high-water", 3) + settled_by_hand(roots) + backstop() # the edge phase: no edges + backstop() # n00 and n01 + late = flow.add("late", tasks.step, "late") + settled_by_hand([late]) + + backstop() # n02, the last node under the mark + assert recorded(roots) == 3 + assert record_of(late.id) == (None, None), ( + "a node added after the pass began was read by that pass" + ) + + ticks = 0 + while record_of(late.id) == (None, None) and ticks < 8: + backstop() + ticks += 1 + assert record_of(late.id) == (FAILED, 0), f"not recorded after {ticks} more ticks" + + +def test_edges_added_after_a_pass_began_wait_for_the_next_pass(monkeypatch): + """ + The same for edges. Three edges from an upstream that succeeded unmarked, + two to a page. The pass marks two, then an edge added after it began + waits: the pass's second edge page stops at the mark it began with. + + Shortcut: settled_by_hand() writes SUCCESSFUL, which marks nothing. + """ + monkeypatch.setattr(workflows, "BACKSTOP_PAGE", 2) + monkeypatch.setattr(workflows, "VISIT_PAGES", 1, raising=False) + flow = workflows.create("edge-high-water") + root = flow.add("root", tasks.step, "root") + children = [ + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="root") for i in range(3) + ] + settled_by_hand([root], to=SUCCESSFUL) + backstop() # c0 and c1 + assert [edge_marks(child) for child in children] == [[True], [True], [False]] + + other = flow.add("other", tasks.step, "other") + late = flow.add("late", tasks.step, "late", depends_on="other") + settled_by_hand([other], to=SUCCESSFUL) + backstop() # c2, the last edge under the mark + assert edge_marks(children[2]) == [True] + assert edge_marks(late) == [False], ( + "an edge added after the pass began was read by that pass" + ) + + ticks = 0 + while edge_marks(late) == [False] and ticks < 8: + backstop() + ticks += 1 + assert edge_marks(late) == [True], f"not marked after {ticks} more ticks" + assert status(late.id) == READY + + +def test_a_visit_pages_on_while_its_pages_come_back_full(monkeypatch): + """ + Five open nodes at two a page is three pages a phase. One visit takes them + all, within VISIT_PAGES, and the pass ends inside it, so the place it + stored after its full pages is back at rest. + """ + monkeypatch.setattr(workflows, "BACKSTOP_PAGE", 2) + flow, roots = open_roots("busy", 5) + settled_by_hand(roots) + + backstop() + assert recorded(roots) == 5 + place = ( + OxpullWorkflowScan.objects.filter(workflow_id=flow.id) + .values_list("phase", "position", "edge_hwm", "node_hwm") + .first() + ) + assert place == (EDGES, 0, None, None) + + +def test_a_tick_that_spends_its_budget_leaves_the_next_workflow_to_the_next_tick( + monkeypatch, +): + """ + With no budget each tick runs its one guaranteed page, so ticks take turns + over the workflows, and each one gets its visit in every round. + """ + monkeypatch.setattr(workflows, "BACKSTOP_BUDGET", 0.0, raising=False) + first, (a,) = open_roots("first", 1) + second, (b,) = open_roots("second", 1) + settled_by_hand([a, b]) + in_order = sorted([first.id, second.id]) + + positions = [] + for _ in range(4): + backstop() + positions.append( + OxpullWorkflowRound.objects.values_list("workflow_id", flat=True).first() + ) + assert positions == [in_order[0], in_order[1], in_order[0], in_order[1]] + assert (record_of(a.id)[0], record_of(b.id)[0]) == (FAILED, FAILED) + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_node_row_another_transaction_holds_is_queued_and_passed(monkeypatch): + """ + The first node's row is held. The page records the rest, queues the held + one, and moves past it. Once the lock is gone the queue gets it. + """ + monkeypatch.setattr(workflows, "BACKSTOP_PAGE", 2) + flow, roots = open_roots("held-node", 3) + settled_by_hand(roots) + held = OxpullWorkflowNode.objects.get(task_id=roots[0].id) + + with holding_node_row(held.pk): + backstop() + records = [record_of(root.id)[0] for root in roots] + queued = skip_queue(flow) + assert records == [None, FAILED, FAILED] + assert queued == [str(roots[0].id)] + + backstop() + assert record_of(roots[0].id) == (FAILED, 0) + assert skip_queue(flow) == [] + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_node_row_held_across_ticks_is_tried_every_tick_and_others_are_served(): + """ + While a node row stays held, every tick tries it again with a lock wait up + to its bound, and a timed-out wait keeps it queued. Another workflow's node + is recorded in the same ticks. Once the lock is gone, the node is recorded. + """ + flow, (a,) = open_roots("held-across-ticks", 1) + settled_by_hand([a]) + _, (b,) = open_roots("served-meanwhile", 1) + held = OxpullWorkflowNode.objects.get(task_id=a.id) + waited = [] + + with holding_node_row(held.pk): + backstop() + assert skip_queue(flow) == [str(a.id)] + settled_by_hand([b]) + for _ in range(2): + started = time.monotonic() + backstop() + waited.append(time.monotonic() - started) + assert skip_queue(flow) == [str(a.id)] + assert record_of(a.id) == (None, None) + assert record_of(b.id) == (FAILED, 0) + assert min(waited) >= workflows.BACKSTOP_LOCK_TIMEOUT * 0.9, waited + + backstop() + assert record_of(a.id) == (FAILED, 0) + assert skip_queue(flow) == [] + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_an_edge_row_another_transaction_holds_is_queued_and_passed(): + """ + The edge into c0 is held. The page marks the edge into c1 and releases c1 + without waiting, queues c0, and moves past it. Once the lock is gone, the + queue gets c0. + """ + flow = workflows.create("held-edge") + flow.add("root", tasks.step, "root") + c0 = flow.add("c0", tasks.step, "c0", depends_on="root") + c1 = flow.add("c1", tasks.step, "c1", depends_on="root") + assert free_worker().run_once() is True + held = OxpullWorkflowEdge.objects.get(downstream_task_id=c0.id) + done = threading.Event() + locked = threading.Event() + + def hold() -> None: + try: + with transaction.atomic(): + list( + OxpullWorkflowEdge.objects.select_for_update() + .filter(pk=held.pk) + .values_list("pk", flat=True) + ) + locked.set() + done.wait(120) + finally: + locked.set() + connections.close_all() + + thread = threading.Thread(target=hold) + thread.start() + try: + assert locked.wait(30) + backstop() + during = (edge_marks(c0), status(c0.id), edge_marks(c1), status(c1.id)) + queued = skip_queue(flow) + finally: + done.set() + thread.join(60) + assert during == ([False], WAITING, [True], READY) + assert queued == [str(c0.id)] + + backstop() + assert (edge_marks(c0), status(c0.id), skip_queue(flow)) == ([True], READY, []) + + +@pytest.mark.django_db(transaction=True) +def test_the_backstop_releases_only_after_its_marks_commit(monkeypatch): + flow = workflows.create("marks-then-release") + flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + assert free_worker().run_once() is True + assert edge_marks(child) == [False] + seen = [] + real = _waiting.release_many + + def release_many(rows, *, using): + rows = list(rows) + seen.append(on_another_connection(lambda: edge_marks(child))) + return real(rows, using=using) + + monkeypatch.setattr(workflows._waiting, "release_many", release_many) + backstop() + assert seen == [[True]] + assert status(child.id) == READY + + +# -- completion ------------------------------------------------------------------ + + +def test_completion_records_outcomes_first_and_completes_on_the_next_tick(): + flow = workflows.create("records-first") + root = flow.add("root", tasks.step, "root") + flow.seal() + assert free_worker().run_once() is True + alias = oxpull_db_alias() + + assert workflows._complete_if_finished(flow.id, alias) is False + assert record_of(root.id)[0] == SUCCESSFUL + assert stored(flow).status == OxpullWorkflow.Status.SEALED + + assert workflows._complete_if_finished(flow.id, alias) is True + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +def test_completion_writes_what_it_counted_under_its_generation(): + flow = workflows.create( + "counted", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + flow.add("root", tasks.step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.add("bad", tasks.failing_step, "bad") + flow.seal() + drain(flow_worker()) + + assert reconcile() == 1 + row = stored(flow) + assert list(row.completions) == ["0"] + entry = row.completions["0"] + counts = { + key: entry[key] + for key in ("total", "succeeded", "failed", "cancelled", "unattributed") + } + assert counts == { + "total": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "unattributed": 0, + } + assert entry["callback_task_id"] == str(row.callback_task_id) + assert entry["success_task_id"] is None + written = datetime.fromisoformat(entry["completed_at"]) + assert abs((written - row.completed_at).total_seconds()) < 1 + + +def test_completion_never_rewrites_a_generations_entry(): + flow = workflows.create("kept") + flow.add("root", tasks.step, "root") + flow.seal() + drain(flow_worker()) + OxpullWorkflow.objects.filter(pk=flow.id).update(completions={"0": {"kept": True}}) + + assert reconcile() == 1 + assert stored(flow).completions == {"0": {"kept": True}} + + +def test_progress_counts_a_pruned_failure_as_completion_does(): + """ + A failure recorded before ox_prune deleted its row is still a failure, in + progress() as in the completion that follows it. + """ + flow = workflows.create("pruned-failure-progress") + bad = flow.add("bad", tasks.failing_step, "bad") + flow.add("good", tasks.step, "good") + flow.seal() + drain(flow_worker()) + OxTask.objects.filter(pk=bad.id).update( + finished_at=timezone.now() - timedelta(days=8) + ) + call_command("ox_prune", "--include-failed", stdout=StringIO()) + assert not OxTask.objects.filter(pk=bad.id).exists() + + progress = flow.progress() + assert (progress.succeeded, progress.failed, progress.unattributed) == (1, 1, 0) + assert reconcile() == 1 + assert stored(flow).failed_count == progress.failed + + +def test_a_node_pruned_before_anything_recorded_it_does_not_hold_completion(): + """ + root ran on an OxpullWorker, which recorded it. leaf ran on a django-ox + worker, which records nothing, and ox_prune deleted both rows before any + tick. root still counts as succeeded by its record. leaf is finished and + its outcome unknown: it does not hold the workflow, and on_success is + withheld. + """ + flow = workflows.create( + "unknown-leaf", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + on_success=tasks.record_callback, + on_success_args=["success"], + ) + root = flow.add("root", tasks.step, "root") + leaf = flow.add("leaf", tasks.step.using(priority=-5), "leaf") + flow.seal() + assert flow_worker().run_once() is True + assert free_worker().run_once() is True + assert order() == ["root", "leaf"] + call_command("ox_prune", "--older-than", "0", stdout=StringIO()) + assert not OxTask.objects.filter(pk__in=[root.id, leaf.id]).exists() + assert (record_of(root.id)[0], record_of(leaf.id)) == (SUCCESSFUL, (None, None)) + + assert reconcile() == 1 + row = stored(flow) + assert row.status == OxpullWorkflow.Status.COMPLETE + assert row.callback_task_id is not None and row.success_task_id is None + entry = row.completions["0"] + assert (entry["succeeded"], entry["failed"], entry["unattributed"]) == (1, 0, 1) + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_retry_racing_the_completion_waits_for_it(monkeypatch): + """ + django-ox's retry takes no workflow lock. It moves the FAILED leaf back to + READY while completion is deciding. The completion holds the leaf's row, + so the retry lands after it commits, never inside it. + """ + flow = workflows.create( + "retry-races-completion", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + leaf = flow.add("leaf", tasks.failing_step, "leaf") + flow.seal() + drain(flow_worker()) + assert status(leaf.id) == FAILED + step: dict[str, Any] = {} + real = workflows._enqueue_callback + + def enqueue_callback(*args, **kwargs): + if "thread" not in step: + thread = threading.Thread( + target=lambda: step.setdefault( + "retried", on_another_connection(lambda: actions.retry(leaf.id)) + ) + ) + step["thread"] = thread + thread.start() + thread.join(2.0) + step["finished_inside"] = not thread.is_alive() + return real(*args, **kwargs) + + monkeypatch.setattr(workflows, "_enqueue_callback", enqueue_callback) + completed = reconcile() + step["thread"].join(60) + facts = ( + f"completed={completed} retry finished inside the completion=" + f"{step['finished_inside']} retried={step.get('retried')} " + f"workflow={stored(flow).status} leaf={status(leaf.id)}" + ) + assert not (step["finished_inside"] and completed == 1), facts + assert completed == 1, facts + assert step.get("retried") is True, facts + assert status(leaf.id) == READY, facts + + +# -- cancellation ------------------------------------------------------------------ + + +def test_the_cancellation_records_each_node_it_discards(): + flow = workflows.create("recorded-cancellation") + flow.add("root", tasks.failing_step, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + leaf = flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.seal() + drain(flow_worker()) + + assert reconcile() == 1 + for node in (mid, leaf): + epoch = OxTask.objects.get(pk=node.id).lease_epoch + assert (status(node.id), record_of(node.id)) == (DISCARDED, (DISCARDED, epoch)) + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +AFTER_THE_FIRST_READ = "after the first read" +AFTER_THE_LOCK = "after the lock" + + +class CancellationPause: + """ + Run `step` once on its own thread and connection, inside the reconciler's + first cancellation transaction, and wait up to `wait` seconds for it. + + The transaction is found by what it sends: its first statement is the + UPDATE of the workflow row. AFTER_THE_FIRST_READ is after the first SELECT + that follows it, AFTER_THE_LOCK after the first SELECT ... FOR UPDATE on the + task table that follows it. + """ + + def __init__(self, point: str, step: Callable[[], Any], wait: float = 3.0): + self.point = point + self.step = step + self.wait = wait + self.opened = False + self.fired = False + self.finished_inside: bool | None = None + self.result: Any = None + self.error: BaseException | None = None + self.thread: threading.Thread | None = None + + def _run(self) -> None: + try: + self.result = self.step() + except BaseException as exc: + self.error = exc + finally: + connections.close_all() + + def __call__(self, execute, sql, params, many, context): + result = execute(sql, params, many, context) + text = " ".join(sql.split()).upper() + if not self.opened: + self.opened = text.startswith("UPDATE") and WORKFLOW_TABLE in text + return result + if self.fired or not text.startswith("SELECT"): + return result + if self.point == AFTER_THE_LOCK and not ( + "FOR UPDATE" in text and TASK_TABLE in text + ): + return result + self.fired = True + self.thread = threading.Thread(target=self._run) + self.thread.start() + self.thread.join(self.wait) + self.finished_inside = not self.thread.is_alive() + return result + + def join(self) -> None: + if self.thread is not None: + self.thread.join(60) + assert not self.thread.is_alive() + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_an_upstream_retried_before_the_cancellation_locks_it_keeps_its_child(): + """ + x and y both failed for good, each with a child. The operator retries x + after the cancellation has read its candidates and before it locks them. + The lock finds x READY, so only y's child is cancelled, and x's child runs + once x succeeds. + """ + flow = workflows.create("retried-before-the-lock") + x = flow.add("x", flagged_step, "x") + d = flow.add("d", tasks.step, "d", depends_on="x") + flow.add("y", tasks.failing_step, "y") + e = flow.add("e", tasks.step, "e", depends_on="y") + flow.seal() + tasks.STATE["fail:x"] = True + drain(flow_worker()) + assert (status(x.id), status(d.id), status(e.id)) == (FAILED, WAITING, WAITING) + + pause = CancellationPause(AFTER_THE_FIRST_READ, lambda: actions.retry(x.id)) + with connection.execute_wrapper(pause): + reconcile() + pause.join() + after = (status(x.id), status(d.id), status(e.id)) + tasks.STATE["fail:x"] = False + drain(flow_worker()) + reconcile() + facts = ( + f"retry finished inside the pass={pause.finished_inside} " + f"returned={pause.result!r} error={pause.error!r}; after the pass " + f"(x, d, e)={after}; at the end d={status(d.id)} order={order()} " + f"workflow={stored(flow).status}" + ) + assert after[1] != DISCARDED, facts + assert pause.fired and pause.finished_inside and pause.result is True, facts + assert after[2] == DISCARDED, facts + assert status(d.id) == SUCCESSFUL, facts + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_an_upstream_dead_before_the_pass_makes_a_retry_wait_for_the_cancellation(): + """ + The interlock: the retry of an upstream the cancellation holds waits for + it, so the child is cancelled and the retry lands afterwards. + """ + flow = workflows.create("dead-before-the-pass") + x = flow.add("x", tasks.failing_step, "x") + d = flow.add("d", tasks.step, "d", depends_on="x") + flow.seal() + drain(flow_worker()) + assert status(x.id) == FAILED + + pause = CancellationPause(AFTER_THE_LOCK, lambda: actions.retry(x.id)) + with connection.execute_wrapper(pause): + reconcile() + pause.join() + facts = ( + f"fired={pause.fired} retry finished inside the pass=" + f"{pause.finished_inside} returned={pause.result!r} error={pause.error!r} " + f"x={status(x.id)} d={status(d.id)}" + ) + assert pause.fired and pause.finished_inside is False, facts + assert pause.error is None and pause.result is True, facts + assert (status(d.id), status(x.id)) == (DISCARDED, READY), facts + + +# The orders that meet at the downstream row: the cancellation's discard and a +# release, each pinned to the lease epoch its writer read. Under the release rule +# no oxpull release moves a node with an unmarked edge from a dead upstream, so +# the release below stands in for one that is not oxpull's (a faulty release or +# an application's UPDATE). A claim that lands first is +# test_workflows_hazard_cancel_cas's case. + + +def doomed_child(name: str): + flow = workflows.create(name) + x = flow.add("x", tasks.failing_step, "x") + d = flow.add("d", tasks.step, "d", depends_on="x") + flow.seal() + drain(flow_worker()) + assert (status(x.id), status(d.id)) == (FAILED, WAITING) + return flow, x, d, OxTask.objects.get(pk=d.id).lease_epoch + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_release_that_lands_before_the_cancellations_discard_is_cancelled_anyway( + monkeypatch, +): + """ + The cancellation has read d WAITING at epoch e. A release moves d to READY at + e before the discard. The discard is pinned to e, and a release leaves the + epoch alone, so it still matches READY at e and d is cancelled. + """ + flow, _, d, epoch = doomed_child("release-first") + real = _waiting.cancel_many + landed = [] + + def cancel_many(rows, *, using): + rows = list(rows) + landed.append( + on_another_connection( + lambda: _waiting.release(d.id, lease_epoch=epoch, using=using) + ) + ) + landed.append(on_another_connection(lambda: status(d.id))) + return real(rows, using=using) + + monkeypatch.setattr(workflows._waiting, "cancel_many", cancel_many) + workflows._cancel_unreachable(flow.id, oxpull_db_alias()) + assert landed == [True, READY] + assert (status(d.id), record_of(d.id)) == (DISCARDED, (DISCARDED, epoch)) + + +def test_a_release_pinned_before_the_cancellation_matches_nothing_after_it(): + """ + Cancel first: a release decided while d waited at epoch e lands after d was + cancelled at e, and moves nothing. + """ + flow, _, d, epoch = doomed_child("cancel-first") + workflows._cancel_unreachable(flow.id, oxpull_db_alias()) + assert status(d.id) == DISCARDED + + assert _waiting.release_many([(d.id, epoch)], using=oxpull_db_alias()) == (0, 1) + assert (status(d.id), record_of(d.id)) == (DISCARDED, (DISCARDED, epoch)) + + +def test_a_release_pinned_before_a_cancellation_and_a_revival_matches_nothing(): + """ + Cancel and revive first: d is WAITING again at epoch e + 1, so the release + decided at e moves nothing, and the next cancellation of d records its own + epoch. + """ + flow, _, d, epoch = doomed_child("cancel-and-revive-first") + alias = oxpull_db_alias() + workflows._cancel_unreachable(flow.id, alias) + revived = _waiting.revive_many([(d.id, epoch)], using=alias) + assert revived == {uuid.UUID(str(d.id)): _waiting.Revival.REVIVED} + + assert _waiting.release_many([(d.id, epoch)], using=alias) == (0, 1) + assert OxTask.objects.values_list("status", "lease_epoch").get(pk=d.id) == ( + WAITING, + epoch + 1, + ) + workflows._cancel_unreachable(flow.id, alias) + assert (status(d.id), record_of(d.id)) == (DISCARDED, (DISCARDED, epoch + 1)) + + +def test_every_id_the_cancellation_names_is_a_uuid(): + """ + The pairs handed to django-ox are (UUID, int), whatever the database returns + for a UUID column. + """ + flow = workflows.create("pairs") + flow.add("root", tasks.failing_step, "root") + flow.add("child", tasks.step, "child", depends_on="root") + flow.seal() + drain(flow_worker()) + handed = [] + real = _waiting.cancel_many + + def cancel_many(rows, *, using): + rows = list(rows) + handed.extend(rows) + return real(rows, using=using) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(workflows._waiting, "cancel_many", cancel_many) + reconcile() + assert handed + assert all(isinstance(pk, uuid.UUID) and isinstance(e, int) for pk, e in handed) + + +# -- lock order ---------------------------------------------------------------------- + +NODE_TABLE = OxpullWorkflowNode._meta.db_table +EDGE_TABLE = OxpullWorkflowEdge._meta.db_table + + +@contextmanager +def every_statement() -> Iterator[list[str]]: + """Every statement this thread's connection runs in the block, in order.""" + log: list[str] = [] + + def wrapper(execute, sql, params, many, context): + log.append(" ".join(sql.split())) + return execute(sql, params, many, context) + + with connection.execute_wrapper(wrapper): + yield log + + +def ordered_by_key(sql: str, table: str) -> bool: + """ + Whether a SELECT of `table` is ordered by its key. Django orders by position + when the key is the first column it selects. + """ + quoted = connection.ops.quote_name + key = f"{quoted(table)}.{quoted('id')}" + return f"FROM {quoted(table)}" in sql and ( + f"ORDER BY {key} ASC" in sql + or (sql.startswith(f"SELECT {key}") and "ORDER BY 1 ASC" in sql) + ) + + +def ordered_lock_of(sql: str, table: str) -> bool: + """Whether a statement is a locking read of `table` in key order.""" + return ordered_by_key(sql, table) and " FOR UPDATE" in sql + + +def multi_row_writes_lock_in_key_order(log: list[str], table: str) -> int: + """ + Check that every UPDATE of several rows of `table` by primary key comes after + a locking read of that table in key order, and return how many there were. + """ + quoted = connection.ops.quote_name + update = f"UPDATE {quoted(table)} " + by_keys = f"{quoted(table)}.{quoted('id')} IN (" + checked = 0 + for at, sql in enumerate(log): + if not (sql.startswith(update) and by_keys in sql): + continue + checked += 1 + assert any(ordered_lock_of(s, table) for s in log[:at]), ( + f"an UPDATE of {table} by keys took no lock in key order first: {sql}" + ) + return checked + + +def _chain_run_by_a_free_worker(name: str): + flow = workflows.create(name) + root = flow.add("root", tasks.step, "root") + children = [ + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="root") for i in range(3) + ] + assert free_worker().run_once() is True + return flow, root, children + + +def _lock_order_hook(): + flow = workflows.create("hook") + flow.add("root", tasks.step, "root") + for i in range(3): + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="root") + with every_statement() as log: + assert flow_worker().run_once() is True + return log, EDGE_TABLE + + +def _lock_order_backstop_edges(): + _chain_run_by_a_free_worker("backstop-edges") + with every_statement() as log: + backstop() + return log, EDGE_TABLE + + +def _lock_order_backstop_nodes(): + _, roots = open_roots("backstop-nodes", 3) + settled_by_hand(roots) + with every_statement() as log: + backstop() + return log, NODE_TABLE + + +def _lock_order_repair_nodes(): + flow = workflows.create("repair-nodes") + roots = [flow.add(f"n{i}", tasks.step, f"n{i}") for i in range(3)] + flow.seal() + settled_by_hand(roots) + with every_statement() as log: + workflows._complete_if_finished(flow.id, oxpull_db_alias()) + return log, NODE_TABLE + + +def _lock_order_repair_edges(): + flow, _, _ = _chain_run_by_a_free_worker("repair-edges") + flow.seal() + with every_statement() as log: + workflows._complete_if_finished(flow.id, oxpull_db_alias()) + return log, EDGE_TABLE + + +def _lock_order_cancellation(): + flow = workflows.create("cancellation") + flow.add("root", tasks.failing_step, "root") + for i in range(3): + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="root") + flow.seal() + drain(flow_worker()) + with every_statement() as log: + workflows._cancel_unreachable(flow.id, oxpull_db_alias()) + return log, NODE_TABLE + + +def _lock_order_add_recheck(): + flow = workflows.create("add-recheck") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b") + real = workflows._succeeded_among + + def sees_nothing_yet(task_ids, db_alias): + # Both upstreams succeed after add() read them and before it commits. + OxTask.objects.filter(pk__in=task_ids).update(status=SUCCESSFUL) + return real([], db_alias) + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(workflows, "_succeeded_among", sees_nothing_yet) + with every_statement() as log: + flow.add("child", tasks.step, "child", depends_on=["a", "b"]) + return log, EDGE_TABLE + + +LOCK_ORDER_PATHS = { + "the outcome hook's marks": _lock_order_hook, + "the backstop's edge page": _lock_order_backstop_edges, + "the backstop's node page": _lock_order_backstop_nodes, + "completion's record repair": _lock_order_repair_nodes, + "completion's marks": _lock_order_repair_edges, + "the cancellation's records": _lock_order_cancellation, + "add()'s check after commit": _lock_order_add_recheck, +} + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("path", list(LOCK_ORDER_PATHS)) +def test_a_multi_row_write_of_node_or_edge_rows_locks_them_in_key_order_first(path): + """ + The lock order: several node or edge rows are locked by a locking read in + key order before they are written, because an UPDATE locks rows in the + order its plan reads them. Each path here writes several rows of one table, + and each such UPDATE must follow a locking read of that table ordered by + its key. The paths run in autocommit, as a worker and the reconciler do. + """ + log, table = LOCK_ORDER_PATHS[path]() + assert multi_row_writes_lock_in_key_order(log, table) >= 1, ( + f"{path} wrote no rows of {table} by key, so there was nothing to check" + ) + + +def test_the_outcome_hook_reads_the_edges_out_of_a_node_in_key_order(): + """ + The hook's read of the edges it marks is ordered by key on every database, + and is the locking read where the database has row locks. + """ + flow = workflows.create("hook-read") + flow.add("root", tasks.step, "root") + for i in range(3): + flow.add(f"c{i}", tasks.step, f"c{i}", depends_on="root") + quoted = connection.ops.quote_name + with every_statement() as log: + assert flow_worker().run_once() is True + reads = [ + sql + for sql in log + if sql.startswith(f"SELECT {quoted(EDGE_TABLE)}.") + and f"{quoted(EDGE_TABLE)}.{quoted('upstream_task_id')} = " in sql + ] + assert reads, log + assert all(ordered_by_key(sql, EDGE_TABLE) for sql in reads), reads + if connection.features.has_select_for_update: + assert all(sql.endswith("FOR UPDATE") for sql in reads), reads + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_node_that_fails_again_inside_the_completion_holds_it_back(): + """ + Both nodes failed for good. Completion takes the workflow row, and just then + an operator retries b and a worker claims it, so completion's read of the + FAILED rows finds only a and locks only a. b's worker then fails it again. + Under READ COMMITTED the classification sees b FAILED and recorded, as if + nothing had happened. Nothing holds b, so an operator's next retry would + land at once and the workflow would complete with b READY. The completion + must notice a FAILED row it did not lock and leave the workflow SEALED for + the next tick. + """ + flow = workflows.create("fails-again-inside") + a = flow.add("a", tasks.failing_step, "a") + b = flow.add("b", tasks.failing_step, "b") + flow.seal() + drain(flow_worker()) + assert (status(a.id), status(b.id)) == (FAILED, FAILED) + steps: dict[str, Any] = {} + + def retry_and_claim_b(): + assert actions.retry(b.id) is True + claimed = flow_worker().claim_one() + assert claimed is not None and str(claimed.id) == str(b.id) + return claimed + + class Pauses: + def __call__(self, execute, sql, params, many, context): + result = execute(sql, params, many, context) + text = " ".join(sql.split()).upper() + if ( + "cas" not in steps + and text.startswith("UPDATE") + and WORKFLOW_TABLE in text + ): + steps["cas"] = True + steps["claimed"] = on_another_connection(retry_and_claim_b) + elif ( + "cas" in steps + and "failed_again" not in steps + and text.startswith("SELECT") + and "FOR UPDATE" in text + and TASK_TABLE in text + ): + steps["failed_again"] = True + on_another_connection( + lambda: flow_worker().execute(steps["claimed"], inline=True) + ) + steps["b_after"] = on_another_connection(lambda: status(b.id)) + return result + + with connection.execute_wrapper(Pauses()): + completed = workflows._complete_if_finished(flow.id, oxpull_db_alias()) + facts = ( + f"completed={completed} steps={sorted(steps)} workflow={stored(flow).status}" + ) + assert steps.get("failed_again") and steps.get("b_after") == FAILED, facts + assert completed is False, facts + assert stored(flow).status == OxpullWorkflow.Status.SEALED, facts + + assert reconcile() == 1 + assert stored(flow).failed_count == 2 + + +def test_add_reads_the_upstream_nodes_it_names_in_key_order(): + """ + add() holds the workflow row, then reads the nodes it depends on. The read is + ordered by key, and is the locking read where the database has row locks. + """ + flow = workflows.create("add-read") + for name in ("c", "a", "b"): + flow.add(name, tasks.step, name) + quoted = connection.ops.quote_name + with every_statement() as log: + flow.add("child", tasks.step, "child", depends_on=["c", "a", "b"]) + reads = [ + sql + for sql in log + if sql.startswith(f"SELECT {quoted(NODE_TABLE)}.") and quoted("name_key") in sql + ] + assert len(reads) == 1, log + assert ordered_by_key(reads[0], NODE_TABLE), reads + if connection.features.has_select_for_update: + assert reads[0].endswith("FOR UPDATE"), reads + + +# -- decisions at both isolation levels ----------------------------------------- + +LEVELS = ["read committed", "repeatable read"] +NODE_TABLE_UPPER = NODE_TABLE.upper() + + +@contextmanager +def every_connection_at(level: str) -> Iterator[None]: + """ + DATABASES OPTIONS["isolation_level"] for every connection opened in the + block, in any thread, as a project that sets it runs. PostgreSQL and MySQL. + """ + alias = oxpull_db_alias() + targets = { + id(options): options + for options in ( + connections.settings[alias].setdefault("OPTIONS", {}), + connections[alias].settings_dict.setdefault("OPTIONS", {}), + ) + } + missing = object() + saved = { + key: options.get("isolation_level", missing) for key, options in targets.items() + } + isolation.outlives_the_transaction(level) + value: Any = level + if connection.vendor == "postgresql": + from psycopg import IsolationLevel + + value = ( + IsolationLevel.REPEATABLE_READ + if level == "repeatable read" + else IsolationLevel.READ_COMMITTED + ) + for options in targets.values(): + options["isolation_level"] = value + connections[alias].close() + try: + assert level_in_use() == level + yield + finally: + for key, options in targets.items(): + if saved[key] is missing: + options.pop("isolation_level", None) + else: + options["isolation_level"] = saved[key] + connections[alias].close() + + +def serialization_failures_at(level: str) -> bool: + """Whether a row changed after the snapshot fails a lock at this level.""" + return connection.vendor == "postgresql" and level == "repeatable read" + + +def events_named(caplog, name: str) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if getattr(record, "event", "") == name + ] + + +def level_in_use() -> str: + with transaction.atomic(), connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SHOW transaction_isolation") + else: + cursor.execute("SELECT @@SESSION.transaction_isolation") + return str(cursor.fetchone()[0]).replace("-", " ").lower() + + +class RetryInside: + """ + Run django-ox's retry of one task on another connection, once, right after + the completing transaction sends the statement `after` picks out, and wait + up to three seconds for it. The transaction is found by its compare-and-set, + the UPDATE of the workflow row to COMPLETE. + """ + + def __init__(self, task_id: Any, after: Callable[[str], bool]): + self.task_id = task_id + self.after = after + self.cas = False + self.fired = False + self.finished_inside: bool | None = None + self.result: Any = None + self.thread: threading.Thread | None = None + + def _run(self) -> None: + try: + self.result = actions.retry(self.task_id) + finally: + connections.close_all() + + def __call__(self, execute, sql, params, many, context): + result = execute(sql, params, many, context) + text = " ".join(sql.split()).upper() + if self.fired: + return result + if not self.cas: + self.cas = ( + text.startswith("UPDATE") + and WORKFLOW_TABLE in text + and OxpullWorkflow.Status.COMPLETE in [str(p) for p in params or ()] + ) + return result + if self.after(text): + self.fired = True + self.thread = threading.Thread(target=self._run) + self.thread.start() + self.thread.join(3.0) + self.finished_inside = not self.thread.is_alive() + return result + + def join(self) -> None: + if self.thread is not None: + self.thread.join(60) + assert not self.thread.is_alive() + + +def the_read_of_dead_rows(text: str) -> bool: + return text.startswith("SELECT") and TASK_TABLE in text and "FOR UPDATE" not in text + + +def the_classification(text: str) -> bool: + return text.startswith("SELECT") and NODE_TABLE_UPPER in text and "GROUP BY" in text + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_retry_after_completions_read_of_the_failed_rows_holds_the_workflow(level): + """ + The completing transaction has read the FAILED leaf at its epoch and not yet + locked it. An operator's retry commits in that moment, so the leaf is READY. + The lock then finds the leaf in another status than the read saw, and the + completion rolls back. At MySQL REPEATABLE READ the classification that + follows answers from the read view the first read opened, where the leaf is + still FAILED, so that comparison is the only thing that stops a COMPLETE + workflow with a READY leaf. At PostgreSQL REPEATABLE READ the lock raises a + serialization failure, and the transaction runs again and finds the leaf + pending. + """ + flow = workflows.create( + f"retry-after-the-read-{level}", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + leaf = flow.add("leaf", tasks.failing_step, "leaf") + flow.add("other", tasks.step, "other") + flow.seal() + drain(flow_worker()) + assert status(leaf.id) == FAILED + + pause = RetryInside(leaf.id, the_read_of_dead_rows) + with every_connection_at(level), connection.execute_wrapper(pause): + completed = workflows._complete_if_finished(flow.id, oxpull_db_alias()) + pause.join() + facts = ( + f"level={level} completed={completed} retry finished inside=" + f"{pause.finished_inside} returned={pause.result!r} " + f"workflow={stored(flow).status} leaf={status(leaf.id)}" + ) + assert pause.fired and pause.finished_inside and pause.result is True, facts + assert completed is False, facts + assert (stored(flow).status, status(leaf.id)) == ( + OxpullWorkflow.Status.SEALED, + READY, + ), facts + assert "complete" not in tasks.STATE, facts + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_retry_of_a_lost_leaf_waits_for_the_completion(level): + """ + The leaf's worker died and the reaper wrote LOST. The completing transaction + locks the LOST row with the FAILED ones. An operator's retry of it after the + classification therefore waits for the commit and lands on a COMPLETE + workflow, never inside the completion. + """ + flow = workflows.create( + f"lost-leaf-{level}", + on_complete=tasks.record_callback, + on_complete_args=["complete"], + ) + leaf = flow.add("leaf", tasks.step, "leaf") + reaper = flow_worker() + for _ in range(3): + claimed = flow_worker().claim_one() + assert claimed is not None and str(claimed.id) == str(leaf.id) + assert actions.expire_lease(leaf.id) is True + assert reaper.reap() == 1 + assert status(leaf.id) == OxTask.Status.LOST + flow.add("other", tasks.step, "other") + flow.seal() + drain(flow_worker()) + assert workflows._complete_if_finished(flow.id, oxpull_db_alias()) is False + assert record_of(leaf.id)[0] == OxTask.Status.LOST + + pause = RetryInside(leaf.id, the_classification) + with every_connection_at(level), connection.execute_wrapper(pause): + completed = workflows._complete_if_finished(flow.id, oxpull_db_alias()) + pause.join() + facts = ( + f"level={level} completed={completed} retry finished inside=" + f"{pause.finished_inside} returned={pause.result!r} " + f"workflow={stored(flow).status} leaf={status(leaf.id)}" + ) + assert pause.fired and pause.result is True, facts + assert pause.finished_inside is False, facts + assert completed is True, facts + assert (stored(flow).status, status(leaf.id)) == ( + OxpullWorkflow.Status.COMPLETE, + READY, + ), facts + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_an_upstream_retried_before_the_lock_keeps_its_child_at_either_level( + caplog, level +): + """ + The retry lands after the cancellation's first read and before its lock, as + in the test above, at a level a project sets. The cone is taken from the + upstreams the lock found dead and nothing else. At MySQL REPEATABLE READ a + fresh read of the edges answers from the read view the first read opened, + where x is still FAILED, and would cancel x's child. At PostgreSQL + REPEATABLE READ the lock raises a serialization failure, and the + cancellation runs its transaction again in the same tick. + """ + flow = workflows.create(f"retried-before-the-lock-{level}") + x = flow.add("x", flagged_step, "x") + d = flow.add("d", tasks.step, "d", depends_on="x") + flow.add("y", tasks.failing_step, "y") + e = flow.add("e", tasks.step, "e", depends_on="y") + flow.seal() + tasks.STATE["fail:x"] = True + drain(flow_worker()) + assert (status(x.id), status(d.id), status(e.id)) == (FAILED, WAITING, WAITING) + + pause = CancellationPause(AFTER_THE_FIRST_READ, lambda: actions.retry(x.id)) + with ( + every_connection_at(level), + caplog.at_level("INFO", logger="oxpull"), + connection.execute_wrapper(pause), + ): + reconcile() + pause.join() + failures = events_named(caplog, "workflow_reconcile_failed") + reissued = events_named(caplog, "workflow_reconcile_conflict_retry") + after = (status(x.id), status(d.id), status(e.id)) + facts = ( + f"level={level} retry finished inside the pass={pause.finished_inside} " + f"returned={pause.result!r} error={pause.error!r} (x, d, e)={after} " + f"failures={failures} reissued={reissued}" + ) + assert pause.fired and pause.finished_inside and pause.result is True, facts + assert after == (READY, WAITING, DISCARDED), facts + assert failures == [], facts + if serialization_failures_at(level): + assert len(reissued) == 1, facts + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_record_repair_that_waited_on_another_writer_runs_again(caplog, level): + """ + Completion repairs a record a writer of another version left behind. Another + transaction is writing that same record and holds the node row. The repair's + lock waits for it. At PostgreSQL REPEATABLE READ the lock then raises a + serialization failure, because the row changed after the repair's snapshot. + The repair runs its transaction again, finds the record written, and the + tick goes on. Nothing is logged as a failed workflow, and the next tick + completes it. + """ + flow = workflows.create(f"repair-waits-{level}") + root = flow.add("root", tasks.step, "root") + flow.seal() + settled_by_hand([root]) + epoch = OxTask.objects.get(pk=root.id).lease_epoch + node_pk = OxpullWorkflowNode.objects.get(task_id=root.id).pk + held = threading.Event() + release = threading.Event() + errors: list[BaseException] = [] + + def write_the_same_record() -> None: + try: + with transaction.atomic(): + assert workflows.record_outcome( + node_pk, FAILED, epoch, oxpull_db_alias() + ) + held.set() + release.wait(60) + except BaseException as exc: + errors.append(exc) + held.set() + finally: + connections.close_all() + + def commit_the_writer_while_the_repair_waits(execute, sql, params, many, context): + text = " ".join(sql.split()).upper() + if ( + not release.is_set() + and text.startswith("SELECT") + and NODE_TABLE_UPPER in text + and "FOR UPDATE" in text + and "SKIP LOCKED" not in text + ): + threading.Timer(0.5, release.set).start() + return execute(sql, params, many, context) + + with every_connection_at(level): + writer = threading.Thread(target=write_the_same_record) + writer.start() + assert held.wait(30) and not errors, errors + try: + with ( + caplog.at_level("INFO", logger="oxpull"), + connection.execute_wrapper(commit_the_writer_while_the_repair_waits), + ): + reconcile() + finally: + release.set() + writer.join(60) + failures = events_named(caplog, "workflow_reconcile_failed") + reissued = events_named(caplog, "workflow_reconcile_conflict_retry") + assert not errors, errors + assert failures == [], failures + if serialization_failures_at(level): + assert len(reissued) == 1, reissued + assert record_of(root.id) == (FAILED, epoch) + assert reconcile() == 1 + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +# -- deadlocks ------------------------------------------------------------------- + + +def lose_a_deadlock_once(monkeypatch, name: str) -> list[int]: + """ + Make the first call of workflows., one attempt of an owned + transaction, lose a deadlock with this database's own code before it sends + anything: InnoDB's 1213 on MySQL, 40P01 on PostgreSQL. The server would + have rolled the transaction back. + """ + real = getattr(workflows, name) + calls: list[int] = [] + + def attempt(*args, **kwargs): + calls.append(1) + if len(calls) == 1: + raise conflict(DEADLOCK_CODE[connection.vendor]) + return real(*args, **kwargs) + + monkeypatch.setattr(workflows, name, attempt) + return calls + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +def test_a_record_repair_that_loses_a_deadlock_runs_again(monkeypatch, caplog): + """ + The record repair owns its transaction. One that loses a deadlock runs it + again, records the outcome no worker recorded, and the next tick completes + the workflow. + """ + flow = workflows.create("repair-deadlock") + root = flow.add("root", tasks.step, "root") + flow.seal() + settled_by_hand([root]) + calls = lose_a_deadlock_once(monkeypatch, "_repair_records_once") + + with caplog.at_level("INFO", logger="oxpull"): + assert workflows._complete_if_finished(flow.id, oxpull_db_alias()) is False + + conflicts = [ + record.conflict + for record in caplog.records + if getattr(record, "event", "") == "workflow_reconcile_conflict_retry" + ] + assert (len(calls), conflicts) == (2, ["deadlock"]) + assert record_of(root.id)[0] == FAILED + assert reconcile() == 1 + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +def failed_upstream_with_a_child_below_it(label: str): + """ + A sealed workflow whose FAILED x has a WAITING child d whose key sorts below + x's. Task ids are random, so children are added until one does, in a new + workflow each time eight children did not. Earlier workflows stay OPEN and + are never reconciled here. + """ + for attempt in range(8): + flow = workflows.create(f"{label}-{attempt}") + x = flow.add("x", tasks.failing_step, f"{label}-{attempt}-x") + for i in range(8): + d = flow.add(f"d{i}", tasks.step, f"{label}-{attempt}-d{i}", depends_on="x") + if uuid.UUID(str(d.id)) < uuid.UUID(str(x.id)): + flow.seal() + drain(flow_worker()) + assert (status(x.id), status(d.id)) == (FAILED, WAITING) + return flow, x, d + raise AssertionError("no child sorted below its upstream") + + +DEADLOCK_WEIGHT_TABLE = "oxpull_test_deadlock_weight" + + +@needs_row_locks +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("level", LEVELS) +def test_a_cancellation_that_loses_a_real_deadlock_runs_again(caplog, level): + """ + The cancellation holds FAILED x's row and next locks its child d, whose key + is lower. Another transaction took django-ox's ordered locking read of d and + x together, so it holds d and waits for x. That is a deadlock, and the + database picks the cancellation as its victim. On MySQL the other transaction + first writes rows, so InnoDB rolls back the lighter cancellation. On + PostgreSQL it waits longer before it looks for the cycle, so the + cancellation finds it first. The cancellation owns its transaction, so it + runs it again once the other one ends, and d is cancelled and recorded. + """ + flow, x, d = failed_upstream_with_a_child_below_it(f"deadlock-{level}") + if connection.vendor == "mysql": + with connection.cursor() as cursor: + cursor.execute(f"DROP TABLE IF EXISTS {DEADLOCK_WEIGHT_TABLE}") + cursor.execute(f"CREATE TABLE {DEADLOCK_WEIGHT_TABLE} (id INT PRIMARY KEY)") + other: dict[str, Any] = {} + + class RollBack(Exception): + pass + + def lock_d_then_x() -> None: + try: + with transaction.atomic(): + with connection.cursor() as cursor: + if connection.vendor == "postgresql": + cursor.execute("SET LOCAL deadlock_timeout = '20s'") + else: + cursor.executemany( + f"INSERT INTO {DEADLOCK_WEIGHT_TABLE} (id) VALUES (%s)", # noqa: S608 + [(i,) for i in range(300)], + ) + other["locked"] = list( + OxTask.objects.filter(pk__in=[x.id, d.id]) + .order_by("pk") + .select_for_update() + .values_list("pk", flat=True) + ) + raise RollBack + except RollBack: + pass + + pause = CancellationPause(AFTER_THE_LOCK, lock_d_then_x, wait=1.0) + try: + with ( + every_connection_at(level), + caplog.at_level("INFO", logger="oxpull"), + connection.execute_wrapper(pause), + ): + workflows._cancel_unreachable(flow.id, oxpull_db_alias()) + pause.join() + finally: + if connection.vendor == "mysql": + with connection.cursor() as cursor: + cursor.execute(f"DROP TABLE IF EXISTS {DEADLOCK_WEIGHT_TABLE}") + conflicts = [ + record.conflict + for record in caplog.records + if getattr(record, "event", "") == "workflow_reconcile_conflict_retry" + ] + facts = ( + f"level={level} fired={pause.fired} inside={pause.finished_inside} " + f"error={pause.error!r} locked={other.get('locked')} d={status(d.id)} " + f"conflicts={conflicts}" + ) + assert pause.fired and pause.finished_inside is False, facts + assert pause.error is None and len(other["locked"]) == 2, facts + assert conflicts == ["deadlock"], facts + assert status(d.id) == DISCARDED, facts + assert record_of(d.id)[0] == DISCARDED, facts + assert status(x.id) == FAILED, facts diff --git a/tests_oxpull/test_workflows_statistics.py b/tests_oxpull/test_workflows_statistics.py new file mode 100644 index 0000000..ac56f1d --- /dev/null +++ b/tests_oxpull/test_workflows_statistics.py @@ -0,0 +1,307 @@ +""" +Planner statistics taken during a wide fan-in do not take the workflow paths off +their indexes. + +A 20,000-way fan-in fills the edge table with edges into one node. Statistics +gathered then say a downstream id matches thousands of edges. SQLite then +planned the claim's dependency check as a scan of the whole edge table for each +candidate row, and PostgreSQL and MySQL read the whole edge table once per claim. +A candidate has no edges in, or a few, so the claim cost the fan-in's width. + +The test builds the fan-in through Workflow.add(), gathers statistics the way +each database does (ANALYZE, ANALYZE TABLE), and then runs the real paths: + +- the claim of a map, through OxpullWorker.claim_one(); +- a map's run through OxpullWorker.run_once(): its outcome, its record, the + mark on its edge, and the release it tries after the commit; +- release_downstream() of the node the fan-in waits on; +- the cancellation, once a map has failed for good. + +Every statement they send to a workflow or task table is explained with the +database's own EXPLAIN, and no plan may read a whole table where a row is looked +up for each row of another: inside a correlated subquery, on the inner side of a +join, or materialised for a join. The task table is never read whole. A read of +one workflow's own edges, or of every edge into one node, may read the table +when that is most of it, as here. + +The plans are printed, so a failure shows the plan that read the table. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +import pytest +from django.db import connection, transaction +from django.test.utils import CaptureQueriesContext + +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.models import OxpullWorkflowEdge, OxpullWorkflowNode +from oxpull.routing import oxpull_db_alias + +from . import tasks +from .test_workflows import drain, flow_worker +from .test_workflows_cost import claim_candidates + +pytestmark = pytest.mark.django_db(transaction=True) + +WIDTH = 20000 +TASK = OxTask._meta.db_table +EDGE = OxpullWorkflowEdge._meta.db_table +NODE = OxpullWorkflowNode._meta.db_table +TABLES = (TASK, EDGE, NODE) + + +def gather_statistics() -> None: + with connection.cursor() as cursor: + if connection.vendor == "sqlite": + cursor.execute("ANALYZE") + return + for table in TABLES: + if connection.vendor == "postgresql": + cursor.execute(f"ANALYZE {connection.ops.quote_name(table)}") + else: + cursor.execute(f"ANALYZE TABLE {connection.ops.quote_name(table)}") + cursor.fetchall() + + +@dataclass(frozen=True) +class WholeRead: + """A plan step that reads a whole table.""" + + table: str + repeated: bool + step: str + + +def whole_reads(sql: str) -> tuple[list[WholeRead], str]: + """The whole-table reads in this statement's plan, and the plan as text.""" + with connection.cursor() as cursor: + if connection.vendor == "sqlite": + cursor.execute("EXPLAIN QUERY PLAN " + sql) + return _sqlite_reads(cursor.fetchall(), sql) + if connection.vendor == "postgresql": + cursor.execute("EXPLAIN (FORMAT JSON) " + sql) + plan = cursor.fetchone()[0] + if isinstance(plan, str): + plan = json.loads(plan) + return _postgresql_reads(plan[0]["Plan"]), json.dumps(plan, indent=1) + cursor.execute("EXPLAIN FORMAT=TREE " + sql) + text = cursor.fetchone()[0] + return _mysql_reads(text, sql), text + + +def _aliases(sql: str, quote: str) -> dict[str, str]: + """Table aliases in the SQL, mapped to the table, and each table to itself.""" + found = {table: table for table in TABLES} + for table, alias in re.findall( + rf"{quote}(\w+){quote}(?: AS)? ({quote}?[A-Za-z_]\w*{quote}?)", sql + ): + if table in TABLES: + found[alias.strip(quote)] = table + return found + + +def _sqlite_reads(rows, sql) -> tuple[list[WholeRead], str]: + """ + EXPLAIN QUERY PLAN rows are (id, parent, unused, detail). A step runs again + for each row of another when it sits under a CORRELATED subquery, or when + an earlier step at its level reads a table, which makes it a join's inner + side. + """ + aliases = _aliases(sql, '"') + parents = {row[0]: row[1] for row in rows} + details = {row[0]: row[3] for row in rows} + reads = [] + for node_id, parent, _, detail in rows: + match = re.match(r"SCAN (\w+)", detail) + if not match or match.group(1) not in aliases: + continue + repeated = False + up = parent + while up in details: + repeated = repeated or details[up].startswith("CORRELATED") + up = parents[up] + earlier = [row for row in rows if row[1] == parent and row[0] < node_id] + repeated = repeated or any(re.match(r"(SCAN|SEARCH) ", r[3]) for r in earlier) + reads.append(WholeRead(aliases[match.group(1)], repeated, detail)) + text = "\n".join(f"{row[0]}<{row[1]} {row[3]}" for row in rows) + return reads, text + + +def _postgresql_reads(node, *, repeated=False) -> list[WholeRead]: + reads = [] + kind = node.get("Node Type", "") + if kind == "Seq Scan" and node.get("Relation Name") in TABLES: + reads.append(WholeRead(node["Relation Name"], repeated, kind)) + children = node.get("Plans", []) + for child in children: + relation = child.get("Parent Relationship", "") + again = ( + repeated + or relation == "SubPlan" + or (relation == "Inner" and kind.startswith("Nested Loop")) + or kind == "Materialize" + or (kind == "Hash" and node.get("Parent Relationship") == "Inner") + ) + reads.extend(_postgresql_reads(child, repeated=again)) + return reads + + +def _mysql_reads(text: str, sql: str) -> list[WholeRead]: + aliases = _aliases(sql, "`") + lines = text.splitlines() + reads = [] + for index, line in enumerate(lines): + match = re.search(r"-> (Table scan|Index scan) on (\w+)", line) + if not match or match.group(2) not in aliases: + continue + depth = len(line) - len(line.lstrip()) + repeated = False + for above in reversed(lines[:index]): + above_depth = len(above) - len(above.lstrip()) + if above_depth >= depth: + continue + depth = above_depth + if "dependent)" in above or "Materialize" in above or "antijoin" in above: + repeated = True + reads.append(WholeRead(aliases[match.group(2)], repeated, line.strip())) + return reads + + +def explained(label: str, captured) -> list[tuple[str, WholeRead]]: + """Explain every workflow or task statement captured, and return the bad reads.""" + bad = [] + print(f"\n==== {label} ({connection.vendor})") # noqa: T201 + for query in captured.captured_queries: + sql = query["sql"] + if not sql.lstrip().upper().startswith(("SELECT", "UPDATE", "DELETE")): + continue + if not any(table in sql for table in TABLES): + continue + reads, text = whole_reads(sql) + print(" SQL:", " ".join(sql.split())[:300]) # noqa: T201 + print(" " + text.replace("\n", "\n ")[:4000]) # noqa: T201 + for read in reads: + if read.repeated or read.table == TASK: + bad.append((" ".join(sql.split())[:200], read)) + return bad + + +def test_statistics_from_a_wide_fan_in_keep_every_path_on_its_indexes(): + flow = workflows.create("statistics") + names = [f"m{i:05d}" for i in range(WIDTH)] + with transaction.atomic(): + # The first map runs first and fails for good, for the cancellation. + flow.add(names[0], tasks.failing_step.using(priority=10), names[0]) + for name in names[1:]: + flow.add(name, tasks.step, name) + waiting = flow.add("reduce", tasks.step, "reduce", depends_on=names) + flow.seal() + gather_statistics() + alias = oxpull_db_alias() + worker = flow_worker() + bad = [] + + with CaptureQueriesContext(connection) as captured: + claimed = worker.claim_one() + assert claimed is not None and claimed.args == [names[0]] + bad += explained("the claim", captured) + worker.execute(claimed, inline=True) + assert drain(worker, limit=2) == 2 + assert OxTask.objects.get(pk=claimed.id).status == OxTask.Status.FAILED + + with CaptureQueriesContext(connection) as captured: + assert worker.run_once() is True + bad += explained("a map's run, record, mark and release", captured) + + with CaptureQueriesContext(connection) as captured: + assert workflows.release_downstream([waiting.id], alias) == 0 + bad += explained("release_downstream of the node the fan-in waits on", captured) + + with CaptureQueriesContext(connection) as captured: + assert workflows._cancel_unreachable(flow.id, alias) == 1 + bad += explained("the cancellation", captured) + assert OxTask.objects.get(pk=waiting.id).status == OxTask.Status.DISCARDED + + assert bad == [], "\n".join(f"{read} in {sql}" for sql, read in bad) + + +def test_the_claim_names_the_edge_index_and_postgresql_keeps_its_estimate(): + """ + The cheap half of the proof, for every run: the claim's SQL carries the + hint on SQLite and MySQL, and the PostgreSQL column carries its n_distinct. + """ + tasks.step.enqueue("plain") + with CaptureQueriesContext(connection) as captured: + assert flow_worker().claim_one() is not None + edge_reads = [q["sql"] for q in captured.captured_queries if EDGE in q["sql"]] + assert edge_reads, captured.captured_queries + index = connection.ops.quote_name(workflows.EDGE_DOWNSTREAM_INDEX) + if connection.vendor == "sqlite": + assert f"INDEXED BY {index}" in edge_reads[0], edge_reads[0] + elif connection.vendor == "mysql": + assert f"FORCE INDEX ({index})" in edge_reads[0], edge_reads[0] + assert "EXISTS(SELECT /*+ NO_SEMIJOIN() */ " in edge_reads[0], edge_reads[0] + else: + with connection.cursor() as cursor: + cursor.execute( + "SELECT attoptions FROM pg_attribute WHERE attrelid = %s::regclass" + " AND attname = 'downstream_task_id'", + [EDGE], + ) + assert cursor.fetchone()[0] == ["n_distinct=-0.5"] + + +def index_hint() -> str: + """The hint this connection's engine takes, unquoted by any surrounding.""" + index = connection.ops.quote_name(workflows.EDGE_DOWNSTREAM_INDEX) + if connection.vendor == "sqlite": + return f"INDEXED BY {index}" + return f"FORCE INDEX ({index})" + + +@pytest.mark.skipif( + connection.vendor == "postgresql", reason="PostgreSQL takes no index hint" +) +def test_the_edge_hint_comes_last_in_the_from_item_it_applies_to(): + """ + Where the hint sits, which the check above cannot see. + + Both engines read the hint as part of a FROM item, so it has to follow + the table and any alias on it. Django renders that alias differently + across versions: quoted on some, bare on others. A hint written into + the gap between table and alias is a syntax error, and the claim then + takes nothing at all. Looking for the hint's text anywhere in the + statement stays green while that is true, so this reads the FROM item + itself and asserts the hint ends it. + + The SQL is compiled and not run, so a failure names the bad FROM item + rather than the error the engine raises on it. + """ + sql = str(claim_candidates(gated=True).query) + opening = f"FROM {connection.ops.quote_name(EDGE)}" + start = sql.index(opening) + item = sql[start : sql.index(" WHERE ", start)] + assert item.endswith(" " + index_hint()), item + + +@pytest.mark.skipif( + connection.vendor == "postgresql", reason="PostgreSQL takes no index hint" +) +def test_an_edge_table_with_no_alias_keeps_the_hint_out_of_the_where(): + """ + A FROM item with no alias takes the hint too, and keeps its WHERE. + + Nothing compiles this shape today, because the probe's subquery is + always aliased. It is asserted so that a later reading of the alias + cannot quietly treat the WHERE keyword as one. + """ + table = connection.ops.quote_name(EDGE) + # S608: text handed to a string function, and never sent anywhere. + sql = f'EXISTS(SELECT 1 AS "a" FROM {table} WHERE ("d" = 1) LIMIT 1)' # noqa: S608 + pinned = workflows.pin_edge_probe(sql, connection) + assert f"FROM {table} {index_hint()} WHERE " in pinned, pinned diff --git a/tests_oxpull/test_workflows_waiting.py b/tests_oxpull/test_workflows_waiting.py new file mode 100644 index 0000000..fbc0235 --- /dev/null +++ b/tests_oxpull/test_workflows_waiting.py @@ -0,0 +1,1171 @@ +""" +Workflow nodes that wait in django-ox's WAITING status. + +A node with dependencies is inserted WAITING, which no worker claims, and +it moves to READY only once every edge into it is marked satisfied. Only a +writer that saw the upstream row SUCCESSFUL marks an edge. The claim still +checks the edges, and a READY row behind an unmarked edge whose upstream row +is absent or not SUCCESSFUL is not claimed. + +These tests pin that and the schema it rests on. The helpers django-ox +exposes for WAITING are private, so their signatures are pinned here too. +""" + +import hashlib +import inspect +import logging +import threading +import time +import uuid +from datetime import timedelta + +import pytest +from django.conf import settings +from django.db import ( + DatabaseError, + OperationalError, + connection, + connections, + transaction, +) +from django.db.migrations.exceptions import IrreversibleError +from django.db.migrations.executor import MigrationExecutor +from django.db.migrations.recorder import MigrationRecorder +from django.db.models import ProtectedError +from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from django.utils import timezone + +from django_ox import _waiting, actions +from django_ox.exceptions import TaskTimeout +from django_ox.models import OxTask +from oxpull import workflows +from oxpull.exceptions import UnknownWorkflowNode, WorkflowNodeExists +from oxpull.models import ( + OxpullWorkflow, + OxpullWorkflowEdge, + OxpullWorkflowNode, + OxpullWorkflowScan, +) +from oxpull.routing import oxpull_db_alias +from oxpull.worker import OxpullWorker + +from . import tasks +from .test_batches import session_isolation +from .test_workflows import drain, flow_worker, free_worker, order, reconcile, stored +from .test_workflows_hazard_postgresql_repeatable_read_concurrent_add import ( + level_of_a_new_transaction, + new_connections_at, + wait_for_lock_waiters, +) + +pytestmark = pytest.mark.django_db + +READY = OxTask.Status.READY +WAITING = OxTask.Status.WAITING + + +def status_of(result): + return OxTask.objects.get(pk=result.id).status + + +def chain(name="chain", **using): + """parent -> child on the "flows" backend, left OPEN.""" + flow = workflows.create(name) + parent = flow.add("parent", tasks.step, "parent") + child_task = tasks.step.using(**using) if using else tasks.step + child = flow.add("child", child_task, "child", depends_on="parent") + return flow, parent, child + + +def edge_marks(result): + return sorted( + OxpullWorkflowEdge.objects.filter(downstream_task_id=result.id).values_list( + "satisfied", flat=True + ) + ) + + +# -- the django-ox helpers this package calls ----------------------------- + +# Match django-ox's waiting-status definitions. django_ox._waiting is a private +# module, so changes there may be unannounced. This compatibility test must catch +# a mismatch before it reaches an application. +WAITING_HELPER_SIGNATURES = { + "enqueue": ( + "(task: 'Task[P, R]', args: 'Sequence[Any]', kwargs: 'Mapping[str, Any]', " + "*, using: 'str') -> 'TaskResult[P, R]'" + ), + "release": "(task_id: 'TaskId', *, lease_epoch: 'int', using: 'str') -> 'bool'", + "release_many": ( + "(rows: 'Iterable[tuple[TaskId, int]]', *, using: 'str') -> 'tuple[int, int]'" + ), + "cancel_many": "(rows: 'Iterable[tuple[TaskId, int]]', *, using: 'str') -> 'int'", + "revive_many": ( + "(rows: 'Iterable[tuple[TaskId, int]]', *, using: 'str') " + "-> 'dict[uuid.UUID, Revival]'" + ), +} + + +@pytest.mark.parametrize("name", sorted(WAITING_HELPER_SIGNATURES)) +def test_the_waiting_helpers_keep_the_signatures_this_package_calls(name): + assert ( + str(inspect.signature(getattr(_waiting, name))) + == (WAITING_HELPER_SIGNATURES[name]) + ) + + +def test_every_status_django_ox_writes_is_pending_or_settled(): + """ + WAITING, the django-ox status for blocked workflow tasks, holds a workflow open. + """ + assert WAITING in workflows.PENDING_STATUSES + assert set(workflows.PENDING_STATUSES) | set(workflows.SETTLED_STATUSES) == set( + OxTask.Status + ) + + +# -- add() ------------------------------------------------------------------ + + +def test_a_root_is_enqueued_ready_and_a_node_with_dependencies_waiting(): + flow, parent, child = chain() + assert (status_of(parent), status_of(child)) == (READY, WAITING) + assert edge_marks(child) == [False] + assert flow.progress().blocked == 1 + + +def test_a_node_whose_upstreams_all_succeeded_is_released_inside_the_add(): + """ + Both upstreams have succeeded before the child is added. The child's + edges go in marked, and the child is READY before add()'s transaction + commits, so it never needs the check after commit. + """ + flow = workflows.create("late-add") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b") + assert drain(flow_worker()) == 2 + with transaction.atomic(): + child = flow.add("child", tasks.step, "child", depends_on=["a", "b"]) + assert status_of(child) == READY + assert edge_marks(child) == [True, True] + assert flow_worker().run_once() is True + assert order() == ["a", "b", "child"] + + +def test_a_node_waits_until_its_last_upstream_succeeds(): + flow = workflows.create("one-of-two") + flow.add("a", tasks.step.using(priority=5), "a") + flow.add("b", tasks.step.using(priority=0), "b") + assert flow_worker().run_once() is True # a + child = flow.add("child", tasks.step, "child", depends_on=["a", "b"]) + assert status_of(child) == WAITING + assert edge_marks(child) == [False, True] + assert flow_worker().run_once() is True # b + assert status_of(child) == READY + assert edge_marks(child) == [True, True] + assert flow_worker().run_once() is True + assert order() == ["a", "b", "child"] + + +def settle_without_success(result, how): + if how == "discarded": + actions.discard(result.id) + assert status_of(result) == OxTask.Status.DISCARDED + return + OxTask.objects.filter(pk=result.id).update( + status=OxTask.Status.FAILED if how == "failed" else OxTask.Status.LOST, + finished_at=timezone.now(), + ) + + +@pytest.mark.parametrize("how", ["discarded", "failed", "lost"]) +def test_a_node_added_after_its_upstream_did_not_succeed_is_never_released(how): + """ + add() marks an edge satisfied only when it reads the upstream row + SUCCESSFUL. An upstream that was discarded, failed or lost leaves the + edge unmarked, so the node waits and no worker runs it. A marked edge + would release the node inside the add and pass the claim's check too. + """ + flow = workflows.create(f"after-{how}") + parent = flow.add("parent", tasks.step, "parent") + settle_without_success(parent, how) + child = flow.add("child", tasks.step, "child", depends_on="parent") + assert status_of(child) == WAITING + assert edge_marks(child) == [False] + assert flow_worker().run_once() is False + assert free_worker().run_once() is False + flow.seal() + reconcile() + assert status_of(child) != READY + assert order() == [] + if how != "lost": + assert status_of(child) == OxTask.Status.DISCARDED + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +def first_statement(statements, prefix): + return next( + (index for index, sql in enumerate(statements) if sql.startswith(prefix)), + None, + ) + + +@pytest.mark.parametrize("depends", [False, True], ids=["root", "dependent"]) +def test_add_writes_the_workflow_row_before_it_inserts_the_task_row(depends): + """ + Every workflow operation locks the workflow row first. add() takes that + lock with its first write, so it inserts no task row while seal() or + another add() holds the workflow. + """ + flow = workflows.create("lock-order") + flow.add("parent", tasks.step, "parent") + quote = connection.ops.quote_name + with CaptureQueriesContext(connection) as captured: + flow.add("node", tasks.step, "node", depends_on="parent" if depends else None) + statements = [query["sql"] for query in captured.captured_queries] + workflow_write = first_statement( + statements, f"UPDATE {quote(OxpullWorkflow._meta.db_table)}" + ) + task_insert = first_statement( + statements, f"INSERT INTO {quote(OxTask._meta.db_table)}" + ) + assert workflow_write is not None and task_insert is not None, statements + assert workflow_write < task_insert, statements + + +def test_a_callers_run_after_stays_on_the_waiting_row_and_through_the_release(): + later = timezone.now() + timedelta(hours=1) + _, _, child = chain("scheduled", run_after=later) + row = OxTask.objects.get(pk=child.id) + assert row.status == WAITING + assert abs(row.run_after - later) < timedelta(seconds=1) + + assert flow_worker().run_once() is True # the parent + row = OxTask.objects.get(pk=child.id) + assert row.status == READY + assert abs(row.run_after - later) < timedelta(seconds=1) + assert flow_worker().run_once() is False + assert free_worker().run_once() is False + + +def test_add_counts_each_node_it_commits_on_the_workflow_row(): + flow = workflows.create("counted") + flow.add("a", tasks.step, "a") + flow.add("b", tasks.step, "b", depends_on="a") + with pytest.raises(WorkflowNodeExists): + flow.add("a", tasks.step, "again") + with pytest.raises(UnknownWorkflowNode): + flow.add("c", tasks.step, "c", depends_on="missing") + nodes = OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() + assert (stored(flow).node_count, nodes) == (2, 2) + + +def test_a_node_name_is_its_exact_spelling(): + """ + name_key is the SHA-256 digest of the name's UTF-8 bytes, so names that + differ by case, an accent or a trailing space are different nodes on + every database, and depends_on finds the one spelled that way. + """ + names = ["Extract", "extract", "resume", "résumé", "load", "load "] + flow = workflows.create("spelling") + added = {name: flow.add(name, tasks.step, name) for name in names} + child = flow.add("child", tasks.step, "child", depends_on="load ") + keys = dict( + OxpullWorkflowNode.objects.filter( + workflow_id=flow.id, name__in=names + ).values_list("task_id", "name_key") + ) + assert {str(task_id): key for task_id, key in keys.items()} == { + str(added[name].id): hashlib.sha256(name.encode("utf-8")).hexdigest() + for name in names + } + edge = OxpullWorkflowEdge.objects.get(downstream_task_id=child.id) + assert str(edge.upstream_task_id) == str(added["load "].id) + + +@pytest.mark.django_db(transaction=True) +def test_an_upstream_success_the_add_did_not_see_is_caught_after_commit(monkeypatch): + """ + add() reads its upstreams inside its own transaction, and an upstream can + succeed after that read and before the node's edges are visible to the + upstream's worker. A read that saw nothing stands in for that one here. + The node commits WAITING, and the check after commit marks the edge and + releases it. + """ + flow = workflows.create("stale-read") + flow.add("parent", tasks.step, "parent") + assert flow_worker().run_once() is True + monkeypatch.setattr(workflows, "_succeeded_among", lambda ids, alias: set()) + with transaction.atomic(): + child = flow.add("child", tasks.step, "child", depends_on="parent") + assert status_of(child) == WAITING + assert status_of(child) == READY + assert edge_marks(child) == [True] + assert flow_worker().run_once() is True + assert order() == ["parent", "child"] + + +# -- the claim -------------------------------------------------------------- + + +class SelectThenUpdateWorker(OxpullWorker): + """ + The same dependency check through the SELECT ... FOR UPDATE SKIP LOCKED + claim, which PostgreSQL uses when a worker's claim_filter_sql() is empty. + """ + + def claim_filter_sql(self): + return "", {} + + +WORKERS = { + "default-claim": flow_worker, + "select-then-update-claim": lambda: SelectThenUpdateWorker( + backend_alias="flows", backoff_initial=0, poll_interval=0.01 + ), +} + + +@pytest.mark.parametrize("claimer", sorted(WORKERS)) +def test_a_ready_node_whose_upstream_row_is_gone_is_not_claimed(claimer): + """ + Nothing recorded that the upstream succeeded, and its row is gone. The + node has been put back to READY, the shape a django-ox retry or an + application's own UPDATE leaves. A missing row says nothing about how + the upstream finished, so the claim refuses the node. + """ + _, parent, child = chain(f"pruned-{claimer}") + OxTask.objects.filter(pk=parent.id).delete() + OxTask.objects.filter(pk=child.id).update(status=READY) + assert WORKERS[claimer]().claim_one() is None + assert status_of(child) == READY + + +@pytest.mark.parametrize("claimer", sorted(WORKERS)) +def test_a_node_whose_edge_was_marked_runs_after_the_upstream_row_is_gone(claimer): + _, parent, child = chain(f"marked-{claimer}") + assert WORKERS[claimer]().run_once() is True # parent, which marks and releases + assert edge_marks(child) == [True] + OxTask.objects.filter(pk=parent.id).delete() + assert WORKERS[claimer]().run_once() is True + assert order() == ["parent", "child"] + + +# -- completion, cancellation and the reconciler ---------------------------- + + +def test_a_sealed_workflow_whose_unfinished_node_waits_stays_sealed(): + """ + The upstream row is gone with nothing recorded, so the child waits and + nothing releases it. WAITING is pending: the workflow holds, and the + waiting node shows in progress(). + """ + flow, parent, child = chain("held") + flow.seal() + OxTask.objects.filter(pk=parent.id).delete() + reconcile() + reconcile() + assert stored(flow).status == OxpullWorkflow.Status.SEALED + assert status_of(child) == WAITING + progress = flow.progress() + assert (progress.pending, progress.blocked) == (1, 1) + assert free_worker().claim_one() is None + + +def test_the_reconciler_cancels_a_waiting_cone(): + flow = workflows.create("doomed-waiting") + flow.add("root", tasks.failing_step, "root") + mid = flow.add("mid", tasks.step, "mid", depends_on="root") + leaf = flow.add("leaf", tasks.step, "leaf", depends_on="mid") + flow.seal() + worker = flow_worker() + for _ in range(3): + assert worker.run_once() is True + assert (status_of(mid), status_of(leaf)) == (WAITING, WAITING) + assert reconcile() == 1 + assert (status_of(mid), status_of(leaf)) == ( + OxTask.Status.DISCARDED, + OxTask.Status.DISCARDED, + ) + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + + +def test_the_reconciler_releases_a_node_no_worker_released(): + """ + A SUCCESSFUL written by something other than an OxpullWorker releases + nothing on its own. One reconciler pass marks the edge and releases the + node, in an OPEN workflow as well as a sealed one. + """ + _, parent, child = chain("out-of-band") + OxTask.objects.filter(pk=parent.id).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + assert status_of(child) == WAITING + assert flow_worker().run_once() is False + reconcile() + assert edge_marks(child) == [True] + assert status_of(child) == READY + assert flow_worker().run_once() is True + assert order() == ["child"] + + +def test_a_release_that_fails_after_the_outcome_is_left_to_the_reconciler( + monkeypatch, caplog +): + _, parent, child = chain("release-fails") + + def fails(task_ids, db_alias): + raise DatabaseError("simulated: the release statement failed") + + with monkeypatch.context() as patch: + patch.setattr("oxpull.worker.release_downstream", fails) + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert flow_worker().run_once() is True + assert status_of(parent) == OxTask.Status.SUCCESSFUL + assert status_of(child) == WAITING + # The edge was marked with the outcome. Only the release after it failed. + assert edge_marks(child) == [True] + assert [ + record + for record in caplog.records + if getattr(record, "event", "") == "workflow_release_failed" + ] + reconcile() + assert status_of(child) == READY + + +def test_the_reconciler_holds_a_waiting_node_whose_upstream_row_is_gone(): + _, parent, child = chain("unknown") + OxTask.objects.filter(pk=parent.id).delete() + reconcile() + assert edge_marks(child) == [False] + assert status_of(child) == WAITING + + +# -- a release that loses a lock conflict ------------------------------------- + + +class SimulatedDriverError(Exception): + """The driver error a lock conflict carries as its cause.""" + + +def lock_conflict(): + """An OperationalError this database's release reads as a lost lock conflict.""" + if connection.vendor == "mysql": + return OperationalError(1213, "Deadlock found when trying to get lock") + cause = SimulatedDriverError("simulated lock conflict") + if connection.vendor == "postgresql": + cause.sqlstate = "40001" + else: + cause.sqlite_errorcode = 5 # SQLITE_BUSY + error = OperationalError("simulated lock conflict") + error.__cause__ = cause + return error + + +def not_a_lock_conflict(): + return OperationalError("simulated: not a lock conflict") + + +def events(caplog, name): + return [r for r in caplog.records if getattr(r, "event", "") == name] + + +def children_ready_to_release(name, count): + """`count` WAITING children of one SUCCESSFUL parent, every edge marked.""" + flow = workflows.create(name) + parent = flow.add("parent", tasks.step, "parent") + children = [ + flow.add(f"c{index}", tasks.step, f"c{index}", depends_on="parent") + for index in range(count) + ] + OxTask.objects.filter(pk=parent.id).update( + status=OxTask.Status.SUCCESSFUL, finished_at=timezone.now() + ) + OxpullWorkflowEdge.objects.filter(upstream_task_id=parent.id).update(satisfied=True) + assert {status_of(child) for child in children} == {WAITING} + return sorted(uuid.UUID(str(child.id)) for child in children) + + +def conflicting_release_many(monkeypatch, losing, conflicts, error=lock_conflict): + """ + django-ox's release_many, except that a call carrying any id in `losing` + raises `error()` instead, the first `conflicts` times. Returns the ids of + every call. + """ + real = _waiting.release_many + calls = [] + left = [conflicts] + + def release_many(rows, *, using): + rows = list(rows) + ids = {uuid.UUID(str(pk)) for pk, _ in rows} + calls.append(ids) + if ids & losing and left[0] > 0: + left[0] -= 1 + raise error() + return real(rows, using=using) + + monkeypatch.setattr(workflows._waiting, "release_many", release_many) + return calls + + +def statuses(ids): + found = dict(OxTask.objects.filter(pk__in=ids).values_list("pk", "status")) + return [found[pk] for pk in ids] + + +@pytest.mark.django_db(transaction=True) +def test_a_release_that_loses_a_lock_conflict_decides_again_from_fresh_reads( + monkeypatch, caplog +): + ids = children_ready_to_release("conflict-once", 1) + calls = conflicting_release_many(monkeypatch, set(ids), conflicts=1) + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + assert calls == [set(ids), set(ids)] + assert statuses(ids) == [READY] + assert not events(caplog, "workflow_release_deferred") + + +@pytest.mark.django_db(transaction=True) +def test_a_release_chunk_that_loses_twice_is_left_to_the_reconciler( + monkeypatch, caplog +): + monkeypatch.setattr(workflows, "ID_CHUNK_SIZE", 1) + ids = children_ready_to_release("conflict-twice", 2) + calls = conflicting_release_many(monkeypatch, {ids[0]}, conflicts=2) + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert workflows.release_downstream(ids, oxpull_db_alias()) == 1 + assert calls == [{ids[0]}, {ids[0]}, {ids[1]}] + assert statuses(ids) == [WAITING, READY] + assert len(events(caplog, "workflow_release_deferred")) == 1 + reconcile() + assert statuses(ids) == [READY, READY] + + +def skip_queue(flow): + """The ids this workflow's backstop is holding to try again.""" + row = OxpullWorkflowScan.objects.filter(workflow_id=flow.id).first() + return [] if row is None else list(row.skipped) + + +def record_of(result): + return OxpullWorkflowNode.objects.values_list( + "outcome", "outcome_epoch", "succeeded" + ).get(task_id=result.id) + + +@pytest.mark.django_db(transaction=True) +def test_a_workers_release_that_loses_twice_keeps_everything_but_the_release( + monkeypatch, caplog +): + """ + The deferral as a worker meets it, which is the only way a user meets + it. The test above calls the release directly; this one runs a task. + + A parent with two children, one id to a chunk, and the chunk that would + free c1 loses both of its tries. Everything except that release has to + survive it: the task body ran, its outcome is committed on the row, the + node's record carries it, both edges are marked satisfied, c0 is READY + from the chunk that won and c1 is WAITING, c1's id is in its workflow's + skip queue for the reconciler, and the product logged one deferral + naming how many rows it could not release. + + A chunk that lost a lock conflict is not a chunk that took the work down + with it, and a mixed call is not all-or-nothing: that is what the two + statuses side by side say. + """ + monkeypatch.setattr(workflows, "ID_CHUNK_SIZE", 1) + flow = workflows.create("worker-defers") + parent = flow.add("parent", tasks.step, "parent") + c0 = flow.add("c0", tasks.step, "c0", depends_on="parent") + c1 = flow.add("c1", tasks.step, "c1", depends_on="parent") + flow.seal() + losing = uuid.UUID(str(c1.id)) + calls = conflicting_release_many(monkeypatch, {losing}, conflicts=2) + + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert flow_worker().run_once() is True + + assert order() == ["parent"] + assert status_of(parent) == OxTask.Status.SUCCESSFUL + epoch = OxTask.objects.get(pk=parent.id).lease_epoch + assert record_of(parent) == (OxTask.Status.SUCCESSFUL, epoch, True) + assert (edge_marks(c0), edge_marks(c1)) == ([True], [True]) + # The chunk that won released its node; the chunk that lost did not. + assert (status_of(c0), status_of(c1)) == (READY, WAITING) + assert len(calls) == 3 and calls.count({losing}) == 2, calls + deferred = events(caplog, "workflow_release_deferred") + assert len(deferred) == 1, deferred + assert deferred[0].task_count == 1 + assert skip_queue(flow) == [str(c1.id)] + + +# Reconciler passes the recovery below is allowed, with a task run after each. +# One is what it takes, from the skip queue and from the round alike, and both +# are measured: the bound passes with this set to 1. The second is slack for a +# tick that stopped between phases, and nothing here is large enough to make a +# tick stop at all. +RECOVERY_PASSES = 2 + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("queued", [True, False], ids=["queued", "dropped"]) +def test_the_reconciler_recovers_a_deferred_release_and_runs_nothing_twice( + monkeypatch, caplog, queued +): + """ + The other half of the contract: what the reconciler owes a deferral. + + A worker's release loses both tries against both children at once, so + they stay WAITING with their edges marked. The contention then goes away, + as it does the moment the other writer commits, and nothing but the + ordinary reconciler and an ordinary worker runs after that: no hand + release, no repair, no unbounded polling. Within RECOVERY_PASSES passes + every eligible node is runnable and has run, exactly once each and after + its parent. + + `dropped` empties the skip queue first. The product documents that a + queue at its cap drops ids with a log line, so a node whose only route + back was that queue is a node the reconciler can lose forever. Run that + way, this test passes only if the round finds the node from the persisted + edge marks alone. + + A node that is genuinely blocked stays blocked throughout: its upstream + is running elsewhere, so no pass may release it and no pass may cancel + it either. + """ + flow = workflows.create("recovers") + flow.add("parent", tasks.step, "parent") + c0 = flow.add("c0", tasks.step, "c0", depends_on="parent") + c1 = flow.add("c1", tasks.step, "c1", depends_on="parent") + stuck = flow.add("stuck", tasks.step, "stuck") + blocked = flow.add("blocked", tasks.step, "blocked", depends_on="stuck") + flow.seal() + OxTask.objects.filter(pk=stuck.id).update( + status=OxTask.Status.RUNNING, locked_by="elsewhere", locked_at=timezone.now() + ) + children = {uuid.UUID(str(c0.id)), uuid.UUID(str(c1.id))} + # One chunk holds both children, and it loses both of its tries. + conflicting_release_many(monkeypatch, children, conflicts=2) + + with caplog.at_level(logging.WARNING, logger="oxpull"): + assert flow_worker().run_once() is True + assert len(events(caplog, "workflow_release_deferred")) == 1 + assert (status_of(c0), status_of(c1)) == (WAITING, WAITING) + assert sorted(skip_queue(flow)) == sorted([str(c0.id), str(c1.id)]) + if not queued: + OxpullWorkflowScan.objects.filter(workflow_id=flow.id).update(skipped=[]) + + passes = 0 + while {status_of(c0), status_of(c1)} != {OxTask.Status.SUCCESSFUL}: + assert passes < RECOVERY_PASSES, ( + f"after {passes} reconciler passes c0 is {status_of(c0)} and c1 is " + f"{status_of(c1)}, with their edges marked " + f"{(edge_marks(c0), edge_marks(c1))} and the skip queue holding " + f"{skip_queue(flow)}" + ) + reconcile() + drain(flow_worker()) + passes += 1 + + ran = order() + assert sorted(ran) == ["c0", "c1", "parent"], ran + assert ran[0] == "parent", ran + assert skip_queue(flow) == [] + # The blocked node is still blocked: not released, and not cancelled + # either, because its upstream has not settled. + assert (status_of(blocked), edge_marks(blocked)) == (WAITING, [False]) + assert status_of(stuck) == OxTask.Status.RUNNING + # Nothing is owed any more, and another pass finds nothing to redo. + assert workflows._release_backstop(oxpull_db_alias()) == 0 + assert drain(flow_worker()) == 0 + assert order() == ran + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize("inside_a_transaction", [True, False]) +def test_a_release_decides_again_only_after_a_lock_conflict_in_its_own_transaction( + monkeypatch, inside_a_transaction +): + """ + Inside a caller's transaction a lock conflict propagates at once, because + the caller's work is lost with it. An error that is not a lock conflict + propagates at once everywhere. + """ + ids = children_ready_to_release("conflict-propagates", 1) + error = lock_conflict if inside_a_transaction else not_a_lock_conflict + calls = conflicting_release_many(monkeypatch, set(ids), conflicts=2, error=error) + with pytest.raises(OperationalError): + if inside_a_transaction: + with transaction.atomic(): + workflows.release_downstream(ids, oxpull_db_alias()) + else: + workflows.release_downstream(ids, oxpull_db_alias()) + assert calls == [set(ids)] + assert statuses(ids) == [WAITING] + + +@pytest.mark.django_db(transaction=True) +def test_at_repeatable_read_a_release_that_waited_on_another_release_still_releases( + monkeypatch, +): + """ + Two releases at REPEATABLE READ. One has released a row and not yet + committed. The other's chunk holds that row and a second one, and waits on + the first's lock. On PostgreSQL the waiting chunk loses a serialization + failure when the first commits. It decides again from fresh reads and + releases the second row. On MySQL the locking read is a current read, and + the chunk goes through on its first attempt. + """ + if connection.vendor not in ("postgresql", "mysql"): + pytest.skip("SQLite has no isolation level and no row locks") + # The first release commits only once the second is seen waiting, however + # long a busy machine takes to see it, so the release's two bounds are + # moved out of the way. The wait is inside a locking read, which the + # statement bound cuts short as the lock wait bound does, on both engines. + # Past either one the chunk is decided again with the first release still + # uncommitted. test_workflows_outcomes.py tests the bounds. + monkeypatch.setattr(workflows, "RELEASE_LOCK_TIMEOUT", 30.0) + monkeypatch.setattr(workflows, "RELEASE_STATEMENT_TIMEOUT", 30.0) + alias = oxpull_db_alias() + ids = children_ready_to_release("two-releases", 2) + epoch = OxTask.objects.get(pk=ids[0]).lease_epoch + calls = conflicting_release_many(monkeypatch, set(), conflicts=0) + held, go = threading.Event(), threading.Event() + box = {} + + def hold(): + try: + box["holder level"] = level_of_a_new_transaction() + with transaction.atomic(using=alias): + box["held"] = _waiting.release(ids[0], lease_epoch=epoch, using=alias) + held.set() + go.wait(30) + except BaseException as exc: + box["holder error"] = exc + finally: + held.set() + connections.close_all() + + def release(): + try: + box["releaser level"] = level_of_a_new_transaction() + box["released"] = workflows.release_downstream(ids, alias) + except BaseException as exc: + box["releaser error"] = exc + finally: + connections.close_all() + + with new_connections_at("repeatable-read-options"): + holder = threading.Thread(target=hold) + holder.start() + assert held.wait(30) + releaser = threading.Thread(target=release) + releaser.start() + waiters = wait_for_lock_waiters(1) + go.set() + holder.join(30) + releaser.join(30) + + assert "holder error" not in box and "releaser error" not in box, box + assert box["holder level"] == box["releaser level"] == "repeatable read", box + assert box["held"] is True + assert waiters >= 1, "the second release never waited on the first" + assert box["released"] == 1 + assert statuses(ids) == [READY, READY] + if connection.vendor == "postgresql": + assert calls == [set(ids), {ids[1]}], calls + else: + assert calls == [set(ids)], calls + + +def doomed_sealed_workflow(name): + flow = workflows.create(name) + root = flow.add("root", tasks.step, "root") + child = flow.add("child", tasks.step, "child", depends_on="root") + OxTask.objects.filter(pk=root.id).update( + status=OxTask.Status.FAILED, finished_at=timezone.now() + ) + flow.seal() + return flow, child + + +def test_a_tick_whose_release_backstop_fails_still_cancels_and_completes( + monkeypatch, caplog +): + flow, child = doomed_sealed_workflow("beside-a-failed-backstop") + + def fails(db_alias): + raise DatabaseError("simulated: the release backstop failed") + + monkeypatch.setattr(workflows, "_release_backstop", fails) + with caplog.at_level(logging.ERROR, logger="oxpull"): + # The root was failed by hand, so nothing recorded it. The first tick + # cancels the child and records the root. The second completes. + assert reconcile() == 0 + assert status_of(child) == OxTask.Status.DISCARDED + assert reconcile() == 1 + assert stored(flow).status == OxpullWorkflow.Status.COMPLETE + assert len(events(caplog, "workflow_release_backstop_failed")) == 2 + + +def test_a_task_timeout_in_the_release_backstop_ends_the_tick(monkeypatch): + _flow, child = doomed_sealed_workflow("timed-out-backstop") + + def times_out(db_alias): + raise TaskTimeout("simulated") + + monkeypatch.setattr(workflows, "_release_backstop", times_out) + with pytest.raises(TaskTimeout): + reconcile() + assert status_of(child) == WAITING + + +def test_a_workflow_row_that_has_nodes_cannot_be_deleted(): + flow, _, _ = chain("protected") + with pytest.raises(ProtectedError): + OxpullWorkflow.objects.filter(pk=flow.id).delete() + assert OxpullWorkflowNode.objects.filter(workflow_id=flow.id).count() == 2 + assert OxpullWorkflowEdge.objects.filter(workflow_id=flow.id).count() == 1 + + +# -- the second workflow backend ---------------------------------------------- + + +def test_the_second_workflow_backend_holds_a_node_like_the_first(): + flow = workflows.create("leftover") + flow.add("parent", tasks.deferred_step, "parent") + child = flow.add("child", tasks.deferred_step, "child", depends_on="parent") + row = OxTask.objects.get(pk=child.id) + assert (row.status, row.run_after) == (WAITING, None) + + +def database_check(databases): + """ + oxpull's database-tagged check, called as `check --database` calls it. + + Called directly rather than through checks.run_checks(), whose other + registered checks depend on which modules earlier tests imported. + """ + from oxpull.checks import check_workflow_schema + + return check_workflow_schema(databases=databases) + + +def test_waiting_nodes_with_no_workflows_backend_are_a_database_check_warning(): + _, _, child = chain("orphaned") + assert database_check(None) == [] + assert not [m for m in database_check(["default"]) if m.id == "oxpull.W007"] + without = { + alias: { + **config, + "OPTIONS": { + key: value + for key, value in config.get("OPTIONS", {}).items() + if key != "WORKFLOWS" + }, + } + for alias, config in settings.TASKS.items() + } + with override_settings(TASKS=without): + found = [m for m in database_check(["default"]) if m.id == "oxpull.W007"] + assert status_of(child) == WAITING + assert len(found) == 1 + assert found[0].msg.startswith("1 workflow node(s)") + + +# -- migrating back --------------------------------------------------------- + + +def _migrate_to(targets): + executor = MigrationExecutor(connection) + executor.loader.build_graph() + executor.migrate(targets) + + +def _leaves(): + executor = MigrationExecutor(connection) + executor.loader.build_graph() + return [ + (app, name) + for app, name in executor.loader.graph.leaf_nodes() + if app in ("oxpull", "django_ox") + ] + + +WORKFLOW_TABLES = [ + model._meta.db_table + for model in (OxpullWorkflow, OxpullWorkflowEdge, OxpullWorkflowNode) +] + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "target", + [("oxpull", "0003_oxpullratewindow"), ("django_ox", "0007_oxschedule")], + ids=lambda target: target[0], +) +def test_migrating_back_past_workflows_refuses_while_a_workflow_exists(target): + """ + Migrating either app back unapplies oxpull's 0005 and then 0004 first. + Each refuses while any workflow row exists, before it undoes anything, so + no table or index is dropped and every migration stays applied. + """ + leaves = _leaves() + workflows.create("keeps-its-tables") + try: + with pytest.raises(IrreversibleError, match=r"0004_workflows"): + _migrate_to([target]) + tables = connection.introspection.table_names() + assert [table for table in WORKFLOW_TABLES if table not in tables] == [] + applied = MigrationRecorder(connection).applied_migrations() + assert ("oxpull", "0005_workflow_edge_upstream_idx") in applied + with connection.cursor() as cursor: + constraints = connection.introspection.get_constraints( + cursor, OxpullWorkflowEdge._meta.db_table + ) + assert "oxpull_wf_edge_up_idx" in constraints + finally: + _migrate_to(leaves) + + +@pytest.mark.django_db(transaction=True) +def test_migrating_back_to_0004_names_what_unapplying_0005_drops(): + """ + A target of 0004_workflows unapplies only 0005. Its refusal says what 0005 + drops, the index, and does not say the tables go. + """ + leaves = _leaves() + workflows.create("keeps-its-index") + try: + with pytest.raises(IrreversibleError) as refused: + _migrate_to([("oxpull", "0004_workflows")]) + message = str(refused.value) + assert message.startswith( + "Cannot migrate oxpull back past 0005_workflow_edge_upstream_idx." + ), message + assert "Unapplying it drops the index" in message, message + applied = MigrationRecorder(connection).applied_migrations() + assert ("oxpull", "0005_workflow_edge_upstream_idx") in applied + finally: + _migrate_to(leaves) + + +@pytest.mark.django_db(transaction=True) +def test_0004_refuses_on_its_own_once_0005_is_unapplied(): + """ + 0005's refusal fires first on the way back, so 0004's is reached only when + 0005 was unapplied before the first workflow. Then 0004 still refuses, and + the tables stay. + """ + leaves = _leaves() + try: + _migrate_to([("oxpull", "0004_workflows")]) + workflows.create("after-0005-went") + with pytest.raises(IrreversibleError) as refused: + _migrate_to([("oxpull", "0003_oxpullratewindow")]) + message = str(refused.value) + assert message.startswith( + "Cannot migrate oxpull back past 0004_workflows. 1 workflow(s)" + ), message + tables = connection.introspection.table_names() + assert [table for table in WORKFLOW_TABLES if table not in tables] == [] + applied = MigrationRecorder(connection).applied_migrations() + assert ("oxpull", "0004_workflows") in applied + finally: + _migrate_to(leaves) + + +@pytest.mark.django_db(transaction=True) +def test_migrating_back_past_workflows_is_allowed_before_the_first_workflow(): + leaves = _leaves() + try: + _migrate_to([("oxpull", "0003_oxpullratewindow")]) + tables = connection.introspection.table_names() + assert [table for table in WORKFLOW_TABLES if table in tables] == [] + finally: + _migrate_to(leaves) + + +# -- MySQL REPEATABLE READ: a held claim and the WAITING key range -------------- + +BLOCKED = 400 + + +@pytest.mark.django_db(transaction=True) +def test_on_mysql_repeatable_read_a_held_claim_locks_no_waiting_row_and_add_does_not_wait(): # noqa: E501 + """ + A claim at REPEATABLE READ holds next-key locks on every index entry it + scans until its transaction ends. Blocked nodes that were READY sat in + that range, so the claim locked them all and an add() inserting another + one waited on it. WAITING rows are outside the range every claim scans. + + The claim is held open while performance_schema.data_locks is read from + another connection, and while add() calls on other connections insert + WAITING nodes ahead of, inside and behind the existing ones with a 2 s + lock wait. + """ + if connection.vendor != "mysql": + pytest.skip("MySQL only: data_locks and next-key locks are InnoDB's") + flow = workflows.create("locks") + root = flow.add("root", tasks.step, "root") + # Running elsewhere, so nothing downstream of it can be released. + OxTask.objects.filter(pk=root.id).update( + status=OxTask.Status.RUNNING, locked_by="elsewhere" + ) + blocked = [ + flow.add( + f"b{i:04d}", tasks.step.using(priority=10), f"b{i:04d}", depends_on="root" + ) + for i in range(BLOCKED) + ] + for i in range(3): + tasks.step.using(priority=0).enqueue(f"runnable-{i}") + with connection.cursor() as cursor: + cursor.execute( + f"ANALYZE TABLE {OxTask._meta.db_table}, " + f"{OxpullWorkflowEdge._meta.db_table}, {OxpullWorkflowNode._meta.db_table}" + ) + blocked_hex = {uuid.UUID(str(result.id)).hex for result in blocked} + schema = connection.settings_dict["NAME"] + + def locks_now(): + observer = connections.create_connection("default") + try: + with observer.cursor() as cursor: + cursor.execute( + "SELECT INDEX_NAME, LOCK_MODE, LOCK_DATA FROM " + "performance_schema.data_locks WHERE OBJECT_SCHEMA = %s " + "AND OBJECT_NAME = %s", + [schema, OxTask._meta.db_table], + ) + return cursor.fetchall() + finally: + observer.close() + + def add_with_a_short_wait(name, priority): + outcome = {} + + def run(): + try: + with connections["default"].cursor() as cursor: + cursor.execute("SET SESSION innodb_lock_wait_timeout = 2") + started = time.monotonic() + try: + flow.add( + name, + tasks.step.using(priority=priority), + name, + depends_on="root", + ) + outcome["result"] = "added" + except OperationalError as exc: + outcome["result"] = f"{exc.args[0]} {exc}" + outcome["seconds"] = round(time.monotonic() - started, 2) + finally: + connections.close_all() + + thread = threading.Thread(target=run) + thread.start() + thread.join(30) + return outcome + + worker = flow_worker() + with session_isolation("REPEATABLE READ"), transaction.atomic(): + claimed = worker.claim_one() + held = locks_now() + adds = { + priority: add_with_a_short_wait(f"late-{priority}", priority) + for priority in (100, 10, -100) + } + with connection.cursor() as cursor: + cursor.execute("SELECT @@transaction_isolation") + (isolation,) = cursor.fetchone() + + assert isolation.upper().replace("_", "-") == "REPEATABLE-READ" + assert claimed is not None and claimed.args[0].startswith("runnable-"), claimed + on_blocked = [ + row for row in held if any(key in (row[2] or "") for key in blocked_hex) + ] + on_waiting = [row for row in held if "'WAITING'" in (row[2] or "")] + assert (on_blocked, on_waiting) == ([], []), ( + f"the held claim locked {len(on_blocked)} of {BLOCKED} blocked rows and " + f"{len(on_waiting)} WAITING index entries; all its locks: {held[:12]}" + ) + assert {priority: outcome.get("result") for priority, outcome in adds.items()} == { + 100: "added", + 10: "added", + -100: "added", + }, adds + + +# -- MySQL: the task-id columns follow OxTask.id's collation --------------------- + + +def _mysql_collation(table, column): + with connection.cursor() as cursor: + cursor.execute( + "SELECT COLLATION_NAME FROM information_schema.COLUMNS" + " WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s" + " AND COLUMN_NAME = %s", + [table, column], + ) + row = cursor.fetchone() + return None if row is None else row[0] + + +@pytest.mark.django_db(transaction=True) +def test_on_mysql_migrate_puts_the_task_id_columns_on_the_task_ids_collation(): + """ + The database's default collation is changed after django-ox's tables + were created, as a dump restored into a newer server leaves it, and then + `migrate oxpull` creates the workflow tables. Every column that holds a + task id ends up on OxTask.id's collation, so no claim meets MySQL's + illegal mix of collations. + """ + if connection.vendor != "mysql": + pytest.skip("MySQL only: a uuid column has no collation elsewhere") + from django.core.management import call_command + + from .test_upgrade import _latest_migration, _migrate + + task_table = OxTask._meta.db_table + task_collation = _mysql_collation(task_table, OxTask._meta.pk.column) + new_default = ( + "utf8mb4_0900_ai_ci" + if task_collation != "utf8mb4_0900_ai_ci" + else "utf8mb4_general_ci" + ) + name = connection.settings_dict["NAME"] + with connection.cursor() as cursor: + cursor.execute( + "SELECT DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA" + " WHERE SCHEMA_NAME = DATABASE()" + ) + (original_default,) = cursor.fetchone() + columns = [ + (OxpullWorkflowEdge._meta.db_table, "upstream_task_id"), + (OxpullWorkflowEdge._meta.db_table, "downstream_task_id"), + (OxpullWorkflowNode._meta.db_table, "task_id"), + (OxpullWorkflowNode._meta.db_table, "cancelled_by_task_id"), + ] + latest = _latest_migration() + _migrate("0003_oxpullratewindow") + try: + with connection.cursor() as cursor: + cursor.execute(f"ALTER DATABASE `{name}` COLLATE {new_default}") + call_command("migrate", "oxpull", skip_checks=True, verbosity=0) + found = {f"{t}.{c}": _mysql_collation(t, c) for t, c in columns} + finally: + _migrate("0003_oxpullratewindow") + with connection.cursor() as cursor: + cursor.execute(f"ALTER DATABASE `{name}` COLLATE {original_default}") + _migrate(latest) + assert found == dict.fromkeys(found, task_collation), ( + f"the database defaulted to {new_default} and task ids are on " + f"{task_collation}; migrate left the workflow columns on {found}" + ) diff --git a/tests_oxpull/workers.py b/tests_oxpull/workers.py new file mode 100644 index 0000000..674989c --- /dev/null +++ b/tests_oxpull/workers.py @@ -0,0 +1,134 @@ +""" +Running real workers at each other, for the rate-limit tests. + +racing.py races one callable; a limiter's subject is a Worker running its +claim loop against a shared budget, which that harness cannot express. So +this sits beside it rather than replacing it. +""" + +import threading +import time + +from django.db import connections + +from oxpull.worker import OxpullWorker + +from .racing import pool_for + + +def run_workers( + count, + *, + seconds, + worker_kwargs=None, + worker_class=OxpullWorker, + until_idle=False, +): + """ + Run `count` worker_class instances against the queue at once, each on + its own thread with its own connection, all released from one barrier, + each claiming and executing until `seconds` have elapsed. Returns the + monotonic instant of every claim, per worker. + + With until_idle the run ends when every worker is at rest on an empty + queue instead, and `seconds` becomes a ceiling that a passing run never + waits out and that fails the run when it is reached. That is the shape + a test needs when it counts what the workers took before they stopped: + what they took is the limiter's business, how long they needed is the + machine's. + + A worker comes to rest when a claim found nothing and nothing was taken + while that claim ran, and it stays there until another worker takes + something. So the last worker to come to rest finds every other one + already resting: no claim is in flight, each of them has been refused + since the last task was taken, and a claim refused with no other claim + in its way was refused by the limiter. That is the moment the run ends. + Resting on the tally rather than on an empty claim alone is what makes + it sound: a compare-and-set claim comes back empty when other workers + took every candidate it had fetched, and a SKIP LOCKED claim when the + rows it wanted were held by a claim still in flight, and both of those + move the tally. + + Threads, not processes: a thread with its own connection is the same + claimer to the database, and it is what a test can join. The W in the + guarantee counts worker processes, which is why the bound assertions + are written against the number of workers started here rather than + against a constant. + """ + kwargs = {"backoff_initial": 0, "poll_interval": 0.01, **(worker_kwargs or {})} + # Each worker process has a connection pool of its own. These threads + # share this process's, so it is sized for them. + with pool_for(count): + return _run_workers(count, seconds, kwargs, worker_class, until_idle) + + +def _run_workers(count, seconds, kwargs, worker_class, until_idle): + barrier = threading.Barrier(count) + claims = [[] for _ in range(count)] + quiet = threading.Condition() + # True while a worker is waiting out a claim that found nothing on a + # tally that did not move while it ran. + resting = [False] * count + taken = 0 + idle = threading.Event() + errors = [] + + def run(index): + nonlocal taken + try: + worker = worker_class(**kwargs) + barrier.wait(timeout=10) + deadline = time.monotonic() + seconds + while time.monotonic() < deadline and not idle.is_set(): + with quiet: + began_at = taken + db_task = worker.claim_one() + if db_task is None: + with quiet: + if until_idle and began_at == taken: + resting[index] = True + if all(resting): + idle.set() + quiet.notify_all() + break + woken = quiet.wait_for( + lambda: idle.is_set() or not resting[index], + timeout=max(0.0, deadline - time.monotonic()), + ) + if not woken: + # Out of time, not back at work: leave the + # run one worker short of idle so it fails. + resting[index] = False + continue + # Idle rather than spin: the point of the run is what + # the limiter admits, not how hard a blocked worker + # can hammer the database. + time.sleep(0.005) + continue + claims[index].append(time.monotonic()) + with quiet: + taken += 1 + # What every resting worker is waiting for. + resting[:] = [False] * count + quiet.notify_all() + worker.execute(db_task) + except BaseException as exc: + errors.append(exc) + finally: + connections.close_all() + + threads = [threading.Thread(target=run, args=(index,)) for index in range(count)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=seconds + 30) + assert not thread.is_alive() + assert not errors, errors + if until_idle: + assert idle.is_set(), f"the workers were still claiming after {seconds}s" + return claims + + +def admissions(claims): + """Every claim across every worker, in the order they happened.""" + return sorted(instant for worker in claims for instant in worker) diff --git a/tests_wheel/__init__.py b/tests_wheel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests_wheel/conftest.py b/tests_wheel/conftest.py new file mode 100644 index 0000000..0da092d --- /dev/null +++ b/tests_wheel/conftest.py @@ -0,0 +1,90 @@ +""" +Installed-wheel tests. Opt-in: they build wheels, create virtualenvs and run +pip, so they are not in the default run (testpaths is tests) and every test +skips unless the run asks for the marker: + + python -m pytest tests_wheel -m wheel + +Django's wheels come from the index once per run (or from OX_WHEELHOUSE, a +directory of wheels, for an offline run). Nothing else leaves the machine. +""" + +import os +import sys + +import pytest + +from . import wheelkit + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "wheel: builds wheels and virtualenvs and runs pip; run with -m wheel", + ) + + +def pytest_collection_modifyitems(config, items): + selected = "wheel" in (config.option.markexpr or "") + skip = pytest.mark.skip(reason="installed-wheel test: run with -m wheel") + for item in items: + item.add_marker(pytest.mark.wheel) + if not selected: + item.add_marker(skip) + + +@pytest.fixture(scope="session") +def artifacts(tmp_path_factory): + return wheelkit.build_artifacts(tmp_path_factory.mktemp("artifacts")) + + +def _download(dest, requirements): + dest.mkdir(parents=True, exist_ok=True) + done = wheelkit.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--dest", + dest, + *requirements, + ] + ) + if done.returncode != 0: + pytest.skip(f"cannot download {requirements}: {done.stderr[-300:]}") + return dest + + +@pytest.fixture(scope="session") +def wheelhouse(tmp_path_factory): + """Django 6.0 and its dependencies.""" + if os.environ.get("OX_WHEELHOUSE"): + from pathlib import Path + + return Path(os.environ["OX_WHEELHOUSE"]) + return _download(tmp_path_factory.mktemp("wheelhouse"), ["Django>=6.0,<6.1"]) + + +@pytest.fixture(scope="session") +def wheelhouse_52(tmp_path_factory): + """Django 5.2 and the Tasks backport.""" + return _download( + tmp_path_factory.mktemp("wheelhouse52"), + ["Django>=5.2,<5.3", "django-tasks>=0.12"], + ) + + +@pytest.fixture +def make_env(tmp_path, artifacts, wheelhouse): + count = 0 + + def make(wheelhouse_dir=None): + nonlocal count + count += 1 + return wheelkit.Env( + tmp_path, f"env{count}", wheelhouse_dir or wheelhouse, artifacts + ) + + return make diff --git a/tests_wheel/test_wheel_transitions.py b/tests_wheel/test_wheel_transitions.py new file mode 100644 index 0000000..22dd653 --- /dev/null +++ b/tests_wheel/test_wheel_transitions.py @@ -0,0 +1,341 @@ +""" +What happens to an installed project when the combined django-ox arrives. + +These tests cover package contents, upgrade ordering and repair of +overlapping installations. +The tests build a stand-in earlier `oxpull` wheel from the current source, +versioned as an earlier release, rather than using the published 1.8.0 wheel. +Two kinds of test live here: + +- the supported path (back up, uninstall oxpull, install django-ox, migrate) + must leave every oxpull row, the schema, the migration state and the queued + work exactly as it was; +- the unsafe paths do not fail loudly in pip. They are pinned as they behave, + so that a change in pip, in the packaging or in what the wheel ships shows + up as a failed test and a decision, not as a user report. Each says + what the repair is. + +Run with: python -m pytest tests_wheel -m wheel +""" + +import tarfile +import zipfile + +import pytest + +from . import wheelkit + +pytestmark = pytest.mark.wheel + + +def ok(done): + assert done.returncode == 0, f"rc={done.returncode}\n{done.stdout}\n{done.stderr}" + return done + + +# ---------------------------------------------------------------- the artifacts + + +def test_the_wheel_and_sdist_carry_oxpull_and_pass_the_release_check(artifacts): + wheel = wheelkit.read_wheel(artifacts.combined) + names = set(wheel) + for needed in ( + "django_ox/__init__.py", + "oxpull/__init__.py", + "oxpull/backend.py", + "oxpull/models.py", + "oxpull/py.typed", + "oxpull/_django_gate.py", + "oxpull/_requires_django_6.py", + ): + assert needed in names, needed + migrations = {n for n in names if n.startswith("oxpull/migrations/0")} + assert len(migrations) == 5 + # Nothing but the two packages and their metadata. + tops = {n.split("/")[0] for n in names} + assert tops == {"django_ox", "oxpull", f"django_ox-{artifacts.version}.dist-info"} + licences = [n for n in names if n.endswith("LICENSE")] + assert licences and b"BSD 3-Clause" in wheel[licences[0]] + with tarfile.open(artifacts.sdist) as tar: + members = tar.getnames() + assert any("/src/oxpull/backend.py" in m for m in members) + assert any("/src/oxpull/migrations/0005_" in m for m in members) + assert not any(m.endswith(".gitignore") for m in members) + assert not any("/tests_oxpull/" in m or "/tests_wheel/" in m for m in members) + # The release check opens the same archives, as it does for a tag. + from tools.check_release import check_dist + + problems, _ = check_dist(artifacts.dist_dir, artifacts.version) + assert problems == [] + + +def test_twine_accepts_the_combined_distribution_and_the_metapackage(artifacts): + done = wheelkit.run( + [ + wheelkit.sys.executable, + "-m", + "twine", + "check", + "--strict", + artifacts.combined, + artifacts.sdist, + artifacts.meta, + ] + ) + if "No module named twine" in done.stderr: + pytest.skip("twine is not installed") + ok(done) + + +def test_the_metapackage_ships_no_files(artifacts): + with zipfile.ZipFile(artifacts.meta) as z: + names = z.namelist() + metadata = z.read(next(n for n in names if n.endswith("METADATA"))).decode() + assert all(".dist-info/" in n for n in names), names + assert f"Requires-Dist: django-ox>={artifacts.version}" in metadata + + +# ----------------------------------------------------------------- a fresh install + + +def test_a_fresh_install_runs_a_task_through_each_backend(make_env, artifacts): + # Installing django-ox without an earlier `oxpull` distribution. + env = make_env() + ok(env.install(artifacts.combined)) + ok(env.check()) + done = env.import_oxpull() + assert done.stdout.strip() == artifacts.version + ok(env.manage("migrate")) + ok(env.manage("makemigrations", "--check", "--dry-run")) + assert env.enqueue("through-oxpull", "oxpull").stdout.strip().endswith("READY") + assert env.enqueue("through-django-ox", "free").stdout.strip().endswith("READY") + ok(env.run_worker()) + assert sorted(env.marks()) == ["through-django-ox", "through-oxpull"] + + +# --------------------------------------------------- the stub that holds the name + + +def test_the_stub_then_the_combined_wheel_leaves_the_combined_init(make_env, artifacts): + # Upgrade from the earlier module-bearing `oxpull` stub. + env = make_env() + ok(env.install(artifacts.stub)) + ok(env.install(artifacts.combined)) + ok(env.check()) + assert env.import_oxpull().stdout.strip() == artifacts.version + + +def test_removing_the_stub_after_the_combined_wheel_takes_oxpull_init_with_it( + make_env, artifacts +): + # The earlier stub's RECORD lists oxpull/__init__.py, so uninstalling it + # deletes the file django-ox wrote over it. `oxpull` survives as a namespace + # package. The apps keep working, but __version__ is gone. + env = make_env() + ok(env.install(artifacts.stub)) + ok(env.install(artifacts.combined)) + ok(env.uninstall("oxpull")) + assert not (env.oxpull_dir() / "__init__.py").exists() + assert (env.oxpull_dir() / "backend.py").exists() + assert env.import_oxpull().stdout.strip() == "NO __version__" + ok(env.manage("check")) + ok(env.manage("migrate")) + # Repair. + ok(env.install("--force-reinstall", "--no-deps", artifacts.combined)) + assert env.import_oxpull().stdout.strip() == artifacts.version + + +def test_removing_the_stub_first_is_clean(make_env, artifacts): + env = make_env() + ok(env.install(artifacts.stub)) + ok(env.uninstall("oxpull")) + ok(env.install(artifacts.combined)) + ok(env.check()) + assert (env.oxpull_dir() / "__init__.py").exists() + assert env.import_oxpull().stdout.strip() == artifacts.version + ok(env.uninstall("oxpull")) if "oxpull" in env.freeze() else None + assert env.import_oxpull().stdout.strip() == artifacts.version + + +# --------------------------- an earlier privately distributed oxpull installation + + +def legacy_oxpull_site(make_env, artifacts): + """A project on legacy django-ox plus legacy oxpull, with rows and queued work.""" + env = make_env() + ok( + env.install( + artifacts.legacy_free, + artifacts.legacy_oxpull, + extra_links=[artifacts.legacy_oxpull.parent], + ) + ) + ok(env.check()) + ok(env.manage("migrate")) + ok(env.py("seed.py")) + ok(env.enqueue("before-upgrade-1", "oxpull")) + ok(env.run_worker()) + # still READY when the packages change + ok(env.enqueue("before-upgrade-2", "oxpull")) + assert env.marks() == ["before-upgrade-1"] + return env + + +def test_the_supported_procedure_changes_nothing_in_the_database(make_env, artifacts): + # Back up, uninstall the earlier `oxpull` distribution, install django-ox, migrate. + env = legacy_oxpull_site(make_env, artifacts) + before = env.snapshot() + assert before["rows"]["oxpull_oxpullbatch"] and before["tasks"] + env.backup() + ok(env.uninstall("oxpull")) + ok(env.install(artifacts.combined)) + ok(env.check()) + done = ok(env.manage("migrate")) + assert "No migrations to apply" in done.stdout + assert env.snapshot() == before + assert env.import_oxpull().stdout.strip() == artifacts.version + ok(env.run_worker()) # the task that was READY before the change runs + ok(env.enqueue("after-upgrade", "oxpull")) + ok(env.run_worker()) + assert env.marks() == ["before-upgrade-1", "before-upgrade-2", "after-upgrade"] + + +def test_installing_over_old_oxpull_succeeds_in_pip_and_pip_check_fails( + make_env, artifacts +): + # Installing django-ox over the earlier distribution exits 0 with a resolver + # complaint on stderr. Both claim oxpull/*, and `pip check` fails until the + # earlier `oxpull` distribution is removed. + env = legacy_oxpull_site(make_env, artifacts) + done = ok(env.install(artifacts.combined)) + assert "requires django-ox==" in done.stderr + assert env.check().returncode == 1 + assert ( + env.import_oxpull().stdout.strip() == artifacts.version + ) # last writer: the combined file + ok(env.manage("check")) + done = ok(env.manage("migrate")) + assert "No migrations to apply" in done.stdout + + +def test_uninstalling_old_oxpull_after_installing_over_it_deletes_oxpull( + make_env, artifacts +): + # Uninstalling the earlier distribution after the overlap reports success + # and makes `pip check` pass, but removes the package files. Data is untouched. + env = legacy_oxpull_site(make_env, artifacts) + before = env.snapshot() + ok(env.install(artifacts.combined)) + ok(env.uninstall("oxpull")) + ok(env.check()) + assert not env.oxpull_dir().exists() + failed = env.import_oxpull() + assert failed.returncode != 0 and "No module named 'oxpull'" in failed.stderr + assert env.snapshot() == before + # Repair: one command, then the same checks as the supported path. + ok(env.install("--force-reinstall", "--no-deps", artifacts.combined)) + assert env.import_oxpull().stdout.strip() == artifacts.version + assert "No migrations to apply" in ok(env.manage("migrate")).stdout + assert env.snapshot() == before + ok(env.run_worker()) + assert env.marks()[-1] == "before-upgrade-2" + + +def test_pip_install_upgrade_oxpull_over_old_oxpull_deletes_oxpull_and_reports_success( + make_env, artifacts +): + # The metapackage upgrade removes the earlier `oxpull` distribution after + # unpacking django-ox, so that uninstall can remove shared package files. + env = legacy_oxpull_site(make_env, artifacts) + before = env.snapshot() + done = ok( + env.install( + "-U", "oxpull", extra_links=[artifacts.meta.parent, artifacts.dist_dir] + ) + ) + assert "Successfully installed" in done.stdout + ok(env.check()) + assert env.freeze()["oxpull"] == artifacts.version + assert not env.oxpull_dir().exists() + assert "No module named 'oxpull'" in env.import_oxpull().stderr + assert env.snapshot() == before + ok(env.install("--force-reinstall", "--no-deps", artifacts.combined)) + assert env.import_oxpull().stdout.strip() == artifacts.version + assert env.snapshot() == before + ok(env.run_worker()) + + +def test_uninstalling_old_oxpull_before_the_metapackage_upgrade_is_safe( + make_env, artifacts +): + # Remove stale distribution metadata, then restore django-ox's files. + env = legacy_oxpull_site(make_env, artifacts) + before = env.snapshot() + ok(env.uninstall("oxpull")) + ok( + env.install( + "-U", "oxpull", extra_links=[artifacts.meta.parent, artifacts.dist_dir] + ) + ) + ok(env.check()) + assert env.import_oxpull().stdout.strip() == artifacts.version + assert "No migrations to apply" in ok(env.manage("migrate")).stdout + assert env.snapshot() == before + + +# ----------------------------------------------------------------- the metapackage + + +def test_the_metapackage_after_the_combined_wheel_collides_with_nothing( + make_env, artifacts +): + # The `oxpull` metapackage contains no importable module. + env = make_env() + ok(env.install(artifacts.combined)) + ok(env.install(artifacts.meta, extra_links=[artifacts.dist_dir])) + ok(env.check()) + assert env.freeze()["oxpull"] == artifacts.version + ok(env.uninstall("oxpull")) + ok(env.check()) + assert env.import_oxpull().stdout.strip() == artifacts.version + + +def test_the_metapackage_before_the_combined_wheel_collides_with_nothing( + make_env, artifacts +): + # The metapackage depends on django-ox for the importable `oxpull` package. + env = make_env() + ok(env.install("--no-deps", artifacts.meta)) + assert env.check().returncode == 1 # django-ox missing, as pip says + ok(env.install(artifacts.combined)) + ok(env.check()) + assert env.import_oxpull().stdout.strip() == artifacts.version + + +def test_the_metapackage_alone_installs_the_combined_wheel(make_env, artifacts): + # Uninstalling the metapackage leaves django-ox's module files in place. + env = make_env() + ok(env.install("oxpull", extra_links=[artifacts.meta.parent, artifacts.dist_dir])) + ok(env.check()) + assert env.freeze()["django-ox"] == artifacts.version + assert env.import_oxpull().stdout.strip() == artifacts.version + + +# ----------------------------------------------------------------------- Django 5.2 + + +def test_the_installed_wheel_refuses_oxpull_on_django_5_2_with_the_fix( + make_env, artifacts, wheelhouse_52 +): + # The check must tell the project what to do rather than raise + # ModuleNotFoundError from inside the check. + env = make_env(wheelhouse_52) + ok(env.install(artifacts.combined, "django-tasks>=0.12")) + env.oxpull_app = False + ok(env.manage("check")) # django-ox alone is fine on 5.2 + ok(env.manage("migrate")) + env.oxpull_app = True + refused = env.manage("check") + assert refused.returncode != 0 + assert "oxpull needs Django 6.0 or later" in refused.stderr + assert "remove 'oxpull' from INSTALLED_APPS" in refused.stderr diff --git a/tests_wheel/wheelkit.py b/tests_wheel/wheelkit.py new file mode 100644 index 0000000..39e8d35 --- /dev/null +++ b/tests_wheel/wheelkit.py @@ -0,0 +1,427 @@ +""" +Build the artifacts the transition tests install, and run pip against them. + +The combined wheel and sdist are built from this tree. Everything older is +rebuilt from that wheel, so the tests need no package index for these wheels +and no network beyond Django itself: + +- the legacy django-ox wheel is the combined wheel without oxpull/; +- the earlier separately installed oxpull wheel contains only oxpull/, with the literal + __version__ the earlier `oxpull` distribution had and an exact pin on the + legacy django-ox. Its other files are the combined wheel's, so it runs + against the legacy django-ox; it is the file layout, not the old code, that + the tests need; +- the PyPI stub is the placeholder that holds the name today; +- the metapackage is a real hatchling build of a wheel with no files. + +Their versions sit below the combined one, which is the order the release +puts them in. File ownership is what the pip behaviour depends on, and these +have the same file names as the real ones. +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import subprocess +import sys +import textwrap +import zipfile +from dataclasses import dataclass +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +LEGACY = "0.9.0" + +STUB_INIT = '''\ +"""Placeholder for the Oxpull commercial package.""" + +__version__ = "0.0.1" + + +def __getattr__(name: str) -> object: + raise AttributeError("The 'oxpull' package on PyPI is a name placeholder.") +''' + +LEGACY_INIT = f'__version__ = "{LEGACY}"\n\n__all__ = ["__version__"]\n' + + +def run(argv, *, cwd=None, env=None, check=False): + return subprocess.run( + [str(a) for a in argv], + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=check, + ) + + +def _record_line(name: str, data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=") + return f"{name},sha256={digest.decode()},{len(data)}" + + +def write_wheel( + dest: Path, + name: str, + version: str, + files: dict[str, bytes], + requires: tuple[str, ...] = (), + extra_metadata: str = "", +) -> Path: + """A pure-Python wheel with the given files, named the way pip expects.""" + norm = name.replace("-", "_") + info = f"{norm}-{version}.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n" + metadata += "".join(f"Requires-Dist: {r}\n" for r in requires) + metadata += extra_metadata + members = dict(files) + members[f"{info}/METADATA"] = metadata.encode() + members[f"{info}/WHEEL"] = ( + b"Wheel-Version: 1.0\nGenerator: tests_wheel\n" + b"Root-Is-Purelib: true\nTag: py3-none-any\n" + ) + record = [_record_line(n, d) for n, d in members.items()] + record.append(f"{info}/RECORD,,") + members[f"{info}/RECORD"] = ("\n".join(record) + "\n").encode() + dest.mkdir(parents=True, exist_ok=True) + path = dest / f"{norm}-{version}-py3-none-any.whl" + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: + for member, data in members.items(): + z.writestr(member, data) + return path + + +def read_wheel(path: Path) -> dict[str, bytes]: + with zipfile.ZipFile(path) as z: + return {n: z.read(n) for n in z.namelist() if not n.endswith("/")} + + +@dataclass +class Artifacts: + """Where each kind of artifact lives, and the version it carries.""" + + root: Path + version: str + combined: Path + sdist: Path + legacy_free: Path + legacy_oxpull: Path + stub: Path + meta: Path + + @property + def dist_dir(self) -> Path: + return self.root / "combined" + + +def build_combined(root: Path) -> tuple[Path, Path, str]: + out = root / "combined" + out.mkdir(parents=True) + done = run([sys.executable, "-m", "build", "--no-isolation", "--outdir", out, REPO]) + assert done.returncode == 0, done.stdout + done.stderr + (wheel,) = out.glob("*.whl") + (sdist,) = out.glob("*.tar.gz") + version = wheel.name.split("-")[1] + return wheel, sdist, version + + +def build_meta(root: Path, version: str) -> Path: + src = root / "meta-src" + src.mkdir(parents=True) + (src / "README.md").write_text("Installs django-ox, which contains oxpull.\n") + (src / "pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [build-system] + requires = ["hatchling>=1.27"] + build-backend = "hatchling.build" + + [project] + name = "oxpull" + version = "{version}" + description = "Installs django-ox, which now contains oxpull." + readme = "README.md" + requires-python = ">=3.12" + dependencies = ["django-ox>={version}"] + + [tool.hatch.build.targets.wheel] + bypass-selection = true + """ + ) + ) + out = root / "meta" + done = run( + [ + sys.executable, + "-m", + "build", + "--no-isolation", + "--wheel", + "--outdir", + out, + src, + ] + ) + assert done.returncode == 0, done.stdout + done.stderr + (wheel,) = out.glob("*.whl") + return wheel + + +def build_artifacts(root: Path) -> Artifacts: + combined, sdist, version = build_combined(root) + members = read_wheel(combined) + + def dist_info(name: str) -> bool: + return name.split("/")[0].endswith(".dist-info") + + free_files = { + n: d + for n, d in members.items() + if not n.startswith("oxpull/") and not dist_info(n) + } + legacy_free = write_wheel( + root / "legacy", "django-ox", LEGACY, free_files, ("Django>=5.2",) + ) + oxpull_files = {n: d for n, d in members.items() if n.startswith("oxpull/")} + oxpull_files["oxpull/__init__.py"] = LEGACY_INIT.encode() + legacy_oxpull = write_wheel( + root / "legacy", "oxpull", LEGACY, oxpull_files, (f"django-ox=={LEGACY}",) + ) + stub = write_wheel( + root / "legacy", "oxpull", "0.0.1", {"oxpull/__init__.py": STUB_INIT.encode()} + ) + return Artifacts( + root=root, + version=version, + combined=combined, + sdist=sdist, + legacy_free=legacy_free, + legacy_oxpull=legacy_oxpull, + stub=stub, + meta=build_meta(root, version), + ) + + +SETTINGS = """\ +import os + +SECRET_KEY = "x" +USE_TZ = True +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" +INSTALLED_APPS = ["django_ox"] +if os.environ.get("SITE_OXPULL") == "1": + INSTALLED_APPS.append("oxpull") +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.environ["SITE_DB"], + "OPTIONS": {"timeout": 20}, + } +} +_free = {"BACKEND": "django_ox.backend.OxBackend", "QUEUES": ["default"]} +_oxpull = {"BACKEND": "oxpull.backend.OxpullBackend", "QUEUES": ["default"]} +TASKS = {"default": _free, "free": _free} +if os.environ.get("SITE_OXPULL") == "1": + TASKS["oxpull"] = _oxpull +""" + +TASKS = """\ +import os + +from django.tasks import task + + +@task() +def hello(tag): + with open(os.environ["SITE_MARK"], "a") as f: + f.write(tag + "\\n") + return tag +""" + +MANAGE = """\ +import os, sys +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "wheelsite.settings") +from django.core.management import execute_from_command_line +execute_from_command_line(sys.argv) +""" + +SEED = """\ +import os, uuid +import django +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "wheelsite.settings") +django.setup() +from django.utils import timezone +from oxpull.models import OxpullBatch, OxpullRateWindow, OxpullUniqueLock +now = timezone.now() +OxpullBatch.objects.create(name="seed-batch", created_at=now) +OxpullRateWindow.objects.create( + limit_name="seed-limit", window_id=7, window_started_at=now, used=3 +) +OxpullUniqueLock.objects.create( + key="k" * 64, + task_id=uuid.uuid4(), + task_path="wheelsite.tasks.hello", + created_at=now, +) +""" + +ENQUEUE = """\ +import os, sys +import django +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "wheelsite.settings") +django.setup() +from wheelsite.tasks import hello +print(hello.using(backend=sys.argv[2]).enqueue(sys.argv[1]).status) +""" + + +class Env: + """A virtualenv with a one-file Django site next to it.""" + + def __init__(self, root: Path, name: str, wheelhouse: Path, artifacts: Artifacts): + self.dir = root / name + self.dir.mkdir(parents=True) + self.wheelhouse = wheelhouse + self.artifacts = artifacts + done = run([sys.executable, "-m", "venv", self.dir / "venv"]) + assert done.returncode == 0, done.stderr + self.python = self.dir / "venv" / "bin" / "python" + self.site = self.dir / "site" + (self.site / "wheelsite").mkdir(parents=True) + (self.site / "wheelsite" / "__init__.py").write_text("") + (self.site / "wheelsite" / "settings.py").write_text(SETTINGS) + (self.site / "wheelsite" / "tasks.py").write_text(TASKS) + (self.site / "manage.py").write_text(MANAGE) + (self.site / "seed.py").write_text(SEED) + (self.site / "enqueue.py").write_text(ENQUEUE) + self.db = self.dir / "db.sqlite3" + self.mark = self.dir / "mark.txt" + self.oxpull_app = True + + # pip ------------------------------------------------------------------ + def pip(self, *args, extra_links=()): + links = [self.wheelhouse, *extra_links] + argv = [self.python, "-m", "pip", "--disable-pip-version-check"] + argv += [a for a in args if a != "--offline"] + if "--offline" in args: + argv += ["--no-index"] + for link in links: + argv += ["--find-links", link] + return run(argv) + + def install(self, *specs, extra_links=(), flags=()): + return self.pip( + "install", + "--offline", + *flags, + *[str(s) for s in specs], + extra_links=extra_links, + ) + + def uninstall(self, name): + return self.pip("uninstall", "-y", name) + + def check(self): + return self.pip("check") + + def freeze(self) -> dict[str, str]: + done = self.pip("list", "--format=freeze") + return dict(line.split("==") for line in done.stdout.split()) + + @property + def packages(self) -> Path: + done = run( + [ + self.python, + "-c", + "import sysconfig;print(sysconfig.get_paths()['purelib'])", + ] + ) + return Path(done.stdout.strip()) + + # the site ------------------------------------------------------------- + def env(self, **extra): + env = { + **os.environ, + "SITE_DB": str(self.db), + "SITE_MARK": str(self.mark), + "SITE_OXPULL": "1" if self.oxpull_app else "0", + "PYTHONPATH": str(self.site), + "PYTHONDONTWRITEBYTECODE": "1", + } + env.pop("DJANGO_SETTINGS_MODULE", None) + env.update(extra) + return env + + def py(self, *args): + return run([self.python, *args], cwd=self.site, env=self.env()) + + def code(self, source: str): + return self.py("-c", source) + + def manage(self, *args): + return run([self.python, "manage.py", *args], cwd=self.site, env=self.env()) + + def enqueue(self, tag: str, backend: str = "default"): + return self.py("enqueue.py", tag, backend) + + def run_worker(self): + return self.manage("ox_worker", "--batch") + + def marks(self) -> list[str]: + return self.mark.read_text().split() if self.mark.exists() else [] + + def oxpull_dir(self) -> Path: + return self.packages / "oxpull" + + def import_oxpull(self): + return self.code( + "import oxpull; print(getattr(oxpull, '__version__', 'NO __version__'))" + ) + + def snapshot(self) -> dict: + """The oxpull rows and schema, applied migrations and task rows.""" + import sqlite3 + + c = sqlite3.connect(self.db) + try: + tables = [ + r[0] + for r in c.execute( + "select name from sqlite_master" + " where type='table' and name like 'oxpull_%' order by 1" + ) + ] + rows = {t: list(c.execute(f"select * from {t} order by 1")) for t in tables} + schema = list( + c.execute( + "select sql from sqlite_master" + " where name like 'oxpull_%' order by name" + ) + ) + applied = { + app: [ + r[0] + for r in c.execute( + "select name from django_migrations where app=? order by id", + (app,), + ) + ] + for app in ("oxpull", "django_ox") + } + tasks = list( + c.execute("select status,count(*) from django_ox_oxtask group by 1") + ) + finally: + c.close() + return {"rows": rows, "schema": schema, "applied": applied, "tasks": tasks} + + def backup(self, suffix="backup") -> Path: + import shutil + + target = self.dir / f"db.{suffix}.sqlite3" + shutil.copy(self.db, target) + return target diff --git a/tools/build_llms_full.py b/tools/build_llms_full.py index 7f98957..4a0007f 100644 --- a/tools/build_llms_full.py +++ b/tools/build_llms_full.py @@ -21,8 +21,8 @@ OUTPUT = DOCS / "llms-full.txt" SITE_URL = "https://oxpull.com/django-ox/" -# mkdocs.yml uses a `!relative` tag that yaml.safe_load rejects, and the nav is -# a flat list, so a line match is simpler and has no dependency. +# mkdocs.yml uses a `!relative` tag that yaml.safe_load rejects, so a line +# match is simpler and has no dependency. NAV_ENTRY = re.compile(r"^\s+- (?P[^:]+): (?P<file>\S+\.md)\s*$") SNIPPET = re.compile(r'^--8<-- "(?P<file>[^"]+)"\s*$') @@ -46,7 +46,11 @@ def nav_pages() -> list[tuple[str, str]]: def page_url(filename: str) -> str: stem = filename.removesuffix(".md") - return SITE_URL if stem == "index" else f"{SITE_URL}{stem}/" + if stem == "index": + return SITE_URL + if stem.endswith("/index"): + return f"{SITE_URL}{stem.removesuffix('index')}" + return f"{SITE_URL}{stem}/" def page_text(filename: str) -> str: diff --git a/tools/check-policy-pair b/tools/check-policy-pair deleted file mode 100755 index 61427db..0000000 --- a/tools/check-policy-pair +++ /dev/null @@ -1,653 +0,0 @@ -#!/usr/bin/env python3 -""" -The acceptance gate for a django-ox release paired with its Oxpull release. - -django-ox and Oxpull ship as an exact-pinned pair, so two green suites are -not a verdict on their own: each could have imported the other package from -somewhere else (an editable install, a wheel left in the venv, PyPI's copy of -the pinned version), and skips hide what a leg did not run. This runs both -suites on one leg and fails unless all of the following hold: - -- `django_ox` is imported from the django-ox checkout under test, and - `oxpull` from the Oxpull checkout, in the environment each suite runs in. - For the Oxpull suite that includes the interpreter on its own, with - PYTHONPATH cleared, because Oxpull's subprocess tests start children that - build their own path: those children must find the same `django_ox`, and - the settings module, without the parent's PYTHONPATH. -- Oxpull's pyproject pins exactly the `django_ox.__version__` under test. -- Neither suite has a failure or an error, and pytest exits 0. -- Every skip is explained. django-ox's explanations are listed below, each - for the databases (and Django versions) where it is expected, so a - PostgreSQL leg that skips the connection-pool tests because psycopg_pool is - missing fails here rather than passing with fewer tests. Oxpull's come - from a file (--pro-skips), because they belong to that repository. -- Every xfail is known: its test id is in the baseline file (--xfails, - --pro-xfails). django-ox has none. A test that now passes where the - baseline expected a failure is reported, not failed. - -One leg is one interpreter and one settings module per suite. Name them with -flags, or list several in a JSON file (--legs) to run them all in one -command: a list of objects whose keys are the long flag names without the -dashes and with underscores, for example - - [{"name": "dj61-pg", "python": ".../dj61/bin/python", - "settings": "tests.settings_postgres", - "pro_python": ".../paired61/bin/python", - "pro_settings": "tests.settings_postgres"}] - -A key a leg leaves out takes the flag's value. --record DIR writes the skips -and xfails each suite showed, in the files' own format, to review and pass -back next time; the run still fails on any failure. - -The suites run from their checkouts with PYTHONPATH built here: the django-ox -checkout's src and its root (its tests import as `tests`) for the django-ox -suite, and Oxpull's src plus django-ox's src, never django-ox's root, for the -Oxpull suite, whose own tests package is also `tests`. --pythonpath entries -(a settings directory, say) are appended to both. Everything else in the -environment passes through, so a variable a settings module reads (a test -database name, say) is set on this command. - -Exit status: 0 when every leg passed, 1 when a leg failed the gate, 2 when -the command or the environment is wrong before any suite ran. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import shlex -import subprocess -import sys -import tempfile -import time -import xml.etree.ElementTree as ET -from dataclasses import dataclass, field -from pathlib import Path - -HERE = Path(__file__).resolve().parent -DEFAULT_DJANGO_OX = HERE.parent - -#: Why a django-ox test may skip, and where. `databases` None means on every -#: database; `django` names the only Django version (major.minor) where it is -#: expected. A skip matching none of these is unexplained. Add a line here in -#: the same change that adds the skip, saying where it belongs. -DJANGO_OX_SKIPS = [ - # Run without --cov; test_timeouts.py checks the tool id coverage holds. - {"reason": r"this session is not measured by coverage"}, - {"reason": r"there is no process clock on this path to skew"}, - {"reason": r"Django 6\.0's @task takes no policy", "django": "6.0"}, - { - "reason": r"no gap on this backend: RETURNING or a row lock covers it", - "databases": {"postgresql", "mysql"}, - }, - { - "reason": r"this database claims with SKIP LOCKED, not compare-and-set", - "databases": {"postgresql", "mysql"}, - }, - { - "reason": r"innodb_lock_wait_timeout is MySQL's", - "databases": {"postgresql", "sqlite"}, - }, - {"reason": r"SQLite's parameter limit", "databases": {"postgresql", "mysql"}}, - { - "reason": r"Django's connection pool is PostgreSQL only", - "databases": {"sqlite", "mysql"}, - }, - {"reason": r"counts pg_stat_activity sessions", "databases": {"sqlite", "mysql"}}, - { - "reason": r"buffers per claim are PostgreSQL's reading", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"only PostgreSQL reports rows actually read", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"only PostgreSQL claims in a single statement", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"Only PostgreSQL raises a serialization failure here", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"stops a worker with SIGSTOP, run on PostgreSQL", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"uses pg_terminate_backend or SELECT \.\.\. FOR UPDATE NOWAIT", - "databases": {"sqlite", "mysql"}, - }, - { - "reason": r"has the server end a connection, with pg_terminate_backend or KILL", - "databases": {"sqlite"}, - }, - { - "reason": r"closes the database to new connections, with ALTER DATABASE", - "databases": {"sqlite", "mysql"}, - }, - {"reason": r"this database has no SKIP LOCKED", "databases": {"sqlite"}}, - {"reason": r"SQLite has no locking read to lose on", "databases": {"sqlite"}}, - { - "reason": r"SQLite has no row locks to take in any order", - "databases": {"sqlite"}, - }, - {"reason": r"SQLite has no row locks to deadlock over", "databases": {"sqlite"}}, - { - "reason": r"SQLite has no row locks, and its prune sends no locking read", - "databases": {"sqlite"}, - }, - { - "reason": r"SQLite lets one writer in at a time, whatever the queue", - "databases": {"sqlite"}, - }, - { - "reason": r"SQLite has one writer; this is its lock", - "databases": {"postgresql", "mysql"}, - }, - { - "reason": r"SQLite has one writer; the next test covers it", - "databases": {"sqlite"}, - }, - {"reason": r"SQLite's single writer", "databases": {"postgresql", "mysql"}}, - {"reason": r"ends a connection from the server's side", "databases": {"sqlite"}}, - { - "reason": r"ends backends by application name", - "databases": {"sqlite", "mysql"}, - }, -] - -PYTEST_ARGS = ["-q", "-p", "no:cacheprovider", "-rfEsxX"] - -# Run by each suite's interpreter, in that suite's environment. Prints what -# the gate checks, as JSON on the last line. -PROBE = r""" -import json, os, sys -out = {"executable": sys.executable, "python": sys.version.split()[0]} -for name in ("django_ox", "oxpull", "django"): - try: - module = __import__(name) - except Exception as exc: - out[name] = {"error": f"{type(exc).__name__}: {exc}"} - else: - out[name] = { - "file": getattr(module, "__file__", None), - "version": getattr(module, "__version__", None), - } -settings = os.environ.get("DJANGO_SETTINGS_MODULE") -if settings: - try: - import django - django.setup() - from django.db import connections - out["database"] = connections["default"].vendor - except Exception as exc: - out["database"] = {"error": f"{type(exc).__name__}: {exc}"} -print(json.dumps(out)) -""" - - -class GateError(Exception): - """The environment is wrong before any suite has run.""" - - -@dataclass -class Leg: - name: str - django_ox: Path - pro: Path | None - python: str - settings: str | None - pro_python: str | None - pro_settings: str | None - pythonpath: list[str] - skips: Path | None - xfails: Path | None - pro_skips: Path | None - pro_xfails: Path | None - pytest_args: list[str] - pro_pytest_args: list[str] - only: str - - -@dataclass -class Outcome: - suite: str - exit_code: int - log: Path - counts: dict[str, int] = field(default_factory=dict) - problems: list[str] = field(default_factory=list) - notes: list[str] = field(default_factory=list) - skips: list[tuple[str, str]] = field(default_factory=list) - xfails: list[str] = field(default_factory=list) - - -def _under(path: str | None, root: Path) -> bool: - if not path: - return False - return root.resolve() in Path(path).resolve().parents - - -def _probe(python: str, cwd: Path, env: dict[str, str]) -> dict: - completed = subprocess.run( # noqa: S603 - [python, "-c", PROBE], - cwd=cwd, - env=env, - capture_output=True, - text=True, - check=False, - ) - lines = completed.stdout.strip().splitlines() - if completed.returncode != 0 or not lines: - raise GateError( - f"{python} could not run the import probe in {cwd}:\n" - f"{completed.stderr.strip()}" - ) - return json.loads(lines[-1]) - - -def _base_env(settings: str | None, pythonpath: list[str]) -> dict[str, str]: - env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} - if settings: - env["DJANGO_SETTINGS_MODULE"] = settings - else: - env.pop("DJANGO_SETTINGS_MODULE", None) - if pythonpath: - env["PYTHONPATH"] = os.pathsep.join(pythonpath) - return env - - -def _pinned_version(pro: Path) -> str | None: - text = (pro / "pyproject.toml").read_text() - found = re.search(r"""["']django-ox\s*==\s*([^"',;\s]+)""", text) - return found.group(1) if found else None - - -def _read_list(path: Path | None) -> list[str]: - if path is None: - return [] - lines = path.read_text().splitlines() - return [line.strip() for line in lines if line.strip() and not line.startswith("#")] - - -def _scoped(line: str) -> tuple[set[str] | None, str]: - """`[sqlite,mysql] text` -> ({"sqlite", "mysql"}, "text").""" - found = re.match(r"\[([a-z,\s]+)\]\s*(.*)", line) - if not found: - return None, line - return {part.strip() for part in found.group(1).split(",")}, found.group(2) - - -def _explained(reason: str, database: str, django: str, allowed: list[dict]) -> bool: - for entry in allowed: - databases = entry.get("databases") - if databases is not None and database not in databases: - continue - version = entry.get("django") - if version is not None and not django.startswith(version + "."): - continue - if re.search(entry["reason"], reason): - return True - return False - - -def _file_skips(path: Path | None) -> list[dict]: - allowed = [] - for line in _read_list(path): - databases, reason = _scoped(line) - allowed.append({"reason": reason, "databases": databases}) - return allowed - - -def _file_xfails(path: Path | None, database: str) -> set[str]: - known = set() - for line in _read_list(path): - databases, test_id = _scoped(line) - if databases is None or database in databases: - known.add(test_id) - return known - - -def _junit(path: Path) -> tuple[dict[str, int], list[str], list[tuple], list[str]]: - counts = {"passed": 0, "failed": 0, "errors": 0, "skipped": 0, "xfailed": 0} - failed: list[str] = [] - skips: list[tuple[str, str]] = [] - xfails: list[str] = [] - for case in ET.parse(path).getroot().iter("testcase"): # noqa: S314 - test_id = f"{case.get('classname')}::{case.get('name')}" - if case.find("failure") is not None: - counts["failed"] += 1 - failed.append(test_id) - elif case.find("error") is not None: - counts["errors"] += 1 - failed.append(test_id) - elif (skipped := case.find("skipped")) is not None: - if skipped.get("type") == "pytest.xfail": - counts["xfailed"] += 1 - xfails.append(test_id) - else: - counts["skipped"] += 1 - skips.append((test_id, skipped.get("message") or "")) - else: - counts["passed"] += 1 - return counts, failed, skips, xfails - - -def _run_suite( - *, - suite: str, - python: str, - cwd: Path, - env: dict[str, str], - args: list[str], - out: Path, - database: str, - django: str, - allowed_skips: list[dict], - known_xfails: set[str], -) -> Outcome: - junit = out / f"{suite}.xml" - log = out / f"{suite}.log" - command = [python, "-m", "pytest", *PYTEST_ARGS, f"--junitxml={junit}", *args] - print(f" running {suite}: {shlex.join(command)}", flush=True) - print(f" in {cwd}, log {log}", flush=True) - started = time.monotonic() - with log.open("w") as stream: - completed = subprocess.run( # noqa: S603 - command, - cwd=cwd, - env=env, - stdout=stream, - stderr=subprocess.STDOUT, - check=False, - ) - outcome = Outcome(suite=suite, exit_code=completed.returncode, log=log) - if not junit.exists(): - outcome.problems.append( - f"pytest exited {completed.returncode} without a report; see {log}" - ) - return outcome - counts, failed, skips, xfails = _junit(junit) - outcome.counts = counts - outcome.skips = skips - outcome.xfails = xfails - outcome.notes.append(f"{time.monotonic() - started:.0f}s") - if completed.returncode != 0: - outcome.problems.append(f"pytest exited {completed.returncode}") - outcome.problems.extend(f"failed: {test_id}" for test_id in failed) - for test_id, reason in skips: - if not _explained(reason, database, django, allowed_skips): - outcome.problems.append( - f"unexplained skip on {database}: {test_id}: {reason}" - ) - for test_id in xfails: - if test_id not in known_xfails: - outcome.problems.append(f"new xfail on {database}: {test_id}") - xpassed = re.findall(r"^XPASS (\S+)", log.read_text(), flags=re.MULTILINE) - outcome.notes.extend(f"now passes, baseline expects xfail: {t}" for t in xpassed) - return outcome - - -def _record(outcome: Outcome, database: str, directory: Path) -> None: - """ - Add what `outcome` showed to the files in `directory`, merged with what - they hold already, so a suite run in parts records the whole of it. - """ - directory.mkdir(parents=True, exist_ok=True) - found = { - "skips": {f"[{database}] {re.escape(reason)}" for _, reason in outcome.skips}, - "xfails": {f"[{database}] {test_id}" for test_id in outcome.xfails}, - } - for kind, lines in found.items(): - path = directory / f"{outcome.suite}-{database}.{kind}" - lines |= set(_read_list(path)) if path.exists() else set() - path.write_text("".join(f"{line}\n" for line in sorted(lines))) - - -def run_leg(leg: Leg, out: Path, record: Path | None) -> list[Outcome]: - out.mkdir(parents=True, exist_ok=True) - dox_src = leg.django_ox / "src" - outcomes: list[Outcome] = [] - print(f"leg {leg.name}", flush=True) - - if leg.only in ("both", "django-ox"): - env = _base_env( - leg.settings, [str(dox_src), str(leg.django_ox), *leg.pythonpath] - ) - seen = _probe(leg.python, leg.django_ox, env) - imported = seen["django_ox"].get("file") - if not _under(imported, dox_src): - raise GateError( - f"{leg.name}: the django-ox suite imports django_ox from " - f"{imported or seen['django_ox']}, not from {dox_src}" - ) - database = seen.get("database") - if not isinstance(database, str): - raise GateError( - f"{leg.name}: no database for the django-ox suite: {database}" - ) - django = seen["django"]["version"] - print( - f" django-ox suite: django_ox {imported}, Django {django}, " - f"Python {seen['python']}, {database}", - flush=True, - ) - outcome = _run_suite( - suite="django-ox", - python=leg.python, - cwd=leg.django_ox, - env=env, - args=leg.pytest_args, - out=out, - database=database, - django=django, - allowed_skips=DJANGO_OX_SKIPS + _file_skips(leg.skips), - known_xfails=_file_xfails(leg.xfails, database), - ) - outcomes.append(outcome) - if record is not None: - _record(outcome, database, record) - - if leg.only in ("both", "pro"): - if leg.pro is None or leg.pro_python is None: - raise GateError( - f"{leg.name}: the Oxpull suite needs --pro and --pro-python" - ) - pro_src = leg.pro / "src" - env = _base_env(leg.pro_settings, [str(pro_src), str(dox_src), *leg.pythonpath]) - seen = _probe(leg.pro_python, leg.pro, env) - for name, root in (("django_ox", dox_src), ("oxpull", pro_src)): - imported = seen[name].get("file") - if not _under(imported, root): - raise GateError( - f"{leg.name}: the Oxpull suite imports {name} from " - f"{imported or seen[name]}, not from {root}" - ) - pinned = _pinned_version(leg.pro) - version = seen["django_ox"]["version"] - if pinned != version: - raise GateError( - f"{leg.name}: Oxpull pins django-ox=={pinned}, and the django-ox " - f"under test is {version}; the pair is not exact-pinned" - ) - # What Oxpull's subprocess tests' children see: the interpreter's own - # path, plus Oxpull's src and the checkout root, which they put on - # PYTHONPATH themselves. django_ox is not there: a child that runs a - # released Oxpull gets none, so the interpreter has to supply it. - child = _base_env(leg.pro_settings, [str(pro_src), str(leg.pro)]) - alone = _probe(leg.pro_python, leg.pro, child) - imported = alone["django_ox"].get("file") - if not _under(imported, dox_src): - raise GateError( - f"{leg.name}: without PYTHONPATH, {leg.pro_python} imports " - f"django_ox from {imported or alone['django_ox']}, not from " - f"{dox_src}; Oxpull's subprocess tests would run another " - "django-ox. Put a .pth naming it in that venv's site-packages, " - "or pip install --no-deps -e it there." - ) - if not isinstance(alone.get("database"), str): - raise GateError( - f"{leg.name}: without PYTHONPATH, {leg.pro_python} cannot load " - f"the settings {leg.pro_settings!r}: {alone.get('database')}. " - "Oxpull's subprocess tests import it from the interpreter's own " - "path; put its directory in a .pth in that venv." - ) - database = seen.get("database") - if not isinstance(database, str): - raise GateError(f"{leg.name}: no database for the Oxpull suite: {database}") - django = seen["django"]["version"] - print( - f" Oxpull suite: oxpull {seen['oxpull']['file']}, django_ox " - f"{seen['django_ox']['file']} {version} (pinned {pinned}), " - f"Django {django}, Python {seen['python']}, {database}", - flush=True, - ) - outcome = _run_suite( - suite="oxpull", - python=leg.pro_python, - cwd=leg.pro, - env=env, - args=leg.pro_pytest_args, - out=out, - database=database, - django=django, - allowed_skips=_file_skips(leg.pro_skips), - known_xfails=_file_xfails(leg.pro_xfails, database), - ) - outcomes.append(outcome) - if record is not None: - _record(outcome, database, record) - return outcomes - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=__doc__.split("\n\n")[1], - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--name", default="leg", help="a name for the leg") - parser.add_argument( - "--django-ox", - type=Path, - default=DEFAULT_DJANGO_OX, - help="the django-ox checkout (default: the one holding this script)", - ) - parser.add_argument("--pro", type=Path, help="the Oxpull checkout") - parser.add_argument( - "--python", default=sys.executable, help="interpreter for the django-ox suite" - ) - parser.add_argument( - "--settings", - default="tests.settings", - help="DJANGO_SETTINGS_MODULE for django-ox (default: %(default)s)", - ) - parser.add_argument("--pro-python", help="interpreter for the Oxpull suite") - parser.add_argument( - "--pro-settings", - default="tests.settings", - help="DJANGO_SETTINGS_MODULE for Oxpull (default: %(default)s)", - ) - parser.add_argument( - "--pythonpath", - action="append", - default=[], - help="a directory added to both suites' PYTHONPATH (repeatable)", - ) - parser.add_argument("--skips", type=Path, help="extra django-ox skip reasons") - parser.add_argument("--xfails", type=Path, help="known django-ox xfails") - parser.add_argument("--pro-skips", type=Path, help="explained Oxpull skip reasons") - parser.add_argument("--pro-xfails", type=Path, help="known Oxpull xfails") - parser.add_argument( - "--pytest-args", - default="", - help=( - "more arguments for the django-ox pytest, as one shell-quoted " - "string; write --pytest-args=... when it starts with a dash" - ), - ) - parser.add_argument( - "--pro-pytest-args", - default="", - help="more arguments for the Oxpull pytest, the same way", - ) - parser.add_argument( - "--only", - choices=("both", "django-ox", "pro"), - default="both", - help="run one suite of the pair (the gate needs both)", - ) - parser.add_argument("--legs", type=Path, help="a JSON list of legs to run") - parser.add_argument("--out", type=Path, help="where reports and logs go") - parser.add_argument( - "--record", type=Path, help="write each suite's skips and xfails here" - ) - return parser - - -def _legs(args: argparse.Namespace) -> list[Leg]: - defaults = vars(args) - specs = json.loads(args.legs.read_text()) if args.legs else [{}] - legs = [] - for spec in specs: - unknown = set(spec) - set(defaults) - if unknown: - raise GateError(f"unknown keys in a leg: {sorted(unknown)}") - merged = {**defaults, **spec} - - def path(key: str, merged: dict = merged) -> Path | None: - value = merged.get(key) - return None if value in (None, "") else Path(value).resolve() - - legs.append( - Leg( - name=merged["name"], - django_ox=path("django_ox"), - pro=path("pro"), - python=merged["python"], - settings=merged["settings"], - pro_python=merged["pro_python"], - pro_settings=merged["pro_settings"], - pythonpath=[str(Path(p).resolve()) for p in merged["pythonpath"]], - skips=path("skips"), - xfails=path("xfails"), - pro_skips=path("pro_skips"), - pro_xfails=path("pro_xfails"), - pytest_args=shlex.split(merged["pytest_args"]), - pro_pytest_args=shlex.split(merged["pro_pytest_args"]), - only=merged["only"], - ) - ) - return legs - - -def main(argv: list[str] | None = None) -> int: - args = _parser().parse_args(argv) - out = args.out or Path(tempfile.mkdtemp(prefix="check-policy-pair-")) - try: - legs = _legs(args) - results = [(leg, run_leg(leg, out / leg.name, args.record)) for leg in legs] - except GateError as exc: - print(f"check-policy-pair: {exc}", file=sys.stderr) - return 2 - failed = False - print(f"\nreports and logs: {out}") - for leg, outcomes in results: - for outcome in outcomes: - counts = ", ".join(f"{v} {k}" for k, v in outcome.counts.items()) - verdict = "FAIL" if outcome.problems else "ok" - print(f"{verdict} {leg.name} {outcome.suite}: {counts} ({outcome.log})") - for note in outcome.notes: - print(f" {note}") - for problem in outcome.problems: - print(f" {problem}") - failed = failed or bool(outcome.problems) - if leg.only != "both": - print(f" {leg.name} ran --only {leg.only}: not a pair verdict") - failed = True - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/check_release.py b/tools/check_release.py index 48d8476..2e389a6 100644 --- a/tools/check_release.py +++ b/tools/check_release.py @@ -82,6 +82,28 @@ def read_dunder_version(repo: Path = REPO) -> str | None: return match.group(1) if match else None +OXPULL_VERSION_IMPORT = re.compile(r"^from django_ox import __version__\s*$", re.M) +OXPULL_VERSION_LITERAL = re.compile(r"^__version__\s*=", re.M) + + +def check_oxpull_version(repo: Path = REPO) -> tuple[list[str], list[str]]: + """oxpull ships in the django-ox distribution, so its __version__ must be + django-ox's own and not a second literal that drifts at release.""" + path = repo / "src" / "oxpull" / "__init__.py" + if not path.is_file(): + return [], [] + text = path.read_text(encoding="utf-8") + if OXPULL_VERSION_LITERAL.search(text) or not OXPULL_VERSION_IMPORT.search(text): + return [ + fail( + "src/oxpull/__init__.py does not take __version__ from django_ox. " + "Use `from django_ox import __version__`: one distribution has " + "one version, and a second literal drifts." + ) + ], [] + return [], ["oxpull.__version__ is django_ox.__version__"] + + def read_changelog(repo: Path = REPO) -> tuple[str | None, set[str]]: text = (repo / "CHANGELOG.md").read_text(encoding="utf-8") headings = re.findall(r"^##\s*\[([^\]]+)\]", text, re.M) @@ -109,6 +131,16 @@ def read_migrations(repo: Path = REPO) -> set[str]: } +def read_oxpull_migrations(repo: Path = REPO) -> set[str]: + """The oxpull migration filenames the source tree declares.""" + directory = repo / "src" / "oxpull" / "migrations" + if not directory.is_dir(): + return set() + return { + path.name for path in directory.glob("[0-9]*.py") if path.name != "__init__.py" + } + + def check_urls(urls: dict[str, str]) -> list[str]: problems: list[str] = [] missing = EXPECTED_URL_KEYS - urls.keys() @@ -191,6 +223,10 @@ def check_source( else: notes.append(f"__version__ and pyproject agree on {version}") + oxpull_problems, oxpull_notes = check_oxpull_version(repo) + problems.extend(oxpull_problems) + notes.extend(oxpull_notes) + if newest is None: problems.append(fail("CHANGELOG has no released version heading")) elif newest != version: @@ -250,7 +286,10 @@ def check_sdist_ignore_files(path: Path, members: list[str]) -> list[str]: def check_dist( - dist: Path, version: str, migrations: set[str] | None = None + dist: Path, + version: str, + migrations: set[str] | None = None, + oxpull_migrations: set[str] | None = None, ) -> tuple[list[str], list[str]]: """Everything only the built archive can show. Returns (problems, notes).""" problems: list[str] = [] @@ -260,6 +299,9 @@ def check_dist( return [fail(f"{dist} does not exist; run python -m build first")], notes expected = read_migrations() if migrations is None else migrations + expected_oxpull = ( + read_oxpull_migrations() if oxpull_migrations is None else oxpull_migrations + ) for kind, pattern in ( ("wheel", "django_ox-*.whl"), @@ -306,6 +348,29 @@ def check_dist( for tail in tails if tail.startswith("migrations/") and tail != "migrations/__init__.py" } + for required in ("__init__.py", "backend.py", "models.py", "py.typed"): + if f"oxpull/{required}" not in { + name.split("src/", 1)[-1] for name in members + }: + problems.append( + fail(f"{path.name} does not contain oxpull/{required}") + ) + shipped_oxpull = { + name.split("oxpull/migrations/", 1)[-1] + for name in members + if "oxpull/migrations/" in name + and not name.endswith("oxpull/migrations/__init__.py") + and name.endswith(".py") + } + for missing in sorted(expected_oxpull - shipped_oxpull): + problems.append( + fail( + f"{path.name} is missing oxpull migration {missing}. " + "An upgrade against it leaves the table without the " + "column the code expects." + ) + ) + for missing in sorted(expected - shipped): problems.append( fail( diff --git a/tools/ci/changes.py b/tools/ci/changes.py new file mode 100644 index 0000000..6201079 --- /dev/null +++ b/tools/ci/changes.py @@ -0,0 +1,183 @@ +""" +Decide which parts of the oxpull feature suite (tests_oxpull/) a CI run needs. + +Run by the `changes` job in .github/workflows/ci.yml: + + python tools/ci/changes.py # reads the environment, writes $GITHUB_OUTPUT + python tools/ci/changes.py --files F # dry run: the changed paths come from file F + # ("-" for stdin), the event from --event + +Outputs, one `name=value` line each: + + feature "true" unless every changed path is known to be inert for the suite + hazard "true" when the hazard tests (marker `hazard`) must run too + full "true" when the hazard suite must run on the whole matrix + sqlite_cells, postgres_cells the matrix cells, as JSON + +Every rule fails towards running more. A path no rule knows runs the feature +suite. A diff that cannot be computed turns everything on. The only way to a +`false` is a changed-path list in which every path matched an inert pattern +(feature) or matched no source path at all (hazard). + +Why the hazard tests are not simply always on: they are about 78 percent of +the feature suite's time (PostgreSQL: 2,219 of 2,855 measured seconds), and +most changes cannot reach what they pin. They run on main, nightly, on a +release candidate and when a change touches the code they pin. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +# Paths that cannot change what tests_oxpull does. README.md is NOT here: a +# hazard test reads it. tests/ is not here for the opposite reason in a +# different column: tests_oxpull imports nothing from it, so a change that only +# touches the base suite does not need the feature suite (see FREE_ONLY). +INERT = re.compile( + r"""^( + docs/ | overrides/ | benchmarks/ | brand/ + | mkdocs\.yml | context7\.json | LICENSE | CHANGELOG\.md | CODE_OF_CONDUCT\.md + | CONTRIBUTING\.md | SECURITY\.md | AGENTS\.md + | \.github/ISSUE_TEMPLATE/ | \.github/PULL_REQUEST_TEMPLATE\.md + | \.github/FUNDING\.yml | \.github/dependabot\.yml + | tests/ + )""", + re.VERBOSE, +) + +# Skip change-selected hazard jobs only for the modules below, which are not +# reached by the hazard suite's static reference scan. Imports, command calls, +# and URL includes count; mounting django_ox.urls also reaches its views. +# tests/test_ci_changes.py guards this list against new hazard references. +# This is a static check, not proof of runtime isolation. Nightly, rc-tag, +# and manual-dispatch runs still execute every hazard test. +SRC_NOT_HAZARD = re.compile( + r"""^src/django_ox/( + testing\.py + | management/_beat_.* | management/commands/ox_import_beat_schedules\.py + )$""", + re.VERBOSE, +) + +# Paths that change what the hazard tests run against or how. +HAZARD_PATHS = re.compile( + r"""^( + src/(django_ox|oxpull)/.* | pyproject\.toml | \.github/workflows/ + | tools/ci/ | README\.md + | tests_oxpull/(?!test_) # conftest, settings, helpers, fixtures + | tests_oxpull/test_[^/]*hazard + )""", + re.VERBOSE, +) + +CORNERS = [ + {"python": "3.12", "django": "6.0"}, + {"python": "3.14", "django": "6.1"}, +] +FULL_GRID = [ + {"python": py, "django": dj} + for py in ("3.12", "3.13", "3.14") + for dj in ("6.0", "6.1") +] + + +def classify(event: str, ref: str, paths: list[str] | None) -> dict[str, bool]: + """ + `paths` is the changed-path list of a pull request, or None when it could + not be computed (or, for any other event, is not meaningful). + """ + if event in ("schedule", "workflow_dispatch"): + return {"feature": True, "hazard": True, "full": True} + if event == "push": + released = ref.startswith(("refs/tags/rc", "refs/tags/v")) + return {"feature": True, "hazard": True, "full": released} + # pull_request, or anything unknown. + if event != "pull_request" or paths is None: + return {"feature": True, "hazard": True, "full": False} + feature = any(not INERT.match(p) for p in paths) + hazard = False + for p in paths: + if p.startswith("src/") and SRC_NOT_HAZARD.match(p): + continue + if HAZARD_PATHS.match(p): + hazard = True + # A workflow or script change reaches everything below it. + return {"feature": feature or hazard, "hazard": hazard, "full": False} + + +def cells(*, full: bool) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + plain = "tests_oxpull.settings_postgres" + pooled = "tests_oxpull.settings_postgres_pooled" + sqlite = FULL_GRID if full else CORNERS + newest = {"python": "3.14", "django": "6.1"} + postgres = [{**newest, "settings": plain, "label": "postgres"}] + if full: + postgres = [ + {**cell, "settings": plain, "label": "postgres"} for cell in FULL_GRID + ] + [{**newest, "settings": pooled, "label": "postgres pooled"}] + return sqlite, postgres + + +def changed_paths(base: str) -> list[str] | None: + try: + out = subprocess.run( # noqa: S603 + ["git", "diff", "--name-only", f"{base}...HEAD"], # noqa: S607 + check=True, + capture_output=True, + text=True, + ).stdout + except (subprocess.CalledProcessError, OSError): + return None + return [line for line in out.splitlines() if line] + + +def render(flags: dict[str, bool]) -> dict[str, str]: + sqlite, postgres = cells(full=flags["full"]) + return { + "feature": str(flags["feature"]).lower(), + "hazard": str(flags["hazard"]).lower(), + "full": str(flags["full"]).lower(), + "sqlite_cells": json.dumps(sqlite, separators=(",", ":")), + "postgres_cells": json.dumps(postgres, separators=(",", ":")), + } + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--event", default=os.environ.get("EVENT", "")) + parser.add_argument("--ref", default=os.environ.get("REF", "")) + parser.add_argument("--base", default=os.environ.get("BASE_SHA", "")) + parser.add_argument( + "--files", help="dry run: read changed paths from this file, - for stdin" + ) + args = parser.parse_args(argv) + + paths: list[str] | None + if args.files: + text = sys.stdin.read() if args.files == "-" else Path(args.files).read_text() + paths = [line.strip() for line in text.splitlines() if line.strip()] + elif args.event == "pull_request" and args.base: + paths = changed_paths(args.base) + else: + paths = None + + outputs = render(classify(args.event, args.ref, paths)) + if paths is not None: + print("changed paths:", *paths, sep="\n ", file=sys.stderr) + lines = [f"{key}={value}" for key, value in outputs.items()] + target = os.environ.get("GITHUB_OUTPUT") + if target and not args.files: + with Path(target).open("a") as handle: + handle.write("\n".join(lines) + "\n") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/ci/shard.py b/tools/ci/shard.py new file mode 100644 index 0000000..6dcc593 --- /dev/null +++ b/tools/ci/shard.py @@ -0,0 +1,117 @@ +""" +The six shards of the oxpull feature suite (tests_oxpull/). + + python tools/ci/shard.py check every test file is in exactly one shard + python tools/ci/shard.py run shard3 SETTINGS [--hazard true|false] + run one shard under pytest + python tools/ci/shard.py files shard3 [all|hazard|not-hazard] + the files of one shard, one line + +A file belongs to the hazard group when "hazard" is in its name, the rule +tests_oxpull/conftest.py marks tests by. A mode of "not-hazard" drops those +files, so a shard made of nothing but hazard files prints nothing and the job +skips pytest instead of failing on its exit status 5 (no tests collected). + +Whole files stay together because a file's tests share fixtures and, in a few +files, state that one test leaves for the next. Membership is in shards.json, +fitted to measured durations. The check is what stops a new test file from +falling out of every shard: CI fails until the file is added to one. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SHARDS = ROOT / "tools" / "ci" / "shards.json" +TESTS = ROOT / "tests_oxpull" + + +def load() -> dict[str, list[str]]: + shards: dict[str, list[str]] = json.loads(SHARDS.read_text())["shards"] + return shards + + +def problems(shards: dict[str, list[str]]) -> list[str]: + on_disk = {f"tests_oxpull/{p.name}" for p in TESTS.glob("test_*.py")} + seen: dict[str, str] = {} + found: list[str] = [] + for name, files in shards.items(): + for f in files: + if f in seen: + found.append(f"{f} is in {seen[f]} and {name}") + seen[f] = name + found += [ + f"{f} is in no shard: add it to tools/ci/shards.json" + for f in sorted(on_disk - set(seen)) + ] + found += [f"{f} is in a shard but not on disk" for f in sorted(set(seen) - on_disk)] + return found + + +def select(shards: dict[str, list[str]], name: str, mode: str) -> list[str]: + return [ + f + for f in shards[name] + if mode == "all" or ("hazard" in Path(f).name) == (mode == "hazard") + ] + + +def run(shards: dict[str, list[str]], name: str, settings: str, *, hazard: bool) -> int: + """ + Replace this process with pytest on one shard's files. + + exec, not a child process, so a SIGABRT from `timeout -s ABRT` reaches + pytest and faulthandler dumps its threads. Without the hazard flag the + hazard files are not passed at all, and `-m "not hazard"` stays on as the + second guard; a shard of hazard files only then runs nothing and passes, + where pytest alone would exit 5. + """ + files = select(shards, name, "all" if hazard else "not-hazard") + if not files: + print(f"{name}: no files to run without the hazard suite") + return 0 + argv = [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider"] + argv += [f"--ds={settings}", *files] + if not hazard: + argv += ["-m", "not hazard"] + os.chdir(ROOT) + os.execv(sys.executable, argv) # noqa: S606 - the point is to be pytest + return 0 # pragma: no cover + + +def main(argv: list[str]) -> int: + shards = load() + if argv[:1] == ["check"]: + found = problems(shards) + if found: + print("\n".join(found), file=sys.stderr) + return 1 + print(f"{sum(map(len, shards.values()))} files in {len(shards)} shards") + return 0 + if argv[:1] == ["files"] and 2 <= len(argv) <= 3 and argv[1] in shards: + mode = argv[2] if len(argv) == 3 else "all" + if mode not in ("all", "hazard", "not-hazard"): + print(__doc__, file=sys.stderr) + return 2 + files = select(shards, argv[1], mode) + if files: + print(*files) + return 0 + if argv[:1] == ["run"] and len(argv) in (3, 5) and argv[1] in shards: + hazard = len(argv) == 5 and argv[3] == "--hazard" and argv[4] == "true" + if len(argv) == 5 and ( + argv[3] != "--hazard" or argv[4] not in ("true", "false") + ): + print(__doc__, file=sys.stderr) + return 2 + return run(shards, argv[1], argv[2], hazard=hazard) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/ci/shards.json b/tools/ci/shards.json new file mode 100644 index 0000000..9c3e606 --- /dev/null +++ b/tools/ci/shards.json @@ -0,0 +1,92 @@ +{ + "basis": "Longest-processing-time on per-file loaded seconds (SQLite plus PostgreSQL summed), whole files kept together, six shards. Measured on a loaded Mac from the 1.8.0 oxpull suite, so rebalance from CI timings once they exist.", + "shards": { + "shard1": [ + "tests_oxpull/test_workflows_hazard_in_flight_fan_in_edges_make_the_gate_cost.py" + ], + "shard2": [ + "tests_oxpull/test_heartbeat_file.py", + "tests_oxpull/test_policy.py", + "tests_oxpull/test_workflows_deferred.py", + "tests_oxpull/test_workflows_guarantees.py", + "tests_oxpull/test_workflows_hazard_cancel_cas.py", + "tests_oxpull/test_workflows_hazard_a_correctly_configured_worker_on_another.py", + "tests_oxpull/test_workflows_hazard_a_downstream_node_can_start_while_an_earlier.py", + "tests_oxpull/test_workflows_hazard_every_reconciler_tick_row_locks_every_dead.py", + "tests_oxpull/test_workflows_hazard_mysql_edge_columns_created_on_a_different.py", + "tests_oxpull/test_workflows_hazard_a_workflow_callback_that_becomes_unique.py", + "tests_oxpull/test_workflows_hazard_a_unique_enqueue_inside_a_transaction_can.py", + "tests_oxpull/test_workflows_hazard_unique_nodes.py", + "tests_oxpull/test_workflows_statistics.py" + ], + "shard3": [ + "tests_oxpull/test_run_tasks_integration.py", + "tests_oxpull/test_schedule_isolation.py", + "tests_oxpull/test_upgrade.py", + "tests_oxpull/test_workflows.py", + "tests_oxpull/test_workflows_cost.py", + "tests_oxpull/test_workflows_hazard_discard_promise.py", + "tests_oxpull/test_workflows_hazard_reconcile_schedule.py", + "tests_oxpull/test_workflows_hazard_cancellation_locks_the_upstreams_that_are.py", + "tests_oxpull/test_workflows_hazard_defer_blocked_nodes_is_decided_by_the_node.py", + "tests_oxpull/test_workflows_hazard_the_cancellation_transaction_s_lock_order.py", + "tests_oxpull/test_workflows_hazard_under_use_tz_false_on_sqlite_or_mysql_every.py", + "tests_oxpull/test_workflows_hazard_defer_blocked_nodes_does_not_take_nodes.py", + "tests_oxpull/test_workflows_hazard_mysql_repeatable_read_the_cancellation_s.py", + "tests_oxpull/test_workflows_hazard_a_dag_that_runs_longer_than_ox_prune_older.py", + "tests_oxpull/test_workflows_hazard_add_fires_task_enqueued_before_it_validates.py", + "tests_oxpull/test_workflows_hazard_unknown_status.py", + "tests_oxpull/test_workflows_one_key_statistics.py" + ], + "shard4": [ + "tests_oxpull/test_unique.py", + "tests_oxpull/test_worker_subclass.py", + "tests_oxpull/test_workflows_at_scale.py", + "tests_oxpull/test_workflows_hazard_lost_straggler.py", + "tests_oxpull/test_workflows_hazard_prune_membership.py", + "tests_oxpull/test_workflows_hazard_a_node_that_builds_workflow_structure_is.py", + "tests_oxpull/test_workflows_hazard_deleting_a_workflow_row_cascades_its_edges.py", + "tests_oxpull/test_workflows_hazard_rolling_back_migrate_oxpull_0003_or_downgrade.py", + "tests_oxpull/test_workflows_hazard_the_reconciler_s_per_workflow_except_exception.py", + "tests_oxpull/test_workflows_hazard_a_stored_admin_edited_reconciler_schedule.py", + "tests_oxpull/test_workflows_hazard_a_workflow_build_that_raises_part_way_leaves.py", + "tests_oxpull/test_workflows_hazard_mysql_repeatable_read_inside_a_caller_s.py", + "tests_oxpull/test_workflows_hazard_ox_prune_include_failed_deletes_a_workflow.py", + "tests_oxpull/test_workflows_outcome_reconnect.py", + "tests_oxpull/test_workflows_reconciler.py" + ], + "shard5": [ + "tests_oxpull/test_rate_limit_names.py", + "tests_oxpull/test_rate_limits.py", + "tests_oxpull/test_workflows_hazard_prune_ungates.py", + "tests_oxpull/test_workflows_hazard_on_postgresql_a_worker_class_subclass_of.py", + "tests_oxpull/test_workflows_hazard_unique_nodes_let_two_workflows_build_a_dependency.py", + "tests_oxpull/test_workflows_hazard_mysql_node_name_uniqueness_follows_the_column.py", + "tests_oxpull/test_workflows_hazard_mysql_release_dependents_and_the_cancellation.py", + "tests_oxpull/test_workflows_hazard_nodes_the_reconciler_cancels_emit_no_task.py", + "tests_oxpull/test_workflows_hazard_seal_lock.py", + "tests_oxpull/test_workflows_outcomes.py", + "tests_oxpull/test_workflows_waiting.py" + ], + "shard6": [ + "tests_oxpull/test_batches.py", + "tests_oxpull/test_concurrency.py", + "tests_oxpull/test_keys.py", + "tests_oxpull/test_prune.py", + "tests_oxpull/test_workflows_callers.py", + "tests_oxpull/test_workflows_concurrency.py", + "tests_oxpull/test_workflows_hazard_add_mysql.py", + "tests_oxpull/test_workflows_hazard_mixed_fleet_migration.py", + "tests_oxpull/test_workflows_hazard_django_ox_stats_ox_health_and_the_prometheus.py", + "tests_oxpull/test_workflows_hazard_mysql_a_gated_claim_whose_scan_spans_an.py", + "tests_oxpull/test_workflows_hazard_retrying_a_failed_node_after_one_reconciler.py", + "tests_oxpull/test_workflows_hazard_the_reconciler_s_release_sweep_reads_every.py", + "tests_oxpull/test_workflows_hazard_postgresql_repeatable_read_concurrent_add.py", + "tests_oxpull/test_workflows_hazard_the_stored_cancelled_count_counts_every.py", + "tests_oxpull/test_workflows_hazard_with_defer_blocked_nodes_releasing_a_node.py", + "tests_oxpull/test_workflows_hazard_a_support_engineer_sets_an_abandoned_open.py", + "tests_oxpull/test_workflows_hazard_following_w003_s_hint_verbatim_stops_workers.py", + "tests_oxpull/test_workflows_hazard_mariadb_after_a_server_upgrade_from_10_6.py" + ] + } +} diff --git a/tools/mixed_fleet_policy.py b/tools/mixed_fleet_policy.py index 9666811..7eba933 100644 --- a/tools/mixed_fleet_policy.py +++ b/tools/mixed_fleet_policy.py @@ -31,10 +31,11 @@ E. Both workers on one queue at once, rows enqueued from both sides: every row spends exactly its stored budget, whichever worker claims each attempt. -D is why the rollout order is: the new django-ox (and its oxpull pair) -everywhere first, declarations second. While a fleet is genuinely mixed, a -declaration has to be import-compatible (``mixapp/compat.py`` in PROJECT below -is one way) and neither the per-task timeout nor the backoff can be relied on. +D is why the rollout order is: deploy the new django-ox release everywhere +first, then add declarations. While a fleet runs mixed versions, declarations +must remain import-compatible (``mixapp/compat.py`` in PROJECT below shows +one approach). Do not rely on the per-task timeout or backoff until rollout +is complete. Before anything runs, the script establishes which code each side imports: the 1.4.0 side must be the PyPI release (no ``direct_url.json``, and when the