From 96947c633c303e0435cc439253ae8940f164d035 Mon Sep 17 00:00:00 2001 From: PhiLily <252857470+PhiLily@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:00:18 +0300 Subject: [PATCH] Release 1.9.1 **What and why:** Warn at worker start on PostgreSQL when `CONN_MAX_AGE` is 0 and no `OPTIONS["pool"]` is set, because every task then opens a new database connection. Rewrite "Retaining worker connections" in the production guide to give both settings and the pool size to use. Retry an outcome write whose connection is refused, with waits of 50 ms doubling to 1.6 s. The waits total at most a tenth of `LOCK_TIMEOUT`. A worker told to stop ends a wait at once. Replace the benchmarks page with a run of django-ox 1.9.0 against django-tasks-db 0.13.0 at the same connection settings: 1.4 to 1.7 times the tasks per second with pooled or persistent connections. The rows are in `benchmarks/results-raw-2026-10-14-matched.json`, and `benchmarks/render_results.py` checks every number on the page against them. Rewrite sentences on the README and the front-door docs pages to state the point directly. **Operator action:** none. A worker on PostgreSQL with `CONN_MAX_AGE = 0` and no pool logs one warning at start. Set `CONN_MAX_AGE = 600`, or set `OPTIONS["pool"]` and leave `CONN_MAX_AGE` at 0. No database migration is required. --- CHANGELOG.md | 25 + README.md | 45 +- benchmarks/README.md | 12 +- benchmarks/render_results.py | 427 +- .../results-raw-2026-10-14-matched.json | 6831 +++++++++++++++++ docs/agents.md | 110 +- docs/background-tasks.md | 19 +- docs/benchmarks.md | 84 +- docs/choosing.md | 7 +- docs/llms-full.txt | 329 +- docs/migrating.md | 29 +- docs/monitoring.md | 2 +- docs/patterns.md | 12 +- docs/production.md | 25 +- docs/stability.md | 2 +- docs/use-cases.md | 6 +- docs/wagtail.md | 8 +- pyproject.toml | 2 +- src/django_ox/__init__.py | 2 +- src/django_ox/worker.py | 200 +- tests/test_benchmarks_page.py | 221 +- tests/test_connection_count.py | 18 +- tests/test_connection_warning.py | 102 + tests/test_outcome_connect_failure.py | 247 + tests/test_supervisor.py | 5 +- 25 files changed, 8321 insertions(+), 449 deletions(-) create mode 100644 benchmarks/results-raw-2026-10-14-matched.json create mode 100644 tests/test_connection_warning.py create mode 100644 tests/test_outcome_connect_failure.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c2bc7d..c9b92a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] - 2026-10-17 + +**Upgrading:** No database migration is required. + +### Changed + +- `worker_connection_hint` is now a WARNING on PostgreSQL. A worker with + `CONN_MAX_AGE = 0` and no `OPTIONS["pool"]` opens a new connection for every + task, about 5 ms each on a local server, and logs one line at startup saying + so. To remove it, set `CONN_MAX_AGE = 600` or configure Django's pool and + leave `CONN_MAX_AGE` at 0. MySQL keeps the INFO line, and SQLite logs + nothing. "Retaining worker connections" in `docs/production.md` + lists both settings and the pool size to use. +- The benchmarks page now reports django-ox 1.9.0 against django-tasks-db 0.13.0 + at the same connection settings. + +### Fixed + +- A worker whose database refuses the connection for an outcome write, as a + host out of ephemeral ports does under `CONN_MAX_AGE=0`, now tries the write + again after waits of 50 ms doubling to 1.6 s, 3.15 s in all, and the waits + total at most a tenth of `LOCK_TIMEOUT`. A worker told to stop ends a wait at + once. + ## [1.9.0] - 2026-10-12 **Upgrading:** No database migration is required. @@ -2418,6 +2442,7 @@ Initial release. the public API surface, the pre-1.0 SemVer rule, the deprecation window, and the supported Python and Django matrix. +[1.9.1]: https://github.com/oxpull/django-ox/compare/v1.9.0...v1.9.1 [1.9.0]: https://github.com/oxpull/django-ox/compare/v1.8.0...v1.9.0 [1.7.0]: https://github.com/oxpull/django-ox/compare/v1.6.0...v1.7.0 [1.8.0]: https://github.com/oxpull/django-ox/compare/v1.7.0...v1.8.0 diff --git a/README.md b/README.md index 7289688..8cb55d0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ # Django tasks in your database -Stop running Redis to send an email. django-ox runs Django's Tasks framework in your existing database, where a dead worker doesn't mean lost tasks. Run a worker process, not a separate broker. +Stop running Redis to send an email. django-ox runs Django's Tasks framework in your existing database. When a worker dies, another worker picks up its tasks. You run one process: the worker. Supports Django 5.2 LTS through the `django-tasks` backport, and Django 6.0 / 6.1 through core `django.tasks`. Requires Python 3.12+. The optional `oxpull` app provides task-coordination features and requires Django 6.0+. @@ -56,7 +56,9 @@ Inspect attempts and retry or discard tasks in Django admin. Open **Queue overvi For fleet alerting, `ox_health` turns queue thresholds into an exit code. For local loop-liveness probes, pair `ox_worker --heartbeat-file PATH` with `ox_health --heartbeat-file PATH`, using the same process count. Both health modes support `--format json`. Monitor through `django_ox.stats` or `/ox/metrics`, and clear finished rows with `ox_prune`. Use `--database` on `ox_worker`, `ox_prune` and the database mode of `ox_health` to select the database alias. -In the [published benchmarks](https://oxpull.com/django-ox/benchmarks/), 2,000 of 2,000 tasks finished after 20 worker kills per trial, queue drain was up to 20% faster than django-tasks-db, and enqueueing 10,000 tasks took 0.71 s. +With pooled or persistent connections, django-ox delivered 1.4 to 1.7× django-tasks-db's throughput at matched Django connection settings. See the [published benchmarks](https://oxpull.com/django-ox/benchmarks/). + +PostgreSQL 16; no-op tasks; django-ox 1.9.0 vs django-tasks-db 0.13.0; medians of five runs. CI covers Python 3.12 to 3.14, Django 5.2 to 6.1, PostgreSQL, MySQL and SQLite. Open source under the BSD 3-Clause licence. @@ -70,7 +72,7 @@ Documentation: Add `"oxpull"` to `INSTALLED_APPS`, run migrations, and use `"oxpull.backend.OxpullBackend"` for the task aliases that need its features. No separate package installation is needed. -Unique tasks deduplicate pending tasks at enqueue time. Batches also need a reconciler schedule. Rate limits need `OPTIONS["RATE_LIMITS"]` and `"oxpull.worker.OxpullWorker"` as the alias's `OPTIONS["WORKER_CLASS"]`. Workflows need `OPTIONS["WORKFLOWS"] = True`, `OxpullWorker` and a reconciler schedule. Workflows are created at runtime, not declared as workflow definitions in settings. +Unique tasks deduplicate pending tasks at enqueue time. Batches also need a reconciler schedule. Rate limits need `OPTIONS["RATE_LIMITS"]` and `"oxpull.worker.OxpullWorker"` as the alias's `OPTIONS["WORKER_CLASS"]`. Workflows need `OPTIONS["WORKFLOWS"] = True`, `OxpullWorker` and a reconciler schedule. Workflows are created at runtime. See the [feature setup](https://oxpull.com/django-ox/features/) and, for an existing installation, [Earlier `oxpull` installs](https://oxpull.com/django-ox/migrating/#earlier-oxpull-installs). @@ -105,7 +107,7 @@ limit, keeping the full traceback of every attempt. [The benchmarks page](https://oxpull.com/django-ox/benchmarks/) compares django-ox 1.3.0 with `django-tasks-db` 0.13.0, another database backend for the Tasks framework, in runs dated 2026-09-19. Both use Django core -`django.tasks`, PostgreSQL 16 in Docker on an Apple M1 Max, no-op task +`django.tasks`, PostgreSQL 16 in Docker, no-op task bodies and package defaults except `LOCK_TIMEOUT` of 15 seconds for the kill test. It covers backlog drain at two queue depths with one and four workers per backend, bulk enqueue of 10,000 rows in one transaction, @@ -255,10 +257,10 @@ This mode checks file metadata without querying the database. | `--processes N` | `1` | Expected worker process count. At 1, check `PATH`; above 1, require `PATH.supervisor` and every slot file from `PATH.0` through `PATH.(N-1)` to pass. Requires `--heartbeat-file`. | A passing file probe means the expected controlling loops have advanced -recently, not that tasks are progressing. A hung database can make every -file stale and trigger fleet-wide liveness restarts. Use an existing, -writable directory private to the container, allow for startup and slot -replacement, and keep project Django startup database-free. See the +recently. It does not mean that tasks are progressing. A hung database can +make every file stale and trigger fleet-wide liveness restarts. Use an +existing, writable directory private to the container. Allow for startup and +slot replacement, and keep project Django startup database-free. See the [probe guidance](https://oxpull.com/django-ox/monitoring/#health-checks-ox_health) before enabling automatic restarts. @@ -274,11 +276,12 @@ actions. The same operations are `django_ox.actions.retry(result_id)` and FAILED or LOST task; a discard closes a READY, WAITING, FAILED or LOST task without running it. Neither touches a running task. -Open **Queue overview** from the task change list to compare retained status -counts, eligible READY tasks, the oldest eligible task's age, five-minute -throughput and failure rate, and time since the last claim. READY includes -deferred tasks; Eligible ready excludes them. Status totals are retained -rows, not lifetime counts. +Open **Queue overview** from the task change list. It shows retained status +counts, eligible READY tasks and the oldest eligible task's age. It also +shows five-minute throughput and failure rate, and time since the last +claim. READY includes +deferred tasks; Eligible ready excludes them. Status totals count the rows +still in the table. Each visit scans retained task rows, so cost grows with retention. The page does not refresh automatically. It uses the database alias selected by @@ -349,16 +352,16 @@ failure is logged as `schedule_dispatch_error`. After rollback, the worker continues to later schedules if the same database connection remains usable. Missed ticks: if every worker was down when a tick passed, the latest -missed tick fires once on recovery and older ones are skipped, so a -nightly job still runs after an unlucky deploy window but a backlog never -stampedes. A newly deployed schedule waits for its next tick rather than -firing for a time before it existed. +missed tick fires once on recovery. Older missed ticks are skipped. A +nightly job still runs after an unlucky deploy window, and a backlog never +stampedes. A newly deployed schedule waits for its next tick. It does not +fire for a time before it existed. Schedules can also live in the database and be edited in the Django admin -without a deploy, for the cases where whoever needs to pause a job cannot -ship one. A row names a task the code has exposed rather than an import -path, so admin access does not become permission to run anything. See -[Schedules in the database](https://oxpull.com/django-ox/stored-schedules/). +without a deploy. That is for the cases where whoever needs to pause a job +cannot ship a deploy. A row names a task the code has exposed. Import paths +are not accepted, so admin access does not become permission to run anything. +See [Schedules in the database](https://oxpull.com/django-ox/stored-schedules/). ## Behavior details diff --git a/benchmarks/README.md b/benchmarks/README.md index d5a5bb7..4aee469 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -221,7 +221,6 @@ Recorded automatically into the raw results JSON at run time (`collect_environment()` in `bench.py`). The machine used for the published results: -- Apple M1 Max, 10 logical cores, 64 GiB RAM - macOS, Python, Django, psycopg and PostgreSQL versions: the `environment` block of the published raw file, recorded at run time - PostgreSQL 16 (official `postgres:16` image) in Docker Desktop, @@ -288,7 +287,7 @@ all four ran interleaved, run by run, in one window. Read this before quoting any number. -- **Single machine, single run day.** One Mac, one OS state, no controlled +- **Single machine, single run day.** One machine, one OS state, no controlled thermal or background-load environment. Numbers describe relative behaviour on this hardware; absolute performance elsewhere will differ. - **Localhost database.** PostgreSQL runs in Docker on the same machine @@ -446,9 +445,14 @@ Disclosure, all of it: after the clock stopped; 0 is a clean stop on SIGTERM) and `stderr_tail`. Files from before 2026-09-19 carry `concurrency` in place of `processes`, and their concurrency-4 cell is the legacy cell. +- `results-raw-2026-10-14-matched.json`: the matched-connection run + (django-ox 1.9.0 against django-tasks-db 0.13.0): one summary row per + non-warmup run, with the failed and invalid rows and their flags. Not + produced by `bench.py`; it was condensed from the per-run files of a + separate harness. - `render_results.py`: recomputes the results page's figures from - `results-raw-.json` and `results-kill-.json`; `--check PAGE` - verifies every number on the page against them. + `results-raw-.json`, `results-kill-.json` and the matched file; + `--check PAGE` verifies every number on the page against them. - `docs/benchmarks.md` (in the repository root's docs): the results page. - `soak.py` / `soaksite/`: soak and chaos harness (see above). - `SOAK-.md` / `soak-results-raw-.json`: its results. diff --git a/benchmarks/render_results.py b/benchmarks/render_results.py index 3e35601..075af1f 100644 --- a/benchmarks/render_results.py +++ b/benchmarks/render_results.py @@ -25,6 +25,15 @@ that mode stops a figure drifting to another value that happens to exist elsewhere in the files, and only that mode reads the order of a series. + The matched-connection run (django-ox 1.9.0 against django-tasks-db 0.13.0, + `results-raw-2026-10-14-matched.json`, one summary row per run) is read the + same way: per-cell medians, medians of per-round ratios, ranges, CPU per + task, drain times, and the wording claims about them. A table row on the + page is read cell by cell, and a default-settings figure above 4 workers is + refused. The 1.3.0 drain cells and the 100,000-task probe are no longer on + the page, so their numbers are no longer sourced; they stay in the old raw + file. `--matched FILE` names the matched file. + Usage: python render_results.py python render_results.py --raw results-raw-2026-09-19.json \ @@ -56,9 +65,11 @@ import argparse import json +import math import re import statistics import sys +from collections.abc import Callable from datetime import datetime from pathlib import Path @@ -636,12 +647,6 @@ def render(self) -> str: # arithmetic of a percentage, not results. ("1", "median - 1) x 100", "constant in the printed rate-difference formula"), ("100", "median - 1) x 100", "constant in the printed rate-difference formula"), - # The drain headline, "Up to 20% faster queue drain". It is a rounded claim, - # not a computed figure, and no row of the drain table reaches it. The cell - # behind it is the 100,000-task probe, `e2e_probe` in the raw file: 115.4 - # against 92.7 tasks/s, 24.5% on those rates. Tied to the headline's words - # because the probe band is also written 20%. - ("20%", "faster queue drain", "drain headline, under the 100,000-task probe cell (e2e_probe, 115.4 against 92.7 tasks/s)"), ] @@ -676,13 +681,146 @@ def has_figure(unit: str, figure: str) -> bool: return re.search(pattern, unit.lower()) is not None +# -- the matched-connection run --------------------------------------------- + +SETTING_LABEL = { + "P": "Persistent", + "PH": "Persistent, health checks", + "G": "Pooled", + "GH": "Pooled, health checks", + "D": "Defaults", +} +RETAINED = ("P", "PH", "G", "GH") +OX, TD = "django-ox", "django-tasks-db" + + +def outward(low: float, high: float) -> tuple[float, float]: + """The one-decimal range that contains [low, high]: floor of the low end, ceiling of the high end.""" + return math.floor(low * 10 + 1e-9) / 10, math.ceil(high * 10 - 1e-9) / 10 + + +def row_complete(r: dict) -> bool: + """Every task finished: the run is ok, all tasks completed and persisted, nothing ran twice, workers exited 0.""" + return bool( + r["status"] == "ok" + and r["completed"] == r["tasks"] + and r["after_successful"] == r["tasks"] + and r["duplicate_starts"] == 0 + and r["duplicate_dones"] == 0 + and r["worker_exit_codes"] + and all(c == 0 for c in r["worker_exit_codes"]) + ) + + +class Matched: + """The matched-connection run, results-raw-2026-10-14-matched.json: one row per run.""" + + def __init__(self, data: dict): + self.data = data + self.rows: list[dict] = data["rows"] + self.cells: list[str] = list(data["cells"]) + self.by: dict[tuple[str, str, str], dict[int, dict]] = {} + for r in self.rows: + self.by.setdefault((r["cell"], r["setting"], r["backend"]), {})[r["run"]] = r + + def workload(self, cell: str) -> str: + r = next(r for r in self.rows if r["cell"] == cell) + return f"{r['tasks']:,} tasks, {r['workers']} workers" + + def workers(self, cell: str) -> int: + return int(next(r for r in self.rows if r["cell"] == cell)["workers"]) + + def rates(self, cell: str, setting: str, backend: str) -> dict[int, float]: + return { + run: r["tasks_per_sec"] + for run, r in self.by[(cell, setting, backend)].items() + if r["tasks_per_sec"] is not None + } + + def medians(self, cell: str, setting: str) -> tuple[float, float]: + return ( + median(list(self.rates(cell, setting, OX).values())), + median(list(self.rates(cell, setting, TD).values())), + ) + + def round_ratios(self, cell: str, setting: str) -> list[float]: + ox, td = self.rates(cell, setting, OX), self.rates(cell, setting, TD) + return [ox[run] / td[run] for run in sorted(set(ox) & set(td))] + + def ratio(self, cell: str, setting: str) -> float: + """The median of the per-round ratios.""" + return median(self.round_ratios(cell, setting)) + + def shown(self) -> list[tuple[str, str]]: + """The (cell, setting) pairs whose throughput the page may show: retained connections everywhere, defaults at the first cell.""" + return [(c, s) for c in self.cells for s in RETAINED] + [(self.cells[0], "D")] + + def shown_rows(self) -> list[dict]: + keep = set(self.shown()) + return [r for r in self.rows if (r["cell"], r["setting"]) in keep] + + def ratio_range(self) -> tuple[float, float]: + """Outward-rounded range over every retained cell, by the median of per-round ratios (the figure the table shows).""" + values = [self.ratio(c, s) for c in self.cells for s in RETAINED] + return outward(min(values), max(values)) + + def default_range(self) -> tuple[float, float]: + c = self.cells[0] + ox, td = self.medians(c, "D") + return outward(*sorted([self.ratio(c, "D"), ox / td])) + + def cpu_range(self, backend: str) -> tuple[float, float]: + values = [ + r["worker_cpu_ms_per_task"] + for r in self.rows + if r["backend"] == backend and r["setting"] == "P" and r["worker_cpu_ms_per_task"] is not None + ] + return min(values), max(values) + + def elapsed(self, cell: str, setting: str, backend: str) -> list[float]: + return [r["elapsed_s"] for r in self.by[(cell, setting, backend)].values()] + + def spread(self, backend: str) -> float: + """Standard deviation of the default-settings rates over their mean, in percent.""" + values = list(self.rates(self.cells[0], "D", backend).values()) + return statistics.pstdev(values) / statistics.mean(values) * 100.0 + + def port_rate(self) -> list[float]: + """New connections per second in the default-settings rows above 4 workers that opened any.""" + return [ + r["sessions_opened"] / r["elapsed_s"] + for r in self.rows + if r["setting"] == "D" and r["workers"] > 4 and r["sessions_opened"] and r["elapsed_s"] + ] + + def interleaved(self) -> tuple[bool, str]: + """Five runs per backend and setting, and within each cell the rounds follow one another.""" + five = all(sorted(runs) == [1, 2, 3, 4, 5] for runs in self.by.values()) + in_order = True + for cell in self.cells: + rounds = [r["run"] for r in sorted((r for r in self.rows if r["cell"] == cell), key=lambda r: r["started_utc"])] + in_order = in_order and rounds == sorted(rounds) + return five and in_order, f"five runs in all {len(self.by)} series: {five}; rounds in time order in every cell: {in_order}" + + def balanced(self) -> tuple[bool, str]: + """No arm kept one position: each arm held at least three different positions over its five rounds.""" + spots: dict[tuple[str, str, str], set[int]] = {} + for r in self.rows: + spots.setdefault((r["cell"], r["setting"], r["backend"]), set()).add(r["position"]) + fewest = min(len(v) for v in spots.values()) + return fewest >= 3, f"fewest distinct positions held by one arm over its five rounds: {fewest}" + + class PageCheck: """Every value the page may quote, computed from the two raw files.""" - def __init__(self, raw: dict, kill: dict, flags: set[str]): + def __init__(self, raw: dict, kill: dict, matched: dict, flags: set[str]): self.raw = raw self.kill = kill + self.m = Matched(matched) self.flags = flags + # (regex over the page text, check of one match -> (holds, what was computed)) + self.patterns: list[tuple[re.Pattern[str], Callable[[re.Match[str]], tuple[bool, str]]]] = [] self.pool: dict[str, list[str]] = {} # (label, figures that must share one unit, text that switches the # requirement on, whether every unit carrying that text must comply) @@ -692,6 +830,7 @@ def __init__(self, raw: dict, kill: dict, flags: set[str]): self.environment() self.performance() self.failure() + self.matched_section() def add(self, text: str, label: str) -> None: labels = self.pool.setdefault(text, []) @@ -727,54 +866,9 @@ def performance(self) -> None: params = raw["parameters"] r = Renderer(raw, self.kill, draft=False, flags=set()) - drains: dict[tuple[int, str], tuple[list[float], list[float]]] = {} - for depth in params["depths"]: - self.add(f"{depth:,}", "queued tasks in a drain cell, parameters.depths") - for topology in params["topologies"]: - procs = topology.split("v")[0] - self.add(procs, f"worker processes per backend ({topology}), parameters.topologies") - ox = r.e2e_values(depth, topology, "ox", "tasks_per_sec") - td = r.e2e_values(depth, topology, "tasksdb", "tasks_per_sec") - drains[(depth, topology)] = (ox, td) - where = f"drain {depth:,} {topology}" - for arm, values in (("django-ox", ox), ("django-tasks-db", td)): - self.add(fmt(median(values), 1), f"{where} {arm} median tasks/s") - self.add(fmt(min(values), 1), f"{where} {arm} slowest run") - self.add(fmt(max(values), 1), f"{where} {arm} fastest run") - pct = rate_difference(ox, td) - pct_text = f"{abs(pct):.1f}%" - self.add(pct_text, f"{where} rate difference from the unrounded medians") - self.need( - f"{where} row", - f"{depth:,}", cell(ox, 1), cell(td, 1), f"{pct_text} {'faster' if pct > 0 else 'slower'}", - ) - clear = [k for k, (ox, td) in drains.items() if not ranges_overlap(ox, td)] - self.statements.append(( - "No run of one backend overlaps", - len(clear) == len(drains), - f"runs do not overlap in {len(clear)} of {len(drains)} drain cells", - )) - ahead = [k for k, (ox, td) in drains.items() if min(ox) > max(td)] - self.statements.append(( - "every django-ox run outperformed every django-tasks-db run", - len(ahead) == len(drains), - f"django-ox's slowest run beats django-tasks-db's fastest in {len(ahead)} of {len(drains)} drain cells", - )) - ordered = sorted(raw["e2e"], key=lambda e: parse_ts(e["started_at"])) - runs = [e["run"] for e in ordered] - per_run = {run: {(e["depth"], e["topology"]) for e in ordered if e["run"] == run} for run in runs} - self.statements.append(( - "ran interleaved, run by run", - runs == sorted(runs) and all(cells == set(drains) for cells in per_run.values()), - f"every run holds all {len(drains)} drain cells and the runs follow one another, " - f"{ordered[0]['started_at']} to {ordered[-1]['finished_at']}", - )) - self.statements.append(( - "ran interleaved in one window", - runs == sorted(runs) and all(cells == set(drains) for cells in per_run.values()), - f"every run holds all {len(drains)} drain cells and the runs follow one another, " - f"{ordered[0]['started_at']} to {ordered[-1]['finished_at']}", - )) + # The 1.3.0 drain cells and the 100,000-task probe left the page when the + # matched-connection run replaced them. Their numbers stay in the old raw + # file and are no longer sourced here, so none of them can come back. self.add(str(params["latency_count"]), "latency samples per run, parameters.latency_count") for q in ("p50", "p95"): @@ -816,36 +910,6 @@ def performance(self) -> None: self.add(attempt, f"retry {ARM[backend]} attempt on which every task ended") self.need(f"retry {ARM[backend]} row", f"{count} of {rc} {status} on attempt {attempt}") - probe = raw["probe"] - entries = {e["backend"]: e for e in raw["e2e_probe"]} - self.add(f"{probe['probe_depth']:,}", "probe depth, probe.probe_depth") - self.add(f"{probe['band'] * 100:.0f}%", "probe band, probe.band") - figures = [] - for backend in ("ox", "tasksdb"): - e = entries[backend] - self.add(fmt(e["tasks_per_sec"], 1), f"probe {ARM[backend]} tasks/s") - self.add(fmt(e["seconds"], 1), f"probe {ARM[backend]} seconds") - self.add(f"{probe['arms'][backend]['ratio']:.2f}", f"probe {ARM[backend]} rate over its own {probe['deepest_cell_depth']:,}-task median") - figures += [ - f"{fmt(e['tasks_per_sec'], 1)} tasks/s", - f"{fmt(e['seconds'], 1)} s", - f"= {probe['arms'][backend]['ratio']:.2f}", - ] - self.need("probe figures", *figures, when=f"{probe['probe_depth']:,}-task probe") - probe_backends = sorted(e["backend"] for e in raw["e2e_probe"]) - drain_runs = {len(ox) for ox, _td in drains.values()} | {len(td) for _ox, td in drains.values()} - self.statements.append(( - f"the {probe['probe_depth']:,}-task run was one pair, not five", - probe_backends == ["ox", "tasksdb"] and drain_runs == {5}, - f"e2e_probe holds {len(probe_backends)} entries ({', '.join(probe_backends)}); " - f"runs per backend in the drain cells: {', '.join(map(str, sorted(drain_runs)))}", - )) - self.statements.append(( - f"held within {probe['band'] * 100:.0f}%", - bool(probe["rate_holds_for_both_arms"]), - f"probe.rate_holds_for_both_arms is {probe['rate_holds_for_both_arms']}", - )) - def failure(self) -> None: kill = self.kill p = kill["parameters"] @@ -938,6 +1002,184 @@ def status(arm: str, s: str) -> list[int]: listed = " and ".join([", ".join(map(str, flagged[:-1])), str(flagged[-1])]) if len(flagged) > 1 else str(flagged[0]) self.need("flagged kill trials, from --flag", f"trials {listed}", when="load flag", every=True) + def matched_section(self) -> None: + m = self.m + env = m.data["environment"] + self.add(m.data["date"], "matched run date, matched file") + assert env["django_tasks_db"] == self.raw["environment"]["packages"]["django-tasks-db"] + for key, label in ( + ("django_ox", "django-ox"), + ("django_tasks_db", "django-tasks-db"), + ("django", "Django"), + ("psycopg", "psycopg"), + ): + self.add(env[key], f"{label} version, matched environment") + self.add(".".join(env["django"].split(".")[:2]), "Django series, matched environment.django") + self.add(str(env["payload_bytes"]), "task payload in bytes, matched environment.payload_bytes") + for key in ("host", "postgres"): + for match in TOKEN.finditer(env[key]): + self.add(match.group(), f"matched environment.{key}") + for setting, text in m.data["settings"].items(): + for match in TOKEN.finditer(text): + self.add(match.group(), f"matched settings.{setting}") + + for cell in m.cells: + r = next(r for r in m.rows if r["cell"] == cell) + self.add(f"{r['tasks']:,}", f"tasks in matched cell {cell}") + self.add(str(r["workers"]), f"workers in matched cell {cell}") + + lo, hi = m.ratio_range() + self.add(f"{lo:.1f}", "headline ratio range, low end rounded down, matched rows") + self.add(f"{hi:.1f}", "headline ratio range, high end rounded up, matched rows") + dlo, dhi = m.default_range() + self.add(f"{dlo:.1f}", "default-settings ratio range, low end rounded down") + self.add(f"{dhi:.1f}", "default-settings ratio range, high end rounded up") + self.default_text = f"{dlo:.1f} to {dhi:.1f}" + + for cell, setting in m.shown(): + ox, td = m.medians(cell, setting) + ratio = m.ratio(cell, setting) + where = f"matched {cell} {setting}" + self.add(fmt(ox, 0), f"{where} django-ox median tasks/s") + self.add(fmt(td, 0), f"{where} django-tasks-db median tasks/s") + self.add(f"{ratio:.2f}", f"{where} median of per-round ratios") + tail = [f"{dlo:.1f}", f"{dhi:.1f}"] if setting == "D" else [f"{ratio:.2f}"] + self.need(f"{where} row", m.workload(cell), fmt(ox, 0), fmt(td, 0), *tail) + + last_cell = m.cells[-1] + for setting in ("P", "G"): + for backend in (OX, TD): + values = m.elapsed(last_cell, setting, backend) + self.add(fmt(min(values), 1), f"{last_cell} {setting} {backend} fastest drain, s") + self.add(fmt(max(values), 1), f"{last_cell} {setting} {backend} slowest drain, s") + self.add(f"{round(median(values) / 10) * 10}", f"{last_cell} {setting} {backend} median drain to the nearest 10 s") + for backend in (OX, TD): + low, high = m.cpu_range(backend) + self.add(f"{low:.1f}", f"{backend} worker CPU per task, lowest, persistent connections, ms") + self.add(f"{high:.1f}", f"{backend} worker CPU per task, highest, persistent connections, ms") + for backend in (OX, TD): + self.add(f"{round(m.spread(backend))}%", f"{backend} default-settings run-to-run spread (sd/mean)") + self.add(f"{round(median(m.port_rate()) / 50) * 50}", "new connections per second at the port ceiling, nearest 50") + loads = [r["load_1m_before"] for r in m.rows] + self.add(f"{min(loads):.1f}", "lowest one-minute load before a run") + self.add(f"{max(loads):.1f}", "highest one-minute load before a run") + + shown = m.shown_rows() + self.statements.append(( + "Every task finished in these runs, with zero duplicate starts or completions", + all(row_complete(r) for r in shown), + f"{sum(row_complete(r) for r in shown)} of {len(shown)} runs shown completed every task, " + "persisted every outcome, repeated none and exited 0", + )) + ok, why = m.interleaved() + self.statements.append(("five interleaved runs", ok, why)) + ok, why = m.balanced() + self.statements.append(("order balanced across rounds", ok, why)) + + def num(x: str) -> float: + return float(x) + + def headline(g: re.Match[str]) -> tuple[bool, str]: + return (num(g[1]), num(g[2])) == (lo, hi), f"matched ratios, outward-rounded: {lo:.1f} to {hi:.1f}" + + self.patterns.append((re.compile(r"(\d\.\d) to (\d\.\d)× (?:the throughput|django-tasks-db)"), headline)) + + def workers(g: re.Match[str]) -> tuple[bool, str]: + said = {int(x) for x in re.findall(r"\d+", g[1])} + have = {m.workers(c) for c in m.cells} + return said == have, f"workers in the matched cells: {sorted(have)}" + + self.patterns.append((re.compile(r"across ([\d, ]+ and \d+) workers"), workers)) + + def cpu(g: re.Match[str]) -> tuple[bool, str]: + a, b = m.cpu_range(OX), m.cpu_range(TD) + want = (f"{a[0]:.1f}", f"{a[1]:.1f}", f"{b[0]:.1f}", f"{b[1]:.1f}") + return g.groups() == want, f"worker CPU ms per task, persistent connections: {want}" + + self.patterns.append((re.compile( + r"worker CPU per task was (\d\.\d) to (\d\.\d) ms for django-ox versus (\d\.\d) to (\d\.\d) ms for django-tasks-db"), cpu)) + + def drain(g: re.Match[str]) -> tuple[bool, str]: + tasks_ = next(r for r in m.rows if r["cell"] == last_cell)["tasks"] + said = g.groups() + ranges = {} + for setting in ("P", "G"): + for backend in (OX, TD): + v = m.elapsed(last_cell, setting, backend) + ranges[(setting, backend)] = (f"{min(v):.1f}", f"{max(v):.1f}", round(median(v) / 10) * 10) + want = [ + f"{tasks_:,}", str(m.workers(last_cell)), + str(ranges[("P", OX)][2]), *ranges[("P", OX)][:2], + str(ranges[("P", TD)][2]), *ranges[("P", TD)][:2], + *ranges[("G", OX)][:2], *ranges[("G", TD)][:2], + ] + same = ranges[("P", OX)][2] == ranges[("G", OX)][2] and ranges[("P", TD)][2] == ranges[("G", TD)][2] + return list(said) == want and same, f"{last_cell} drain: {want}" + + self.patterns.append((re.compile( + r"draining ([\d,]+) tasks at (\d+) workers took about (\d+) s for django-ox \((\d+\.\d) to (\d+\.\d) s over five runs\) " + r"and about (\d+) s for django-tasks-db \((\d+\.\d) to (\d+\.\d) s\) with persistent connections; " + r"with a pool (\d+\.\d) to (\d+\.\d) s vs (\d+\.\d) to (\d+\.\d) s", re.I), drain)) + + def spread(g: re.Match[str]) -> tuple[bool, str]: + a, b = round(m.spread(OX)), round(m.spread(TD)) + return int(g[1]) == a == b, f"run-to-run spread, standard deviation over mean: {m.spread(OX):.1f}% and {m.spread(TD):.1f}%" + + self.patterns.append((re.compile(r"spread between runs is (\d+)% on both"), spread)) + + def about_default(g: re.Match[str]) -> tuple[bool, str]: + return num(g[1]) == dlo, f"default-settings ratio, rounded down: {dlo:.1f}" + + self.patterns.append((re.compile(r"treat this as about (\d\.\d)×"), about_default)) + + def ports(g: re.Match[str]) -> tuple[bool, str]: + rates = m.port_rate() + n = int(g[1]) + return bool(rates) and n == round(median(rates) / 50) * 50 and all(0.9 * n <= x <= 1.1 * n for x in rates), \ + f"new connections per second, default settings above 4 workers: {min(rates):.0f} to {max(rates):.0f}, median {median(rates):.0f}" + + self.patterns.append((re.compile(r"about (\d+) new connections a second"), ports)) + + def no_throughput(g: re.Match[str]) -> tuple[bool, str]: + n = int(g[1]) + above = [r for r in m.rows if r["setting"] == "D" and r["workers"] > n] + below = [r for r in m.rows if r["setting"] == "D" and r["workers"] <= n] + bad = sum(row_complete(r) for r in above) + return bad == 0 and all(row_complete(r) for r in below) and bool(above), \ + f"default-settings runs above {n} workers that completed cleanly: {bad} of {len(above)}; at or below: {sum(row_complete(r) for r in below)} of {len(below)}" + + self.patterns.append((re.compile(r"no default-settings throughput above (\d+) workers"), no_throughput)) + + def load(g: re.Match[str]) -> tuple[bool, str]: + loads = [r["load_1m_before"] for r in m.rows] + return (num(g[1]), num(g[2])) == (round(min(loads), 1), round(max(loads), 1)), \ + f"one-minute load before a run: {min(loads):.2f} to {max(loads):.2f}" + + self.patterns.append((re.compile(r"load average (\d+\.\d) to (\d+\.\d) before a run"), load)) + + def table_statements(self, units: list[tuple[int, str]]) -> list[tuple[str, bool, str]]: + """Read every matched-run table row: a known workload, a known setting, the figures that go with them.""" + m = self.m + by_label = {m.workload(c): c for c in m.cells} + by_setting = {v: k for k, v in SETTING_LABEL.items()} + out: list[tuple[str, bool, str]] = [] + for line, unit in units: + if not unit.startswith("|"): + continue + cells = [c.strip().strip("*").strip() for c in unit.strip("|").split("|")] + if len(cells) != 5 or cells[0] not in by_label: + continue + cell, setting = by_label[cells[0]], by_setting.get(cells[1]) + name = f"line {line}: {cells[0]} / {cells[1]}" + if setting is None or (cell, setting) not in m.shown(): + out.append((name, False, "this workload and setting is not one the page may show a throughput for")) + continue + ox, td = m.medians(cell, setting) + ratio = self.default_text + "×" if setting == "D" else f"{m.ratio(cell, setting):.2f}×" + want = [fmt(ox, 0), fmt(td, 0), ratio] + out.append((name, cells[2:] == want, f"expected {' | '.join(want)}")) + return out + # -- the page against them ------------------------------------------- def run(self, page: Path, *, require_complete: bool = False) -> int: @@ -996,9 +1238,14 @@ def run(self, page: Path, *, require_complete: bool = False) -> int: print() print("Statements the page makes about the figures:") failed = 0 - for claim, holds, computed in self.statements: - if claim.lower() not in text.lower(): - continue + flat = " ".join(text.split()) + found_statements = [s for s in self.statements if s[0].lower() in flat.lower()] + for pattern, check_one in self.patterns: + for match in pattern.finditer(flat): + holds, computed = check_one(match) + found_statements.append((match.group(), holds, computed)) + found_statements += self.table_statements(units) + for claim, holds, computed in found_statements: failed += 0 if holds else 1 print(f" {'ok ' if holds else 'FALSE '} \"{claim}\": {computed}") @@ -1020,6 +1267,7 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--raw", default=str(HERE / "results-raw-2026-09-19.json")) parser.add_argument("--kill", default=str(HERE / "results-kill-2026-09-19.json")) + parser.add_argument("--matched", default=str(HERE / "results-raw-2026-10-14-matched.json")) parser.add_argument("--draft", action="store_true", help="prefix rows with review markers") parser.add_argument( "--flag", @@ -1043,6 +1291,7 @@ def main() -> int: parser.error("--require-complete goes with --check PAGE") raw = json.loads(Path(args.raw).read_text()) kill = json.loads(Path(args.kill).read_text()) + matched = json.loads(Path(args.matched).read_text()) flags = {f for f in args.flag.split(",") if f} unknown = sorted(flags - flag_keys(kill)) if unknown: @@ -1051,7 +1300,7 @@ def main() -> int: "Drain cells take no flag: they ran interleaved in one window." ) if args.check: - return PageCheck(raw, kill, flags).run(Path(args.check), require_complete=args.require_complete) + return PageCheck(raw, kill, matched, flags).run(Path(args.check), require_complete=args.require_complete) print(Renderer(raw, kill, draft=args.draft, flags=flags).render(), end="") return 0 diff --git a/benchmarks/results-raw-2026-10-14-matched.json b/benchmarks/results-raw-2026-10-14-matched.json new file mode 100644 index 0000000..630f65f --- /dev/null +++ b/benchmarks/results-raw-2026-10-14-matched.json @@ -0,0 +1,6831 @@ +{ + "date": "2026-10-14", + "environment": { + "host": "single test machine", + "postgres": "PostgreSQL 16 (Docker)", + "django": "6.0.8", + "psycopg": "3.3.6", + "django_ox": "1.9.0", + "django_tasks_db": "0.13.0", + "task": "no-op tasks", + "payload_bytes": 256, + "runs_per_cell": "five, interleaved, order balanced across rounds", + "run_window_utc": "2026-10-14 16:36 to 22:53", + "note": "Identical Django database options for both backends; each package keeps its own defaults otherwise. The Mac carried other load (one-minute load average 1.9 to 6.8 before a run, from the host load average recorded in each row)." + }, + "settings": { + "P": "CONN_MAX_AGE = 600", + "PH": "CONN_MAX_AGE = 600 with CONN_HEALTH_CHECKS", + "G": "OPTIONS['pool'] with CONN_MAX_AGE = 0", + "GH": "G with CONN_HEALTH_CHECKS", + "D": "CONN_MAX_AGE = 0, no pool (default settings)" + }, + "cells": { + "S1": "20,000 tasks, 4 workers", + "S2.W8": "20,000 tasks, 8 workers", + "S2.W16": "20,000 tasks, 16 workers", + "S3": "100,000 tasks, 16 workers" + }, + "rows": [ + { + "cell": "S1", + "backend": "django-ox", + "setting": "P", + "run": 1, + "position": 10, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1780.83, + "elapsed_s": 11.231, + "completed": 20000, + "worker_cpu_ms_per_task": 1.007, + "postgres_cpu_ms_per_task": 0.47, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 5.86, + "started_utc": "2026-10-14T17:15:05Z", + "finished_utc": "2026-10-14T17:15:19Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "P", + "run": 2, + "position": 8, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1814.37, + "elapsed_s": 11.023, + "completed": 20000, + "worker_cpu_ms_per_task": 1.012, + "postgres_cpu_ms_per_task": 0.42, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.71, + "started_utc": "2026-10-14T17:29:39Z", + "finished_utc": "2026-10-14T17:29:53Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "P", + "run": 3, + "position": 6, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1771.42, + "elapsed_s": 11.29, + "completed": 20000, + "worker_cpu_ms_per_task": 1.011, + "postgres_cpu_ms_per_task": 0.479, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.97, + "started_utc": "2026-10-14T17:47:53Z", + "finished_utc": "2026-10-14T17:48:07Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "P", + "run": 4, + "position": 4, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1734.92, + "elapsed_s": 11.528, + "completed": 20000, + "worker_cpu_ms_per_task": 1.004, + "postgres_cpu_ms_per_task": 0.492, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.07, + "started_utc": "2026-10-14T18:06:43Z", + "finished_utc": "2026-10-14T18:06:57Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "P", + "run": 5, + "position": 2, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1771.13, + "elapsed_s": 11.292, + "completed": 20000, + "worker_cpu_ms_per_task": 1.006, + "postgres_cpu_ms_per_task": 0.402, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.67, + "started_utc": "2026-10-14T18:20:09Z", + "finished_utc": "2026-10-14T18:20:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "P", + "run": 1, + "position": 1, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1165.88, + "elapsed_s": 17.154, + "completed": 20000, + "worker_cpu_ms_per_task": 1.513, + "postgres_cpu_ms_per_task": 0.632, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.39, + "started_utc": "2026-10-14T16:36:05Z", + "finished_utc": "2026-10-14T16:36:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "P", + "run": 2, + "position": 3, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1056.37, + "elapsed_s": 18.933, + "completed": 20000, + "worker_cpu_ms_per_task": 1.691, + "postgres_cpu_ms_per_task": 0.675, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.94, + "started_utc": "2026-10-14T17:21:51Z", + "finished_utc": "2026-10-14T17:22:12Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "P", + "run": 3, + "position": 5, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1213.79, + "elapsed_s": 16.477, + "completed": 20000, + "worker_cpu_ms_per_task": 1.492, + "postgres_cpu_ms_per_task": 0.544, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.12, + "started_utc": "2026-10-14T17:45:53Z", + "finished_utc": "2026-10-14T17:46:12Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "P", + "run": 4, + "position": 7, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1226.45, + "elapsed_s": 16.307, + "completed": 20000, + "worker_cpu_ms_per_task": 1.463, + "postgres_cpu_ms_per_task": 0.544, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.99, + "started_utc": "2026-10-14T18:12:44Z", + "finished_utc": "2026-10-14T18:13:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "P", + "run": 5, + "position": 9, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1211.23, + "elapsed_s": 16.512, + "completed": 20000, + "worker_cpu_ms_per_task": 1.491, + "postgres_cpu_ms_per_task": 0.566, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.74, + "started_utc": "2026-10-14T18:32:29Z", + "finished_utc": "2026-10-14T18:32:48Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "PH", + "run": 1, + "position": 9, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1525.1, + "elapsed_s": 13.114, + "completed": 20000, + "worker_cpu_ms_per_task": 1.108, + "postgres_cpu_ms_per_task": 0.559, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.4, + "started_utc": "2026-10-14T17:10:01Z", + "finished_utc": "2026-10-14T17:10:17Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "PH", + "run": 2, + "position": 10, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1620.79, + "elapsed_s": 12.34, + "completed": 20000, + "worker_cpu_ms_per_task": 1.083, + "postgres_cpu_ms_per_task": 0.476, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.27, + "started_utc": "2026-10-14T17:32:44Z", + "finished_utc": "2026-10-14T17:33:00Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "PH", + "run": 3, + "position": 8, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1590.48, + "elapsed_s": 12.575, + "completed": 20000, + "worker_cpu_ms_per_task": 1.079, + "postgres_cpu_ms_per_task": 0.494, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.23, + "started_utc": "2026-10-14T17:51:53Z", + "finished_utc": "2026-10-14T17:52:09Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "PH", + "run": 4, + "position": 6, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1572.21, + "elapsed_s": 12.721, + "completed": 20000, + "worker_cpu_ms_per_task": 1.082, + "postgres_cpu_ms_per_task": 0.49, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.57, + "started_utc": "2026-10-14T18:10:43Z", + "finished_utc": "2026-10-14T18:11:00Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "PH", + "run": 5, + "position": 4, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1592.67, + "elapsed_s": 12.557, + "completed": 20000, + "worker_cpu_ms_per_task": 1.072, + "postgres_cpu_ms_per_task": 0.495, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.79, + "started_utc": "2026-10-14T18:23:54Z", + "finished_utc": "2026-10-14T18:24:10Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "PH", + "run": 1, + "position": 8, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1094.29, + "elapsed_s": 18.277, + "completed": 20000, + "worker_cpu_ms_per_task": 1.582, + "postgres_cpu_ms_per_task": 0.613, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.26, + "started_utc": "2026-10-14T17:06:11Z", + "finished_utc": "2026-10-14T17:06:33Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "PH", + "run": 2, + "position": 6, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1128.83, + "elapsed_s": 17.718, + "completed": 20000, + "worker_cpu_ms_per_task": 1.584, + "postgres_cpu_ms_per_task": 0.581, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.51, + "started_utc": "2026-10-14T17:26:48Z", + "finished_utc": "2026-10-14T17:27:09Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "PH", + "run": 3, + "position": 4, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1122.69, + "elapsed_s": 17.814, + "completed": 20000, + "worker_cpu_ms_per_task": 1.572, + "postgres_cpu_ms_per_task": 0.577, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.42, + "started_utc": "2026-10-14T17:43:47Z", + "finished_utc": "2026-10-14T17:44:08Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "PH", + "run": 4, + "position": 2, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 842.68, + "elapsed_s": 23.734, + "completed": 20000, + "worker_cpu_ms_per_task": 2.06, + "postgres_cpu_ms_per_task": 0.836, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.59, + "started_utc": "2026-10-14T18:01:13Z", + "finished_utc": "2026-10-14T18:01:39Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "PH", + "run": 5, + "position": 1, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1127.53, + "elapsed_s": 17.738, + "completed": 20000, + "worker_cpu_ms_per_task": 1.547, + "postgres_cpu_ms_per_task": 0.62, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.48, + "started_utc": "2026-10-14T18:18:34Z", + "finished_utc": "2026-10-14T18:18:54Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "G", + "run": 1, + "position": 2, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1680.56, + "elapsed_s": 11.901, + "completed": 20000, + "worker_cpu_ms_per_task": 1.07, + "postgres_cpu_ms_per_task": 0.415, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.86, + "started_utc": "2026-10-14T16:38:55Z", + "finished_utc": "2026-10-14T16:39:11Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "G", + "run": 2, + "position": 1, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1734.57, + "elapsed_s": 11.53, + "completed": 20000, + "worker_cpu_ms_per_task": 1.052, + "postgres_cpu_ms_per_task": 0.417, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.26, + "started_utc": "2026-10-14T17:16:24Z", + "finished_utc": "2026-10-14T17:16:39Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "G", + "run": 3, + "position": 3, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1659.16, + "elapsed_s": 12.054, + "completed": 20000, + "worker_cpu_ms_per_task": 1.087, + "postgres_cpu_ms_per_task": 0.475, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.12, + "started_utc": "2026-10-14T17:38:52Z", + "finished_utc": "2026-10-14T17:39:08Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "G", + "run": 4, + "position": 5, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1718.3, + "elapsed_s": 11.639, + "completed": 20000, + "worker_cpu_ms_per_task": 1.035, + "postgres_cpu_ms_per_task": 0.497, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.67, + "started_utc": "2026-10-14T18:08:43Z", + "finished_utc": "2026-10-14T18:08:57Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "G", + "run": 5, + "position": 7, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1728.09, + "elapsed_s": 11.573, + "completed": 20000, + "worker_cpu_ms_per_task": 1.047, + "postgres_cpu_ms_per_task": 0.496, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.38, + "started_utc": "2026-10-14T18:29:44Z", + "finished_utc": "2026-10-14T18:29:58Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "G", + "run": 1, + "position": 3, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1094.74, + "elapsed_s": 18.269, + "completed": 20000, + "worker_cpu_ms_per_task": 1.62, + "postgres_cpu_ms_per_task": 0.598, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.7, + "started_utc": "2026-10-14T16:43:11Z", + "finished_utc": "2026-10-14T16:43:33Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "G", + "run": 2, + "position": 5, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1235.82, + "elapsed_s": 16.184, + "completed": 20000, + "worker_cpu_ms_per_task": 1.516, + "postgres_cpu_ms_per_task": 0.539, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.76, + "started_utc": "2026-10-14T17:25:24Z", + "finished_utc": "2026-10-14T17:25:43Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "G", + "run": 3, + "position": 7, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1225.27, + "elapsed_s": 16.323, + "completed": 20000, + "worker_cpu_ms_per_task": 1.492, + "postgres_cpu_ms_per_task": 0.535, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.12, + "started_utc": "2026-10-14T17:49:43Z", + "finished_utc": "2026-10-14T17:50:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "G", + "run": 4, + "position": 9, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1216.11, + "elapsed_s": 16.446, + "completed": 20000, + "worker_cpu_ms_per_task": 1.512, + "postgres_cpu_ms_per_task": 0.537, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.41, + "started_utc": "2026-10-14T18:15:44Z", + "finished_utc": "2026-10-14T18:16:04Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "G", + "run": 5, + "position": 10, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1212.64, + "elapsed_s": 16.493, + "completed": 20000, + "worker_cpu_ms_per_task": 1.501, + "postgres_cpu_ms_per_task": 0.551, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.5, + "started_utc": "2026-10-14T18:33:54Z", + "finished_utc": "2026-10-14T18:34:13Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "GH", + "run": 1, + "position": 7, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1526.13, + "elapsed_s": 13.105, + "completed": 20000, + "worker_cpu_ms_per_task": 1.136, + "postgres_cpu_ms_per_task": 0.542, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.5, + "started_utc": "2026-10-14T17:04:51Z", + "finished_utc": "2026-10-14T17:05:07Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "GH", + "run": 2, + "position": 9, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1589.02, + "elapsed_s": 12.586, + "completed": 20000, + "worker_cpu_ms_per_task": 1.115, + "postgres_cpu_ms_per_task": 0.486, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.57, + "started_utc": "2026-10-14T17:31:02Z", + "finished_utc": "2026-10-14T17:31:18Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "GH", + "run": 3, + "position": 10, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1127.11, + "elapsed_s": 17.744, + "completed": 20000, + "worker_cpu_ms_per_task": 1.498, + "postgres_cpu_ms_per_task": 0.639, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.64, + "started_utc": "2026-10-14T17:55:53Z", + "finished_utc": "2026-10-14T17:56:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "GH", + "run": 4, + "position": 8, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1561.21, + "elapsed_s": 12.811, + "completed": 20000, + "worker_cpu_ms_per_task": 1.118, + "postgres_cpu_ms_per_task": 0.499, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.14, + "started_utc": "2026-10-14T18:14:08Z", + "finished_utc": "2026-10-14T18:14:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "GH", + "run": 5, + "position": 6, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1571.63, + "elapsed_s": 12.726, + "completed": 20000, + "worker_cpu_ms_per_task": 1.118, + "postgres_cpu_ms_per_task": 0.491, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.44, + "started_utc": "2026-10-14T18:27:39Z", + "finished_utc": "2026-10-14T18:27:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "GH", + "run": 1, + "position": 5, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1085.65, + "elapsed_s": 18.422, + "completed": 20000, + "worker_cpu_ms_per_task": 1.619, + "postgres_cpu_ms_per_task": 0.6, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.37, + "started_utc": "2026-10-14T17:01:21Z", + "finished_utc": "2026-10-14T17:01:43Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "GH", + "run": 2, + "position": 7, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1126.89, + "elapsed_s": 17.748, + "completed": 20000, + "worker_cpu_ms_per_task": 1.599, + "postgres_cpu_ms_per_task": 0.58, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.36, + "started_utc": "2026-10-14T17:28:13Z", + "finished_utc": "2026-10-14T17:28:34Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "GH", + "run": 3, + "position": 9, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1133.11, + "elapsed_s": 17.65, + "completed": 20000, + "worker_cpu_ms_per_task": 1.571, + "postgres_cpu_ms_per_task": 0.604, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 2.04, + "started_utc": "2026-10-14T17:53:43Z", + "finished_utc": "2026-10-14T17:54:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "GH", + "run": 4, + "position": 10, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1113.28, + "elapsed_s": 17.965, + "completed": 20000, + "worker_cpu_ms_per_task": 1.586, + "postgres_cpu_ms_per_task": 0.566, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 3.92, + "started_utc": "2026-10-14T18:17:09Z", + "finished_utc": "2026-10-14T18:17:30Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "GH", + "run": 5, + "position": 8, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 1129.61, + "elapsed_s": 17.705, + "completed": 20000, + "worker_cpu_ms_per_task": 1.574, + "postgres_cpu_ms_per_task": 0.559, + "sessions_per_task": 0.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 10, + "load_1m_before": 4.18, + "started_utc": "2026-10-14T18:31:04Z", + "finished_utc": "2026-10-14T18:31:25Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "D", + "run": 1, + "position": 4, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 429.61, + "elapsed_s": 46.554, + "completed": 20000, + "worker_cpu_ms_per_task": 3.806, + "postgres_cpu_ms_per_task": 3.978, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 3.18, + "started_utc": "2026-10-14T16:50:25Z", + "finished_utc": "2026-10-14T16:51:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "D", + "run": 2, + "position": 2, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 348.15, + "elapsed_s": 57.447, + "completed": 20000, + "worker_cpu_ms_per_task": 4.438, + "postgres_cpu_ms_per_task": 4.918, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 2.74, + "started_utc": "2026-10-14T17:19:45Z", + "finished_utc": "2026-10-14T17:20:45Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "D", + "run": 3, + "position": 1, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 440.36, + "elapsed_s": 45.418, + "completed": 20000, + "worker_cpu_ms_per_task": 3.8, + "postgres_cpu_ms_per_task": 4.026, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 2.74, + "started_utc": "2026-10-14T17:34:03Z", + "finished_utc": "2026-10-14T17:34:52Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "D", + "run": 4, + "position": 3, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 388.86, + "elapsed_s": 51.432, + "completed": 20000, + "worker_cpu_ms_per_task": 4.123, + "postgres_cpu_ms_per_task": 4.444, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 3.3, + "started_utc": "2026-10-14T18:04:23Z", + "finished_utc": "2026-10-14T18:05:18Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-ox", + "setting": "D", + "run": 5, + "position": 5, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 447.06, + "elapsed_s": 44.737, + "completed": 20000, + "worker_cpu_ms_per_task": 3.758, + "postgres_cpu_ms_per_task": 3.915, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 1.92, + "started_utc": "2026-10-14T18:25:44Z", + "finished_utc": "2026-10-14T18:26:32Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "D", + "run": 1, + "position": 6, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 372.87, + "elapsed_s": 53.638, + "completed": 20000, + "worker_cpu_ms_per_task": 4.265, + "postgres_cpu_ms_per_task": 4.523, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 3.35, + "started_utc": "2026-10-14T17:02:51Z", + "finished_utc": "2026-10-14T17:03:47Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "D", + "run": 2, + "position": 4, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 324.25, + "elapsed_s": 61.68, + "completed": 20000, + "worker_cpu_ms_per_task": 4.897, + "postgres_cpu_ms_per_task": 5.254, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 2.21, + "started_utc": "2026-10-14T17:23:16Z", + "finished_utc": "2026-10-14T17:24:20Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "D", + "run": 3, + "position": 2, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 380.79, + "elapsed_s": 52.522, + "completed": 20000, + "worker_cpu_ms_per_task": 4.23, + "postgres_cpu_ms_per_task": 4.415, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 2.87, + "started_utc": "2026-10-14T17:35:57Z", + "finished_utc": "2026-10-14T17:36:52Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "D", + "run": 4, + "position": 1, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 312.26, + "elapsed_s": 64.048, + "completed": 20000, + "worker_cpu_ms_per_task": 4.938, + "postgres_cpu_ms_per_task": 5.476, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 4.7, + "started_utc": "2026-10-14T17:57:23Z", + "finished_utc": "2026-10-14T17:58:29Z", + "valid": true, + "flag": null + }, + { + "cell": "S1", + "backend": "django-tasks-db", + "setting": "D", + "run": 5, + "position": 3, + "tasks": 20000, + "workers": 4, + "status": "ok", + "tasks_per_sec": 385.86, + "elapsed_s": 51.832, + "completed": 20000, + "worker_cpu_ms_per_task": 4.238, + "postgres_cpu_ms_per_task": 4.411, + "sessions_per_task": 1.0005, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0 + ], + "sessions_opened": 20010, + "load_1m_before": 3.64, + "started_utc": "2026-10-14T18:21:44Z", + "finished_utc": "2026-10-14T18:22:38Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "P", + "run": 1, + "position": 2, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2583.52, + "elapsed_s": 7.741, + "completed": 20000, + "worker_cpu_ms_per_task": 1.168, + "postgres_cpu_ms_per_task": 0.61, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.63, + "started_utc": "2026-10-14T18:38:04Z", + "finished_utc": "2026-10-14T18:38:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "P", + "run": 2, + "position": 1, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2493.01, + "elapsed_s": 8.022, + "completed": 20000, + "worker_cpu_ms_per_task": 1.172, + "postgres_cpu_ms_per_task": 0.439, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.16, + "started_utc": "2026-10-14T18:51:50Z", + "finished_utc": "2026-10-14T18:52:01Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "P", + "run": 3, + "position": 3, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2674.01, + "elapsed_s": 7.479, + "completed": 20000, + "worker_cpu_ms_per_task": 1.169, + "postgres_cpu_ms_per_task": 0.549, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.84, + "started_utc": "2026-10-14T19:09:16Z", + "finished_utc": "2026-10-14T19:09:26Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "P", + "run": 4, + "position": 5, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2668.32, + "elapsed_s": 7.495, + "completed": 20000, + "worker_cpu_ms_per_task": 1.164, + "postgres_cpu_ms_per_task": 0.557, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.57, + "started_utc": "2026-10-14T19:26:16Z", + "finished_utc": "2026-10-14T19:26:27Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "P", + "run": 5, + "position": 7, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2685.88, + "elapsed_s": 7.446, + "completed": 20000, + "worker_cpu_ms_per_task": 1.165, + "postgres_cpu_ms_per_task": 0.544, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.14, + "started_utc": "2026-10-14T19:42:37Z", + "finished_utc": "2026-10-14T19:42:47Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "P", + "run": 1, + "position": 5, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1692.7, + "elapsed_s": 11.815, + "completed": 20000, + "worker_cpu_ms_per_task": 1.773, + "postgres_cpu_ms_per_task": 0.639, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.49, + "started_utc": "2026-10-14T18:42:14Z", + "finished_utc": "2026-10-14T18:42:29Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "P", + "run": 2, + "position": 7, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1698.52, + "elapsed_s": 11.775, + "completed": 20000, + "worker_cpu_ms_per_task": 1.781, + "postgres_cpu_ms_per_task": 0.653, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.09, + "started_utc": "2026-10-14T19:00:00Z", + "finished_utc": "2026-10-14T19:00:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "P", + "run": 3, + "position": 9, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1746.75, + "elapsed_s": 11.45, + "completed": 20000, + "worker_cpu_ms_per_task": 1.789, + "postgres_cpu_ms_per_task": 0.637, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.66, + "started_utc": "2026-10-14T19:18:06Z", + "finished_utc": "2026-10-14T19:18:21Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "P", + "run": 4, + "position": 10, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1735.16, + "elapsed_s": 11.526, + "completed": 20000, + "worker_cpu_ms_per_task": 1.771, + "postgres_cpu_ms_per_task": 0.639, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.65, + "started_utc": "2026-10-14T19:33:02Z", + "finished_utc": "2026-10-14T19:33:16Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "P", + "run": 5, + "position": 8, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1740.66, + "elapsed_s": 11.49, + "completed": 20000, + "worker_cpu_ms_per_task": 1.755, + "postgres_cpu_ms_per_task": 0.644, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.26, + "started_utc": "2026-10-14T19:43:52Z", + "finished_utc": "2026-10-14T19:44:07Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "PH", + "run": 1, + "position": 7, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2192.03, + "elapsed_s": 9.124, + "completed": 20000, + "worker_cpu_ms_per_task": 1.292, + "postgres_cpu_ms_per_task": 0.659, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.8, + "started_utc": "2026-10-14T18:46:00Z", + "finished_utc": "2026-10-14T18:46:11Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "PH", + "run": 2, + "position": 9, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2301.2, + "elapsed_s": 8.691, + "completed": 20000, + "worker_cpu_ms_per_task": 1.273, + "postgres_cpu_ms_per_task": 0.562, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.47, + "started_utc": "2026-10-14T19:02:55Z", + "finished_utc": "2026-10-14T19:03:08Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "PH", + "run": 3, + "position": 10, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2348.57, + "elapsed_s": 8.516, + "completed": 20000, + "worker_cpu_ms_per_task": 1.278, + "postgres_cpu_ms_per_task": 0.562, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.43, + "started_utc": "2026-10-14T19:19:26Z", + "finished_utc": "2026-10-14T19:19:38Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "PH", + "run": 4, + "position": 8, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2342.55, + "elapsed_s": 8.538, + "completed": 20000, + "worker_cpu_ms_per_task": 1.266, + "postgres_cpu_ms_per_task": 0.559, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.95, + "started_utc": "2026-10-14T19:30:06Z", + "finished_utc": "2026-10-14T19:30:19Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "PH", + "run": 5, + "position": 6, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2323.6, + "elapsed_s": 8.607, + "completed": 20000, + "worker_cpu_ms_per_task": 1.261, + "postgres_cpu_ms_per_task": 0.561, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.01, + "started_utc": "2026-10-14T19:41:17Z", + "finished_utc": "2026-10-14T19:41:29Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "PH", + "run": 1, + "position": 9, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1507.81, + "elapsed_s": 13.264, + "completed": 20000, + "worker_cpu_ms_per_task": 1.904, + "postgres_cpu_ms_per_task": 0.729, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.87, + "started_utc": "2026-10-14T18:48:35Z", + "finished_utc": "2026-10-14T18:48:52Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "PH", + "run": 2, + "position": 10, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1586.56, + "elapsed_s": 12.606, + "completed": 20000, + "worker_cpu_ms_per_task": 1.871, + "postgres_cpu_ms_per_task": 0.74, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.89, + "started_utc": "2026-10-14T19:04:45Z", + "finished_utc": "2026-10-14T19:05:00Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "PH", + "run": 3, + "position": 8, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1600.66, + "elapsed_s": 12.495, + "completed": 20000, + "worker_cpu_ms_per_task": 1.872, + "postgres_cpu_ms_per_task": 0.732, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.42, + "started_utc": "2026-10-14T19:16:46Z", + "finished_utc": "2026-10-14T19:17:01Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "PH", + "run": 4, + "position": 6, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1593.85, + "elapsed_s": 12.548, + "completed": 20000, + "worker_cpu_ms_per_task": 1.877, + "postgres_cpu_ms_per_task": 0.748, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.49, + "started_utc": "2026-10-14T19:27:31Z", + "finished_utc": "2026-10-14T19:27:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "PH", + "run": 5, + "position": 4, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1606.02, + "elapsed_s": 12.453, + "completed": 20000, + "worker_cpu_ms_per_task": 1.87, + "postgres_cpu_ms_per_task": 0.723, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.4, + "started_utc": "2026-10-14T19:38:32Z", + "finished_utc": "2026-10-14T19:38:47Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "G", + "run": 1, + "position": 6, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2532.9, + "elapsed_s": 7.896, + "completed": 20000, + "worker_cpu_ms_per_task": 1.211, + "postgres_cpu_ms_per_task": 0.614, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.34, + "started_utc": "2026-10-14T18:43:35Z", + "finished_utc": "2026-10-14T18:43:45Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "G", + "run": 2, + "position": 4, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2529.02, + "elapsed_s": 7.908, + "completed": 20000, + "worker_cpu_ms_per_task": 1.214, + "postgres_cpu_ms_per_task": 0.625, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.88, + "started_utc": "2026-10-14T18:55:45Z", + "finished_utc": "2026-10-14T18:55:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "G", + "run": 3, + "position": 2, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2568.83, + "elapsed_s": 7.786, + "completed": 20000, + "worker_cpu_ms_per_task": 1.213, + "postgres_cpu_ms_per_task": 0.436, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.83, + "started_utc": "2026-10-14T19:08:01Z", + "finished_utc": "2026-10-14T19:08:12Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "G", + "run": 4, + "position": 1, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2648.2, + "elapsed_s": 7.552, + "completed": 20000, + "worker_cpu_ms_per_task": 1.212, + "postgres_cpu_ms_per_task": 0.559, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.71, + "started_utc": "2026-10-14T19:20:46Z", + "finished_utc": "2026-10-14T19:20:56Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "G", + "run": 5, + "position": 3, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2647.34, + "elapsed_s": 7.555, + "completed": 20000, + "worker_cpu_ms_per_task": 1.205, + "postgres_cpu_ms_per_task": 0.557, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.42, + "started_utc": "2026-10-14T19:37:17Z", + "finished_utc": "2026-10-14T19:37:27Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "G", + "run": 1, + "position": 8, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1629.42, + "elapsed_s": 12.274, + "completed": 20000, + "worker_cpu_ms_per_task": 1.827, + "postgres_cpu_ms_per_task": 0.674, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.94, + "started_utc": "2026-10-14T18:47:15Z", + "finished_utc": "2026-10-14T18:47:30Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "G", + "run": 2, + "position": 6, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1700.77, + "elapsed_s": 11.759, + "completed": 20000, + "worker_cpu_ms_per_task": 1.805, + "postgres_cpu_ms_per_task": 0.633, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.87, + "started_utc": "2026-10-14T18:58:40Z", + "finished_utc": "2026-10-14T18:58:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "G", + "run": 3, + "position": 4, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1723.95, + "elapsed_s": 11.601, + "completed": 20000, + "worker_cpu_ms_per_task": 1.809, + "postgres_cpu_ms_per_task": 0.639, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 2.89, + "started_utc": "2026-10-14T19:10:31Z", + "finished_utc": "2026-10-14T19:10:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "G", + "run": 4, + "position": 2, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1725.61, + "elapsed_s": 11.59, + "completed": 20000, + "worker_cpu_ms_per_task": 1.774, + "postgres_cpu_ms_per_task": 0.631, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.22, + "started_utc": "2026-10-14T19:22:01Z", + "finished_utc": "2026-10-14T19:22:16Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "G", + "run": 5, + "position": 1, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1748.23, + "elapsed_s": 11.44, + "completed": 20000, + "worker_cpu_ms_per_task": 1.798, + "postgres_cpu_ms_per_task": 0.626, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.41, + "started_utc": "2026-10-14T19:34:22Z", + "finished_utc": "2026-10-14T19:34:37Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "GH", + "run": 1, + "position": 1, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2241.46, + "elapsed_s": 8.923, + "completed": 20000, + "worker_cpu_ms_per_task": 1.325, + "postgres_cpu_ms_per_task": 0.569, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.97, + "started_utc": "2026-10-14T18:36:24Z", + "finished_utc": "2026-10-14T18:36:36Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "GH", + "run": 2, + "position": 3, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2179.96, + "elapsed_s": 9.174, + "completed": 20000, + "worker_cpu_ms_per_task": 1.338, + "postgres_cpu_ms_per_task": 0.627, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.82, + "started_utc": "2026-10-14T18:54:25Z", + "finished_utc": "2026-10-14T18:54:37Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "GH", + "run": 3, + "position": 5, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2282.87, + "elapsed_s": 8.761, + "completed": 20000, + "worker_cpu_ms_per_task": 1.329, + "postgres_cpu_ms_per_task": 0.55, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.9, + "started_utc": "2026-10-14T19:11:56Z", + "finished_utc": "2026-10-14T19:12:08Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "GH", + "run": 4, + "position": 7, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2306.9, + "elapsed_s": 8.67, + "completed": 20000, + "worker_cpu_ms_per_task": 1.308, + "postgres_cpu_ms_per_task": 0.566, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.62, + "started_utc": "2026-10-14T19:28:51Z", + "finished_utc": "2026-10-14T19:29:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "GH", + "run": 5, + "position": 9, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 2269.41, + "elapsed_s": 8.813, + "completed": 20000, + "worker_cpu_ms_per_task": 1.32, + "postgres_cpu_ms_per_task": 0.592, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.94, + "started_utc": "2026-10-14T19:45:12Z", + "finished_utc": "2026-10-14T19:45:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "GH", + "run": 1, + "position": 4, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1541.45, + "elapsed_s": 12.975, + "completed": 20000, + "worker_cpu_ms_per_task": 1.902, + "postgres_cpu_ms_per_task": 0.781, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.23, + "started_utc": "2026-10-14T18:40:54Z", + "finished_utc": "2026-10-14T18:41:09Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "GH", + "run": 2, + "position": 2, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1477.72, + "elapsed_s": 13.534, + "completed": 20000, + "worker_cpu_ms_per_task": 1.957, + "postgres_cpu_ms_per_task": 0.726, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.11, + "started_utc": "2026-10-14T18:53:05Z", + "finished_utc": "2026-10-14T18:53:22Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "GH", + "run": 3, + "position": 1, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1556.19, + "elapsed_s": 12.852, + "completed": 20000, + "worker_cpu_ms_per_task": 1.931, + "postgres_cpu_ms_per_task": 0.743, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.71, + "started_utc": "2026-10-14T19:06:40Z", + "finished_utc": "2026-10-14T19:06:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "GH", + "run": 4, + "position": 3, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1567.86, + "elapsed_s": 12.756, + "completed": 20000, + "worker_cpu_ms_per_task": 1.927, + "postgres_cpu_ms_per_task": 0.757, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 3.31, + "started_utc": "2026-10-14T19:23:21Z", + "finished_utc": "2026-10-14T19:23:36Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "GH", + "run": 5, + "position": 5, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 1582.29, + "elapsed_s": 12.64, + "completed": 20000, + "worker_cpu_ms_per_task": 1.9, + "postgres_cpu_ms_per_task": 0.733, + "sessions_per_task": 0.0007, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 14, + "load_1m_before": 4.01, + "started_utc": "2026-10-14T19:39:57Z", + "finished_utc": "2026-10-14T19:40:12Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "D", + "run": 1, + "position": 3, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 2.41, + "started_utc": "2026-10-14T18:39:19Z", + "finished_utc": "2026-10-14T18:39:51Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "D", + "run": 2, + "position": 5, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 674.26, + "elapsed_s": 29.662, + "completed": 20000, + "worker_cpu_ms_per_task": 4.039, + "postgres_cpu_ms_per_task": 3.786, + "sessions_per_task": 0.8184, + "after_successful": 16353, + "after_running": 3647, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 16367, + "load_1m_before": 3.01, + "started_utc": "2026-10-14T18:57:00Z", + "finished_utc": "2026-10-14T18:57:34Z", + "valid": false, + "flag": "outcome writes lost: 3647 rows left RUNNING" + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "D", + "run": 3, + "position": 7, + "tasks": 20000, + "workers": 8, + "status": "ok", + "tasks_per_sec": 707.19, + "elapsed_s": 28.281, + "completed": 20000, + "worker_cpu_ms_per_task": 4.024, + "postgres_cpu_ms_per_task": 3.845, + "sessions_per_task": 0.8182, + "after_successful": 16351, + "after_running": 3649, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 16365, + "load_1m_before": 4.39, + "started_utc": "2026-10-14T19:14:46Z", + "finished_utc": "2026-10-14T19:15:18Z", + "valid": false, + "flag": "outcome writes lost: 3649 rows left RUNNING" + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "D", + "run": 4, + "position": 9, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 3.69, + "started_utc": "2026-10-14T19:31:26Z", + "finished_utc": "2026-10-14T19:31:57Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W8", + "backend": "django-ox", + "setting": "D", + "run": 5, + "position": 10, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 3.32, + "started_utc": "2026-10-14T19:46:27Z", + "finished_utc": "2026-10-14T19:46:58Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "D", + "run": 1, + "position": 10, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16361, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": 1.0004, + "after_successful": 16361, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": 16367, + "load_1m_before": 3.1, + "started_utc": "2026-10-14T18:50:10Z", + "finished_utc": "2026-10-14T18:50:43Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "D", + "run": 2, + "position": 8, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16360, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": 1.0004, + "after_successful": 16360, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": 16366, + "load_1m_before": 3.14, + "started_utc": "2026-10-14T19:01:20Z", + "finished_utc": "2026-10-14T19:01:51Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "D", + "run": 3, + "position": 6, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16357, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": 1.0004, + "after_successful": 16357, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": 16363, + "load_1m_before": 3.56, + "started_utc": "2026-10-14T19:13:11Z", + "finished_utc": "2026-10-14T19:13:42Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "D", + "run": 4, + "position": 4, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16360, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": 1.0004, + "after_successful": 16360, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": 16366, + "load_1m_before": 3.58, + "started_utc": "2026-10-14T19:24:41Z", + "finished_utc": "2026-10-14T19:25:12Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W8", + "backend": "django-tasks-db", + "setting": "D", + "run": 5, + "position": 2, + "tasks": 20000, + "workers": 8, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16359, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 16359, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 3.36, + "started_utc": "2026-10-14T19:35:42Z", + "finished_utc": "2026-10-14T19:36:13Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "P", + "run": 1, + "position": 6, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3100.66, + "elapsed_s": 6.45, + "completed": 20000, + "worker_cpu_ms_per_task": 1.386, + "postgres_cpu_ms_per_task": 0.573, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.63, + "started_utc": "2026-10-14T19:55:27Z", + "finished_utc": "2026-10-14T19:55:38Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "P", + "run": 2, + "position": 4, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2922.22, + "elapsed_s": 6.844, + "completed": 20000, + "worker_cpu_ms_per_task": 1.409, + "postgres_cpu_ms_per_task": 0.6, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.15, + "started_utc": "2026-10-14T20:07:13Z", + "finished_utc": "2026-10-14T20:07:23Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "P", + "run": 3, + "position": 2, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3064.86, + "elapsed_s": 6.526, + "completed": 20000, + "worker_cpu_ms_per_task": 1.385, + "postgres_cpu_ms_per_task": 0.577, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.18, + "started_utc": "2026-10-14T20:17:54Z", + "finished_utc": "2026-10-14T20:18:05Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "P", + "run": 4, + "position": 1, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3109.86, + "elapsed_s": 6.431, + "completed": 20000, + "worker_cpu_ms_per_task": 1.385, + "postgres_cpu_ms_per_task": 0.571, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.76, + "started_utc": "2026-10-14T20:30:39Z", + "finished_utc": "2026-10-14T20:30:49Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "P", + "run": 5, + "position": 3, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3058.45, + "elapsed_s": 6.539, + "completed": 20000, + "worker_cpu_ms_per_task": 1.38, + "postgres_cpu_ms_per_task": 0.525, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.05, + "started_utc": "2026-10-14T20:46:44Z", + "finished_utc": "2026-10-14T20:46:54Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "P", + "run": 1, + "position": 1, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2009.0, + "elapsed_s": 9.955, + "completed": 20000, + "worker_cpu_ms_per_task": 2.064, + "postgres_cpu_ms_per_task": 0.76, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.88, + "started_utc": "2026-10-14T19:48:42Z", + "finished_utc": "2026-10-14T19:48:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "P", + "run": 2, + "position": 3, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1947.17, + "elapsed_s": 10.271, + "completed": 20000, + "worker_cpu_ms_per_task": 2.063, + "postgres_cpu_ms_per_task": 0.782, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.52, + "started_utc": "2026-10-14T20:05:53Z", + "finished_utc": "2026-10-14T20:06:06Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "P", + "run": 3, + "position": 5, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1939.75, + "elapsed_s": 10.311, + "completed": 20000, + "worker_cpu_ms_per_task": 2.056, + "postgres_cpu_ms_per_task": 0.778, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.85, + "started_utc": "2026-10-14T20:21:58Z", + "finished_utc": "2026-10-14T20:22:11Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "P", + "run": 4, + "position": 7, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1959.47, + "elapsed_s": 10.207, + "completed": 20000, + "worker_cpu_ms_per_task": 2.072, + "postgres_cpu_ms_per_task": 0.772, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.68, + "started_utc": "2026-10-14T20:38:54Z", + "finished_utc": "2026-10-14T20:39:07Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "P", + "run": 5, + "position": 9, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1995.55, + "elapsed_s": 10.022, + "completed": 20000, + "worker_cpu_ms_per_task": 2.05, + "postgres_cpu_ms_per_task": 0.755, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.62, + "started_utc": "2026-10-14T20:55:10Z", + "finished_utc": "2026-10-14T20:55:23Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "PH", + "run": 1, + "position": 10, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1662.39, + "elapsed_s": 12.031, + "completed": 20000, + "worker_cpu_ms_per_task": 1.772, + "postgres_cpu_ms_per_task": 0.769, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.89, + "started_utc": "2026-10-14T20:00:53Z", + "finished_utc": "2026-10-14T20:01:08Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "PH", + "run": 2, + "position": 8, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2698.66, + "elapsed_s": 7.411, + "completed": 20000, + "worker_cpu_ms_per_task": 1.548, + "postgres_cpu_ms_per_task": 0.783, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 2.32, + "started_utc": "2026-10-14T20:12:18Z", + "finished_utc": "2026-10-14T20:12:29Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "PH", + "run": 3, + "position": 6, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2687.45, + "elapsed_s": 7.442, + "completed": 20000, + "worker_cpu_ms_per_task": 1.549, + "postgres_cpu_ms_per_task": 0.855, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 2.95, + "started_utc": "2026-10-14T20:23:55Z", + "finished_utc": "2026-10-14T20:24:05Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "PH", + "run": 4, + "position": 4, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2693.45, + "elapsed_s": 7.425, + "completed": 20000, + "worker_cpu_ms_per_task": 1.553, + "postgres_cpu_ms_per_task": 0.78, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.93, + "started_utc": "2026-10-14T20:34:29Z", + "finished_utc": "2026-10-14T20:34:39Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "PH", + "run": 5, + "position": 2, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2701.11, + "elapsed_s": 7.404, + "completed": 20000, + "worker_cpu_ms_per_task": 1.555, + "postgres_cpu_ms_per_task": 0.755, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.28, + "started_utc": "2026-10-14T20:45:29Z", + "finished_utc": "2026-10-14T20:45:40Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "PH", + "run": 1, + "position": 7, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1775.97, + "elapsed_s": 11.261, + "completed": 20000, + "worker_cpu_ms_per_task": 2.225, + "postgres_cpu_ms_per_task": 0.847, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.16, + "started_utc": "2026-10-14T19:56:42Z", + "finished_utc": "2026-10-14T19:56:57Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "PH", + "run": 2, + "position": 9, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1786.7, + "elapsed_s": 11.194, + "completed": 20000, + "worker_cpu_ms_per_task": 2.197, + "postgres_cpu_ms_per_task": 0.873, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.53, + "started_utc": "2026-10-14T20:13:33Z", + "finished_utc": "2026-10-14T20:13:48Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "PH", + "run": 3, + "position": 10, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1814.55, + "elapsed_s": 11.022, + "completed": 20000, + "worker_cpu_ms_per_task": 2.188, + "postgres_cpu_ms_per_task": 0.993, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.32, + "started_utc": "2026-10-14T20:29:19Z", + "finished_utc": "2026-10-14T20:29:32Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "PH", + "run": 4, + "position": 8, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1791.86, + "elapsed_s": 11.162, + "completed": 20000, + "worker_cpu_ms_per_task": 2.202, + "postgres_cpu_ms_per_task": 0.873, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.36, + "started_utc": "2026-10-14T20:40:14Z", + "finished_utc": "2026-10-14T20:40:29Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "PH", + "run": 5, + "position": 6, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1768.85, + "elapsed_s": 11.307, + "completed": 20000, + "worker_cpu_ms_per_task": 2.193, + "postgres_cpu_ms_per_task": 0.852, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.54, + "started_utc": "2026-10-14T20:50:59Z", + "finished_utc": "2026-10-14T20:51:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "G", + "run": 1, + "position": 8, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2949.96, + "elapsed_s": 6.78, + "completed": 20000, + "worker_cpu_ms_per_task": 1.44, + "postgres_cpu_ms_per_task": 0.57, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 2.98, + "started_utc": "2026-10-14T19:58:03Z", + "finished_utc": "2026-10-14T19:58:12Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "G", + "run": 2, + "position": 6, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3007.5, + "elapsed_s": 6.65, + "completed": 20000, + "worker_cpu_ms_per_task": 1.44, + "postgres_cpu_ms_per_task": 0.566, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.28, + "started_utc": "2026-10-14T20:09:48Z", + "finished_utc": "2026-10-14T20:09:58Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "G", + "run": 3, + "position": 4, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3032.44, + "elapsed_s": 6.595, + "completed": 20000, + "worker_cpu_ms_per_task": 1.433, + "postgres_cpu_ms_per_task": 0.543, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.66, + "started_utc": "2026-10-14T20:20:43Z", + "finished_utc": "2026-10-14T20:20:53Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "G", + "run": 4, + "position": 2, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3018.11, + "elapsed_s": 6.627, + "completed": 20000, + "worker_cpu_ms_per_task": 1.437, + "postgres_cpu_ms_per_task": 0.544, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.94, + "started_utc": "2026-10-14T20:31:54Z", + "finished_utc": "2026-10-14T20:32:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "G", + "run": 5, + "position": 1, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3011.0, + "elapsed_s": 6.642, + "completed": 20000, + "worker_cpu_ms_per_task": 1.437, + "postgres_cpu_ms_per_task": 0.555, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.15, + "started_utc": "2026-10-14T20:44:14Z", + "finished_utc": "2026-10-14T20:44:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "G", + "run": 1, + "position": 4, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1922.88, + "elapsed_s": 10.401, + "completed": 20000, + "worker_cpu_ms_per_task": 2.115, + "postgres_cpu_ms_per_task": 0.799, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.41, + "started_utc": "2026-10-14T19:52:57Z", + "finished_utc": "2026-10-14T19:53:10Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "G", + "run": 2, + "position": 2, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1919.93, + "elapsed_s": 10.417, + "completed": 20000, + "worker_cpu_ms_per_task": 2.124, + "postgres_cpu_ms_per_task": 0.819, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 6.44, + "started_utc": "2026-10-14T20:04:33Z", + "finished_utc": "2026-10-14T20:04:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "G", + "run": 3, + "position": 1, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1938.48, + "elapsed_s": 10.317, + "completed": 20000, + "worker_cpu_ms_per_task": 2.106, + "postgres_cpu_ms_per_task": 0.794, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.63, + "started_utc": "2026-10-14T20:16:28Z", + "finished_utc": "2026-10-14T20:16:41Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "G", + "run": 4, + "position": 3, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1946.06, + "elapsed_s": 10.277, + "completed": 20000, + "worker_cpu_ms_per_task": 2.105, + "postgres_cpu_ms_per_task": 0.785, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.6, + "started_utc": "2026-10-14T20:33:09Z", + "finished_utc": "2026-10-14T20:33:22Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "G", + "run": 5, + "position": 5, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1947.63, + "elapsed_s": 10.269, + "completed": 20000, + "worker_cpu_ms_per_task": 2.125, + "postgres_cpu_ms_per_task": 0.788, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.42, + "started_utc": "2026-10-14T20:49:35Z", + "finished_utc": "2026-10-14T20:49:48Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "GH", + "run": 1, + "position": 5, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2614.91, + "elapsed_s": 7.648, + "completed": 20000, + "worker_cpu_ms_per_task": 1.624, + "postgres_cpu_ms_per_task": 0.883, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.43, + "started_utc": "2026-10-14T19:54:13Z", + "finished_utc": "2026-10-14T19:54:24Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "GH", + "run": 2, + "position": 7, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2624.98, + "elapsed_s": 7.619, + "completed": 20000, + "worker_cpu_ms_per_task": 1.607, + "postgres_cpu_ms_per_task": 0.583, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 2.68, + "started_utc": "2026-10-14T20:11:03Z", + "finished_utc": "2026-10-14T20:11:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "GH", + "run": 3, + "position": 9, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2627.75, + "elapsed_s": 7.611, + "completed": 20000, + "worker_cpu_ms_per_task": 1.61, + "postgres_cpu_ms_per_task": 0.598, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.49, + "started_utc": "2026-10-14T20:28:04Z", + "finished_utc": "2026-10-14T20:28:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "GH", + "run": 4, + "position": 10, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2642.15, + "elapsed_s": 7.57, + "completed": 20000, + "worker_cpu_ms_per_task": 1.612, + "postgres_cpu_ms_per_task": 0.793, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.58, + "started_utc": "2026-10-14T20:42:59Z", + "finished_utc": "2026-10-14T20:43:10Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "GH", + "run": 5, + "position": 8, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2598.34, + "elapsed_s": 7.697, + "completed": 20000, + "worker_cpu_ms_per_task": 1.602, + "postgres_cpu_ms_per_task": 0.587, + "sessions_per_task": 0.0019, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 6.08, + "started_utc": "2026-10-14T20:53:55Z", + "finished_utc": "2026-10-14T20:54:06Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "GH", + "run": 1, + "position": 3, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1788.84, + "elapsed_s": 11.18, + "completed": 20000, + "worker_cpu_ms_per_task": 2.245, + "postgres_cpu_ms_per_task": 0.824, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.79, + "started_utc": "2026-10-14T19:51:37Z", + "finished_utc": "2026-10-14T19:51:52Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "GH", + "run": 2, + "position": 5, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1758.37, + "elapsed_s": 11.374, + "completed": 20000, + "worker_cpu_ms_per_task": 2.25, + "postgres_cpu_ms_per_task": 0.845, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.24, + "started_utc": "2026-10-14T20:08:28Z", + "finished_utc": "2026-10-14T20:08:43Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "GH", + "run": 3, + "position": 7, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1762.9, + "elapsed_s": 11.345, + "completed": 20000, + "worker_cpu_ms_per_task": 2.252, + "postgres_cpu_ms_per_task": 0.85, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.53, + "started_utc": "2026-10-14T20:25:08Z", + "finished_utc": "2026-10-14T20:25:23Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "GH", + "run": 4, + "position": 9, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1813.14, + "elapsed_s": 11.031, + "completed": 20000, + "worker_cpu_ms_per_task": 2.252, + "postgres_cpu_ms_per_task": 0.985, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.38, + "started_utc": "2026-10-14T20:41:34Z", + "finished_utc": "2026-10-14T20:41:47Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "GH", + "run": 5, + "position": 10, + "tasks": 20000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1785.55, + "elapsed_s": 11.201, + "completed": 20000, + "worker_cpu_ms_per_task": 2.256, + "postgres_cpu_ms_per_task": 0.925, + "sessions_per_task": 0.0011, + "after_successful": 20000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.85, + "started_utc": "2026-10-14T20:56:56Z", + "finished_utc": "2026-10-14T20:57:10Z", + "valid": true, + "flag": null + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "D", + "run": 1, + "position": 9, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 3.24, + "started_utc": "2026-10-14T19:59:17Z", + "finished_utc": "2026-10-14T19:59:46Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "D", + "run": 2, + "position": 10, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 4.51, + "started_utc": "2026-10-14T20:14:53Z", + "finished_utc": "2026-10-14T20:15:23Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "D", + "run": 3, + "position": 8, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 4.4, + "started_utc": "2026-10-14T20:26:29Z", + "finished_utc": "2026-10-14T20:26:57Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "D", + "run": 4, + "position": 6, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 4.49, + "started_utc": "2026-10-14T20:37:19Z", + "finished_utc": "2026-10-14T20:37:48Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-ox", + "setting": "D", + "run": 5, + "position": 4, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": null, + "load_1m_before": 2.64, + "started_utc": "2026-10-14T20:47:59Z", + "finished_utc": "2026-10-14T20:48:28Z", + "valid": false, + "flag": "run failed: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "D", + "run": 1, + "position": 2, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 4.2, + "started_utc": "2026-10-14T19:50:02Z", + "finished_utc": "2026-10-14T19:50:31Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "D", + "run": 2, + "position": 1, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": 16360, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": 1.0004, + "after_successful": 16361, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": 16367, + "load_1m_before": 3.63, + "started_utc": "2026-10-14T20:02:53Z", + "finished_utc": "2026-10-14T20:03:26Z", + "valid": false, + "flag": "workers exited early: worker connect error" + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "D", + "run": 3, + "position": 3, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 3.19, + "started_utc": "2026-10-14T20:19:08Z", + "finished_utc": "2026-10-14T20:19:37Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "D", + "run": 4, + "position": 5, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 3.98, + "started_utc": "2026-10-14T20:35:44Z", + "finished_utc": "2026-10-14T20:36:13Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S2.W16", + "backend": "django-tasks-db", + "setting": "D", + "run": 5, + "position": 7, + "tasks": 20000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 4.04, + "started_utc": "2026-10-14T20:52:20Z", + "finished_utc": "2026-10-14T20:52:48Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "P", + "run": 1, + "position": 8, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3357.82, + "elapsed_s": 29.781, + "completed": 100000, + "worker_cpu_ms_per_task": 1.211, + "postgres_cpu_ms_per_task": 0.676, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.7, + "started_utc": "2026-10-14T21:17:50Z", + "finished_utc": "2026-10-14T21:18:31Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "P", + "run": 2, + "position": 6, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3356.46, + "elapsed_s": 29.793, + "completed": 100000, + "worker_cpu_ms_per_task": 1.208, + "postgres_cpu_ms_per_task": 0.679, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.35, + "started_utc": "2026-10-14T21:36:36Z", + "finished_utc": "2026-10-14T21:37:17Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "P", + "run": 3, + "position": 4, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3364.45, + "elapsed_s": 29.723, + "completed": 100000, + "worker_cpu_ms_per_task": 1.216, + "postgres_cpu_ms_per_task": 0.67, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.2, + "started_utc": "2026-10-14T21:56:02Z", + "finished_utc": "2026-10-14T21:56:42Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "P", + "run": 4, + "position": 2, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3325.87, + "elapsed_s": 30.067, + "completed": 100000, + "worker_cpu_ms_per_task": 1.211, + "postgres_cpu_ms_per_task": 0.687, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.96, + "started_utc": "2026-10-14T22:11:47Z", + "finished_utc": "2026-10-14T22:12:28Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "P", + "run": 5, + "position": 1, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3369.19, + "elapsed_s": 29.681, + "completed": 100000, + "worker_cpu_ms_per_task": 1.21, + "postgres_cpu_ms_per_task": 0.661, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.27, + "started_utc": "2026-10-14T22:33:23Z", + "finished_utc": "2026-10-14T22:34:04Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "P", + "run": 1, + "position": 1, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2029.86, + "elapsed_s": 49.264, + "completed": 100000, + "worker_cpu_ms_per_task": 1.896, + "postgres_cpu_ms_per_task": 1.04, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 6.31, + "started_utc": "2026-10-14T21:02:20Z", + "finished_utc": "2026-10-14T21:03:16Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "P", + "run": 2, + "position": 3, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2018.34, + "elapsed_s": 49.546, + "completed": 100000, + "worker_cpu_ms_per_task": 1.893, + "postgres_cpu_ms_per_task": 1.018, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.46, + "started_utc": "2026-10-14T21:30:16Z", + "finished_utc": "2026-10-14T21:31:14Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "P", + "run": 3, + "position": 5, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1964.62, + "elapsed_s": 50.9, + "completed": 100000, + "worker_cpu_ms_per_task": 1.891, + "postgres_cpu_ms_per_task": 1.163, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.5, + "started_utc": "2026-10-14T21:57:47Z", + "finished_utc": "2026-10-14T21:58:45Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "P", + "run": 4, + "position": 7, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1954.34, + "elapsed_s": 51.168, + "completed": 100000, + "worker_cpu_ms_per_task": 1.895, + "postgres_cpu_ms_per_task": 1.184, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.34, + "started_utc": "2026-10-14T22:25:48Z", + "finished_utc": "2026-10-14T22:26:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "P", + "run": 5, + "position": 9, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1988.75, + "elapsed_s": 50.283, + "completed": 100000, + "worker_cpu_ms_per_task": 1.895, + "postgres_cpu_ms_per_task": 1.111, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.97, + "started_utc": "2026-10-14T22:50:59Z", + "finished_utc": "2026-10-14T22:51:57Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "PH", + "run": 1, + "position": 6, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2883.88, + "elapsed_s": 34.676, + "completed": 100000, + "worker_cpu_ms_per_task": 1.378, + "postgres_cpu_ms_per_task": 0.81, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.12, + "started_utc": "2026-10-14T21:13:55Z", + "finished_utc": "2026-10-14T21:14:40Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "PH", + "run": 2, + "position": 4, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2860.4, + "elapsed_s": 34.96, + "completed": 100000, + "worker_cpu_ms_per_task": 1.378, + "postgres_cpu_ms_per_task": 0.838, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.99, + "started_utc": "2026-10-14T21:32:56Z", + "finished_utc": "2026-10-14T21:33:41Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "PH", + "run": 3, + "position": 2, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2883.13, + "elapsed_s": 34.684, + "completed": 100000, + "worker_cpu_ms_per_task": 1.374, + "postgres_cpu_ms_per_task": 0.806, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 6.82, + "started_utc": "2026-10-14T21:51:01Z", + "finished_utc": "2026-10-14T21:51:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "PH", + "run": 4, + "position": 1, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2883.53, + "elapsed_s": 34.68, + "completed": 100000, + "worker_cpu_ms_per_task": 1.377, + "postgres_cpu_ms_per_task": 0.82, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.97, + "started_utc": "2026-10-14T22:09:57Z", + "finished_utc": "2026-10-14T22:10:42Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "PH", + "run": 5, + "position": 3, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2888.12, + "elapsed_s": 34.625, + "completed": 100000, + "worker_cpu_ms_per_task": 1.378, + "postgres_cpu_ms_per_task": 0.819, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.16, + "started_utc": "2026-10-14T22:37:58Z", + "finished_utc": "2026-10-14T22:38:43Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "PH", + "run": 1, + "position": 10, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1831.6, + "elapsed_s": 54.597, + "completed": 100000, + "worker_cpu_ms_per_task": 2.034, + "postgres_cpu_ms_per_task": 1.113, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.68, + "started_utc": "2026-10-14T21:21:26Z", + "finished_utc": "2026-10-14T21:22:28Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "PH", + "run": 2, + "position": 8, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1840.84, + "elapsed_s": 54.323, + "completed": 100000, + "worker_cpu_ms_per_task": 2.032, + "postgres_cpu_ms_per_task": 1.145, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.05, + "started_utc": "2026-10-14T21:40:01Z", + "finished_utc": "2026-10-14T21:41:03Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "PH", + "run": 3, + "position": 6, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1816.4, + "elapsed_s": 55.054, + "completed": 100000, + "worker_cpu_ms_per_task": 2.037, + "postgres_cpu_ms_per_task": 1.234, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.13, + "started_utc": "2026-10-14T21:59:52Z", + "finished_utc": "2026-10-14T22:00:54Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "PH", + "run": 4, + "position": 4, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1791.26, + "elapsed_s": 55.827, + "completed": 100000, + "worker_cpu_ms_per_task": 2.029, + "postgres_cpu_ms_per_task": 1.244, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.61, + "started_utc": "2026-10-14T22:17:47Z", + "finished_utc": "2026-10-14T22:18:52Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "PH", + "run": 5, + "position": 2, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1792.57, + "elapsed_s": 55.786, + "completed": 100000, + "worker_cpu_ms_per_task": 2.031, + "postgres_cpu_ms_per_task": 1.243, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 3.95, + "started_utc": "2026-10-14T22:35:48Z", + "finished_utc": "2026-10-14T22:36:53Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "G", + "run": 1, + "position": 3, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3296.04, + "elapsed_s": 30.339, + "completed": 100000, + "worker_cpu_ms_per_task": 1.259, + "postgres_cpu_ms_per_task": 0.67, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.27, + "started_utc": "2026-10-14T21:06:25Z", + "finished_utc": "2026-10-14T21:07:06Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "G", + "run": 2, + "position": 5, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3309.71, + "elapsed_s": 30.214, + "completed": 100000, + "worker_cpu_ms_per_task": 1.256, + "postgres_cpu_ms_per_task": 0.686, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.24, + "started_utc": "2026-10-14T21:34:51Z", + "finished_utc": "2026-10-14T21:35:32Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "G", + "run": 3, + "position": 7, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3285.71, + "elapsed_s": 30.435, + "completed": 100000, + "worker_cpu_ms_per_task": 1.268, + "postgres_cpu_ms_per_task": 0.669, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.38, + "started_utc": "2026-10-14T22:02:02Z", + "finished_utc": "2026-10-14T22:02:43Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "G", + "run": 4, + "position": 9, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3283.46, + "elapsed_s": 30.456, + "completed": 100000, + "worker_cpu_ms_per_task": 1.26, + "postgres_cpu_ms_per_task": 0.692, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.3, + "started_utc": "2026-10-14T22:29:58Z", + "finished_utc": "2026-10-14T22:30:39Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "G", + "run": 5, + "position": 10, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 3289.46, + "elapsed_s": 30.4, + "completed": 100000, + "worker_cpu_ms_per_task": 1.259, + "postgres_cpu_ms_per_task": 0.68, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.95, + "started_utc": "2026-10-14T22:53:03Z", + "finished_utc": "2026-10-14T22:53:44Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "G", + "run": 1, + "position": 7, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1936.84, + "elapsed_s": 51.631, + "completed": 100000, + "worker_cpu_ms_per_task": 1.944, + "postgres_cpu_ms_per_task": 1.146, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.17, + "started_utc": "2026-10-14T21:15:45Z", + "finished_utc": "2026-10-14T21:16:46Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "G", + "run": 2, + "position": 9, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1947.81, + "elapsed_s": 51.34, + "completed": 100000, + "worker_cpu_ms_per_task": 1.947, + "postgres_cpu_ms_per_task": 1.155, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.07, + "started_utc": "2026-10-14T21:42:56Z", + "finished_utc": "2026-10-14T21:43:55Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "G", + "run": 3, + "position": 10, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2003.22, + "elapsed_s": 49.92, + "completed": 100000, + "worker_cpu_ms_per_task": 1.945, + "postgres_cpu_ms_per_task": 1.019, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.13, + "started_utc": "2026-10-14T22:07:17Z", + "finished_utc": "2026-10-14T22:08:15Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "G", + "run": 4, + "position": 8, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1952.32, + "elapsed_s": 51.221, + "completed": 100000, + "worker_cpu_ms_per_task": 1.941, + "postgres_cpu_ms_per_task": 1.186, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.64, + "started_utc": "2026-10-14T22:27:53Z", + "finished_utc": "2026-10-14T22:28:51Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "G", + "run": 5, + "position": 6, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1958.85, + "elapsed_s": 51.05, + "completed": 100000, + "worker_cpu_ms_per_task": 1.947, + "postgres_cpu_ms_per_task": 1.134, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.83, + "started_utc": "2026-10-14T22:44:58Z", + "finished_utc": "2026-10-14T22:45:57Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "GH", + "run": 1, + "position": 9, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2822.68, + "elapsed_s": 35.427, + "completed": 100000, + "worker_cpu_ms_per_task": 1.43, + "postgres_cpu_ms_per_task": 0.835, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.74, + "started_utc": "2026-10-14T21:19:35Z", + "finished_utc": "2026-10-14T21:20:20Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "GH", + "run": 2, + "position": 10, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2848.57, + "elapsed_s": 35.105, + "completed": 100000, + "worker_cpu_ms_per_task": 1.428, + "postgres_cpu_ms_per_task": 0.819, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.74, + "started_utc": "2026-10-14T21:45:56Z", + "finished_utc": "2026-10-14T21:46:41Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "GH", + "run": 3, + "position": 8, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2827.06, + "elapsed_s": 35.372, + "completed": 100000, + "worker_cpu_ms_per_task": 1.426, + "postgres_cpu_ms_per_task": 0.839, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 4.06, + "started_utc": "2026-10-14T22:03:47Z", + "finished_utc": "2026-10-14T22:04:32Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "GH", + "run": 4, + "position": 6, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2844.6, + "elapsed_s": 35.154, + "completed": 100000, + "worker_cpu_ms_per_task": 1.427, + "postgres_cpu_ms_per_task": 0.817, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 3.86, + "started_utc": "2026-10-14T22:23:18Z", + "finished_utc": "2026-10-14T22:24:02Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "GH", + "run": 5, + "position": 4, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 2833.44, + "elapsed_s": 35.293, + "completed": 100000, + "worker_cpu_ms_per_task": 1.43, + "postgres_cpu_ms_per_task": 0.832, + "sessions_per_task": 0.0004, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 38, + "load_1m_before": 5.64, + "started_utc": "2026-10-14T22:39:48Z", + "finished_utc": "2026-10-14T22:40:33Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "GH", + "run": 1, + "position": 2, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1824.26, + "elapsed_s": 54.817, + "completed": 100000, + "worker_cpu_ms_per_task": 2.092, + "postgres_cpu_ms_per_task": 1.102, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 6.65, + "started_utc": "2026-10-14T21:04:20Z", + "finished_utc": "2026-10-14T21:05:22Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "GH", + "run": 2, + "position": 1, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1815.13, + "elapsed_s": 55.092, + "completed": 100000, + "worker_cpu_ms_per_task": 2.075, + "postgres_cpu_ms_per_task": 1.176, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.66, + "started_utc": "2026-10-14T21:23:56Z", + "finished_utc": "2026-10-14T21:24:58Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "GH", + "run": 3, + "position": 3, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1792.33, + "elapsed_s": 55.793, + "completed": 100000, + "worker_cpu_ms_per_task": 2.071, + "postgres_cpu_ms_per_task": 1.224, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 2.33, + "started_utc": "2026-10-14T21:53:47Z", + "finished_utc": "2026-10-14T21:54:51Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "GH", + "run": 4, + "position": 5, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1810.17, + "elapsed_s": 55.243, + "completed": 100000, + "worker_cpu_ms_per_task": 2.082, + "postgres_cpu_ms_per_task": 1.173, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 4.86, + "started_utc": "2026-10-14T22:19:58Z", + "finished_utc": "2026-10-14T22:21:00Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "GH", + "run": 5, + "position": 7, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 1829.7, + "elapsed_s": 54.654, + "completed": 100000, + "worker_cpu_ms_per_task": 2.075, + "postgres_cpu_ms_per_task": 1.125, + "sessions_per_task": 0.0002, + "after_successful": 100000, + "after_running": 0, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "sessions_opened": 22, + "load_1m_before": 5.5, + "started_utc": "2026-10-14T22:47:03Z", + "finished_utc": "2026-10-14T22:48:06Z", + "valid": true, + "flag": null + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "D", + "run": 1, + "position": 4, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 811.64, + "elapsed_s": 123.207, + "completed": 100000, + "worker_cpu_ms_per_task": 4.182, + "postgres_cpu_ms_per_task": 3.79, + "sessions_per_task": 0.6569, + "after_successful": 65655, + "after_running": 34345, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 65690, + "load_1m_before": 4.62, + "started_utc": "2026-10-14T21:08:10Z", + "finished_utc": "2026-10-14T21:10:24Z", + "valid": false, + "flag": "outcome writes lost: 34345 rows left RUNNING" + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "D", + "run": 2, + "position": 2, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 811.99, + "elapsed_s": 123.154, + "completed": 100000, + "worker_cpu_ms_per_task": 4.167, + "postgres_cpu_ms_per_task": 3.786, + "sessions_per_task": 0.6543, + "after_successful": 65397, + "after_running": 34603, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 65431, + "load_1m_before": 3.64, + "started_utc": "2026-10-14T21:26:56Z", + "finished_utc": "2026-10-14T21:29:10Z", + "valid": false, + "flag": "outcome writes lost: 34603 rows left RUNNING" + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "D", + "run": 3, + "position": 1, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 817.67, + "elapsed_s": 122.298, + "completed": 100000, + "worker_cpu_ms_per_task": 4.165, + "postgres_cpu_ms_per_task": 3.781, + "sessions_per_task": 0.6542, + "after_successful": 65387, + "after_running": 34613, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 65422, + "load_1m_before": 3.89, + "started_utc": "2026-10-14T21:47:46Z", + "finished_utc": "2026-10-14T21:49:58Z", + "valid": false, + "flag": "outcome writes lost: 34613 rows left RUNNING" + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "D", + "run": 4, + "position": 3, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 811.46, + "elapsed_s": 123.235, + "completed": 100000, + "worker_cpu_ms_per_task": 4.186, + "postgres_cpu_ms_per_task": 3.818, + "sessions_per_task": 0.6542, + "after_successful": 65384, + "after_running": 34616, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 65416, + "load_1m_before": 4.58, + "started_utc": "2026-10-14T22:13:47Z", + "finished_utc": "2026-10-14T22:16:01Z", + "valid": false, + "flag": "outcome writes lost: 34616 rows left RUNNING" + }, + { + "cell": "S3", + "backend": "django-ox", + "setting": "D", + "run": 5, + "position": 5, + "tasks": 100000, + "workers": 16, + "status": "ok", + "tasks_per_sec": 809.25, + "elapsed_s": 123.571, + "completed": 100000, + "worker_cpu_ms_per_task": 4.172, + "postgres_cpu_ms_per_task": 3.825, + "sessions_per_task": 0.6542, + "after_successful": 65386, + "after_running": 34614, + "duplicate_starts": 0, + "duplicate_dones": 0, + "worker_exit_codes": [ + 0 + ], + "sessions_opened": 65423, + "load_1m_before": 4.61, + "started_utc": "2026-10-14T22:41:38Z", + "finished_utc": "2026-10-14T22:43:52Z", + "valid": false, + "flag": "outcome writes lost: 34614 rows left RUNNING" + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "D", + "run": 1, + "position": 5, + "tasks": 100000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 6.23, + "started_utc": "2026-10-14T21:11:30Z", + "finished_utc": "2026-10-14T21:12:04Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "D", + "run": 2, + "position": 7, + "tasks": 100000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 5.38, + "started_utc": "2026-10-14T21:38:21Z", + "finished_utc": "2026-10-14T21:38:55Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "D", + "run": 3, + "position": 9, + "tasks": 100000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 3.8, + "started_utc": "2026-10-14T22:05:37Z", + "finished_utc": "2026-10-14T22:06:11Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "D", + "run": 4, + "position": 10, + "tasks": 100000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 4.85, + "started_utc": "2026-10-14T22:31:43Z", + "finished_utc": "2026-10-14T22:32:17Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + }, + { + "cell": "S3", + "backend": "django-tasks-db", + "setting": "D", + "run": 5, + "position": 8, + "tasks": 100000, + "workers": 16, + "status": "error", + "tasks_per_sec": null, + "elapsed_s": null, + "completed": null, + "worker_cpu_ms_per_task": null, + "postgres_cpu_ms_per_task": null, + "sessions_per_task": null, + "after_successful": 0, + "after_running": 0, + "duplicate_starts": null, + "duplicate_dones": null, + "worker_exit_codes": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sessions_opened": null, + "load_1m_before": 4.05, + "started_utc": "2026-10-14T22:49:10Z", + "finished_utc": "2026-10-14T22:49:44Z", + "valid": false, + "flag": "workers exited early: no new connection available (Mac out of ports)" + } + ] +} \ No newline at end of file diff --git a/docs/agents.md b/docs/agents.md index 54bdccb..1d194f9 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -73,8 +73,8 @@ OK: backlog=0 oldest_age=none last_claim_age=none `manage.py check` reports schedule and backend policy errors through `django_ox.E002` to `E005`, `E010` and `E011`, and warnings through `django_ox.W003` and `django_ox.W004`, before deployment. Database acceptance of schedule arguments is determined at dispatch. Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. Review `django_ox.W003` and `django_ox.W004` warnings too; `W004` asks for a non-bool integer from 1 to 32767 while retaining compatible legacy values. Invalid per-task fields raise `InvalidTask` (`InvalidTaskError` on the -backport) when the task is built, normally at import, rather than appearing -as a system-check message. +backport) when the task is built, normally at import. They do not appear +as system-check messages. Start a worker in its own process, next to the web server, under the same supervisor: @@ -212,9 +212,8 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv ## Facts to get right -- `QUEUES` sits beside `OPTIONS`, not inside it. Inside `OPTIONS` it is - ignored without warning; the symptom is `InvalidTask: Queue 'X' is not - valid for backend.` +- `QUEUES` sits beside `OPTIONS`. Inside `OPTIONS` it is ignored without + warning; the symptom is `InvalidTask: Queue 'X' is not valid for backend.` - Tasks use the Tasks-framework API. Import `task` from `django.tasks` on Django 6.0+, or `django_tasks` on 5.2 LTS, decorate with `@task`, and call `.enqueue(...)`. No django-ox import is needed for declarations. @@ -227,8 +226,8 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv substitution. Keep `OxBackend` and use the public, provisional `django_ox.testing.run_tasks()` helper to test queued execution. - The worker imports a task by its dotted path, so the module must be - importable in the worker process and the worker runs the same code as the - producer; nothing is registered and there is no autodiscovery. `async def` + importable in the worker process. The worker runs the same code as the + producer. Nothing is registered and there is no autodiscovery. `async def` tasks run. - Priority and deferral are Django API: `task.using(priority=N)` with N from -100 to 100, higher first, and `task.using(run_after=...)` with a timedelta @@ -239,18 +238,18 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv A second signal forces exit 130; a process that cannot act on signals needs SIGKILL. - `enqueue()` is one INSERT on the database the router sends `OxTask` to, - `default` unless you wrote a router. Two things make the commit joint: a - `transaction.atomic()` opened on that database, because a bare `atomic()` - opens on `default`; and the caller's own rows written there too. With both, - the task and those rows commit or roll back together, and the task is - visible to workers only after commit. Do not add `transaction.on_commit()` - around it. Rows written on another connection give two transactions, not - one. + `default` unless you wrote a router. Two things make the commit joint. The + first is a `transaction.atomic()` opened on that database, because a bare + `atomic()` opens on `default`. The second is the caller's own rows written + there too. With both, the task and those rows commit or roll back together, + and the task is visible to workers only after commit. Do not add + `transaction.on_commit()` around it. Rows written on another connection + give two transactions. - Many calls of one task go through `django_ox.bulk.enqueue_many(task, [(args, kwargs), ...])`: one INSERT per 1,000 rows, one transaction, results in input order. Set queue, priority and `run_after` once with `.using(...)`. - Execution is at-least-once. Write tasks to be safe to run twice: guard on - state already in the database, not on a flag in memory. + state already in the database. Do not guard on a flag in memory. - The task function runs outside any transaction. Open `transaction.atomic()` inside the task when it needs `select_for_update()`. - An attempt is consumed at claim time, so a worker dying mid-run uses one. @@ -261,22 +260,21 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv delays are not capped. - The worker schedules lease renewal every `LOCK_TIMEOUT / 3` seconds; task length is not bounded by `LOCK_TIMEOUT` while renewals succeed. - Renewal needs a database connection: if the worker cannot refresh its + Renewal needs a database connection. If the worker cannot refresh its lease for `LOCK_TIMEOUT`, the reaper can hand the task to another worker even while it is alive. - With Django's PostgreSQL pool, provide at least `concurrency + 1` pooled - connections per worker process; add a spare pooled connection if fallback + connections per worker process. Add a spare pooled connection if fallback must work under full load. Budget up to `max_size + 2` server connections - per process and database alias in normal operation, including any added spare: - one private renewal connection - and one possible watchdog connection. A worker whose tasks never use - timeouts needs only the first, for `max_size + 1` in normal operation. - A worker stopping with recovery pending opens one more private connection - for its stop-time recovery look: budget up to `max_size + 3` during that - look, or `max_size + 2` if its tasks never use timeouts. - An absent timeout in `OPTIONS` does not - establish that, because tasks can declare their own. Account for all - processes, aliases, other clients and reserved slots. + per process and database alias in normal operation, including any added + spare. The 2 covers one private renewal connection and one possible + watchdog connection. A worker whose tasks never use timeouts needs only + the first, for `max_size + 1` in normal operation. A worker stopping with + recovery pending opens one more private connection for its stop-time + recovery look. Budget up to `max_size + 3` during that look, or + `max_size + 2` if its tasks never use timeouts. An absent timeout in + `OPTIONS` does not establish that, because tasks can declare their own. + Account for all processes, aliases, other clients and reserved slots. - The worker polls; `--interval` (default 1.0 s) is the idle sleep, so a task starts within one interval of its commit. There is no LISTEN/NOTIFY. - Each attempt's timeout comes from the task's `timeout`, then its queue's @@ -288,31 +286,32 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv later is recorded as failed and the worker exits 75 so its supervisor restarts it. The stuck-thread path uses the worker's backoff without calling user callbacks. `django_ox.remaining()` reads the seconds left - from inside a task, including for a task-declared timeout. On a thread - a coverage tool or a debugger is watching (a `sys.settrace` hook, or a - `sys.monitoring` tool with events enabled) nothing is raised inside a - sync task: the worker logs `timeouts_backstop_only`, a task that returns - within `TASK_TIMEOUT_GRACE` is recorded as whatever it did, and one still - running then is recorded as failed and recycles the worker. An async task - is cancelled at the deadline either way. -- Claiming: one `UPDATE ... SKIP LOCKED ... RETURNING` statement on - PostgreSQL; `SELECT ... FOR UPDATE SKIP LOCKED` on MySQL 8+; an atomic - compare-and-set UPDATE on SQLite and other databases without `SKIP LOCKED`. + from inside a task, including for a task-declared timeout. A coverage + tool or a debugger may be watching a thread (a `sys.settrace` hook, or a + `sys.monitoring` tool with events enabled). On such a thread nothing is + raised inside a sync task. The worker logs `timeouts_backstop_only`. A + task that returns within `TASK_TIMEOUT_GRACE` is recorded as whatever it + did. One still running then is recorded as failed and recycles the + worker. An async task is cancelled at the deadline either way. +- Claiming is one `UPDATE ... SKIP LOCKED ... RETURNING` statement on + PostgreSQL. On MySQL 8+ it is `SELECT ... FOR UPDATE SKIP LOCKED`. On + SQLite and other databases without `SKIP LOCKED` it is an atomic + compare-and-set UPDATE. - `--concurrency N` is a thread pool in one process. `--processes N` runs N - such workers under one supervisor; a worker process that dies is restarted - after one second, doubling to 30 s, and more than five deaths of one slot - in a minute stops the supervisor with exit 1. CPU-bound work wants + such workers under one supervisor. A worker process that dies is restarted + after one second, doubling to 30 s. More than five deaths of one slot in a + minute stops the supervisor with exit 1. CPU-bound work wants `--processes N --concurrency 1`. -- Recurring tasks go in `OPTIONS["SCHEDULES"]`, with either `cron` or `every`, - not both. Every worker dispatches them; there is no scheduler process to - start. A tick that passes while every worker is down is enqueued once on - recovery, and older missed ticks are skipped. +- Recurring tasks go in `OPTIONS["SCHEDULES"]`, with `cron` or `every`. Each + schedule takes exactly one of the two. Every worker dispatches them; there + is no scheduler process to start. A tick that passes while every worker is + down is enqueued once on recovery, and older missed ticks are skipped. - Schedules can live in the database instead, editable in the admin. Set `OPTIONS["SCHEDULE_SOURCE"]` to `"django_ox.stored.DatabaseScheduleSource"`. - A row names a key registered with `@schedulable`, never an import path, so - admin access does not become permission to run arbitrary code. The decorator - takes effect only when its module is imported, and django-ox imports each - installed app's `tasks` module. Write rows with + A row names a key registered with `@schedulable`. Import paths are not + accepted, so admin access does not become permission to run arbitrary + code. The decorator takes effect only when its module is imported, and + django-ox imports each installed app's `tasks` module. Write rows with `django_ox.stored.create_schedule`, `create_schedules` or `update_schedule`; `objects.create()` skips validation and leaves the activation boundary stale. Retiming a stored schedule reschedules it from @@ -325,15 +324,15 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv field and message. A non-mapping row or unsupported key raises `TypeError` immediately. Permission refusals raise a separate `PermissionDenied`, naming every - denied row. A database refusal at the check or write raises for one row; - a concurrent name conflict raises `IntegrityError` and rolls back the + denied row. A database refusal at the check or write raises for one row. + A concurrent name conflict raises `IntegrityError` and rolls back the entire call. See [Writing schedules from code](stored-schedules.md#writing-schedules-from-code). - `ox_import_beat_schedules` reads django-celery-beat's database scheduler tables: `PeriodicTask`, `CrontabSchedule` and `IntervalSchedule`. It prints code for review and application. First shipped in 1.2.0 and - rebuilt in 1.8.0, it now prints one all-or-nothing `create_schedules` call - and lists each row it cannot translate with a named reason. It reads + rebuilt in 1.8.0, it now prints one all-or-nothing `create_schedules` call. + It lists each row it cannot translate with a named reason. It reads crontabs with Celery's grammar. Use `--beat-timezone ZONE` to supply beat's timezone. Its `--database` flag selects the alias holding the django-celery-beat @@ -363,7 +362,7 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv - Heartbeats are updated at each poll and drain pass head and before every claim attempt. A busy pass can make up to `--concurrency` claims, but updates occur between claims. Budget for reap and dispatch work plus one - claim, the poll interval and scheduling margin, not concurrency + claim, the poll interval and scheduling margin. Do not budget concurrency multiplied by claim latency. With Django's PostgreSQL connection pool, a refused database can hold a pass for the pool's `timeout`, 30 seconds by default. Include that wait in the freshness budget. @@ -378,11 +377,12 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv [heartbeat liveness](monitoring.md#freshness-and-database-isolation) for freshness sizing and database-stall restart tradeoffs. Docker and Compose outside Swarm mark a container unhealthy when its healthcheck fails. Their - restart policies act on process exit rather than healthcheck status. + restart policies act on process exit. A failed healthcheck does not + restart the container. - `path("ox/", include("django_ox.urls"))` mounts `GET /ox/metrics`, the queue stats as Prometheus gauges. It has no authentication of its own; wrap it with `login_required` or restrict it by network. -- Run `migrate` before rolling workers, not from the worker. +- Run `migrate` before rolling workers. Do not run it from the worker. - `django_ox.actions.retry(result_id)` requeues a FAILED or LOST task for one more attempt. `django_ox.actions.discard(result_id)` closes a READY, WAITING, FAILED or LOST task without running it. Neither touches a RUNNING task. diff --git a/docs/background-tasks.md b/docs/background-tasks.md index f547240..bcb83a3 100644 --- a/docs/background-tasks.md +++ b/docs/background-tasks.md @@ -143,7 +143,7 @@ def send_welcome_email(user_id): Nothing here comes from django-ox. `@task` is the Tasks framework, and the function is ordinary Django code. Arguments are stored as JSON, so pass the -user's id rather than the user. +user's id. Try it from the shell before wiring up a view: @@ -554,12 +554,11 @@ ERRORS: System check identified 1 issue (0 silenced). ``` -The worker runs the same validation at startup, so these errors stop startup -rather than skipping dispatch in silence. The checks do not establish database -acceptance of arguments; schedule-scoped failures at dispatch are logged as -`schedule_dispatch_error`. Put `* * * * *` back and restart the worker. -The startup line now says `schedules=1`, and at the next minute boundary -the schedule fires: +The worker runs the same validation at startup, so these errors stop it from +starting. The checks do not establish database acceptance of arguments; +schedule-scoped failures at dispatch are logged as `schedule_dispatch_error`. +Put `* * * * *` back and restart the worker. The startup line now says +`schedules=1`, and at the next minute boundary the schedule fires: ``` 2026-09-19 07:59:12,044 INFO django_ox Worker myhost-23929-gJ7A6QmA starting: queues=['default'] concurrency=1 poll=1.0s schedules=1 @@ -615,7 +614,7 @@ in the admin without a deploy. `ox_health --heartbeat-file PATH`, using the same `--processes N` on both commands. Use an existing, writable directory private to the container. A passing file probe means the expected controlling loops have advanced - recently, not that tasks are progressing. + recently. It does not mean that tasks are progressing. With no flags, `ox_health` checks that the database answers. Keep that dependency check separate from liveness restarts. Queue thresholds turn @@ -657,13 +656,13 @@ in the admin without a deploy. 8 take a `SKIP LOCKED` claim path built for many workers. See [PostgreSQL, MySQL or SQLite](production.md#postgresql-mysql-or-sqlite). - **On a read replica**, django-ox reads its own rows on the database it - writes them to, so a lagging replica never hides a task from the worker, + writes them to. A lagging replica never hides a task from the worker, the result API or `ox_health`. The admin reads the primary too. `ox_worker`, `ox_prune` and `ox_health` take `--database` to name the alias they work on; leave it unset unless you mean it. Details under [Read replicas](configuration.md#read-replicas). - **Run `migrate` before rolling workers**, as a deploy step or an init - container, not from the worker itself. + container. Do not run it from the worker itself. ## Next steps diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 0b90feb..6331172 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,46 +1,88 @@ # Benchmarks -## Queue drain and worker recovery +## 1.4 to 1.7× the throughput of django-tasks-db with retained connections -### 2,000 of 2,000 tasks finished +django-ox is a database-backed worker for Django's Tasks framework. -Every django-ox kill-test trial finished with zero tasks stuck. django-tasks-db left 13 to 19 tasks stuck RUNNING per trial. +### More tasks per second at matched connection settings -Five trials, each with 20 worker kills and replacements, with LOCK_TIMEOUT set to 15 s. +With pooled or persistent connections, django-ox delivered 1.4 to 1.7× django-tasks-db's throughput across 4, 8 and 16 workers. -Execution is at-least-once. The trials recorded 7 repeated effects across 10,000 tasks. Write tasks that can run again safely. +Every task finished in these runs, with zero duplicate starts or completions. + +With persistent connections, worker CPU per task was 1.0 to 1.4 ms for django-ox versus 1.5 to 2.1 ms for django-tasks-db. + +PostgreSQL 16; no-op tasks; django-ox 1.9.0 vs django-tasks-db 0.13.0; medians of five runs. + +Rates are medians of five interleaved runs; ratios are medians of per-round ratios. The test machine had background load; see How we measured. + +Retain worker connections to speed up both backends: `CONN_MAX_AGE = 600`, or `OPTIONS["pool"]` with `CONN_MAX_AGE = 0`. Django 6.0 rejects a pool with nonzero `CONN_MAX_AGE`; see [Retaining worker connections](production.md#retaining-worker-connections). + +Both backends ran the same settings in each row. Rates are tasks per second. + +| Workload | Setting | django-ox, tasks/s | django-tasks-db, tasks/s | Ratio | +|---|---|---:|---:|---:| +| 20,000 tasks, 4 workers | Persistent | 1,771 | 1,211 | **1.46×** | +| 20,000 tasks, 4 workers | Persistent, health checks | 1,590 | 1,123 | **1.42×** | +| 20,000 tasks, 4 workers | Pooled | 1,718 | 1,216 | **1.41×** | +| 20,000 tasks, 4 workers | Pooled, health checks | 1,561 | 1,127 | **1.40×** | +| 20,000 tasks, 8 workers | Persistent | 2,668 | 1,735 | **1.53×** | +| 20,000 tasks, 8 workers | Persistent, health checks | 2,324 | 1,594 | **1.45×** | +| 20,000 tasks, 8 workers | Pooled | 2,569 | 1,724 | **1.51×** | +| 20,000 tasks, 8 workers | Pooled, health checks | 2,269 | 1,556 | **1.47×** | +| 20,000 tasks, 16 workers | Persistent | 3,065 | 1,959 | **1.54×** | +| 20,000 tasks, 16 workers | Persistent, health checks | 2,693 | 1,787 | **1.50×** | +| 20,000 tasks, 16 workers | Pooled | 3,011 | 1,938 | **1.55×** | +| 20,000 tasks, 16 workers | Pooled, health checks | 2,625 | 1,786 | **1.46×** | +| 100,000 tasks, 16 workers | Persistent | 3,358 | 1,989 | **1.69×** | +| 100,000 tasks, 16 workers | Persistent, health checks | 2,884 | 1,816 | **1.59×** | +| 100,000 tasks, 16 workers | Pooled | 3,289 | 1,952 | **1.68×** | +| 100,000 tasks, 16 workers | Pooled, health checks | 2,833 | 1,815 | **1.57×** | + +Persistent: `CONN_MAX_AGE = 600`. Persistent, health checks: the same with `CONN_HEALTH_CHECKS = True`. Pooled: `OPTIONS["pool"]` with `CONN_MAX_AGE = 0`. Pooled, health checks: the same with `CONN_HEALTH_CHECKS = True`. -### Up to 20% faster queue drain than django-tasks-db +Draining 100,000 tasks at 16 workers took about 30 s for django-ox (29.7 to 30.1 s over five runs) and about 50 s for django-tasks-db (49.3 to 51.2 s) with persistent connections; with a pool 30.2 to 30.5 s vs 49.9 to 51.6 s. -Worker counts match in each row. Rates are tasks per second. +### Default connection settings -| Queued tasks | Workers per backend | django-ox, tasks/s | django-tasks-db, tasks/s | django-ox faster | -|---|---:|---:|---:|---:| -| 2,000 | 1 | 118.5 | 103.9 | **14.0%** | -| 2,000 | 4 | 426.3 | 387.1 | **10.1%** | -| 20,000* | 1 | 115.2 | 97.4 | **18.2%** | -| 20,000* | 4 | 461.8 | 401.6 | **15.0%** | +| Workload | Setting | django-ox, tasks/s | django-tasks-db, tasks/s | Ratio | +|---|---|---:|---:|---:| +| 20,000 tasks, 4 workers | Defaults | 430 | 373 | **1.1 to 1.2×** | -In each row, every django-ox run outperformed every django-tasks-db run. Results apply to these workloads, not every application. +`CONN_MAX_AGE = 0` and no pool. Spread is the standard deviation of the five rates divided by their mean. + +The spread between runs is 9% on both, so treat this as about 1.1×. + +## Worker recovery, enqueue and retries (django-ox 1.3.0) + +### 2,000 of 2,000 tasks finished + +Every django-ox 1.3.0 kill-test trial finished with zero tasks stuck. django-tasks-db left 13 to 19 tasks stuck RUNNING per trial. + +Five trials, each with 20 worker kills and replacements, with LOCK_TIMEOUT set to 15 s. The wallpaper process rendered during kill trials 2 and 3 only; kill counts were unaffected. + +Execution is at-least-once. The trials recorded 7 repeated effects across 10,000 tasks. Write tasks that can run again safely. ### Enqueue 10,000 tasks in 0.71 s -`enqueue_many()` inserted all 10,000 rows in one transaction, including COMMIT. Run times ranged from 0.69 to 0.72 s. +In django-ox 1.3.0, `enqueue_many()` inserted all 10,000 rows in one transaction, including COMMIT. Run times ranged from 0.69 to 0.72 s. ### 20 of 20 tasks retried successfully -Each task raised an exception on its first attempt. django-ox retried them automatically, and all succeeded on attempt two, every run. django-tasks-db marked all 20 FAILED after the first exception. +Each task raised an exception on its first attempt. django-ox 1.3.0 retried them automatically, and all succeeded on attempt two, every run. django-tasks-db marked all 20 FAILED after the first exception. ### How we measured -Oxpull, 2026-09-19. django-ox 1.3.0 vs django-tasks-db 0.13.0. +Matched-connection run: Oxpull, 2026-10-14. django-ox 1.9.0 vs django-tasks-db 0.13.0. + +PostgreSQL 16 ran locally in Docker. Django 6.0.8, psycopg 3.3.6. Task bodies were no-ops with a 256-byte payload. Each cell used five interleaved runs per setting and backend, with order balanced across rounds. Both backends got identical Django database options; each package's own defaults applied otherwise. -PostgreSQL 16 ran locally in Docker on an Apple M1 Max. Task bodies were no-ops. Each cell used five interleaved runs per backend. +The test machine had other load (one-minute load average 1.9 to 6.8 before a run), so absolute rates are lower than a quiet machine would give. -Package defaults applied except for the kill test, with LOCK_TIMEOUT set to 15 s. +At one connection per task, the test machine runs out of ports at about 550 new connections a second, so we show no default-settings throughput above 4 workers. -*The wallpaper process rendered during kill trials 2 and 3 only; kill counts were unaffected. The four drain cells (2,000 and 20,000 tasks) ran interleaved in one window; the 100,000-task run was one pair, not five. +Kill, enqueue and retry sections: Oxpull, 2026-09-19. django-ox 1.3.0 vs django-tasks-db 0.13.0. PostgreSQL 16 ran locally in Docker. Task bodies were no-ops. Each cell used five interleaved runs per backend. Package defaults applied except for the kill test, with LOCK_TIMEOUT set to 15 s. -[**Download both raw JSON files**](https://github.com/oxpull/django-ox/tree/main/benchmarks) +[**Download the raw JSON files**](https://github.com/oxpull/django-ox/tree/main/benchmarks) [**Get started**](background-tasks.md) diff --git a/docs/choosing.md b/docs/choosing.md index 46e8ad8..ef1f56c 100644 --- a/docs/choosing.md +++ b/docs/choosing.md @@ -9,9 +9,10 @@ documentation, source or issue tracker, with the link in the footnotes and the date it was read. Where a project's pages do not say, the cell says so rather than guessing. -The comparison is about fit, not ranking. A team that already runs RabbitMQ and -needs to stop running tasks has a different answer from a team that wants one -fewer service. The last section says where django-ox is not the right fit. +The comparison is about fit. It does not rank the tools. A team that already +runs RabbitMQ and needs to stop running tasks has a different answer from one +that wants one fewer service. The last section says where django-ox is +not the right fit. ## The table diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 4bad25f..fa1cbb6 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -383,7 +383,7 @@ def send_welcome_email(user_id): Nothing here comes from django-ox. `@task` is the Tasks framework, and the function is ordinary Django code. Arguments are stored as JSON, so pass the -user's id rather than the user. +user's id. Try it from the shell before wiring up a view: @@ -794,12 +794,11 @@ ERRORS: System check identified 1 issue (0 silenced). ``` -The worker runs the same validation at startup, so these errors stop startup -rather than skipping dispatch in silence. The checks do not establish database -acceptance of arguments; schedule-scoped failures at dispatch are logged as -`schedule_dispatch_error`. Put `* * * * *` back and restart the worker. -The startup line now says `schedules=1`, and at the next minute boundary -the schedule fires: +The worker runs the same validation at startup, so these errors stop it from +starting. The checks do not establish database acceptance of arguments; +schedule-scoped failures at dispatch are logged as `schedule_dispatch_error`. +Put `* * * * *` back and restart the worker. The startup line now says +`schedules=1`, and at the next minute boundary the schedule fires: ``` 2026-09-19 07:59:12,044 INFO django_ox Worker myhost-23929-gJ7A6QmA starting: queues=['default'] concurrency=1 poll=1.0s schedules=1 @@ -855,7 +854,7 @@ in the admin without a deploy. `ox_health --heartbeat-file PATH`, using the same `--processes N` on both commands. Use an existing, writable directory private to the container. A passing file probe means the expected controlling loops have advanced - recently, not that tasks are progressing. + recently. It does not mean that tasks are progressing. With no flags, `ox_health` checks that the database answers. Keep that dependency check separate from liveness restarts. Queue thresholds turn @@ -897,13 +896,13 @@ in the admin without a deploy. 8 take a `SKIP LOCKED` claim path built for many workers. See [PostgreSQL, MySQL or SQLite](production.md#postgresql-mysql-or-sqlite). - **On a read replica**, django-ox reads its own rows on the database it - writes them to, so a lagging replica never hides a task from the worker, + writes them to. A lagging replica never hides a task from the worker, the result API or `ox_health`. The admin reads the primary too. `ox_worker`, `ox_prune` and `ox_health` take `--database` to name the alias they work on; leave it unset unless you mean it. Details under [Read replicas](configuration.md#read-replicas). - **Run `migrate` before rolling workers**, as a deploy step or an init - container, not from the worker itself. + container. Do not run it from the worker itself. ## Next steps @@ -953,7 +952,7 @@ Wagtail 7.3.x declares Django 6.0 support. On Django 6.1, pip permits Wagtail 7.3.x, but Wagtail 7.3.4 fails in Wagtail's own search with `OperationalError: unable to use function MATCH`, both with and without django-ox. -Do not use Wagtail 7.3.0 with this setup on Django 5.2 or 6.0. Its dependency constraints resolve `django-tasks` 0.9.0. On Django 6.0, that version crashes during app loading (`django.setup()`) once `TASKS` selects a backend built on core Django's `Task`. Every `manage.py` command and the web process fail, not just migrations. +Do not use Wagtail 7.3.0 with this setup on Django 5.2 or 6.0. Its dependency constraints resolve `django-tasks` 0.9.0. On Django 6.0, that version crashes during app loading (`django.setup()`) once `TASKS` selects a backend built on core Django's `Task`. Every `manage.py` command and the web process fail. Upgrade Wagtail before installing django-ox into a Wagtail 7.3.0 environment. @@ -997,7 +996,7 @@ TASKS = { } ``` -If `TASKS` already exists, update its `default` entry rather than replacing other backends. Wagtail's bare `@task()` binds to backend alias `default` and queue `default`. +If `TASKS` already exists, update its `default` entry. Leave the other backends in place. Wagtail's bare `@task()` binds to backend alias `default` and queue `default`. Run migrations: @@ -1057,13 +1056,13 @@ Attempts count claims. The default `MAX_ATTEMPTS` is 3. Every running worker runs a reaper every `min(30, max(LOCK_TIMEOUT/2, 1))` seconds. With the default 300-second lease and 30-second reaper interval, recovery can take about 330 seconds after the last lease renewal. Recovery requires at least one running worker. -`LOCK_TIMEOUT` and `MAX_ATTEMPTS` belong inside `TASKS["default"]["OPTIONS"]`, not at the top level of Django settings. See [Backend options](configuration.md#options). +`LOCK_TIMEOUT` and `MAX_ATTEMPTS` belong inside `TASKS["default"]["OPTIONS"]`. Do not put them at the top level of Django settings. See [Backend options](configuration.md#options). ### Timeouts Timeouts are opt-in: `TASK_TIMEOUT` defaults to `None`, meaning no backend-wide limit. `TASK_TIMEOUTS` sets per-queue limits. Both belong in the backend's [options](configuration.md#options). On Django 6.1, or Django 5.2 with django-tasks 0.12+, a task's `timeout` declaration takes precedence over both; see [per-task policy](configuration.md#per-task-policy). -Wagtail's frontend-cache purge makes outbound HTTP calls, so consider an appropriate task timeout rather than assuming one is already active. +Wagtail's frontend-cache purge makes outbound HTTP calls, so consider an appropriate task timeout. Do not assume one is already active. ### Objects deleted before their tasks run @@ -4205,7 +4204,7 @@ database as the `User` row, your default one, so the task becomes visible to workers only when the transaction commits. If `create_user` is rolled back further up, the email task disappears with it. -Pass the id, not the object. Arguments are stored as JSON, and a stale copy of a +Pass the model's id. Arguments are stored as JSON, and a stale copy of a model is a bug waiting to happen. ## Retry a flaky third-party call @@ -4328,7 +4327,7 @@ return render(request, "report_pending.html", {"attempts": result.attempts}) `is_finished` covers the last two. A task whose worker vanished without reporting also reads as `FAILED`; [the reaper](production.md#the-reaper) explains what that record contains. Return values are stored as JSON, so -return a URL or an id rather than a file or a model. +return a URL or an id. ## Defer work to a specific time @@ -4461,10 +4460,10 @@ def charge_order(order_id): order.save(update_fields=["charged_at"]) ``` -Guard on state you have written, not on a flag you set in memory. The row lock -serialises concurrent attempts, and the idempotency key covers the gap where the -charge succeeds but the transaction does not commit. Use one whenever the -external system offers it. +Guard on state you have written. Do not guard on a flag you set in memory. +The row lock serialises concurrent attempts, and the idempotency key covers +the gap where the charge succeeds but the transaction does not commit. Use +one whenever the external system offers it. **Your task is not run inside a transaction.** The worker manages its own for claiming and bookkeeping, but your function is called outside them, so @@ -4682,14 +4681,14 @@ Of `db_worker`'s worker-specific options, `--backend`, `--interval`, - Replace `--queue-name` with `--queues`. - `--batch` maps by name to `--batch`. In `ox_worker`, it ends after an - error-free empty claim pass with no local tasks in flight; it does not + error-free empty claim pass with no local tasks in flight. It does not wait for future tasks, retry backoff or deferred releases. `db_worker` exits with a traceback when the database is unreachable or its tables are missing. `ox_worker` keeps retrying. An abandoned dispatch pass prevents batch completion until a later dispatch pass completes. A schedule-scoped failure is reported as `schedule_dispatch_error` and does not hold the - batch open. Exit 0 means the batch finished, not that every schedule - enqueued. Give the job a timeout, as + batch open. Exit 0 means the batch finished. It does not mean that every + schedule enqueued. Give the job a timeout, as [Running as a job](production.md#running-as-a-job) explains. - `--max-tasks N` maps by name to `--max-tasks N`. In `ox_worker`, every claimed attempt consumes one of N, including failed attempts and repeat @@ -4705,7 +4704,7 @@ Of `db_worker`'s worker-specific options, `--backend`, `--interval`, Use `--queues default` to preserve the old default. Do not use `--queues '*'` to select all queues. `*` is treated as a literal -queue name, not a wildcard. Omit `--queues` to run every configured queue. +queue name. Omit `--queues` to run every configured queue. ## From Celery @@ -4757,8 +4756,8 @@ decorator. See [Per-task policy](configuration.md#per-task-policy). In django-ox, `max_attempts` counts worker claims, including the first. A worker that dies after claiming a task consumes an attempt too. Choose -the budget from that definition rather than copying a retry count from -another task runner. +the budget from that definition. Do not copy a retry count from another +task runner. One difference in behaviour to read before you switch. With a broker, `enqueue` leaves your process immediately. If the surrounding transaction then @@ -4774,7 +4773,7 @@ implement Celery's chain, group or chord APIs, or routing across multiple brokers. [Batches](features/batches.md) provide grouped work, and provisional [workflows](features/workflows.md) provide dependency DAGs. Both require Django 6.0+ and the optional `oxpull` app. Review their -semantics rather than treating them as drop-in Celery replacements. +semantics. They are not drop-in Celery replacements. ## From huey @@ -4932,7 +4931,7 @@ No drain window available? Run both. Old workers keep serving the old table while new work goes to django-ox. They cannot see each other's rows. Both systems run tasks at least once, so your tasks should already be -idempotent. Confirm it before you start rather than halfway through. +idempotent. Confirm it before you start. ## Migrating away @@ -4948,13 +4947,12 @@ compatibility guarantee. In the other direction, django-ox reads policy only from `PolicyTask`; other Task classes inherit its backend defaults when rebound with `.using(backend=...)`. -Transactional enqueue also needs deciding on the way in rather than on -the way out. Enqueueing inside `transaction.atomic()` on the database that -holds `OxTask` ties the task to that transaction, so it disappears on -rollback. A broker-based backend cannot do this: the enqueue leaves your -process the moment you call it. Code that depends on a rollback removing -a task will behave differently once the queue lives in a broker, and it -will do so quietly. +Transactional enqueue also needs deciding on the way in. Enqueueing inside +`transaction.atomic()` on the database that holds `OxTask` ties the task to +that transaction, so it disappears on rollback. A broker-based backend +cannot do this: the enqueue leaves your process the moment you call it. Code +that depends on a rollback removing a task will behave differently once the +queue lives in a broker. It will do so quietly. If you want to keep that option open, wrap enqueues in `transaction.on_commit()`, the way a broker-based backend requires. django-ox @@ -5134,7 +5132,7 @@ def register(username, email): That is the advantage over sending to a separate broker: the enqueue can share your database transaction. django-tasks-db shares this property too. -The transaction covers the database writes, not email delivery. Make the task safe to repeat. +The transaction covers the database writes. Email delivery sits outside it, so make the task safe to repeat. ## Run recurring Django tasks without a scheduler process @@ -5165,7 +5163,7 @@ Schedules need at least one running worker and a usable database. A schedule-sco ## Deduplicate a repeated webhook -A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue rather than making the queue do the same work again. +A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue. Without a unique-task declaration, these calls create separate tasks: @@ -5191,7 +5189,7 @@ The same app provides [batches](features/batches.md) to track grouped work, [rat ## When to choose another tool -Choose another backend if you need to revoke a chosen running task, require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md), not those Celery APIs. +Choose another backend if you need to revoke a chosen running task. Choose another if you require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md). Transactional enqueue requires the task and model rows to share a database. A separate queue database cannot give you that shared commit. @@ -5214,9 +5212,10 @@ documentation, source or issue tracker, with the link in the footnotes and the date it was read. Where a project's pages do not say, the cell says so rather than guessing. -The comparison is about fit, not ranking. A team that already runs RabbitMQ and -needs to stop running tasks has a different answer from a team that wants one -fewer service. The last section says where django-ox is not the right fit. +The comparison is about fit. It does not rank the tools. A team that already +runs RabbitMQ and needs to stop running tasks has a different answer from one +that wants one fewer service. The last section says where django-ox is +not the right fit. ## The table @@ -6242,20 +6241,31 @@ PgBouncer and third-party pools have not been tested. ### Retaining worker connections -With `CONN_MAX_AGE = 0` and no pool, Django closes initialized execution connections before and after each attempt. The next attempt opens another connection; the poll loop keeps its claim connection. Task ORM queries and outcome writing use the same execution connection on the worker's database alias. `worker_connection_hint` reports this setting once at INFO during `Worker.run()` startup on PostgreSQL and MySQL. +Django's default is `CONN_MAX_AGE = 0`. With it and no pool, each task attempt opens a new database connection and closes it after. On a local PostgreSQL that costs about 5 ms a task. The poll loop keeps its own connection. Task queries and the outcome write share the attempt's connection on the worker's database alias. -To retain connections, configure the worker's alias in a dedicated worker settings module. Launch workers with that module to leave web processes unchanged. For PostgreSQL or MySQL without a pool, set a positive connection age and enable health checks: +When a worker starts on PostgreSQL with that setup, it logs one warning, `worker_connection_hint`, naming the alias and the fix. MySQL gets the same event at INFO. SQLite opens a file and logs nothing. + +Pick one of two settings for the worker's alias. Put it in a dedicated worker settings module and launch workers with that module, so web processes stay as they are. To keep connections open between tasks: ```python DATABASES["default"]["CONN_MAX_AGE"] = 600 -DATABASES["default"]["CONN_HEALTH_CHECKS"] = True ``` -For PostgreSQL, Django's pool requires psycopg 3, `psycopg_pool` and `CONN_MAX_AGE = 0`; use the pool example above instead of a positive age. MySQL has no Django pool; setting `OPTIONS["pool"]` does not enable one. +On PostgreSQL you can use Django's pool instead. It needs psycopg 3 and `psycopg_pool`, and `CONN_MAX_AGE` stays at 0: + +```python +DATABASES["default"]["OPTIONS"] = { + "pool": {"min_size": 0, "max_size": 16, "timeout": 30} +} +``` -Allow pool capacity for task threads and polling, plus the separate renewal, watchdog and stop-time recovery connections described above. Each worker process has its own pool. Retained connections carry session state between tasks; follow the responsibilities below. +Django refuses to open a pool when `CONN_MAX_AGE` is above 0, so use one or the other. MySQL has no Django pool, and `OPTIONS["pool"]` does nothing there. -Persistence and pooling remain opt-in. The default connection lifecycle, transactions, delivery and schema are unchanged. +Each worker process has its own pool. Set `max_size` to at least `concurrency + 1`, and budget the extra connections in [Budget server capacity](#budget-server-capacity). A `max_size` of 16 covers up to 15 task threads in a process. + +`CONN_HEALTH_CHECKS = True` tests a kept connection before each use. It costs about 10% throughput. Set it only if your network drops idle connections. + +Kept connections carry session state from one task to the next. Follow the rules in the next section. ### Session state and connection failures @@ -6265,7 +6275,7 @@ Each execution thread keeps its own connection context, separate from other task Measured default runs opened one connection per task attempt plus one for polling; ORM queries added no connections. Persistence or PostgreSQL pooling opened two connections in total: one for execution and one for polling, regardless of task count. These counts are exact on PostgreSQL and SQLite; MySQL counting tests allow one extra connection per thread because Django's MySQL backend reads the server version through a throwaway connection the first time a thread uses it. After a server-ended idle session, querying tasks used one extra attempt, then succeeded on a new session. Query-free tasks reconnected at outcome writing without spending an attempt; every task completed. Workers with and without persistence shared a queue: all 40 tasks succeeded, each running once with one claim and one epoch. Both workers ran tasks. These measurements cover same-release batch workers, PostgreSQL, MySQL and SQLite connection counts, PostgreSQL pooling, and PostgreSQL and MySQL idle-session recovery. -Set `CONN_HEALTH_CHECKS = True`; health checks reduce stale-connection failures but cannot prevent later disconnects. +With `CONN_HEALTH_CHECKS = True`, stale connections fail less often. A health check cannot prevent a disconnect that comes after it. ### psycopg_pool versions and recovery @@ -7759,7 +7769,7 @@ A stopped reconciler sweep is not a completed sweep. Check `visited_count`, `unv | `worker_started` | INFO | The run loop starts. | | `connection_pool_too_small` | WARNING | Once per `Worker.run()`, after `worker_started` and before threads start: the worker alias's effective PostgreSQL pool maximum is below `concurrency + 1`. Each `--processes` child checks separately. The warning does not resize the pool, refuse startup, measure available server slots, or reserve fallback capacity. | | `connection_pool_cannot_drain` | WARNING | Pooled PostgreSQL only. The installed psycopg_pool has no `ConnectionPool.drain()` method, which was added in 3.3.0. Emitted once per `Worker.run()`, after `worker_started` and the `connection_pool_too_small` check, before threads start. Each `--processes` child checks separately. Also emitted at the first sweep of such a pool in a process, for each database alias, unless startup already emitted it in that process. Concurrent first sweeps may emit duplicate warnings. The sweep record is the only notice available to callers of `run_once()` or `run_tasks()`. Idle connections are still tested as in django-ox 1.7.0, preserving restart recovery but also the wait on a connection that stays open without answering. A stop is not read until that test returns. Upgrade psycopg_pool to 3.3.3 or newer. This warning does not refuse startup or change the pool. No traceback. Keys: `event` and `database` always; `worker_id` only on the startup record. | -| `worker_connection_hint` | INFO | Once at `Worker.run()` startup on PostgreSQL and MySQL when `CONN_MAX_AGE = 0` and no PostgreSQL pool is configured. MySQL ignores pool options. Each attempt opens one execution connection for task queries and outcome writing. Carries `worker_id`, `database` and `vendor`. The hint opens no connection and never stops the worker. See [connection guidance](production.md#retaining-worker-connections). | +| `worker_connection_hint` | WARNING on PostgreSQL, INFO on MySQL | Once at `Worker.run()` startup, so once per worker process, on PostgreSQL and MySQL when `CONN_MAX_AGE = 0` and no PostgreSQL pool is configured. MySQL ignores pool options. Each attempt opens one execution connection for task queries and outcome writing. Carries `worker_id`, `database` and `vendor`. The hint opens no connection and never stops the worker. See [connection guidance](production.md#retaining-worker-connections). | | `task_claimed` | DEBUG | A task was claimed from the queue. | | `task_started` | DEBUG | Execution of an attempt begins. | | `task_succeeded` | INFO | The task reached SUCCESSFUL. | @@ -8202,48 +8212,90 @@ also resolve to `other`. # Benchmarks -## Queue drain and worker recovery +## 1.4 to 1.7× the throughput of django-tasks-db with retained connections -### 2,000 of 2,000 tasks finished +django-ox is a database-backed worker for Django's Tasks framework. -Every django-ox kill-test trial finished with zero tasks stuck. django-tasks-db left 13 to 19 tasks stuck RUNNING per trial. +### More tasks per second at matched connection settings -Five trials, each with 20 worker kills and replacements, with LOCK_TIMEOUT set to 15 s. +With pooled or persistent connections, django-ox delivered 1.4 to 1.7× django-tasks-db's throughput across 4, 8 and 16 workers. -Execution is at-least-once. The trials recorded 7 repeated effects across 10,000 tasks. Write tasks that can run again safely. +Every task finished in these runs, with zero duplicate starts or completions. + +With persistent connections, worker CPU per task was 1.0 to 1.4 ms for django-ox versus 1.5 to 2.1 ms for django-tasks-db. + +PostgreSQL 16; no-op tasks; django-ox 1.9.0 vs django-tasks-db 0.13.0; medians of five runs. + +Rates are medians of five interleaved runs; ratios are medians of per-round ratios. The test machine had background load; see How we measured. + +Retain worker connections to speed up both backends: `CONN_MAX_AGE = 600`, or `OPTIONS["pool"]` with `CONN_MAX_AGE = 0`. Django 6.0 rejects a pool with nonzero `CONN_MAX_AGE`; see [Retaining worker connections](production.md#retaining-worker-connections). + +Both backends ran the same settings in each row. Rates are tasks per second. + +| Workload | Setting | django-ox, tasks/s | django-tasks-db, tasks/s | Ratio | +|---|---|---:|---:|---:| +| 20,000 tasks, 4 workers | Persistent | 1,771 | 1,211 | **1.46×** | +| 20,000 tasks, 4 workers | Persistent, health checks | 1,590 | 1,123 | **1.42×** | +| 20,000 tasks, 4 workers | Pooled | 1,718 | 1,216 | **1.41×** | +| 20,000 tasks, 4 workers | Pooled, health checks | 1,561 | 1,127 | **1.40×** | +| 20,000 tasks, 8 workers | Persistent | 2,668 | 1,735 | **1.53×** | +| 20,000 tasks, 8 workers | Persistent, health checks | 2,324 | 1,594 | **1.45×** | +| 20,000 tasks, 8 workers | Pooled | 2,569 | 1,724 | **1.51×** | +| 20,000 tasks, 8 workers | Pooled, health checks | 2,269 | 1,556 | **1.47×** | +| 20,000 tasks, 16 workers | Persistent | 3,065 | 1,959 | **1.54×** | +| 20,000 tasks, 16 workers | Persistent, health checks | 2,693 | 1,787 | **1.50×** | +| 20,000 tasks, 16 workers | Pooled | 3,011 | 1,938 | **1.55×** | +| 20,000 tasks, 16 workers | Pooled, health checks | 2,625 | 1,786 | **1.46×** | +| 100,000 tasks, 16 workers | Persistent | 3,358 | 1,989 | **1.69×** | +| 100,000 tasks, 16 workers | Persistent, health checks | 2,884 | 1,816 | **1.59×** | +| 100,000 tasks, 16 workers | Pooled | 3,289 | 1,952 | **1.68×** | +| 100,000 tasks, 16 workers | Pooled, health checks | 2,833 | 1,815 | **1.57×** | + +Persistent: `CONN_MAX_AGE = 600`. Persistent, health checks: the same with `CONN_HEALTH_CHECKS = True`. Pooled: `OPTIONS["pool"]` with `CONN_MAX_AGE = 0`. Pooled, health checks: the same with `CONN_HEALTH_CHECKS = True`. + +Draining 100,000 tasks at 16 workers took about 30 s for django-ox (29.7 to 30.1 s over five runs) and about 50 s for django-tasks-db (49.3 to 51.2 s) with persistent connections; with a pool 30.2 to 30.5 s vs 49.9 to 51.6 s. -### Up to 20% faster queue drain than django-tasks-db +### Default connection settings -Worker counts match in each row. Rates are tasks per second. +| Workload | Setting | django-ox, tasks/s | django-tasks-db, tasks/s | Ratio | +|---|---|---:|---:|---:| +| 20,000 tasks, 4 workers | Defaults | 430 | 373 | **1.1 to 1.2×** | -| Queued tasks | Workers per backend | django-ox, tasks/s | django-tasks-db, tasks/s | django-ox faster | -|---|---:|---:|---:|---:| -| 2,000 | 1 | 118.5 | 103.9 | **14.0%** | -| 2,000 | 4 | 426.3 | 387.1 | **10.1%** | -| 20,000* | 1 | 115.2 | 97.4 | **18.2%** | -| 20,000* | 4 | 461.8 | 401.6 | **15.0%** | +`CONN_MAX_AGE = 0` and no pool. Spread is the standard deviation of the five rates divided by their mean. -In each row, every django-ox run outperformed every django-tasks-db run. Results apply to these workloads, not every application. +The spread between runs is 9% on both, so treat this as about 1.1×. + +## Worker recovery, enqueue and retries (django-ox 1.3.0) + +### 2,000 of 2,000 tasks finished + +Every django-ox 1.3.0 kill-test trial finished with zero tasks stuck. django-tasks-db left 13 to 19 tasks stuck RUNNING per trial. + +Five trials, each with 20 worker kills and replacements, with LOCK_TIMEOUT set to 15 s. The wallpaper process rendered during kill trials 2 and 3 only; kill counts were unaffected. + +Execution is at-least-once. The trials recorded 7 repeated effects across 10,000 tasks. Write tasks that can run again safely. ### Enqueue 10,000 tasks in 0.71 s -`enqueue_many()` inserted all 10,000 rows in one transaction, including COMMIT. Run times ranged from 0.69 to 0.72 s. +In django-ox 1.3.0, `enqueue_many()` inserted all 10,000 rows in one transaction, including COMMIT. Run times ranged from 0.69 to 0.72 s. ### 20 of 20 tasks retried successfully -Each task raised an exception on its first attempt. django-ox retried them automatically, and all succeeded on attempt two, every run. django-tasks-db marked all 20 FAILED after the first exception. +Each task raised an exception on its first attempt. django-ox 1.3.0 retried them automatically, and all succeeded on attempt two, every run. django-tasks-db marked all 20 FAILED after the first exception. ### How we measured -Oxpull, 2026-09-19. django-ox 1.3.0 vs django-tasks-db 0.13.0. +Matched-connection run: Oxpull, 2026-10-14. django-ox 1.9.0 vs django-tasks-db 0.13.0. -PostgreSQL 16 ran locally in Docker on an Apple M1 Max. Task bodies were no-ops. Each cell used five interleaved runs per backend. +PostgreSQL 16 ran locally in Docker. Django 6.0.8, psycopg 3.3.6. Task bodies were no-ops with a 256-byte payload. Each cell used five interleaved runs per setting and backend, with order balanced across rounds. Both backends got identical Django database options; each package's own defaults applied otherwise. -Package defaults applied except for the kill test, with LOCK_TIMEOUT set to 15 s. +The test machine had other load (one-minute load average 1.9 to 6.8 before a run), so absolute rates are lower than a quiet machine would give. -*The wallpaper process rendered during kill trials 2 and 3 only; kill counts were unaffected. The four drain cells (2,000 and 20,000 tasks) ran interleaved in one window; the 100,000-task run was one pair, not five. +At one connection per task, the test machine runs out of ports at about 550 new connections a second, so we show no default-settings throughput above 4 workers. -[**Download both raw JSON files**](https://github.com/oxpull/django-ox/tree/main/benchmarks) +Kill, enqueue and retry sections: Oxpull, 2026-09-19. django-ox 1.3.0 vs django-tasks-db 0.13.0. PostgreSQL 16 ran locally in Docker. Task bodies were no-ops. Each cell used five interleaved runs per backend. Package defaults applied except for the kill test, with LOCK_TIMEOUT set to 15 s. + +[**Download the raw JSON files**](https://github.com/oxpull/django-ox/tree/main/benchmarks) [**Get started**](background-tasks.md) @@ -8598,7 +8650,7 @@ process that shares a database before anything writes the new status. The release notes name the value, say how it reads through `django.tasks`, and give the upgrade and rollback steps. -Pin accordingly: `django-ox~=1.9.0` accepts patch releases only; +Pin accordingly: `django-ox~=1.9.1` accepts patch releases only; `django-ox~=1.9` accepts the current major line. ## Deprecation policy @@ -8784,8 +8836,8 @@ OK: backlog=0 oldest_age=none last_claim_age=none `manage.py check` reports schedule and backend policy errors through `django_ox.E002` to `E005`, `E010` and `E011`, and warnings through `django_ox.W003` and `django_ox.W004`, before deployment. Database acceptance of schedule arguments is determined at dispatch. Alert on `schedule_dispatch_error`, `schedule_dispatch_failed` and `schedule_tick_unreadable`. For stored schedules, also alert on `schedule_row_skipped`, `schedule_source_unavailable` and `schedule_boundary_heal_failed`. Review `django_ox.W003` and `django_ox.W004` warnings too; `W004` asks for a non-bool integer from 1 to 32767 while retaining compatible legacy values. Invalid per-task fields raise `InvalidTask` (`InvalidTaskError` on the -backport) when the task is built, normally at import, rather than appearing -as a system-check message. +backport) when the task is built, normally at import. They do not appear +as system-check messages. Start a worker in its own process, next to the web server, under the same supervisor: @@ -8923,9 +8975,8 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv ## Facts to get right -- `QUEUES` sits beside `OPTIONS`, not inside it. Inside `OPTIONS` it is - ignored without warning; the symptom is `InvalidTask: Queue 'X' is not - valid for backend.` +- `QUEUES` sits beside `OPTIONS`. Inside `OPTIONS` it is ignored without + warning; the symptom is `InvalidTask: Queue 'X' is not valid for backend.` - Tasks use the Tasks-framework API. Import `task` from `django.tasks` on Django 6.0+, or `django_tasks` on 5.2 LTS, decorate with `@task`, and call `.enqueue(...)`. No django-ox import is needed for declarations. @@ -8938,8 +8989,8 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv substitution. Keep `OxBackend` and use the public, provisional `django_ox.testing.run_tasks()` helper to test queued execution. - The worker imports a task by its dotted path, so the module must be - importable in the worker process and the worker runs the same code as the - producer; nothing is registered and there is no autodiscovery. `async def` + importable in the worker process. The worker runs the same code as the + producer. Nothing is registered and there is no autodiscovery. `async def` tasks run. - Priority and deferral are Django API: `task.using(priority=N)` with N from -100 to 100, higher first, and `task.using(run_after=...)` with a timedelta @@ -8950,18 +9001,18 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv A second signal forces exit 130; a process that cannot act on signals needs SIGKILL. - `enqueue()` is one INSERT on the database the router sends `OxTask` to, - `default` unless you wrote a router. Two things make the commit joint: a - `transaction.atomic()` opened on that database, because a bare `atomic()` - opens on `default`; and the caller's own rows written there too. With both, - the task and those rows commit or roll back together, and the task is - visible to workers only after commit. Do not add `transaction.on_commit()` - around it. Rows written on another connection give two transactions, not - one. + `default` unless you wrote a router. Two things make the commit joint. The + first is a `transaction.atomic()` opened on that database, because a bare + `atomic()` opens on `default`. The second is the caller's own rows written + there too. With both, the task and those rows commit or roll back together, + and the task is visible to workers only after commit. Do not add + `transaction.on_commit()` around it. Rows written on another connection + give two transactions. - Many calls of one task go through `django_ox.bulk.enqueue_many(task, [(args, kwargs), ...])`: one INSERT per 1,000 rows, one transaction, results in input order. Set queue, priority and `run_after` once with `.using(...)`. - Execution is at-least-once. Write tasks to be safe to run twice: guard on - state already in the database, not on a flag in memory. + state already in the database. Do not guard on a flag in memory. - The task function runs outside any transaction. Open `transaction.atomic()` inside the task when it needs `select_for_update()`. - An attempt is consumed at claim time, so a worker dying mid-run uses one. @@ -8972,22 +9023,21 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv delays are not capped. - The worker schedules lease renewal every `LOCK_TIMEOUT / 3` seconds; task length is not bounded by `LOCK_TIMEOUT` while renewals succeed. - Renewal needs a database connection: if the worker cannot refresh its + Renewal needs a database connection. If the worker cannot refresh its lease for `LOCK_TIMEOUT`, the reaper can hand the task to another worker even while it is alive. - With Django's PostgreSQL pool, provide at least `concurrency + 1` pooled - connections per worker process; add a spare pooled connection if fallback + connections per worker process. Add a spare pooled connection if fallback must work under full load. Budget up to `max_size + 2` server connections - per process and database alias in normal operation, including any added spare: - one private renewal connection - and one possible watchdog connection. A worker whose tasks never use - timeouts needs only the first, for `max_size + 1` in normal operation. - A worker stopping with recovery pending opens one more private connection - for its stop-time recovery look: budget up to `max_size + 3` during that - look, or `max_size + 2` if its tasks never use timeouts. - An absent timeout in `OPTIONS` does not - establish that, because tasks can declare their own. Account for all - processes, aliases, other clients and reserved slots. + per process and database alias in normal operation, including any added + spare. The 2 covers one private renewal connection and one possible + watchdog connection. A worker whose tasks never use timeouts needs only + the first, for `max_size + 1` in normal operation. A worker stopping with + recovery pending opens one more private connection for its stop-time + recovery look. Budget up to `max_size + 3` during that look, or + `max_size + 2` if its tasks never use timeouts. An absent timeout in + `OPTIONS` does not establish that, because tasks can declare their own. + Account for all processes, aliases, other clients and reserved slots. - The worker polls; `--interval` (default 1.0 s) is the idle sleep, so a task starts within one interval of its commit. There is no LISTEN/NOTIFY. - Each attempt's timeout comes from the task's `timeout`, then its queue's @@ -8999,31 +9049,32 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv later is recorded as failed and the worker exits 75 so its supervisor restarts it. The stuck-thread path uses the worker's backoff without calling user callbacks. `django_ox.remaining()` reads the seconds left - from inside a task, including for a task-declared timeout. On a thread - a coverage tool or a debugger is watching (a `sys.settrace` hook, or a - `sys.monitoring` tool with events enabled) nothing is raised inside a - sync task: the worker logs `timeouts_backstop_only`, a task that returns - within `TASK_TIMEOUT_GRACE` is recorded as whatever it did, and one still - running then is recorded as failed and recycles the worker. An async task - is cancelled at the deadline either way. -- Claiming: one `UPDATE ... SKIP LOCKED ... RETURNING` statement on - PostgreSQL; `SELECT ... FOR UPDATE SKIP LOCKED` on MySQL 8+; an atomic - compare-and-set UPDATE on SQLite and other databases without `SKIP LOCKED`. + from inside a task, including for a task-declared timeout. A coverage + tool or a debugger may be watching a thread (a `sys.settrace` hook, or a + `sys.monitoring` tool with events enabled). On such a thread nothing is + raised inside a sync task. The worker logs `timeouts_backstop_only`. A + task that returns within `TASK_TIMEOUT_GRACE` is recorded as whatever it + did. One still running then is recorded as failed and recycles the + worker. An async task is cancelled at the deadline either way. +- Claiming is one `UPDATE ... SKIP LOCKED ... RETURNING` statement on + PostgreSQL. On MySQL 8+ it is `SELECT ... FOR UPDATE SKIP LOCKED`. On + SQLite and other databases without `SKIP LOCKED` it is an atomic + compare-and-set UPDATE. - `--concurrency N` is a thread pool in one process. `--processes N` runs N - such workers under one supervisor; a worker process that dies is restarted - after one second, doubling to 30 s, and more than five deaths of one slot - in a minute stops the supervisor with exit 1. CPU-bound work wants + such workers under one supervisor. A worker process that dies is restarted + after one second, doubling to 30 s. More than five deaths of one slot in a + minute stops the supervisor with exit 1. CPU-bound work wants `--processes N --concurrency 1`. -- Recurring tasks go in `OPTIONS["SCHEDULES"]`, with either `cron` or `every`, - not both. Every worker dispatches them; there is no scheduler process to - start. A tick that passes while every worker is down is enqueued once on - recovery, and older missed ticks are skipped. +- Recurring tasks go in `OPTIONS["SCHEDULES"]`, with `cron` or `every`. Each + schedule takes exactly one of the two. Every worker dispatches them; there + is no scheduler process to start. A tick that passes while every worker is + down is enqueued once on recovery, and older missed ticks are skipped. - Schedules can live in the database instead, editable in the admin. Set `OPTIONS["SCHEDULE_SOURCE"]` to `"django_ox.stored.DatabaseScheduleSource"`. - A row names a key registered with `@schedulable`, never an import path, so - admin access does not become permission to run arbitrary code. The decorator - takes effect only when its module is imported, and django-ox imports each - installed app's `tasks` module. Write rows with + A row names a key registered with `@schedulable`. Import paths are not + accepted, so admin access does not become permission to run arbitrary + code. The decorator takes effect only when its module is imported, and + django-ox imports each installed app's `tasks` module. Write rows with `django_ox.stored.create_schedule`, `create_schedules` or `update_schedule`; `objects.create()` skips validation and leaves the activation boundary stale. Retiming a stored schedule reschedules it from @@ -9036,15 +9087,15 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv field and message. A non-mapping row or unsupported key raises `TypeError` immediately. Permission refusals raise a separate `PermissionDenied`, naming every - denied row. A database refusal at the check or write raises for one row; - a concurrent name conflict raises `IntegrityError` and rolls back the + denied row. A database refusal at the check or write raises for one row. + A concurrent name conflict raises `IntegrityError` and rolls back the entire call. See [Writing schedules from code](stored-schedules.md#writing-schedules-from-code). - `ox_import_beat_schedules` reads django-celery-beat's database scheduler tables: `PeriodicTask`, `CrontabSchedule` and `IntervalSchedule`. It prints code for review and application. First shipped in 1.2.0 and - rebuilt in 1.8.0, it now prints one all-or-nothing `create_schedules` call - and lists each row it cannot translate with a named reason. It reads + rebuilt in 1.8.0, it now prints one all-or-nothing `create_schedules` call. + It lists each row it cannot translate with a named reason. It reads crontabs with Celery's grammar. Use `--beat-timezone ZONE` to supply beat's timezone. Its `--database` flag selects the alias holding the django-celery-beat @@ -9074,7 +9125,7 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv - Heartbeats are updated at each poll and drain pass head and before every claim attempt. A busy pass can make up to `--concurrency` claims, but updates occur between claims. Budget for reap and dispatch work plus one - claim, the poll interval and scheduling margin, not concurrency + claim, the poll interval and scheduling margin. Do not budget concurrency multiplied by claim latency. With Django's PostgreSQL connection pool, a refused database can hold a pass for the pool's `timeout`, 30 seconds by default. Include that wait in the freshness budget. @@ -9089,11 +9140,12 @@ and [earlier security advisory](https://github.com/oxpull/django-ox/security/adv [heartbeat liveness](monitoring.md#freshness-and-database-isolation) for freshness sizing and database-stall restart tradeoffs. Docker and Compose outside Swarm mark a container unhealthy when its healthcheck fails. Their - restart policies act on process exit rather than healthcheck status. + restart policies act on process exit. A failed healthcheck does not + restart the container. - `path("ox/", include("django_ox.urls"))` mounts `GET /ox/metrics`, the queue stats as Prometheus gauges. It has no authentication of its own; wrap it with `login_required` or restrict it by network. -- Run `migrate` before rolling workers, not from the worker. +- Run `migrate` before rolling workers. Do not run it from the worker. - `django_ox.actions.retry(result_id)` requeues a FAILED or LOST task for one more attempt. `django_ox.actions.discard(result_id)` closes a READY, WAITING, FAILED or LOST task without running it. Neither touches a RUNNING task. @@ -9214,6 +9266,30 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.1] - 2026-10-17 + +**Upgrading:** No database migration is required. + +### Changed + +- `worker_connection_hint` is now a WARNING on PostgreSQL. A worker with + `CONN_MAX_AGE = 0` and no `OPTIONS["pool"]` opens a new connection for every + task, about 5 ms each on a local server, and logs one line at startup saying + so. To remove it, set `CONN_MAX_AGE = 600` or configure Django's pool and + leave `CONN_MAX_AGE` at 0. MySQL keeps the INFO line, and SQLite logs + nothing. "Retaining worker connections" in `docs/production.md` + lists both settings and the pool size to use. +- The benchmarks page now reports django-ox 1.9.0 against django-tasks-db 0.13.0 + at the same connection settings. + +### Fixed + +- A worker whose database refuses the connection for an outcome write, as a + host out of ephemeral ports does under `CONN_MAX_AGE=0`, now tries the write + again after waits of 50 ms doubling to 1.6 s, 3.15 s in all, and the waits + total at most a tenth of `LOCK_TIMEOUT`. A worker told to stop ends a wait at + once. + ## [1.9.0] - 2026-10-12 **Upgrading:** No database migration is required. @@ -11627,6 +11703,7 @@ Initial release. the public API surface, the pre-1.0 SemVer rule, the deprecation window, and the supported Python and Django matrix. +[1.9.1]: https://github.com/oxpull/django-ox/compare/v1.9.0...v1.9.1 [1.9.0]: https://github.com/oxpull/django-ox/compare/v1.8.0...v1.9.0 [1.7.0]: https://github.com/oxpull/django-ox/compare/v1.6.0...v1.7.0 [1.8.0]: https://github.com/oxpull/django-ox/compare/v1.7.0...v1.8.0 diff --git a/docs/migrating.md b/docs/migrating.md index 49f6707..cedc101 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -52,14 +52,14 @@ Of `db_worker`'s worker-specific options, `--backend`, `--interval`, - Replace `--queue-name` with `--queues`. - `--batch` maps by name to `--batch`. In `ox_worker`, it ends after an - error-free empty claim pass with no local tasks in flight; it does not + error-free empty claim pass with no local tasks in flight. It does not wait for future tasks, retry backoff or deferred releases. `db_worker` exits with a traceback when the database is unreachable or its tables are missing. `ox_worker` keeps retrying. An abandoned dispatch pass prevents batch completion until a later dispatch pass completes. A schedule-scoped failure is reported as `schedule_dispatch_error` and does not hold the - batch open. Exit 0 means the batch finished, not that every schedule - enqueued. Give the job a timeout, as + batch open. Exit 0 means the batch finished. It does not mean that every + schedule enqueued. Give the job a timeout, as [Running as a job](production.md#running-as-a-job) explains. - `--max-tasks N` maps by name to `--max-tasks N`. In `ox_worker`, every claimed attempt consumes one of N, including failed attempts and repeat @@ -75,7 +75,7 @@ Of `db_worker`'s worker-specific options, `--backend`, `--interval`, Use `--queues default` to preserve the old default. Do not use `--queues '*'` to select all queues. `*` is treated as a literal -queue name, not a wildcard. Omit `--queues` to run every configured queue. +queue name. Omit `--queues` to run every configured queue. ## From Celery @@ -127,8 +127,8 @@ decorator. See [Per-task policy](configuration.md#per-task-policy). In django-ox, `max_attempts` counts worker claims, including the first. A worker that dies after claiming a task consumes an attempt too. Choose -the budget from that definition rather than copying a retry count from -another task runner. +the budget from that definition. Do not copy a retry count from another +task runner. One difference in behaviour to read before you switch. With a broker, `enqueue` leaves your process immediately. If the surrounding transaction then @@ -144,7 +144,7 @@ implement Celery's chain, group or chord APIs, or routing across multiple brokers. [Batches](features/batches.md) provide grouped work, and provisional [workflows](features/workflows.md) provide dependency DAGs. Both require Django 6.0+ and the optional `oxpull` app. Review their -semantics rather than treating them as drop-in Celery replacements. +semantics. They are not drop-in Celery replacements. ## From huey @@ -302,7 +302,7 @@ No drain window available? Run both. Old workers keep serving the old table while new work goes to django-ox. They cannot see each other's rows. Both systems run tasks at least once, so your tasks should already be -idempotent. Confirm it before you start rather than halfway through. +idempotent. Confirm it before you start. ## Migrating away @@ -318,13 +318,12 @@ compatibility guarantee. In the other direction, django-ox reads policy only from `PolicyTask`; other Task classes inherit its backend defaults when rebound with `.using(backend=...)`. -Transactional enqueue also needs deciding on the way in rather than on -the way out. Enqueueing inside `transaction.atomic()` on the database that -holds `OxTask` ties the task to that transaction, so it disappears on -rollback. A broker-based backend cannot do this: the enqueue leaves your -process the moment you call it. Code that depends on a rollback removing -a task will behave differently once the queue lives in a broker, and it -will do so quietly. +Transactional enqueue also needs deciding on the way in. Enqueueing inside +`transaction.atomic()` on the database that holds `OxTask` ties the task to +that transaction, so it disappears on rollback. A broker-based backend +cannot do this: the enqueue leaves your process the moment you call it. Code +that depends on a rollback removing a task will behave differently once the +queue lives in a broker. It will do so quietly. If you want to keep that option open, wrap enqueues in `transaction.on_commit()`, the way a broker-based backend requires. django-ox diff --git a/docs/monitoring.md b/docs/monitoring.md index 3fb60c1..5726b96 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -723,7 +723,7 @@ A stopped reconciler sweep is not a completed sweep. Check `visited_count`, `unv | `worker_started` | INFO | The run loop starts. | | `connection_pool_too_small` | WARNING | Once per `Worker.run()`, after `worker_started` and before threads start: the worker alias's effective PostgreSQL pool maximum is below `concurrency + 1`. Each `--processes` child checks separately. The warning does not resize the pool, refuse startup, measure available server slots, or reserve fallback capacity. | | `connection_pool_cannot_drain` | WARNING | Pooled PostgreSQL only. The installed psycopg_pool has no `ConnectionPool.drain()` method, which was added in 3.3.0. Emitted once per `Worker.run()`, after `worker_started` and the `connection_pool_too_small` check, before threads start. Each `--processes` child checks separately. Also emitted at the first sweep of such a pool in a process, for each database alias, unless startup already emitted it in that process. Concurrent first sweeps may emit duplicate warnings. The sweep record is the only notice available to callers of `run_once()` or `run_tasks()`. Idle connections are still tested as in django-ox 1.7.0, preserving restart recovery but also the wait on a connection that stays open without answering. A stop is not read until that test returns. Upgrade psycopg_pool to 3.3.3 or newer. This warning does not refuse startup or change the pool. No traceback. Keys: `event` and `database` always; `worker_id` only on the startup record. | -| `worker_connection_hint` | INFO | Once at `Worker.run()` startup on PostgreSQL and MySQL when `CONN_MAX_AGE = 0` and no PostgreSQL pool is configured. MySQL ignores pool options. Each attempt opens one execution connection for task queries and outcome writing. Carries `worker_id`, `database` and `vendor`. The hint opens no connection and never stops the worker. See [connection guidance](production.md#retaining-worker-connections). | +| `worker_connection_hint` | WARNING on PostgreSQL, INFO on MySQL | Once at `Worker.run()` startup, so once per worker process, on PostgreSQL and MySQL when `CONN_MAX_AGE = 0` and no PostgreSQL pool is configured. MySQL ignores pool options. Each attempt opens one execution connection for task queries and outcome writing. Carries `worker_id`, `database` and `vendor`. The hint opens no connection and never stops the worker. See [connection guidance](production.md#retaining-worker-connections). | | `task_claimed` | DEBUG | A task was claimed from the queue. | | `task_started` | DEBUG | Execution of an attempt begins. | | `task_succeeded` | INFO | The task reached SUCCESSFUL. | diff --git a/docs/patterns.md b/docs/patterns.md index f5b8919..8a02e70 100644 --- a/docs/patterns.md +++ b/docs/patterns.md @@ -31,7 +31,7 @@ database as the `User` row, your default one, so the task becomes visible to workers only when the transaction commits. If `create_user` is rolled back further up, the email task disappears with it. -Pass the id, not the object. Arguments are stored as JSON, and a stale copy of a +Pass the model's id. Arguments are stored as JSON, and a stale copy of a model is a bug waiting to happen. ## Retry a flaky third-party call @@ -154,7 +154,7 @@ return render(request, "report_pending.html", {"attempts": result.attempts}) `is_finished` covers the last two. A task whose worker vanished without reporting also reads as `FAILED`; [the reaper](production.md#the-reaper) explains what that record contains. Return values are stored as JSON, so -return a URL or an id rather than a file or a model. +return a URL or an id. ## Defer work to a specific time @@ -287,10 +287,10 @@ def charge_order(order_id): order.save(update_fields=["charged_at"]) ``` -Guard on state you have written, not on a flag you set in memory. The row lock -serialises concurrent attempts, and the idempotency key covers the gap where the -charge succeeds but the transaction does not commit. Use one whenever the -external system offers it. +Guard on state you have written. Do not guard on a flag you set in memory. +The row lock serialises concurrent attempts, and the idempotency key covers +the gap where the charge succeeds but the transaction does not commit. Use +one whenever the external system offers it. **Your task is not run inside a transaction.** The worker manages its own for claiming and bookkeeping, but your function is called outside them, so diff --git a/docs/production.md b/docs/production.md index ff539ed..5601610 100644 --- a/docs/production.md +++ b/docs/production.md @@ -863,20 +863,31 @@ PgBouncer and third-party pools have not been tested. ### Retaining worker connections -With `CONN_MAX_AGE = 0` and no pool, Django closes initialized execution connections before and after each attempt. The next attempt opens another connection; the poll loop keeps its claim connection. Task ORM queries and outcome writing use the same execution connection on the worker's database alias. `worker_connection_hint` reports this setting once at INFO during `Worker.run()` startup on PostgreSQL and MySQL. +Django's default is `CONN_MAX_AGE = 0`. With it and no pool, each task attempt opens a new database connection and closes it after. On a local PostgreSQL that costs about 5 ms a task. The poll loop keeps its own connection. Task queries and the outcome write share the attempt's connection on the worker's database alias. -To retain connections, configure the worker's alias in a dedicated worker settings module. Launch workers with that module to leave web processes unchanged. For PostgreSQL or MySQL without a pool, set a positive connection age and enable health checks: +When a worker starts on PostgreSQL with that setup, it logs one warning, `worker_connection_hint`, naming the alias and the fix. MySQL gets the same event at INFO. SQLite opens a file and logs nothing. + +Pick one of two settings for the worker's alias. Put it in a dedicated worker settings module and launch workers with that module, so web processes stay as they are. To keep connections open between tasks: ```python DATABASES["default"]["CONN_MAX_AGE"] = 600 -DATABASES["default"]["CONN_HEALTH_CHECKS"] = True ``` -For PostgreSQL, Django's pool requires psycopg 3, `psycopg_pool` and `CONN_MAX_AGE = 0`; use the pool example above instead of a positive age. MySQL has no Django pool; setting `OPTIONS["pool"]` does not enable one. +On PostgreSQL you can use Django's pool instead. It needs psycopg 3 and `psycopg_pool`, and `CONN_MAX_AGE` stays at 0: + +```python +DATABASES["default"]["OPTIONS"] = { + "pool": {"min_size": 0, "max_size": 16, "timeout": 30} +} +``` + +Django refuses to open a pool when `CONN_MAX_AGE` is above 0, so use one or the other. MySQL has no Django pool, and `OPTIONS["pool"]` does nothing there. + +Each worker process has its own pool. Set `max_size` to at least `concurrency + 1`, and budget the extra connections in [Budget server capacity](#budget-server-capacity). A `max_size` of 16 covers up to 15 task threads in a process. -Allow pool capacity for task threads and polling, plus the separate renewal, watchdog and stop-time recovery connections described above. Each worker process has its own pool. Retained connections carry session state between tasks; follow the responsibilities below. +`CONN_HEALTH_CHECKS = True` tests a kept connection before each use. It costs about 10% throughput. Set it only if your network drops idle connections. -Persistence and pooling remain opt-in. The default connection lifecycle, transactions, delivery and schema are unchanged. +Kept connections carry session state from one task to the next. Follow the rules in the next section. ### Session state and connection failures @@ -886,7 +897,7 @@ Each execution thread keeps its own connection context, separate from other task Measured default runs opened one connection per task attempt plus one for polling; ORM queries added no connections. Persistence or PostgreSQL pooling opened two connections in total: one for execution and one for polling, regardless of task count. These counts are exact on PostgreSQL and SQLite; MySQL counting tests allow one extra connection per thread because Django's MySQL backend reads the server version through a throwaway connection the first time a thread uses it. After a server-ended idle session, querying tasks used one extra attempt, then succeeded on a new session. Query-free tasks reconnected at outcome writing without spending an attempt; every task completed. Workers with and without persistence shared a queue: all 40 tasks succeeded, each running once with one claim and one epoch. Both workers ran tasks. These measurements cover same-release batch workers, PostgreSQL, MySQL and SQLite connection counts, PostgreSQL pooling, and PostgreSQL and MySQL idle-session recovery. -Set `CONN_HEALTH_CHECKS = True`; health checks reduce stale-connection failures but cannot prevent later disconnects. +With `CONN_HEALTH_CHECKS = True`, stale connections fail less often. A health check cannot prevent a disconnect that comes after it. ### psycopg_pool versions and recovery diff --git a/docs/stability.md b/docs/stability.md index 0640af8..2677677 100644 --- a/docs/stability.md +++ b/docs/stability.md @@ -345,7 +345,7 @@ process that shares a database before anything writes the new status. The release notes name the value, say how it reads through `django.tasks`, and give the upgrade and rollback steps. -Pin accordingly: `django-ox~=1.9.0` accepts patch releases only; +Pin accordingly: `django-ox~=1.9.1` accepts patch releases only; `django-ox~=1.9` accepts the current major line. ## Deprecation policy diff --git a/docs/use-cases.md b/docs/use-cases.md index e99194e..b994303 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -83,7 +83,7 @@ def register(username, email): That is the advantage over sending to a separate broker: the enqueue can share your database transaction. django-tasks-db shares this property too. -The transaction covers the database writes, not email delivery. Make the task safe to repeat. +The transaction covers the database writes. Email delivery sits outside it, so make the task safe to repeat. ## Run recurring Django tasks without a scheduler process @@ -114,7 +114,7 @@ Schedules need at least one running worker and a usable database. A schedule-sco ## Deduplicate a repeated webhook -A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue rather than making the queue do the same work again. +A payment provider sends the same event twice. Your webhook queues the same invoice sync twice. Stop the duplicate at enqueue. Without a unique-task declaration, these calls create separate tasks: @@ -140,7 +140,7 @@ The same app provides [batches](features/batches.md) to track grouped work, [rat ## When to choose another tool -Choose another backend if you need to revoke a chosen running task, require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md), not those Celery APIs. +Choose another backend if you need to revoke a chosen running task. Choose another if you require Celery's chain, group or chord APIs, or exceed the throughput one database can serve. django-ox provides [batches](features/batches.md) and provisional [dependency workflows](features/workflows.md). Transactional enqueue requires the task and model rows to share a database. A separate queue database cannot give you that shared commit. diff --git a/docs/wagtail.md b/docs/wagtail.md index 03fd8b2..d6684b8 100644 --- a/docs/wagtail.md +++ b/docs/wagtail.md @@ -22,7 +22,7 @@ Wagtail 7.3.x declares Django 6.0 support. On Django 6.1, pip permits Wagtail 7.3.x, but Wagtail 7.3.4 fails in Wagtail's own search with `OperationalError: unable to use function MATCH`, both with and without django-ox. -Do not use Wagtail 7.3.0 with this setup on Django 5.2 or 6.0. Its dependency constraints resolve `django-tasks` 0.9.0. On Django 6.0, that version crashes during app loading (`django.setup()`) once `TASKS` selects a backend built on core Django's `Task`. Every `manage.py` command and the web process fail, not just migrations. +Do not use Wagtail 7.3.0 with this setup on Django 5.2 or 6.0. Its dependency constraints resolve `django-tasks` 0.9.0. On Django 6.0, that version crashes during app loading (`django.setup()`) once `TASKS` selects a backend built on core Django's `Task`. Every `manage.py` command and the web process fail. Upgrade Wagtail before installing django-ox into a Wagtail 7.3.0 environment. @@ -66,7 +66,7 @@ TASKS = { } ``` -If `TASKS` already exists, update its `default` entry rather than replacing other backends. Wagtail's bare `@task()` binds to backend alias `default` and queue `default`. +If `TASKS` already exists, update its `default` entry. Leave the other backends in place. Wagtail's bare `@task()` binds to backend alias `default` and queue `default`. Run migrations: @@ -126,13 +126,13 @@ Attempts count claims. The default `MAX_ATTEMPTS` is 3. Every running worker runs a reaper every `min(30, max(LOCK_TIMEOUT/2, 1))` seconds. With the default 300-second lease and 30-second reaper interval, recovery can take about 330 seconds after the last lease renewal. Recovery requires at least one running worker. -`LOCK_TIMEOUT` and `MAX_ATTEMPTS` belong inside `TASKS["default"]["OPTIONS"]`, not at the top level of Django settings. See [Backend options](configuration.md#options). +`LOCK_TIMEOUT` and `MAX_ATTEMPTS` belong inside `TASKS["default"]["OPTIONS"]`. Do not put them at the top level of Django settings. See [Backend options](configuration.md#options). ### Timeouts Timeouts are opt-in: `TASK_TIMEOUT` defaults to `None`, meaning no backend-wide limit. `TASK_TIMEOUTS` sets per-queue limits. Both belong in the backend's [options](configuration.md#options). On Django 6.1, or Django 5.2 with django-tasks 0.12+, a task's `timeout` declaration takes precedence over both; see [per-task policy](configuration.md#per-task-policy). -Wagtail's frontend-cache purge makes outbound HTTP calls, so consider an appropriate task timeout rather than assuming one is already active. +Wagtail's frontend-cache purge makes outbound HTTP calls, so consider an appropriate task timeout. Do not assume one is already active. ### Objects deleted before their tasks run diff --git a/pyproject.toml b/pyproject.toml index e27a405..fa5cd92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "django-ox" -version = "1.9.0" +version = "1.9.1" description = "Django tasks in your database, with retries and automatic recovery when a worker dies." readme = "README.md" license = "BSD-3-Clause" diff --git a/src/django_ox/__init__.py b/src/django_ox/__init__.py index 342ae0d..d8a96fc 100644 --- a/src/django_ox/__init__.py +++ b/src/django_ox/__init__.py @@ -6,7 +6,7 @@ from .tasks import BackoffCallback, PolicyTask __all__ = ["BackoffCallback", "PolicyTask", "__version__", "deadline", "remaining"] -__version__ = "1.9.0" +__version__ = "1.9.1" def __getattr__(name: str) -> Any: diff --git a/src/django_ox/worker.py b/src/django_ox/worker.py index bc33266..dcf9125 100644 --- a/src/django_ox/worker.py +++ b/src/django_ox/worker.py @@ -140,6 +140,14 @@ # from 3.2. LIBPQ_MIN_CONNECT_TIMEOUT = 2 +# Waits between the tries of an outcome write whose connect was refused, a +# host out of ephemeral ports or a database at its connection limit. Such a +# write finds no connection open and none lost, and a refusal like that clears +# in milliseconds to seconds as other sockets close, so it earns these short, +# doubling waits. The total is also held to a tenth of the lease, so the lease +# the write is fenced by is never the thing it waits out. +OUTCOME_CONNECT_RETRY_DELAYS = (0.05, 0.1, 0.2, 0.4, 0.8, 1.6) + # While lease renewal can get no connection at all, it says so at most this # often, with the number of renewals missed since it last did. MISSED_RENEWAL_REPORT_INTERVAL = 30.0 @@ -1314,6 +1322,31 @@ def _sweep_pool(conn: Any, *, lost: bool = True) -> None: drain() +def _pause(seconds: float, stop: Event) -> None: + """ + The wait between tries of an outcome write. It ends at once when the + worker is told to stop, so a shutdown never sits out the schedule. A seam + for tests. + """ + stop.wait(seconds) + + +def _outcome_connect_delays(lock_timeout: float) -> list[float]: + """ + OUTCOME_CONNECT_RETRY_DELAYS, cut off where their sum would pass a tenth + of the lease. + """ + budget = lock_timeout / 10 + delays: list[float] = [] + for delay in OUTCOME_CONNECT_RETRY_DELAYS: + delay = min(delay, budget) + if delay <= 0: + break + delays.append(delay) + budget -= delay + return delays + + def _close_lost_connection(conn: Any, *, lost: bool | None = None) -> None: """ Close `conn`, which its caller found unusable, then sweep its pool, @@ -3571,7 +3604,17 @@ def _write_outcome_reconnecting( that still answered among them, and does not promise the second try a fresh connection: a replacement may still be connecting, and the database may still be down. The second try then fails as the first - did. There is no third, and no wait before the second. + did. There is no third for a lost connection, and no wait before the + second. + + A write whose connect was refused is the exception, and only it: the + host is out of ephemeral ports, or the database at its connection + limit, and both clear as other sockets close. It finds no connection + open and none lost, and it is tried again after the short waits of + _outcome_connect_delays, doubling from 50 ms and held to a tenth of + the lease in all. The waits are on the outcome write alone, never + the task body, and the fence is the same on every try. A refusal + that outlasts them ends as a second failure does. The second try is earned by a connection that is gone, and the sweep by one that was lost, which is narrower. A write whose @@ -3607,9 +3650,9 @@ def _write_outcome_reconnecting( fenced this attempt. A row that still awaits recovery is the reaper's once its lease expires, as before. The line says as much, which "Unhandled error executing task" did not. Nothing waits - between the two tries, so an outage of any length that spans both - ends here. Whatever the second try raises that is not a database - error is raised. + between the two tries of a lost connection, so an outage of any + length that spans both ends here. Whatever the second try raises that + is not a database error is raised. """ seen = _Seen(self._db_alias) seen.watch() @@ -3630,41 +3673,47 @@ def _write_outcome_reconnecting( # restart left as dead as this one, so the pool is swept first, when # the first try lost a connection and did not merely fail to get # one; even so, the next one is not certain to be alive. - _close_lost_connection(conn, lost=seen.lost()) - try: - written = self._outcome_already_written(db_task, status, fields) - landed = written or self._write_outcome( - db_task, status=status, duration_ms=duration_ms, **fields - ) - except Error as exc: - with suppress(Error): - conn.close() - logger.error( - "Task id=%s path=%s lost its connection to database %r writing " - "the %s outcome of attempt %d/%d (%s), and a new connection " - "failed too (%s: %s). The outcome is unconfirmed, not " - "necessarily absent: the first write may have landed, or " - "another recovery path may already have fenced this attempt. " - "If the row still awaits recovery, the reaper handles it once " - "its lease expires", - db_task.id, - db_task.task_path, - self._db_alias, - status, - db_task.attempts, - db_task.max_attempts, - lost, - type(exc).__qualname__, - _reason(exc), - exc_info=True, - extra=self._log_extra( - "task_outcome_unrecorded", - db_task, - duration_ms=duration_ms, - dropped_status=str(status), - ), - ) - return False + lost_a_connection = seen.lost() + # A connect the server refused: the first try held no connection and + # opened none, so nothing was open and nothing was lost. Such a refusal + # can clear within a moment, so this write waits out a few short + # delays, _outcome_connect_delays, before each try. A try that held + # or opened a connection, lost or not as the driver tells it, goes at + # once, as before. + retry_delays = ( + [] + if lost_a_connection or seen.raws + else _outcome_connect_delays(self.lock_timeout) + ) + if retry_delays and conn.connection is None and not conn.in_atomic_block: + _pause(retry_delays.pop(0), self._stop) + _close_lost_connection(conn, lost=lost_a_connection) + while True: + tried = _Seen(self._db_alias) + tried.watch() + try: + written = self._outcome_already_written(db_task, status, fields) + landed = written or self._write_outcome( + db_task, status=status, duration_ms=duration_ms, **fields + ) + break + except Error as exc: + # Refused only if this try held and opened no connection. + refused = ( + isinstance(exc, InterfaceError | OperationalError) + and not tried.raws + and conn.connection is None + and not conn.in_atomic_block + ) + with suppress(Error): + conn.close() + if refused and retry_delays: + _pause(retry_delays.pop(0), self._stop) + continue + self._log_outcome_unrecorded(db_task, status, duration_ms, lost, exc) + return False + finally: + tried.unwatch() if written: db_task.status = status for name, value in fields.items(): @@ -3695,6 +3744,41 @@ def _write_outcome_reconnecting( ) return landed + def _log_outcome_unrecorded( + self, + db_task: OxTask, + status: OxTask.Status, + duration_ms: int, + lost: str, + exc: Error, + ) -> None: + """The one line for an outcome write that failed on its last try.""" + logger.error( + "Task id=%s path=%s lost its connection to database %r writing " + "the %s outcome of attempt %d/%d (%s), and a new connection " + "failed too (%s: %s). The outcome is unconfirmed, not " + "necessarily absent: the first write may have landed, or " + "another recovery path may already have fenced this attempt. " + "If the row still awaits recovery, the reaper handles it once " + "its lease expires", + db_task.id, + db_task.task_path, + self._db_alias, + status, + db_task.attempts, + db_task.max_attempts, + lost, + type(exc).__qualname__, + _reason(exc), + exc_info=exc, + extra=self._log_extra( + "task_outcome_unrecorded", + db_task, + duration_ms=duration_ms, + dropped_status=str(status), + ), + ) + def _outcome_connection_lost(self) -> bool: """ Whether this thread's connection to the worker's database is gone, @@ -6102,11 +6186,16 @@ def _warn_if_the_connection_pool_is_short(self) -> None: def _hint_if_every_task_reconnects(self) -> None: """ - Report per-attempt execution connections at PostgreSQL and MySQL startup. + Say so at startup when every task attempt opens a new connection. - Log worker_connection_hint once at INFO when CONN_MAX_AGE is zero and - PostgreSQL pooling is absent. Read settings without opening a connection - or preventing worker startup. + That is CONN_MAX_AGE of 0 with no Django pool on PostgreSQL or MySQL. + PostgreSQL gets a warning: a benchmark on a local server measured + about 5 ms per new connection, and enough of them exhaust a host's + ephemeral ports. MySQL stays at INFO, because no such cost has been + measured there. SQLite opens a file and gets nothing. Both lines + are the event worker_connection_hint, once per Worker.run(), so once + per worker process, never per task or thread. + Settings are read and no connection is opened. """ conn = connections[self._db_alias] if conn.vendor not in ("postgresql", "mysql"): @@ -6115,6 +6204,24 @@ def _hint_if_every_task_reconnects(self) -> None: return if conn.vendor == "postgresql" and _pool_options(self._db_alias) is not None: return + extra = { + "event": "worker_connection_hint", + "worker_id": self.worker_id, + "database": self._db_alias, + "vendor": conn.vendor, + } + if conn.vendor == "postgresql": + logger.warning( + "Worker %s: database %r opens a new connection for every task " + "because CONN_MAX_AGE is 0 and OPTIONS['pool'] is not set. " + "Set CONN_MAX_AGE = 600, or set OPTIONS['pool'] and leave " + "CONN_MAX_AGE at 0. See 'Retaining worker connections' in " + "the production guide.", + self.worker_id, + self._db_alias, + extra=extra, + ) + return logger.info( "Worker %s: database %r opens a new execution connection per " "attempt with CONN_MAX_AGE = 0 on PostgreSQL or MySQL. " @@ -6126,12 +6233,7 @@ def _hint_if_every_task_reconnects(self) -> None: "CONN_MAX_AGE = 0.", self.worker_id, self._db_alias, - extra={ - "event": "worker_connection_hint", - "worker_id": self.worker_id, - "database": self._db_alias, - "vendor": conn.vendor, - }, + extra=extra, ) def _warn_if_the_connection_pool_cannot_drain(self) -> None: diff --git a/tests/test_benchmarks_page.py b/tests/test_benchmarks_page.py index 3d2dc28..b3aaf86 100644 --- a/tests/test_benchmarks_page.py +++ b/tests/test_benchmarks_page.py @@ -16,10 +16,16 @@ REPO = Path(__file__).resolve().parent.parent PAGE = REPO / "docs" / "benchmarks.md" SCRIPT = REPO / "benchmarks" / "render_results.py" -RAW = REPO / "benchmarks" / "results-raw-2026-09-19.json" +MATCHED = REPO / "benchmarks" / "results-raw-2026-10-14-matched.json" -OUTPERFORMED = "every django-ox run outperformed every django-tasks-db run" +EVERY_TASK = ( + "Every task finished in these runs, with zero duplicate starts or completions" +) RUNNING_SERIES = "kill trials django-tasks-db RUNNING at the end, in trial order" +CPU = ( + "worker CPU per task was 1.0 to 1.4 ms for django-ox " + "versus 1.5 to 2.1 ms for django-tasks-db" +) def render_results(*args: str) -> subprocess.CompletedProcess[str]: @@ -31,8 +37,24 @@ def render_results(*args: str) -> subprocess.CompletedProcess[str]: ) -def check(page: Path) -> subprocess.CompletedProcess[str]: - return render_results("--check", str(page)) +def check(page: Path, *args: str) -> subprocess.CompletedProcess[str]: + return render_results("--check", str(page), *args) + + +def edited_page(tmp_path, old: str, new: str, *, count: int = 1) -> Path: + text = PAGE.read_text() + assert text.count(old) == count, (old, text.count(old)) + page = tmp_path / "benchmarks.md" + page.write_text(text.replace(old, new)) + return page + + +def edited_matched(tmp_path, change) -> Path: + data = json.loads(MATCHED.read_text()) + change(data["rows"]) + path = tmp_path / "matched.json" + path.write_text(json.dumps(data)) + return path def test_every_number_on_the_page_is_sourced(): @@ -46,42 +68,184 @@ def test_the_statements_on_the_page_are_read(): """A zero for false statements means nothing unless the claims were found.""" result = check(PAGE) for claim in ( - OUTPERFORMED, - "ran interleaved in one window", - "the 100,000-task run was one pair, not five", + EVERY_TASK, + "five interleaved runs", + "order balanced across rounds", + "1.4 to 1.7\u00d7 the throughput", + "across 4, 8 and 16 workers", + CPU, + "spread between runs is 9% on both", + "about 550 new connections a second", + "no default-settings throughput above 4 workers", + "load average 1.9 to 6.8 before a run", + "Draining 100,000 tasks at 16 workers took about 30 s", ): - assert f'ok "{claim}"' in result.stdout, claim + assert f'ok "{claim}' in result.stdout, claim + rows = [ + line + for line in result.stdout.splitlines() + if '"line ' in line and "tasks, " in line + ] + assert len(rows) == 17, rows + assert all(line.strip().startswith("ok") for line in rows) -def test_a_changed_figure_is_reported(tmp_path): +def test_the_retired_drain_section_is_gone(): text = PAGE.read_text() - assert text.count("118.5") == 1 + for gone in ("118.5", "Up to 20%", "up to 20%", "20% faster", "OUTPERFORMED"): + assert gone not in text, gone + + +def test_a_retired_figure_is_not_sourced(tmp_path): page = tmp_path / "benchmarks.md" - page.write_text(text.replace("118.5", "119.5")) + page.write_text(PAGE.read_text() + "\nThe 1.3.0 drain ran at 118.5 tasks/s.\n") result = check(page) assert result.returncode == 1 + assert "NOT SOURCED" in result.stdout + assert "118.5" in result.stdout + + +def test_a_changed_matched_figure_is_reported(tmp_path): + result = check(edited_page(tmp_path, "1,771", "1,781")) + assert result.returncode == 1 assert "NOT SOURCED line" in result.stdout - assert "119.5" in result.stdout + assert "1,781" in result.stdout -def test_a_false_statement_is_reported(tmp_path): - """One django-tasks-db run made the fastest of its cell: the medians, and so - every number on the page, stay as they are, and the claim stops being true.""" - raw = json.loads(RAW.read_text()) - cell = [ - e - for e in raw["e2e"] - if (e["depth"], e["topology"], e["backend"]) == (2000, "1v1", "tasksdb") - ] - max(cell, key=lambda e: e["tasks_per_sec"])["tasks_per_sec"] = 500.0 - changed = tmp_path / "raw.json" - changed.write_text(json.dumps(raw)) - result = render_results("--raw", str(changed), "--check", str(PAGE)) +def test_a_changed_1_3_0_figure_is_reported(tmp_path): + result = check(edited_page(tmp_path, "0.71 s", "0.77 s")) + assert result.returncode == 1 + assert "NOT SOURCED line" in result.stdout + assert "0.77" in result.stdout + + +def test_sourced_figures_in_the_wrong_row_are_reported(tmp_path): + """Both numbers exist in the data, so only the row check can see the swap.""" + page = edited_page(tmp_path, "| 1,771 | 1,211 |", "| 1,211 | 1,771 |") + result = check(page) + assert result.returncode == 1, result.stdout + assert "unsourced 0; false statements 1" in result.stdout + assert "FALSE" in result.stdout + assert "expected 1,771 | 1,211 | 1.46" in result.stdout + + +def test_a_false_sentence_with_sourced_numbers_is_reported(tmp_path): + page = edited_page(tmp_path, "1.0 to 1.4 ms", "1.0 to 1.5 ms") + result = check(page) + assert result.returncode == 1 + assert f'FALSE "{CPU.replace("1.4", "1.5")}' in result.stdout + assert "unsourced 0; false statements 1" in result.stdout + + +def test_a_duplicate_start_makes_every_task_finished_false(tmp_path): + """One repeated start in a shown row; every number on the page stays.""" + + def change(rows): + row = next( + r + for r in rows + if (r["cell"], r["setting"], r["backend"], r["run"]) + == ("S1", "P", "django-ox", 1) + ) + row["duplicate_starts"] = 1 + + result = render_results( + "--matched", str(edited_matched(tmp_path, change)), "--check", str(PAGE) + ) assert result.returncode == 1, result.stdout + result.stderr - assert f'FALSE "{OUTPERFORMED}"' in result.stdout + assert f'FALSE "{EVERY_TASK}' in result.stdout + assert "unsourced 0; false statements 1" in result.stdout + + +def test_a_task_left_running_makes_every_task_finished_false(tmp_path): + def change(rows): + row = next( + r + for r in rows + if (r["cell"], r["setting"], r["backend"], r["run"]) + == ("S3", "G", "django-tasks-db", 2) + ) + row["after_successful"] -= 1 + + result = render_results( + "--matched", str(edited_matched(tmp_path, change)), "--check", str(PAGE) + ) + assert result.returncode == 1 + assert f'FALSE "{EVERY_TASK}' in result.stdout + + +def test_a_worker_exit_code_makes_every_task_finished_false(tmp_path): + def change(rows): + row = next( + r + for r in rows + if (r["cell"], r["setting"], r["backend"], r["run"]) + == ("S2.W8", "PH", "django-ox", 3) + ) + row["worker_exit_codes"] = [1] + + result = render_results( + "--matched", str(edited_matched(tmp_path, change)), "--check", str(PAGE) + ) + assert result.returncode == 1 + assert f'FALSE "{EVERY_TASK}' in result.stdout + + +def test_default_settings_throughput_above_4_workers_is_never_on_the_page(tmp_path): + rows = json.loads(MATCHED.read_text())["rows"] + for r in rows: + if r["setting"] == "D" and r["workers"] > 4 and r["tasks_per_sec"]: + assert f"{round(r['tasks_per_sec']):,}" not in PAGE.read_text() + page = tmp_path / "benchmarks.md" + page.write_text( + PAGE.read_text() + + "\n| 20,000 tasks, 8 workers | Defaults | 691 | 0 | **n/a** |\n" + ) + result = check(page) + assert result.returncode == 1 + assert "not one the page may show a throughput for" in result.stdout + + +def test_a_claim_about_default_settings_above_4_workers_is_checked(tmp_path): + """Had those runs finished cleanly, the sentence hiding them would be false.""" + + def change(rows): + for r in rows: + if r["setting"] == "D" and r["workers"] > 4: + r.update( + status="ok", + completed=r["tasks"], + after_successful=r["tasks"], + duplicate_starts=0, + duplicate_dones=0, + worker_exit_codes=[0], + ) + + result = render_results( + "--matched", str(edited_matched(tmp_path, change)), "--check", str(PAGE) + ) + assert result.returncode == 1 + assert 'FALSE "no default-settings throughput above 4 workers"' in result.stdout + + +def test_a_changed_headline_range_is_reported(tmp_path): + """1.5 is a number the page already holds; only the claim check sees it is wrong.""" + result = check( + edited_page(tmp_path, "delivered 1.4 to 1.7", "delivered 1.5 to 1.7") + ) + assert result.returncode == 1 + assert 'FALSE "1.5 to 1.7\u00d7 django-tasks-db' in result.stdout assert "unsourced 0; false statements 1" in result.stdout +def test_a_changed_spread_claim_is_reported(tmp_path): + result = check( + edited_page(tmp_path, "spread between runs is 9%", "spread between runs is 4%") + ) + assert result.returncode == 1 + assert 'FALSE "spread between runs is 4% on both"' in result.stdout + + def test_figures_the_page_leaves_out_are_listed_and_are_not_an_error(): result = check(PAGE) assert result.returncode == 0, result.stdout + result.stderr @@ -91,11 +255,12 @@ def test_figures_the_page_leaves_out_are_listed_and_are_not_an_error(): def test_require_complete_reports_the_absent_figures(): - result = render_results("--check", str(PAGE), "--require-complete") + result = check(PAGE, "--require-complete") assert result.returncode == 1 - assert "unsourced 0; missing figures 14; false statements 0" in result.stdout + assert "unsourced 0; missing figures 10; false statements 0" in result.stdout assert "MISSING no-recovery window: 291 s" in result.stdout assert f"MISSING {RUNNING_SERIES}" in result.stdout + assert "MISSING matched" not in result.stdout assert "not on the page" not in result.stdout diff --git a/tests/test_connection_count.py b/tests/test_connection_count.py index 4018806..ff60920 100644 --- a/tests/test_connection_count.py +++ b/tests/test_connection_count.py @@ -1,6 +1,6 @@ """ How many database connections `ox_worker` opens, and the hint it logs when -every task attempt opens one. +every task attempt opens one: a warning on PostgreSQL, an INFO line on MySQL. At the default CONN_MAX_AGE of 0 Django closes a thread's connections after each use, and a task attempt is one use: the execution thread opens a new @@ -67,6 +67,13 @@ POOL = {"min_size": 1, "max_size": 6} LIMIT = 90.0 HINT = "worker_connection_hint" +WARNING_TEXT = ( + "Worker %s: database %r opens a new connection for every task " + "because CONN_MAX_AGE is 0 and OPTIONS['pool'] is not set. " + "Set CONN_MAX_AGE = 600, or set OPTIONS['pool'] and leave " + "CONN_MAX_AGE at 0. See 'Retaining worker connections' in " + "the production guide." +) HINT_TEXT = ( "Worker %s: database %r opens a new execution connection per " "attempt with CONN_MAX_AGE = 0 on PostgreSQL or MySQL. " @@ -391,11 +398,16 @@ def test_closing_after_use_without_a_pool_gets_it( needs_driver(vendor) add_alias(ENGINE=engine, CONN_MAX_AGE=0, OPTIONS={}, TEST={}) (record,) = self.hint(caplog) - assert record.levelno == logging.INFO + # PostgreSQL's connect cost is measured and gets a warning; MySQL's is + # not, and stays an INFO line. + assert record.levelno == ( + logging.WARNING if vendor == "postgresql" else logging.INFO + ) assert record.database == ALIAS assert record.vendor == vendor assert record.worker_id - assert record.getMessage() == HINT_TEXT % (record.worker_id, ALIAS) + text = WARNING_TEXT if vendor == "postgresql" else HINT_TEXT + assert record.getMessage() == text % (record.worker_id, ALIAS) def test_a_missing_setting_is_the_default_and_gets_it(self, add_alias, caplog): pytest.importorskip("psycopg") diff --git a/tests/test_connection_warning.py b/tests/test_connection_warning.py new file mode 100644 index 0000000..e79a10e --- /dev/null +++ b/tests/test_connection_warning.py @@ -0,0 +1,102 @@ +""" +The startup warning for a PostgreSQL worker that opens a connection per task. + +These tests run the real `ox_worker --batch` command through +`call_command`, so the warning has to come from the code path the command +reaches and not from a call the test makes itself. They need no PostgreSQL +server: the suite's database is SQLite, and the test presents it to the worker +as PostgreSQL by name only. The worker's own database work is vendor-neutral +on an empty queue, and the warning reads settings and opens no connection. +tests/test_connection_count.py runs the same rule against a real server. +""" + +import logging + +import pytest +from django.core.management import call_command +from django.db import connections + +from django_ox.worker import Worker + +from .connection_count_tasks import tick + +EVENT = "worker_connection_hint" + +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.fixture +def as_postgresql(monkeypatch): + """ + The settings of a PostgreSQL alias with no pool, for the worker's startup + check alone: the test edits the dict this returns. The check is the real + method, reached by the real `Worker.run()`. It is shown the dict and the + name PostgreSQL only for its own call, because the claim queries of an + empty queue and SQLite's connect cannot run under either. + """ + shape = {"ENGINE": "django.db.backends.postgresql", "OPTIONS": {}} + real = Worker._hint_if_every_task_reconnects + + def check(self): + conn = connections[self._db_alias] + with monkeypatch.context() as shown: + shown.setattr(conn, "vendor", "postgresql") + shown.setattr(conn, "settings_dict", shape) + shown.setitem(connections.settings, self._db_alias, shape) + real(self) + + monkeypatch.setattr(Worker, "_hint_if_every_task_reconnects", check) + return shape + + +def run_command(caplog, *flags): + with ( + caplog.at_level(logging.INFO, logger="django_ox"), + pytest.raises(SystemExit) as stopped, + ): + call_command("ox_worker", "--batch", "--interval", "0.05", *flags) + assert stopped.value.code == 0 + return [r for r in caplog.records if getattr(r, "event", None) == EVENT] + + +def test_the_defaults_warn_once_with_the_setting_and_the_fix(as_postgresql, caplog): + as_postgresql["CONN_MAX_AGE"] = 0 + (record,) = run_command(caplog, "--concurrency", "1") + assert record.levelno == logging.WARNING + assert record.database == "default" + assert record.vendor == "postgresql" + message = record.getMessage() + assert "CONN_MAX_AGE" in message + assert "CONN_MAX_AGE = 600" in message + assert "OPTIONS['pool']" in message + + +def test_a_missing_setting_is_the_default_and_warns(as_postgresql, caplog): + as_postgresql.pop("CONN_MAX_AGE", None) + assert len(run_command(caplog, "--concurrency", "1")) == 1 + + +@pytest.mark.parametrize("age", [600, 1, None]) +def test_a_kept_connection_is_silent(as_postgresql, caplog, age): + as_postgresql["CONN_MAX_AGE"] = age + assert not run_command(caplog, "--concurrency", "1") + + +@pytest.mark.parametrize("pool", [True, {"min_size": 0, "max_size": 16, "timeout": 30}]) +def test_a_pool_is_silent(as_postgresql, caplog, pool): + as_postgresql["CONN_MAX_AGE"] = 0 + as_postgresql["OPTIONS"]["pool"] = pool + assert not run_command(caplog, "--concurrency", "1") + + +def test_several_threads_and_many_tasks_warn_once(as_postgresql, caplog, tmp_path): + as_postgresql["CONN_MAX_AGE"] = 0 + for i in range(12): + tick.enqueue(str(tmp_path / f"{i}.jsonl")) + assert len(run_command(caplog, "--concurrency", "4")) == 1 + + +def test_sqlite_is_silent(caplog): + if connections["default"].vendor != "sqlite": + pytest.skip("the default alias is not SQLite in this run") + assert not run_command(caplog, "--concurrency", "1") diff --git a/tests/test_outcome_connect_failure.py b/tests/test_outcome_connect_failure.py new file mode 100644 index 0000000..848817b --- /dev/null +++ b/tests/test_outcome_connect_failure.py @@ -0,0 +1,247 @@ +""" +The outcome write of a finished attempt when the worker cannot open a +connection for it: a host out of ephemeral ports, a database at its +connection limit. The error is raised at connect time, so no connection was +ever lost and none is open. +""" + +import logging +from threading import Event, Thread + +import pytest +from django.db import OperationalError, connection, connections +from django.db.backends.base.base import BaseDatabaseWrapper + +import django_ox.worker as worker_module +from django_ox.models import OxTask +from django_ox.worker import ( + OUTCOME_CONNECT_RETRY_DELAYS, + Worker, + _outcome_connect_delays, +) + +from .tasks import echo, fail_always + +pytestmark = pytest.mark.django_db(transaction=True) + +CANNOT_ASSIGN = ( + "connection is bad: connection to server at 127.0.0.1, port 5432 failed: " + "Can't assign requested address" +) + + +class Pauses(list): + """Stands in for the worker's sleep: records, never waits.""" + + stops: list + + def __init__(self): + super().__init__() + self.stops = [] + + def __call__(self, seconds, stop=None): + self.append(seconds) + self.stops.append(stop) + + +@pytest.fixture +def pauses(monkeypatch): + seen = Pauses() + monkeypatch.setattr(worker_module, "_pause", seen, raising=False) + return seen + + +@pytest.fixture +def connect_fails(monkeypatch): + """ + Once the task body has finished, the worker's connection is gone and the + next `failures` connects raise as psycopg does when no port can be + assigned (None fails them all). `connects` counts the attempts. + """ + state = {"failures": 0, "connects": 0, "armed": False} + real_discard = Worker._discard_unusable_connections + real_connect = BaseDatabaseWrapper.connect + + def dropped(self): + real_discard(self) + connections[self._db_alias].close() + state["armed"] = True + + def connect(self): + if state["armed"]: + state["connects"] += 1 + limit = state["failures"] + if limit is None or state["connects"] <= limit: + raise OperationalError(CANNOT_ASSIGN) + return real_connect(self) + + monkeypatch.setattr(Worker, "_discard_unusable_connections", dropped) + monkeypatch.setattr(BaseDatabaseWrapper, "connect", connect) + yield state + state["armed"] = False + connection.close() + + +def events(caplog, name): + return [r for r in caplog.records if getattr(r, "event", None) == name] + + +@pytest.mark.parametrize("task", [echo, fail_always], ids=["success", "failure"]) +def test_a_connect_refused_twice_then_allowed_records_the_outcome_once( + caplog, connect_fails, pauses, task +): + connect_fails["failures"] = 3 + worker = Worker(backoff_initial=900, backoff_max=900, poll_interval=0.05) + result = task.enqueue(*(("x",) if task is echo else ())) + + with caplog.at_level(logging.INFO, logger="django_ox"): + assert worker.run_once() + + connect_fails["armed"] = False + stored = OxTask.objects.get(id=result.id) + if task is echo: + assert (stored.status, stored.return_value) == ( + OxTask.Status.SUCCESSFUL, + "x", + ) + else: + assert stored.status == OxTask.Status.READY + assert len(stored.errors) == 1 + assert (stored.attempts, stored.lease_epoch) == (1, 1) + assert connect_fails["connects"] == 4 + assert pauses == [0.05, 0.1, 0.2] + assert len(events(caplog, "task_outcome_reconnected")) == 1 + assert events(caplog, "task_outcome_unrecorded") == [] + assert events(caplog, "task_lease_lost") == [] + + +def test_a_connect_that_stays_refused_ends_bounded_and_logs_once( + caplog, connect_fails, pauses +): + connect_fails["failures"] = None + worker = Worker(backoff_initial=0, poll_interval=0.05) + result = echo.enqueue("x") + + with caplog.at_level(logging.INFO, logger="django_ox"): + assert worker.run_once() + + connect_fails["armed"] = False + stored = OxTask.objects.get(id=result.id) + assert stored.status == OxTask.Status.RUNNING + assert connect_fails["connects"] == len(OUTCOME_CONNECT_RETRY_DELAYS) + 1 + assert pauses == list(OUTCOME_CONNECT_RETRY_DELAYS) + (line,) = events(caplog, "task_outcome_unrecorded") + assert line.exc_info is not None + assert events(caplog, "task_outcome_reconnected") == [] + + +@pytest.mark.parametrize("lock_timeout", [0.0, 0.2, 1.0, 5.0, 30.0, 300.0]) +def test_the_waits_never_pass_a_tenth_of_the_lease(lock_timeout): + delays = _outcome_connect_delays(lock_timeout) + assert sum(delays) <= lock_timeout / 10 + 1e-9 + assert all(0 < d <= max(OUTCOME_CONNECT_RETRY_DELAYS) for d in delays) + assert sum(delays) <= sum(OUTCOME_CONNECT_RETRY_DELAYS) + + +def test_the_default_lease_allows_the_whole_schedule(): + assert _outcome_connect_delays(300.0) == list(OUTCOME_CONNECT_RETRY_DELAYS) + assert sum(OUTCOME_CONNECT_RETRY_DELAYS) < 4 + + +def test_a_short_lease_shortens_the_waits_the_worker_makes( + caplog, connect_fails, pauses +): + connect_fails["failures"] = None + worker = Worker(backoff_initial=0, poll_interval=0.05, lock_timeout=1.0) + echo.enqueue("x") + + assert worker.run_once() + + connect_fails["armed"] = False + assert sum(pauses) <= 0.1 + 1e-9 + assert pauses == [0.05, 0.05] + + +def test_a_write_that_loses_a_connection_does_not_wait(monkeypatch, pauses): + """The dead connection is open and fails its probe: tried again at once.""" + real = Worker._write_outcome + wrapper = type(connections["default"]) + real_usable = wrapper.is_usable + calls = [] + + def dies_once(self, db_task, **fields): + calls.append(1) + if len(calls) == 1: + monkeypatch.setattr(wrapper, "is_usable", lambda self: False) + connections["default"].ensure_connection() + raise OperationalError("server closed the connection unexpectedly") + monkeypatch.setattr(wrapper, "is_usable", real_usable) + return real(self, db_task, **fields) + + monkeypatch.setattr(Worker, "_write_outcome", dies_once) + result = echo.enqueue("x") + worker = Worker(backoff_initial=0, poll_interval=0.05) + + assert worker.run_once() + + assert OxTask.objects.get(id=result.id).status == OxTask.Status.SUCCESSFUL + assert pauses == [] + + +def test_a_pause_ends_when_the_worker_is_told_to_stop(): + """A shutdown never sits out the schedule: the wait is on the stop event.""" + stop = Event() + waiter = Thread(target=worker_module._pause, args=(60.0, stop)) + waiter.start() + stop.set() + waiter.join(timeout=10) + assert not waiter.is_alive() + + +def test_the_write_waits_on_the_workers_own_stop_event(connect_fails, pauses): + connect_fails["failures"] = 2 + result = echo.enqueue("x") + worker = Worker(backoff_initial=0, poll_interval=0.05) + + assert worker.run_once() + + connect_fails["armed"] = False + assert OxTask.objects.get(id=result.id).status == OxTask.Status.SUCCESSFUL + assert pauses + assert all(stop is worker._stop for stop in pauses.stops) + + +def test_a_lost_connection_the_driver_does_not_flag_is_not_a_refused_connect( + monkeypatch, caplog, pauses +): + """ + A connection that was open when the write began and died under it is not + a refused connect, whether or not the driver reports it lost: a second try + that fails too is left to the reaper, with no wait, as before. + """ + monkeypatch.setattr(worker_module, "_lost", lambda raw: False) + wrapper = type(connections["default"]) + calls = [] + + def fails_twice(self, db_task, **fields): + calls.append(1) + if len(calls) == 1: + monkeypatch.setattr(wrapper, "is_usable", lambda self: False) + connections["default"].ensure_connection() + else: + # The server drops the session under the second write too, and + # nothing is left open afterwards. + connections["default"].close() + raise OperationalError("Lost connection to the server during query") + + monkeypatch.setattr(Worker, "_write_outcome", fails_twice) + result = echo.enqueue("x") + worker = Worker(backoff_initial=0, poll_interval=0.05) + + with caplog.at_level(logging.INFO, logger="django_ox"): + assert worker.run_once() + + assert len(calls) == 2 + assert pauses == [] + assert len(events(caplog, "task_outcome_unrecorded")) == 1 + assert OxTask.objects.get(id=result.id).status == OxTask.Status.RUNNING diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index f78b081..e1ddf55 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -675,7 +675,10 @@ def test_worker_drains_when_its_parent_changes(self, caplog): thread = start_worker_thread(worker) thread.join(timeout=10) assert not thread.is_alive() - assert [getattr(r, "event", None) for r in caplog.records] == [ + events = [getattr(r, "event", None) for r in caplog.records] + # On PostgreSQL with the default connection settings the worker also + # warns once about a new connection for every task. + assert [e for e in events if e != "worker_connection_hint"] == [ "worker_orphaned" ]