From 4d9d44e3aabadf6594b5fc243b71d8e2533efad3 Mon Sep 17 00:00:00 2001 From: antkryt Date: Mon, 26 Oct 2026 17:24:26 +0200 Subject: [PATCH 1/3] [ENG-12536] Improve dj-celery-panel (#346) * Add unmatched routes panel to dj-celery-panel --- .../admin/dj_celery_panel/queues.html | 54 +++++++++++++++ app/celery_panel.py | 68 ++++++++++++++----- 2 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 addon_service/templates/admin/dj_celery_panel/queues.html diff --git a/addon_service/templates/admin/dj_celery_panel/queues.html b/addon_service/templates/admin/dj_celery_panel/queues.html new file mode 100644 index 00000000..15e7af6f --- /dev/null +++ b/addon_service/templates/admin/dj_celery_panel/queues.html @@ -0,0 +1,54 @@ +{% extends "admin/dj_celery_panel/queues.html" %} +{% load i18n %} + +{% block panel_page_content %} +{{ block.super }} +{% if queues %} +
+
+
+

{% trans "Routing" %}

+

+ {% trans "Tasks routed to each queue, and task_routes entries that match no registered task" %} +

+
+
+
+ + + + + + + + + + {% for queue in queues %} + + + + + + {% endfor %} + +
{% trans "Queue Name" %}{% trans "Routed Tasks" %}{% trans "Unmatched Routing Entries" %}
+ + {{ queue.name }} + + + {% if queue.routed_tasks_count %} + {{ queue.routed_tasks_count }} + {% else %} + 0 + {% endif %} + + {% for entry in queue.unmatched_routing_entries %} + {{ entry }}{% if not forloop.last %}
{% endif %} + {% empty %} + — + {% endfor %} +
+
+
+{% endif %} +{% endblock %} diff --git a/app/celery_panel.py b/app/celery_panel.py index a12eee34..d10d4a98 100644 --- a/app/celery_panel.py +++ b/app/celery_panel.py @@ -1,3 +1,6 @@ +from collections import defaultdict +from fnmatch import fnmatchcase + from dj_celery_panel.celery_utils import ( CeleryQueuesInspectBackend, QueueDetailPage, @@ -6,16 +9,14 @@ class RoutedTasksQueuesBackend(CeleryQueuesInspectBackend): - """dj-celery-panel queues backend that also lists the tasks routed to each queue - and the configured queues that no active worker consumes. + """dj-celery-panel queues backend that also lists the tasks routed to each queue, the + task_routes entries that match no task, and the configured queues that no active worker consumes. Routes are resolved the same way Celery does when sending a task, so explicit queue= overrides passed to apply_async() at call time are not reflected. """ - BACKEND_DESCRIPTION = ( - "Queue bindings, routing configuration, routed tasks and unconsumed queues" - ) + BACKEND_DESCRIPTION = "Queue bindings, routing configuration, routed tasks, unmatched routing entries and unconsumed queues" def get_queues(self): page = super().get_queues() @@ -27,20 +28,31 @@ def get_queues(self): for name, queue in sorted(self.app.amqp.queues.items()) if name not in listed ] - return QueueListPage(queues=page.queues + unconsumed, error=page.error) + routed_tasks = self.get_routed_tasks_by_queue() + unmatched_entries = self.get_unmatched_routing_entries_by_queue() + return QueueListPage( + queues=[ + { + **queue, + "routed_tasks_count": len(routed_tasks[queue["name"]]), + "unmatched_routing_entries": unmatched_entries[queue["name"]], + } + for queue in page.queues + unconsumed + ], + error=page.error, + ) def get_queue_detail(self, queue_name): page = super().get_queue_detail(queue_name) - queue, error = page.queue, page.error + queue = page.queue if queue is None: # membership check first: indexing Queues creates missing ones if queue_name not in self.app.amqp.queues: return page queue = self.describe_unconsumed_queue(self.app.amqp.queues[queue_name]) - error = "No active worker consumes this queue" return QueueDetailPage( queue={**queue, "routed_tasks": self.get_routed_tasks(queue_name)}, - error=error, + error=page.error, ) def describe_unconsumed_queue(self, queue): @@ -62,13 +74,37 @@ def describe_unconsumed_queue(self, queue): } def get_routed_tasks(self, queue_name): + return self.get_routed_tasks_by_queue()[queue_name] + + def get_registered_tasks(self): # AddonServiceConfig.ready() imports every task module, so the local registry is complete + return { + name: task + for name, task in self.app.tasks.items() + if not name.startswith("celery.") + } + + def get_routed_tasks_by_queue(self): router = self.app.amqp.router - routed_tasks = [] - for name, task in self.app.tasks.items(): - if name.startswith("celery."): - continue + routed_tasks = defaultdict(list) + for name, task in sorted(self.get_registered_tasks().items()): options = dict(task._get_exec_options()) - if router.route(options, name)["queue"].name == queue_name: - routed_tasks.append(name) - return sorted(routed_tasks) + routed_tasks[router.route(options, name)["queue"].name].append(name) + return routed_tasks + + def get_unmatched_routing_entries_by_queue(self): + # Celery matches task_routes entries against task names as fnmatch globs, so an entry + # that matches no registered task name routes nothing: the task is gone, renamed, + # or its module isn't in the celery app's include. + # Entries routed to queues outside task_queues (osf.*, website.*) are for OSF tasks + # sent by name with send_task(), which are never registered here, so they are skipped + task_names = self.get_registered_tasks() + router = self.app.amqp.router + unmatched_entries = defaultdict(list) + for entry, route in sorted(self.app.conf.task_routes.items()): + queue_name = router.expand_destination(dict(route))["queue"].name + if queue_name in self.app.amqp.queues and not any( + fnmatchcase(name, entry) for name in task_names + ): + unmatched_entries[queue_name].append(entry) + return unmatched_entries From 97304fba87f677986fc4f732701d11f1f1c9c50b Mon Sep 17 00:00:00 2001 From: Ostap-Zherebetskyi Date: Mon, 9 Nov 2026 17:22:07 +0200 Subject: [PATCH 2/3] [ENG-12495] Upgrade kombu and celery (#349) * Upgrade celery to 5.6.3 and kombu to 5.6.2 in poetry.lock and pyproject.toml --- poetry.lock | 81 +++++++++++++++++++++++++++++++------------------- pyproject.toml | 3 +- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/poetry.lock b/poetry.lock index fdde496d..b778ef79 100644 --- a/poetry.lock +++ b/poetry.lock @@ -306,14 +306,14 @@ crt = ["awscrt (==0.21.2)"] [[package]] name = "celery" -version = "5.5.0" +version = "5.6.3" description = "Distributed Task Queue." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "celery-5.5.0-py3-none-any.whl", hash = "sha256:f4170c6e5952281318448a899d9e9a15b9cbd007e002091766900dc8f71b9394"}, - {file = "celery-5.5.0.tar.gz", hash = "sha256:10d49f9926d16237310109b0e6e1e2f7a2133b84e684bb36534d7663e66919bb"}, + {file = "celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6"}, + {file = "celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912"}, ] [package.dependencies] @@ -322,13 +322,14 @@ click = ">=8.1.2,<9.0" click-didyoumean = ">=0.3.0" click-plugins = ">=1.1.1" click-repl = ">=0.2.0" -kombu = ">=5.5.2,<5.6" +kombu = ">=5.6.0" python-dateutil = ">=2.8.2" +tzlocal = "*" vine = ">=5.1.0,<6.0" [package.extras] arangodb = ["pyArango (>=2.0.2)"] -auth = ["cryptography (==44.0.2)"] +auth = ["cryptography (==46.0.5)"] azureblockblob = ["azure-identity (>=1.19.0)", "azure-storage-blob (>=12.15.0)"] brotli = ["brotli (>=1.0.0) ; platform_python_implementation == \"CPython\"", "brotlipy (>=0.7.0) ; platform_python_implementation == \"PyPy\""] cassandra = ["cassandra-driver (>=3.25.0,<4)"] @@ -338,26 +339,26 @@ couchbase = ["couchbase (>=3.0.0) ; platform_python_implementation != \"PyPy\" a couchdb = ["pycouchdb (==1.16.0)"] django = ["Django (>=2.2.28)"] dynamodb = ["boto3 (>=1.26.143)"] -elasticsearch = ["elastic-transport (<=8.17.1)", "elasticsearch (<=8.17.2)"] +elasticsearch = ["elastic-transport (<=9.2.1)", "elasticsearch (<=9.3.0)"] eventlet = ["eventlet (>=0.32.0) ; python_version < \"3.10\""] -gcs = ["google-cloud-firestore (==2.20.1)", "google-cloud-storage (>=2.10.0)", "grpcio (==1.67.0)"] +gcs = ["google-cloud-firestore (==2.23.0)", "google-cloud-storage (>=2.10.0)", "grpcio (==1.76.0)"] gevent = ["gevent (>=1.5.0)"] librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] memcache = ["pylibmc (==1.6.3) ; platform_system != \"Windows\""] -mongodb = ["pymongo (==4.10.1)"] -msgpack = ["msgpack (==1.1.0)"] -pydantic = ["pydantic (>=2.4)"] +mongodb = ["kombu[mongodb]"] +msgpack = ["kombu[msgpack]"] +pydantic = ["pydantic (>=2.12.0a1) ; python_version >= \"3.14\"", "pydantic (>=2.4) ; python_version < \"3.14\""] pymemcache = ["python-memcached (>=1.61)"] pyro = ["pyro4 (==4.82) ; python_version < \"3.11\""] -pytest = ["pytest-celery[all] (>=1.2.0,<1.3.0)"] -redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] +pytest = ["pytest-celery[all] (>=1.3.0)"] +redis = ["kombu[redis]"] s3 = ["boto3 (>=1.26.143)"] slmq = ["softlayer_messaging (>=1.0.3)"] solar = ["ephem (==4.2) ; platform_python_implementation != \"PyPy\""] -sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] -sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "urllib3 (>=1.26.16)"] -tblib = ["tblib (>=1.3.0) ; python_version < \"3.8.0\"", "tblib (>=1.5.0) ; python_version >= \"3.8.0\""] -yaml = ["PyYAML (>=3.10)"] +sqlalchemy = ["kombu[sqlalchemy]"] +sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.5.0)", "pycurl (>=7.43.0.5,<7.45.4) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\" and python_version < \"3.9\"", "pycurl (>=7.45.4) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "urllib3 (>=1.26.16)"] +tblib = ["tblib (==3.2.2)"] +yaml = ["kombu[yaml]"] zookeeper = ["kazoo (>=1.3.1)"] zstd = ["zstandard (==0.23.0)"] @@ -1339,19 +1340,20 @@ referencing = ">=0.31.0" [[package]] name = "kombu" -version = "5.5.2" +version = "5.6.2" description = "Messaging library for Python." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "kombu-5.5.2-py3-none-any.whl", hash = "sha256:40f3674ed19603b8a771b6c74de126dbf8879755a0337caac6602faa82d539cd"}, - {file = "kombu-5.5.2.tar.gz", hash = "sha256:2dd27ec84fd843a4e0a7187424313f87514b344812cb98c25daddafbb6a7ff0e"}, + {file = "kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93"}, + {file = "kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55"}, ] [package.dependencies] amqp = ">=5.1.1,<6.0.0" -tzdata = {version = "2025.2", markers = "python_version >= \"3.9\""} +packaging = "*" +tzdata = ">=2025.2" vine = "5.1.0" [package.extras] @@ -1359,16 +1361,16 @@ azureservicebus = ["azure-servicebus (>=7.10.0)"] azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"] confluentkafka = ["confluent-kafka (>=2.2.0)"] consul = ["python-consul2 (==0.1.5)"] -gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.67.0)", "protobuf (==4.25.5)"] +gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.75.1)", "protobuf (==6.32.1)"] librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] -mongodb = ["pymongo (>=4.1.1)"] -msgpack = ["msgpack (==1.1.0)"] +mongodb = ["pymongo (==4.15.3)"] +msgpack = ["msgpack (==1.1.2)"] pyro = ["pyro4 (==4.82)"] -qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"] -redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<=5.2.1)"] -slmq = ["softlayer-messaging (>=1.0.3)"] +qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"] +redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"] +slmq = ["softlayer_messaging (>=1.0.3)"] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] -sqs = ["boto3 (>=1.26.143)", "urllib3 (>=1.26.16)"] +sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"] yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=2.8.0)"] @@ -2561,6 +2563,25 @@ files = [ ] markers = {dev = "sys_platform == \"win32\"", release = "sys_platform == \"win32\""} +[[package]] +name = "tzlocal" +version = "5.4.5" +description = "tzinfo object for the local timezone" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "tzlocal-5.4.5-py3-none-any.whl", hash = "sha256:ecab534f6d65d9deb4c4ea52c121ab6dd5958e9b878e71dcb102578269752454"}, + {file = "tzlocal-5.4.5.tar.gz", hash = "sha256:ba976898bad27945178c9ca25c222c19e322b30c10342b9f40b46714f70d2d5f"}, +] + +[package.dependencies] +tzdata = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +devenv = ["zest.releaser"] +testing = ["check_manifest", "pyroma", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "ruff"] + [[package]] name = "uritemplate" version = "4.1.1" @@ -2788,4 +2809,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "2ac648dad93ba8cb5b776c841ba85149f22ea1903b1f8fc716edb6e8ed39c9c1" +content-hash = "e23cad566c2aa30824c6842f71704c099ba60b7f6d9cb809ed828fb3c4688d1f" diff --git a/pyproject.toml b/pyproject.toml index 7778f697..54e17ab0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ djangorestframework = "3.15.2" djangorestframework-jsonapi = "7.1.0" django-cors-headers = "4.7.0" django-filter = "^24.3" -celery = "5.5.0" +celery = "5.6.3" jsonschema = "4.23.0" aiohttp = "3.11.14" pyjwe = "1.0.0" @@ -30,6 +30,7 @@ itsdangerous = "^2.2.0" redis = "^5.2.1" drf-spectacular = "^0.28.0" drf-spectacular-jsonapi = {git = "https://github.com/opaduchak/drf-spectacular-json-api.git", branch = "osf-fixes"} +kombu = "5.6.2" [tool.poetry.group.dev.dependencies] From 646be9b1b21a95dfe5584a16bea7d0451755c0d2 Mon Sep 17 00:00:00 2001 From: antkryt Date: Tue, 17 Nov 2026 15:48:39 +0200 Subject: [PATCH 3/3] [ENG-12619] Make sure tasks are accounted for (Unit Tests IMPL) (#350) * Add unit tests to catch router problems --- addon_service/tests/test_celery_routing.py | 95 ++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 addon_service/tests/test_celery_routing.py diff --git a/addon_service/tests/test_celery_routing.py b/addon_service/tests/test_celery_routing.py new file mode 100644 index 00000000..2cfe7a27 --- /dev/null +++ b/addon_service/tests/test_celery_routing.py @@ -0,0 +1,95 @@ +from fnmatch import fnmatchcase + +from django.test import SimpleTestCase +from kombu import Queue + +from app.celery import app as celery_app + + +# Set on the task itself (e.g. @shared_task(queue=...)), these take precedence over task_routes +ROUTING_ATTRIBUTES = ("queue", "exchange", "routing_key") + + +def _queue_name(queue): + # task_routes entries may give a queue by name or as a kombu Queue + return queue.name if isinstance(queue, Queue) else queue + + +def _routes_to_osf_queue(route): + # Queues with no_declare=True are declared and consumed by osf's celery. They carry the osf tasks + # sent by name with send_task(), which are never registered here + queue = route.get("queue") + return isinstance(queue, Queue) and queue.no_declare + + +class TestCeleryRouting(SimpleTestCase): + maxDiff = None + + def setUp(self): + # AddonServiceConfig.ready() imports celery's `include` modules like the worker does on start, + # so the local registry has every task + self.registered_tasks = { + name: task + for name, task in celery_app.tasks.items() + if not name.startswith("celery.") + } + + def test_every_task_has_a_route(self): + # Celery silently sends a task to task_default_queue when no route matches it or its matching route + # is empty or names no queue, so a task may only end up there when its route names that queue + default_queue = celery_app.conf.task_default_queue + router = celery_app.amqp.router + implicit = [] + for name, task in sorted(self.registered_tasks.items()): + # Resolve the queue like Task.apply_async() does: the task's own options merged over its route + queue = router.route(dict(task._get_exec_options()), name).get("queue") + route = router.lookup_route(name) or {} + if (queue is None or queue.name == default_queue) and _queue_name( + route.get("queue") + ) != default_queue: + implicit.append(name) + self.assertEqual( + implicit, [], "Tasks sent to the default queue without a route naming it" + ) + + def test_every_route_targets_a_declared_queue(self): + # A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it + declared = {queue.name for queue in celery_app.conf.task_queues} + undeclared = { + entry: _queue_name(route.get("queue")) + for entry, route in celery_app.conf.task_routes.items() + if not _routes_to_osf_queue(route) + and _queue_name(route.get("queue")) not in declared + } + self.assertEqual( + undeclared, {}, "Routes to queues missing from celery's task_queues" + ) + + def test_every_route_matches_a_registered_task(self): + # Celery matches task_routes entries against task names as fnmatch globs, so an entry that + # matches no registered task routes nothing: the task is gone, renamed, or not in celery's `include` + unmatched = [ + entry + for entry, route in celery_app.conf.task_routes.items() + if not _routes_to_osf_queue(route) + and not any(fnmatchcase(name, entry) for name in self.registered_tasks) + ] + self.assertEqual(unmatched, [], "Routes matching no registered task") + + def test_no_task_sets_routing_attributes(self): + # Only catches routing set on the task decorator (e.g. @shared_task(queue=...)). Overrides passed at call time, + # e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here + overrides = {} + for name, task in sorted(self.registered_tasks.items()): + attributes = { + attr: getattr(task, attr) + for attr in ROUTING_ATTRIBUTES + if getattr(task, attr, None) is not None + } + if attributes: + overrides[name] = attributes + self.assertEqual( + overrides, + {}, + "Tasks setting routing attributes instead of using celery's task_routes", + )