diff --git a/addon_service/templates/admin/dj_celery_panel/queues.html b/addon_service/templates/admin/dj_celery_panel/queues.html
new file mode 100644
index 00000000..15e7af6f
--- /dev/null
+++ b/addon_service/templates/admin/dj_celery_panel/queues.html
@@ -0,0 +1,54 @@
+{% extends "admin/dj_celery_panel/queues.html" %}
+{% load i18n %}
+
+{% block panel_page_content %}
+{{ block.super }}
+{% if queues %}
+
+{% endif %}
+{% endblock %}
diff --git a/addon_service/tests/test_celery_routing.py b/addon_service/tests/test_celery_routing.py
new file mode 100644
index 00000000..2cfe7a27
--- /dev/null
+++ b/addon_service/tests/test_celery_routing.py
@@ -0,0 +1,95 @@
+from fnmatch import fnmatchcase
+
+from django.test import SimpleTestCase
+from kombu import Queue
+
+from app.celery import app as celery_app
+
+
+# Set on the task itself (e.g. @shared_task(queue=...)), these take precedence over task_routes
+ROUTING_ATTRIBUTES = ("queue", "exchange", "routing_key")
+
+
+def _queue_name(queue):
+ # task_routes entries may give a queue by name or as a kombu Queue
+ return queue.name if isinstance(queue, Queue) else queue
+
+
+def _routes_to_osf_queue(route):
+ # Queues with no_declare=True are declared and consumed by osf's celery. They carry the osf tasks
+ # sent by name with send_task(), which are never registered here
+ queue = route.get("queue")
+ return isinstance(queue, Queue) and queue.no_declare
+
+
+class TestCeleryRouting(SimpleTestCase):
+ maxDiff = None
+
+ def setUp(self):
+ # AddonServiceConfig.ready() imports celery's `include` modules like the worker does on start,
+ # so the local registry has every task
+ self.registered_tasks = {
+ name: task
+ for name, task in celery_app.tasks.items()
+ if not name.startswith("celery.")
+ }
+
+ def test_every_task_has_a_route(self):
+ # Celery silently sends a task to task_default_queue when no route matches it or its matching route
+ # is empty or names no queue, so a task may only end up there when its route names that queue
+ default_queue = celery_app.conf.task_default_queue
+ router = celery_app.amqp.router
+ implicit = []
+ for name, task in sorted(self.registered_tasks.items()):
+ # Resolve the queue like Task.apply_async() does: the task's own options merged over its route
+ queue = router.route(dict(task._get_exec_options()), name).get("queue")
+ route = router.lookup_route(name) or {}
+ if (queue is None or queue.name == default_queue) and _queue_name(
+ route.get("queue")
+ ) != default_queue:
+ implicit.append(name)
+ self.assertEqual(
+ implicit, [], "Tasks sent to the default queue without a route naming it"
+ )
+
+ def test_every_route_targets_a_declared_queue(self):
+ # A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it
+ declared = {queue.name for queue in celery_app.conf.task_queues}
+ undeclared = {
+ entry: _queue_name(route.get("queue"))
+ for entry, route in celery_app.conf.task_routes.items()
+ if not _routes_to_osf_queue(route)
+ and _queue_name(route.get("queue")) not in declared
+ }
+ self.assertEqual(
+ undeclared, {}, "Routes to queues missing from celery's task_queues"
+ )
+
+ def test_every_route_matches_a_registered_task(self):
+ # Celery matches task_routes entries against task names as fnmatch globs, so an entry that
+ # matches no registered task routes nothing: the task is gone, renamed, or not in celery's `include`
+ unmatched = [
+ entry
+ for entry, route in celery_app.conf.task_routes.items()
+ if not _routes_to_osf_queue(route)
+ and not any(fnmatchcase(name, entry) for name in self.registered_tasks)
+ ]
+ self.assertEqual(unmatched, [], "Routes matching no registered task")
+
+ def test_no_task_sets_routing_attributes(self):
+ # Only catches routing set on the task decorator (e.g. @shared_task(queue=...)). Overrides passed at call time,
+ # e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here
+ overrides = {}
+ for name, task in sorted(self.registered_tasks.items()):
+ attributes = {
+ attr: getattr(task, attr)
+ for attr in ROUTING_ATTRIBUTES
+ if getattr(task, attr, None) is not None
+ }
+ if attributes:
+ overrides[name] = attributes
+ self.assertEqual(
+ overrides,
+ {},
+ "Tasks setting routing attributes instead of using celery's task_routes",
+ )
diff --git a/app/celery_panel.py b/app/celery_panel.py
index a12eee34..d10d4a98 100644
--- a/app/celery_panel.py
+++ b/app/celery_panel.py
@@ -1,3 +1,6 @@
+from collections import defaultdict
+from fnmatch import fnmatchcase
+
from dj_celery_panel.celery_utils import (
CeleryQueuesInspectBackend,
QueueDetailPage,
@@ -6,16 +9,14 @@
class RoutedTasksQueuesBackend(CeleryQueuesInspectBackend):
- """dj-celery-panel queues backend that also lists the tasks routed to each queue
- and the configured queues that no active worker consumes.
+ """dj-celery-panel queues backend that also lists the tasks routed to each queue, the
+ task_routes entries that match no task, and the configured queues that no active worker consumes.
Routes are resolved the same way Celery does when sending a task, so explicit
queue= overrides passed to apply_async() at call time are not reflected.
"""
- BACKEND_DESCRIPTION = (
- "Queue bindings, routing configuration, routed tasks and unconsumed queues"
- )
+ BACKEND_DESCRIPTION = "Queue bindings, routing configuration, routed tasks, unmatched routing entries and unconsumed queues"
def get_queues(self):
page = super().get_queues()
@@ -27,20 +28,31 @@ def get_queues(self):
for name, queue in sorted(self.app.amqp.queues.items())
if name not in listed
]
- return QueueListPage(queues=page.queues + unconsumed, error=page.error)
+ routed_tasks = self.get_routed_tasks_by_queue()
+ unmatched_entries = self.get_unmatched_routing_entries_by_queue()
+ return QueueListPage(
+ queues=[
+ {
+ **queue,
+ "routed_tasks_count": len(routed_tasks[queue["name"]]),
+ "unmatched_routing_entries": unmatched_entries[queue["name"]],
+ }
+ for queue in page.queues + unconsumed
+ ],
+ error=page.error,
+ )
def get_queue_detail(self, queue_name):
page = super().get_queue_detail(queue_name)
- queue, error = page.queue, page.error
+ queue = page.queue
if queue is None:
# membership check first: indexing Queues creates missing ones
if queue_name not in self.app.amqp.queues:
return page
queue = self.describe_unconsumed_queue(self.app.amqp.queues[queue_name])
- error = "No active worker consumes this queue"
return QueueDetailPage(
queue={**queue, "routed_tasks": self.get_routed_tasks(queue_name)},
- error=error,
+ error=page.error,
)
def describe_unconsumed_queue(self, queue):
@@ -62,13 +74,37 @@ def describe_unconsumed_queue(self, queue):
}
def get_routed_tasks(self, queue_name):
+ return self.get_routed_tasks_by_queue()[queue_name]
+
+ def get_registered_tasks(self):
# AddonServiceConfig.ready() imports every task module, so the local registry is complete
+ return {
+ name: task
+ for name, task in self.app.tasks.items()
+ if not name.startswith("celery.")
+ }
+
+ def get_routed_tasks_by_queue(self):
router = self.app.amqp.router
- routed_tasks = []
- for name, task in self.app.tasks.items():
- if name.startswith("celery."):
- continue
+ routed_tasks = defaultdict(list)
+ for name, task in sorted(self.get_registered_tasks().items()):
options = dict(task._get_exec_options())
- if router.route(options, name)["queue"].name == queue_name:
- routed_tasks.append(name)
- return sorted(routed_tasks)
+ routed_tasks[router.route(options, name)["queue"].name].append(name)
+ return routed_tasks
+
+ def get_unmatched_routing_entries_by_queue(self):
+ # Celery matches task_routes entries against task names as fnmatch globs, so an entry
+ # that matches no registered task name routes nothing: the task is gone, renamed,
+ # or its module isn't in the celery app's include.
+ # Entries routed to queues outside task_queues (osf.*, website.*) are for OSF tasks
+ # sent by name with send_task(), which are never registered here, so they are skipped
+ task_names = self.get_registered_tasks()
+ router = self.app.amqp.router
+ unmatched_entries = defaultdict(list)
+ for entry, route in sorted(self.app.conf.task_routes.items()):
+ queue_name = router.expand_destination(dict(route))["queue"].name
+ if queue_name in self.app.amqp.queues and not any(
+ fnmatchcase(name, entry) for name in task_names
+ ):
+ unmatched_entries[queue_name].append(entry)
+ return unmatched_entries
diff --git a/poetry.lock b/poetry.lock
index fdde496d..b778ef79 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -306,14 +306,14 @@ crt = ["awscrt (==0.21.2)"]
[[package]]
name = "celery"
-version = "5.5.0"
+version = "5.6.3"
description = "Distributed Task Queue."
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "celery-5.5.0-py3-none-any.whl", hash = "sha256:f4170c6e5952281318448a899d9e9a15b9cbd007e002091766900dc8f71b9394"},
- {file = "celery-5.5.0.tar.gz", hash = "sha256:10d49f9926d16237310109b0e6e1e2f7a2133b84e684bb36534d7663e66919bb"},
+ {file = "celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6"},
+ {file = "celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912"},
]
[package.dependencies]
@@ -322,13 +322,14 @@ click = ">=8.1.2,<9.0"
click-didyoumean = ">=0.3.0"
click-plugins = ">=1.1.1"
click-repl = ">=0.2.0"
-kombu = ">=5.5.2,<5.6"
+kombu = ">=5.6.0"
python-dateutil = ">=2.8.2"
+tzlocal = "*"
vine = ">=5.1.0,<6.0"
[package.extras]
arangodb = ["pyArango (>=2.0.2)"]
-auth = ["cryptography (==44.0.2)"]
+auth = ["cryptography (==46.0.5)"]
azureblockblob = ["azure-identity (>=1.19.0)", "azure-storage-blob (>=12.15.0)"]
brotli = ["brotli (>=1.0.0) ; platform_python_implementation == \"CPython\"", "brotlipy (>=0.7.0) ; platform_python_implementation == \"PyPy\""]
cassandra = ["cassandra-driver (>=3.25.0,<4)"]
@@ -338,26 +339,26 @@ couchbase = ["couchbase (>=3.0.0) ; platform_python_implementation != \"PyPy\" a
couchdb = ["pycouchdb (==1.16.0)"]
django = ["Django (>=2.2.28)"]
dynamodb = ["boto3 (>=1.26.143)"]
-elasticsearch = ["elastic-transport (<=8.17.1)", "elasticsearch (<=8.17.2)"]
+elasticsearch = ["elastic-transport (<=9.2.1)", "elasticsearch (<=9.3.0)"]
eventlet = ["eventlet (>=0.32.0) ; python_version < \"3.10\""]
-gcs = ["google-cloud-firestore (==2.20.1)", "google-cloud-storage (>=2.10.0)", "grpcio (==1.67.0)"]
+gcs = ["google-cloud-firestore (==2.23.0)", "google-cloud-storage (>=2.10.0)", "grpcio (==1.76.0)"]
gevent = ["gevent (>=1.5.0)"]
librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
memcache = ["pylibmc (==1.6.3) ; platform_system != \"Windows\""]
-mongodb = ["pymongo (==4.10.1)"]
-msgpack = ["msgpack (==1.1.0)"]
-pydantic = ["pydantic (>=2.4)"]
+mongodb = ["kombu[mongodb]"]
+msgpack = ["kombu[msgpack]"]
+pydantic = ["pydantic (>=2.12.0a1) ; python_version >= \"3.14\"", "pydantic (>=2.4) ; python_version < \"3.14\""]
pymemcache = ["python-memcached (>=1.61)"]
pyro = ["pyro4 (==4.82) ; python_version < \"3.11\""]
-pytest = ["pytest-celery[all] (>=1.2.0,<1.3.0)"]
-redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"]
+pytest = ["pytest-celery[all] (>=1.3.0)"]
+redis = ["kombu[redis]"]
s3 = ["boto3 (>=1.26.143)"]
slmq = ["softlayer_messaging (>=1.0.3)"]
solar = ["ephem (==4.2) ; platform_python_implementation != \"PyPy\""]
-sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
-sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "urllib3 (>=1.26.16)"]
-tblib = ["tblib (>=1.3.0) ; python_version < \"3.8.0\"", "tblib (>=1.5.0) ; python_version >= \"3.8.0\""]
-yaml = ["PyYAML (>=3.10)"]
+sqlalchemy = ["kombu[sqlalchemy]"]
+sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.5.0)", "pycurl (>=7.43.0.5,<7.45.4) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\" and python_version < \"3.9\"", "pycurl (>=7.45.4) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "urllib3 (>=1.26.16)"]
+tblib = ["tblib (==3.2.2)"]
+yaml = ["kombu[yaml]"]
zookeeper = ["kazoo (>=1.3.1)"]
zstd = ["zstandard (==0.23.0)"]
@@ -1339,19 +1340,20 @@ referencing = ">=0.31.0"
[[package]]
name = "kombu"
-version = "5.5.2"
+version = "5.6.2"
description = "Messaging library for Python."
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "kombu-5.5.2-py3-none-any.whl", hash = "sha256:40f3674ed19603b8a771b6c74de126dbf8879755a0337caac6602faa82d539cd"},
- {file = "kombu-5.5.2.tar.gz", hash = "sha256:2dd27ec84fd843a4e0a7187424313f87514b344812cb98c25daddafbb6a7ff0e"},
+ {file = "kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93"},
+ {file = "kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55"},
]
[package.dependencies]
amqp = ">=5.1.1,<6.0.0"
-tzdata = {version = "2025.2", markers = "python_version >= \"3.9\""}
+packaging = "*"
+tzdata = ">=2025.2"
vine = "5.1.0"
[package.extras]
@@ -1359,16 +1361,16 @@ azureservicebus = ["azure-servicebus (>=7.10.0)"]
azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"]
confluentkafka = ["confluent-kafka (>=2.2.0)"]
consul = ["python-consul2 (==0.1.5)"]
-gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.67.0)", "protobuf (==4.25.5)"]
+gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.75.1)", "protobuf (==6.32.1)"]
librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
-mongodb = ["pymongo (>=4.1.1)"]
-msgpack = ["msgpack (==1.1.0)"]
+mongodb = ["pymongo (==4.15.3)"]
+msgpack = ["msgpack (==1.1.2)"]
pyro = ["pyro4 (==4.82)"]
-qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"]
-redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<=5.2.1)"]
-slmq = ["softlayer-messaging (>=1.0.3)"]
+qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"]
+redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"]
+slmq = ["softlayer_messaging (>=1.0.3)"]
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
-sqs = ["boto3 (>=1.26.143)", "urllib3 (>=1.26.16)"]
+sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"]
yaml = ["PyYAML (>=3.10)"]
zookeeper = ["kazoo (>=2.8.0)"]
@@ -2561,6 +2563,25 @@ files = [
]
markers = {dev = "sys_platform == \"win32\"", release = "sys_platform == \"win32\""}
+[[package]]
+name = "tzlocal"
+version = "5.4.5"
+description = "tzinfo object for the local timezone"
+optional = false
+python-versions = ">=3.10"
+groups = ["main"]
+files = [
+ {file = "tzlocal-5.4.5-py3-none-any.whl", hash = "sha256:ecab534f6d65d9deb4c4ea52c121ab6dd5958e9b878e71dcb102578269752454"},
+ {file = "tzlocal-5.4.5.tar.gz", hash = "sha256:ba976898bad27945178c9ca25c222c19e322b30c10342b9f40b46714f70d2d5f"},
+]
+
+[package.dependencies]
+tzdata = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+devenv = ["zest.releaser"]
+testing = ["check_manifest", "pyroma", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "ruff"]
+
[[package]]
name = "uritemplate"
version = "4.1.1"
@@ -2788,4 +2809,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
[metadata]
lock-version = "2.1"
python-versions = "^3.12"
-content-hash = "2ac648dad93ba8cb5b776c841ba85149f22ea1903b1f8fc716edb6e8ed39c9c1"
+content-hash = "e23cad566c2aa30824c6842f71704c099ba60b7f6d9cb809ed828fb3c4688d1f"
diff --git a/pyproject.toml b/pyproject.toml
index 7778f697..54e17ab0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,7 +15,7 @@ djangorestframework = "3.15.2"
djangorestframework-jsonapi = "7.1.0"
django-cors-headers = "4.7.0"
django-filter = "^24.3"
-celery = "5.5.0"
+celery = "5.6.3"
jsonschema = "4.23.0"
aiohttp = "3.11.14"
pyjwe = "1.0.0"
@@ -30,6 +30,7 @@ itsdangerous = "^2.2.0"
redis = "^5.2.1"
drf-spectacular = "^0.28.0"
drf-spectacular-jsonapi = {git = "https://github.com/opaduchak/drf-spectacular-json-api.git", branch = "osf-fixes"}
+kombu = "5.6.2"
[tool.poetry.group.dev.dependencies]