Repository navigation
Expand file tree
/
Copy pathdocker-compose.yml
More file actions
401 lines (388 loc) · 16.2 KB
/
Copy pathdocker-compose.yml
File metadata and controls
401 lines (388 loc) · 16.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
# Testing environment for the Yggdrasil workflow.
#
# make compose-up # statusdb + yggdrasil + genomics-status
# # (yggdrasil runs the dind-flavoured image with a
# # docker:dind sidecar, so it can run nf-core
# # pipelines; slurmctld + slurmd provide a local
# # single-node SLURM cluster for job submission)
# make compose-up-full # the above + biomate (--profile full)
# make compose-rebuild # rebuild all images without cache (recreate containers)
# make compose-logs # follow yggdrasil logs
# make compose-reset # tear everything down, including volumes
#
# Yggdrasil starts only after its dependencies are healthy (compose
# depends_on) and its entrypoint verifies the seeded databases exist,
# creates its own databases, and (by default) injects a test scenario
# that the daemon then picks up, generates a plan for, and executes.
name: dataflow-testing
services:
statusdb:
image: dataflow-statusdb:latest
build:
context: ./StatusDB_NGI
dockerfile: Dockerfile
environment:
# Stack-specific variable names (STACK_*) so we never pick up
# unrelated COUCHDB_* variables from the ambient shell environment.
COUCHDB_USER: ${STACK_COUCH_USER:-admin}
COUCHDB_PASSWORD: ${STACK_COUCH_PASSWORD:-secret}
ports:
- "5984:5984"
volumes:
- couchdb-data:/opt/couchdb/data
# Test-flowcell seeds live in deploy/seed (not in the submodule —
# they are only needed by this testing environment). The image's
# init-seed-data.sh loads every subdirectory of
# /opt/couchdb/seed_data as a same-named database at first boot,
# so these mounts are picked up automatically. Re-seeding only
# happens after make compose-reset (the INIT_MARKER lives on the
# couchdb-data volume).
# Deliberately not :ro — the image entrypoint chowns everything
# under /opt/couchdb at boot (and aborts on a read-only mount), so
# first boot hands ownership of the mounted files to the image's
# couchdb user (uid 5984); `sudo chown -R $(id -u):$(id -g)` on
# the two directories restores editability.
- ./deploy/seed/demux_sample_info:/opt/couchdb/seed_data/demux_sample_info
- ./deploy/seed/flowcell_status:/opt/couchdb/seed_data/flowcell_status
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:5984/_up"]
interval: 5s
timeout: 5s
retries: 30
start_period: 15s
networks:
- yggdrasil-net
biomate:
image: dataflow-biomate:latest
build:
context: ./BioMate
dockerfile: Dockerfile
# Started only with --profile full (make compose-up-full): nothing at
# runtime depends on the BioMate web interface (the dmx_realm demux
# steps are simulated).
profiles: ["full"]
command: ["biomate", "web-interface", "--host", "0.0.0.0", "--port", "8765"]
user: "${HOST_UID:-1000}:${HOST_GID:-1000}"
ports:
- "8765:8765"
# No curl in this image; use python for the healthcheck.
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8765/', timeout=5)",
]
interval: 10s
timeout: 5s
retries: 18
start_period: 20s
networks:
- yggdrasil-net
genomics-status:
image: dataflow-genomics-status:latest
# Source + configs are baked into the image (see
# deploy/Dockerfile.genomics-status, which builds on top of the image
# produced by the submodule's Dockerfile). No bind mounts: the submodule
# working tree must never be written to at runtime.
build:
context: .
dockerfile: deploy/Dockerfile.genomics-status
# Part of the default stack: the yggdrasil entrypoint waits for this
# UI before starting the daemon (deploy/yggdrasil-entrypoint.sh,
# step 6), so it comes up with the core stack, not only with
# --profile full.
ports:
- "9761:9761"
healthcheck:
test:
[
"CMD",
"/opt/conda/bin/python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:9761/', timeout=5)",
]
interval: 10s
timeout: 5s
retries: 18
start_period: 30s
depends_on:
statusdb:
condition: service_healthy
networks:
- yggdrasil-net
yggdrasil:
# The dind-flavoured image (Nextflow + Docker CLI; see
# deploy/Dockerfile.yggdrasil.dind). Both yggdrasil images are built
# explicitly by the Makefile (target compose-build-yggdrasil), not by
# compose: the dind image is layered on the base image, and compose
# can't express that dependency between one service's build and its
# runtime image. Plain `docker compose up` therefore assumes the
# images exist - use `make compose-up`.
image: dataflow-yggdrasil-dind:latest
# The daemon stops gracefully on SIGTERM (cli.py routes it to the same
# shutdown path as Ctrl+C), but its stop waits for in-flight CouchDB
# changes polls (uncancellable threads) and the plan drain. Each longpoll
# holds up to its configured time plus a 5 s socket margin; both are 5 s in
# deploy/yggdrasil-config/main.json (production default 60 s):
# yggdrasil.plan_watcher_longpoll_timeout_ms for the PlanWatcher and
# external_systems.defaults.couchdb.longpoll_timeout_ms for the realm
# watchers. Give the stop headroom beyond that: it then ends with exit 0
# instead of a SIGKILL (exit 137). Raise this if you raise either key.
stop_grace_period: "30s"
environment:
YGG_COUCH_USER: ${STACK_COUCH_USER:-admin}
YGG_COUCH_PASS: ${STACK_COUCH_PASSWORD:-secret}
YGG_COUCH_URL: http://statusdb:5984
YGG_WORK_ROOT: /work
YGG_EVENT_SPOOL: /events
OPS_DB: yggdrasil_ops
YGG_INJECT_TEST_SCENARIO: ${YGG_INJECT_TEST_SCENARIO:-true}
YGG_SCENARIO_ID: ${YGG_SCENARIO_ID:-test_scenario:compose-demo}
# Where the dmx_realm Nextflow runs execute: "local" (default, a
# subprocess of the daemon) or "slurm" (sbatch job on the SLURM
# node, like a production HPC). The entrypoint materialises the
# choice as dev_dmx_realm.json (see deploy/yggdrasil-entrypoint.sh).
DMX_EXECUTOR: ${DMX_EXECUTOR:-local}
# Put Nextflow's home (pipeline asset clones + caches) on the shared
# ygg-work volume: the dind sidecar mounts the same volume at /work,
# and nested pipeline containers bind-mount repo-internal files
# (e.g. MULTIQC's assets/multiqc_config.yml) through the dind daemon.
# Invisible paths make Nextflow stage dangling symlinks there.
NXF_HOME: /work/nf-nxf
PYTHONUNBUFFERED: "1"
# Rich's console log handler sizes its line wrapping to the console
# width; with stdout as a pipe (detached container) it reads the
# COLUMNS env var instead of a TTY. 200 keeps long log lines
# unbroken in `docker compose logs`.
COLUMNS: "200"
# Ensure Nextflow is in PATH (it should be set by the image, but
# explicitly add it here to be safe)
PATH: /opt/nextflow:/opt/venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# Nextflow configuration for demultiplexing
DMX_NF_CONFIG: /work/demultiplex_local.config
DMX_HPC_BASE_PATH: /work
NXF_VER: 26.10.0
NXF_OPTS: -Ddocker.run.args='--platform linux/amd64'
volumes:
# Bind-mount the demux_realm submodule over the copy baked into the
# image (editable-installed at /app/demux_realm) so local edits are
# picked up on container (re)start without rebuilding the image.
- ./demux_realm:/app/demux_realm
# Same pattern for the configuration directory: host edits to
# main.json / dmx_realm.json are picked up on container (re)start
# without rebuilding the image. Not :ro - the entrypoint writes
# dev_dmx_realm.json there to switch the realm's Nextflow executor
# (DMX_EXECUTOR=slurm). The image-baked copy remains as fallback
# for image-only deployments. (The daemon caches configs per
# process, so a container restart is still required to see edits.)
- ./deploy/yggdrasil-config:/app/yggdrasil_workspace/common/configurations
- ygg-work:/work
- ygg-events:/events
- ygg-logs:/logs/yggdrasil
# The SLURM cluster's munge key (the slurm-munge volume, mounted at
# /etc/munge and shared with the slurmctld/slurmd services): the
# daemon's sbatch/sacct calls authenticate with it (the dind image
# installs the SLURM client; see deploy/Dockerfile.yggdrasil.dind).
- slurm-munge:/etc/munge
# Tiny Illumina BCL run for nf-core/demultiplex test runs (the
# dmx_realm demux workflow; see the README "Firing more work").
- ./BioMate/temp:/data/test_run:ro
depends_on:
# statusdb and dind only. biomate is profile-gated ("full") and
# compose rejects depends_on entries that point at profile-excluded
# services; nothing at runtime needs biomate anyway (the
# dmx_realm demux steps are simulated, not real BioMate runs).
# genomics-status is in the default stack but stays a soft
# entrypoint wait, not a static dependency: it is a UI the daemon
# does not need, and a hard gate would couple daemon startup to its
# slower conda-python boot.
statusdb:
condition: service_healthy
# Awaited: the daemon and the realm talk to the dind daemon via
# DOCKER_HOST=tcp://dind:2375 (baked into the dind image).
dind:
condition: service_healthy
# The local SLURM cluster: the entrypoint waits for the munge key
# these containers share, and the slurm_job test scenario submits
# batch jobs to this node (hard gate: without it every SLURM
# submission in the stack would fail to authenticate).
slurmctld:
condition: service_healthy
slurmd:
condition: service_healthy
networks:
- yggdrasil-net
# Docker daemon-in-docker sidecar: lets the dind-flavoured yggdrasil
# image run nf-core pipeline containers with -profile docker.
# The daemon itself stays arm64 (native runc + network hooks work);
# amd64-only pipeline tool images (e.g. samshee, bcl-convert) run as
# leaf containers translated by Rosetta — requires Docker Desktop's
# "Use Rosetta for x86_64 emulation" to be enabled.
# Part of the default stack (yggdrasil depends on it).
dind:
image: docker:27-dind
privileged: true
# Expose the daemon on 0.0.0.0:2375 (plain TCP; TLS is disabled via
# DOCKER_TLS_CERTDIR="") so the yggdrasil container can reach it at
# $DOCKER_HOST=tcp://dind:2375. The default command only binds the
# unix socket.
command: ["dockerd", "-H", "unix:///var/run/docker.sock", "-H", "tcp://0.0.0.0:2375"]
environment:
# The dind image defaults to TLS on :2376 with auto-generated certs
# (enabled by DOCKER_TLS_CERTDIR); this stack is local-only, so use
# plain TCP on :2375 (matches DOCKER_HOST baked into the
# dind-flavoured yggdrasil image).
DOCKER_TLS_CERTDIR: ""
volumes:
- dind-data:/var/lib/docker
# Share the ygg-work volume: Nextflow work dirs must be visible
# from BOTH containers so the nested daemon can bind-mount them
# into pipeline containers (run nextflow with
# --work-dir /work/...).
- ygg-work:/work
healthcheck:
test: ["CMD-SHELL", "docker info >/dev/null 2>&1"]
interval: 5s
timeout: 10s
retries: 30
start_period: 10s
networks:
- yggdrasil-net
# MariaDB backend for slurmdbd (the SLURM accounting database). Real HPCs
# run slurmctld + slurmdbd + a database; without accounting storage
# sacct is disabled, and Yggdrasil's SlurmJobManager polls sacct.
slurmdbd-db:
image: mariadb:11
environment:
MARIADB_ROOT_PASSWORD: secret
MARIADB_DATABASE: slurm
MARIADB_USER: slurm
MARIADB_PASSWORD: secret
volumes:
- slurmdbd-data:/var/lib/mysql
healthcheck:
test: ["CMD-SHELL", "healthcheck.sh --connect --innodb_initialized"]
interval: 5s
timeout: 10s
retries: 30
start_period: 30s
networks:
- yggdrasil-net
# SLURM accounting daemon (see deploy/Dockerfile.slurmdbd and
# deploy/slurm/slurmdbd.conf). The cluster/user bootstrap is done by
# the slurmdbd-bootstrap service (24.11 has no AutoCreate).
slurmdbd:
image: dataflow-slurmdbd:latest
build:
context: .
dockerfile: deploy/Dockerfile.slurmdbd
hostname: slurmdbd
depends_on:
slurmdbd-db:
condition: service_healthy
volumes:
- slurm-munge:/etc/munge
healthcheck:
# Ready once slurmdbd answers its clients: sacctmgr (installed in
# the image) lists the registered clusters from the database.
test: ["CMD-SHELL", "sacctmgr show clusters >/dev/null 2>&1"]
interval: 10s
timeout: 10s
retries: 30
start_period: 30s
networks:
- yggdrasil-net
# One-shot accounting bootstrap (SLURM 24.11 has no AutoCreate):
# registers the cluster; 24.11's cluster registration also creates the
# root account, the root admin user and their association (see
# deploy/slurm/bootstrap-slurmdbd.sh). Idempotent (checks state before
# adding), exits 0 on success; compose leaves it finished.
slurmdbd-bootstrap:
image: dataflow-slurmdbd:latest
entrypoint: ["/usr/local/bin/bootstrap-slurmdbd.sh"]
depends_on:
slurmdbd:
condition: service_healthy
volumes:
# The shared munge key: the script starts munged with it so
# sacctmgr can authenticate to slurmdbd.
- slurm-munge:/etc/munge
restart: "no"
networks:
- yggdrasil-net
# Single-node SLURM cluster (controller + one compute node): a stand-in
# for the HPC, so the yggdrasil daemon can submit real batch jobs to
# SLURM on the local machine. Real SLURM (Debian trixie package), munge
# auth, nothing leaves the compose network. See deploy/slurm/slurm.conf
# and deploy/Dockerfile.slurm-ctld / .slurmd.
slurmctld:
image: dataflow-slurmctld:latest
build:
context: .
dockerfile: deploy/Dockerfile.slurm-ctld
hostname: slurmctld
depends_on:
slurmdbd:
condition: service_healthy
volumes:
# The cluster's munge key (the volume is mounted at /etc/munge, which
# holds the key in the Debian munge package layout). This container
# starts first in the stack, so its entrypoint is the key authority;
# slurmd and yggdrasil read the same key from the same volume.
- slurm-munge:/etc/munge
# Controller state (saved job state + accounting): persists across
# container (re)creations so job IDs stay valid.
- slurm-state:/var/spool/slurmctld
healthcheck:
test: ["CMD-SHELL", "scontrol ping 2>/dev/null | grep -q 'is UP'"]
interval: 5s
timeout: 10s
retries: 30
start_period: 30s
networks:
- yggdrasil-net
slurmd:
image: dataflow-slurmd:latest
build:
context: .
dockerfile: deploy/Dockerfile.slurmd
hostname: slurmd
depends_on:
slurmctld:
condition: service_healthy
# Batch jobs run Nextflow with -profile docker against the dind
# daemon (DOCKER_HOST in deploy/Dockerfile.slurmd); the realm's
# batch script does not wait for it.
dind:
condition: service_healthy
volumes:
- slurm-munge:/etc/munge
# Shared job workspace: the yggdrasil container writes the batch
# scripts there and the node executes them there.
- ygg-work:/work
healthcheck:
# Ready once the node has registered with the controller and
# reports the idle state.
test: ["CMD-SHELL", "test \"$(sinfo -h -n slurmd -o %t 2>/dev/null)\" = idle"]
interval: 5s
timeout: 10s
retries: 30
start_period: 30s
networks:
- yggdrasil-net
networks:
yggdrasil-net:
volumes:
couchdb-data:
ygg-work:
ygg-events:
ygg-logs:
dind-data:
# SLURM cluster: the shared munge key (mounted at /etc/munge in the
# slurmctld, slurmd, slurmdbd and yggdrasil containers), the controller
# state, and the slurmdbd MariaDB data.
slurm-munge:
slurm-state:
slurmdbd-data: