Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ This creates:
**Supported Launchers:**
- `torchrun` - PyTorch DDP/FSDP
- `deepspeed` - ZeRO optimization
- `megatron` - Megatron-LM training
- `megatron-lm` - Megatron-LM training
- `torchtitan` - LLM pre-training
- `primus` - Primus unified pretrain (Megatron / TorchTitan / MaxText YAML)
- `vllm` - LLM inference
Expand Down
6 changes: 3 additions & 3 deletions examples/k8s-configs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ To validate rendered YAML after a debug run, install [kubeconform](https://githu

### Multi-node DNS (PyTorch vs Ray)

For **PyTorch-native** launchers (`torchrun`, `deepspeed`, `torchtitan`, `megatron`, `primus`), multi-node Jobs use a **headless Service** whose name matches `pod.spec.subdomain`, per Kubernetes DNS rules, so pods get stable per-pod DNS names for rendezvous.
For **PyTorch-native** launchers (`torchrun`, `deepspeed`, `torchtitan`, `megatron-lm`, `primus`), multi-node Jobs use a **headless Service** whose name matches `pod.spec.subdomain`, per Kubernetes DNS rules, so pods get stable per-pod DNS names for rendezvous.

For **Ray-based** multi-node (`vllm`, `sglang`), a headless Service may still be created for networking, but **per-pod DNS via `subdomain` is not applied** the same way as for PyTorch; production multi-node Ray on Kubernetes often uses **KubeRay** (see upstream vLLM / Ray docs). Treat Job-based multi-node Ray as a best-effort path.

Expand Down Expand Up @@ -580,7 +580,7 @@ Configuration for distributed workloads (training and inference):

| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `launcher` | string | - | Launcher type: `torchrun`, `deepspeed`, `torchtitan`, `megatron`, `primus`, `vllm`, `sglang` |
| `launcher` | string | - | Launcher type: `torchrun`, `deepspeed`, `torchtitan`, `megatron-lm`, `primus`, `vllm`, `sglang` |
| `enabled` | boolean | `false` | Enable distributed execution (legacy, prefer `launcher`) |
| `backend` | string | `"nccl"` | `"nccl"`, `"gloo"`, or `"mpi"` |
| `nnodes` | integer | `1` | Number of nodes |
Expand Down Expand Up @@ -679,7 +679,7 @@ Write durable outputs under `/results/<replica-id>/` in the container so each re
**Training Launchers:**
- **torchrun**: Standard PyTorch DDP/FSDP training
- **deepspeed**: ZeRO optimization for memory efficiency
- **megatron**: Megatron-LM tensor and pipeline parallelism
- **megatron-lm**: Megatron-LM tensor and pipeline parallelism
- **torchtitan**: LLM pre-training with multi-dimensional parallelism (FSDP2+TP+PP)
- **primus**: Unified Primus pretrain (Megatron / TorchTitan / MaxText experiment YAML; see [Primus on Kubernetes](#primus-on-kubernetes))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"launcher": "torchrun",
"nnodes": 1,
"nproc_per_node": 2,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"launcher": "torchrun",
"nnodes": 1,
"nproc_per_node": 2,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"launcher": "torchrun",
"nnodes": 2,
"nproc_per_node": 2,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"launcher": "torchrun",
"nnodes": 4,
"nproc_per_node": 2,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"launcher": "torchrun",
"nnodes": 1,
"nproc_per_node": 4,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"nnodes": 1,
"nproc_per_node": 2,

"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
},

"distributed": {
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 4,
"nproc_per_node": 8,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"launcher": "sglang-disagg",
"nnodes": 7,
"nproc_per_node": 8,
"master_port": 29500,
"port": 29500,
"sglang_disagg": {
"prefill_nodes": 4,
"decode_nodes": 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"launcher": "sglang-disagg",
"nnodes": 5,
"nproc_per_node": 8,
"master_port": 29500
"port": 29500
},

"context": {
Expand Down
2 changes: 1 addition & 1 deletion examples/k8s-configs/basic/sglang-multi-node-basic.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"launcher": "sglang",
"nnodes": 2,
"nproc_per_node": 4,
"master_port": 29500
"port": 29500
},

"context": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"launcher": "torchtitan",
"nnodes": 4,
"nproc_per_node": 8,
"master_port": 29500
"port": 29500
},

"context": {
Expand Down
2 changes: 1 addition & 1 deletion examples/k8s-configs/basic/vllm-multi-node-basic.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"launcher": "vllm",
"nnodes": 2,
"nproc_per_node": 4,
"master_port": 29500
"port": 29500
},

"context": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
},

"distributed": {
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 1,
"nproc_per_node": 2
},
Expand Down
2 changes: 1 addition & 1 deletion examples/k8s-configs/minimal/megatron-lm-minimal.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
},

"distributed": {
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 1,
"nproc_per_node": 2
},
Expand Down
4 changes: 2 additions & 2 deletions examples/k8s-configs/minimal/megatron-lm-optimized.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@
"distributed": {
"enabled": true,
"backend": "nccl",
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 1,
"nproc_per_node": 2,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
6 changes: 3 additions & 3 deletions examples/slurm-configs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ madengine uses intelligent multi-layer configuration merging:
```json
{
"distributed": {
"launcher": "torchrun", // Launcher type: torchrun, vllm, sglang, deepspeed, megatron, slurm_multi
"launcher": "torchrun", // Launcher type: torchrun, vllm, sglang, deepspeed, megatron-lm, slurm_multi
"backend": "nccl", // Communication backend (nccl/gloo)
"port": 29500, // Master node port
"nnodes": 2, // Number of nodes (overrides slurm.nodes if set)
Expand All @@ -418,12 +418,12 @@ madengine uses intelligent multi-layer configuration merging:
- `vllm`: vLLM inference engine (TP/PP parallelism)
- `sglang`: SGLang inference engine
- `deepspeed`: DeepSpeed training framework
- `megatron`: Megatron-LM large model training
- `megatron-lm`: Megatron-LM large model training
- `slurm_multi` / `slurm-multi`: Self-managed multi-container topologies (escape hatch)
- Custom: Set environment variables, model script handles launcher

**Note**: For vLLM and SGLang, the model script handles process spawning directly.
For torchrun/deepspeed/megatron, use `$MAD_MULTI_NODE_RUNNER` in your model script.
For torchrun/deepspeed/megatron-lm, use `$MAD_MULTI_NODE_RUNNER` in your model script.
For slurm_multi, the model's `.slurm` script runs on baremetal and manages Docker containers via `srun` internally.

### Environment Variables
Expand Down
4 changes: 2 additions & 2 deletions examples/slurm-configs/basic/09-megatron-lm-multi-node.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@
},

"distributed": {
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 4,
"nproc_per_node": 8,
"master_port": 29500
"port": 29500
},

"env_vars": {
Expand Down
2 changes: 1 addition & 1 deletion examples/slurm-configs/minimal/megatron-lm-minimal.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
},

"distributed": {
"launcher": "megatron",
"launcher": "megatron-lm",
"nnodes": 1,
"nproc_per_node": 2
},
Expand Down
42 changes: 42 additions & 0 deletions src/madengine/cli/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,47 @@ def _validate_gpu_vendor_guest_after_defaults(context: Dict[str, Any]) -> None:
)


def _validate_launcher_after_defaults(context: Dict[str, Any]) -> None:
"""Validate any launcher in the context and rewrite it to its canonical spelling.

Fails here rather than at deploy time: a launcher madengine does not recognize
used to run the model as a plain single-process job and still report SUCCESS,
so the benchmark number was wrong with nothing to indicate it.
"""
from madengine.core.errors import ConfigurationError
from madengine.deployment.common import validate_launcher

launcher_cfg = context.get("launcher")
if launcher_cfg is not None and not isinstance(launcher_cfg, dict):
# A bare string is a natural mistake, since distributed.launcher *is* a
# string. Name both valid shapes rather than only rejecting this one.
console.print(f"❌ Invalid launcher: [red]{launcher_cfg!r}[/red]")
console.print("💡 'launcher' must be an object. Use one of:")
console.print(
' [green]{"launcher": {"type": "torchrun", "nnodes": 2}}[/green]'
)
console.print(
' [green]{"distributed": {"launcher": "torchrun", "nnodes": 2}}[/green]'
)
raise typer.Exit(ExitCode.INVALID_ARGS)

targets = []
distributed = context.get("distributed")
if isinstance(distributed, dict) and "launcher" in distributed:
targets.append((distributed, "launcher"))
if isinstance(launcher_cfg, dict) and "type" in launcher_cfg:
targets.append((launcher_cfg, "type"))

for holder, key in targets:
try:
holder[key] = validate_launcher(holder[key], source="additional_context")
except ConfigurationError as exc:
console.print(f"❌ Invalid launcher: [red]{holder[key]!r}[/red]")
for suggestion in exc.suggestions or []:
console.print(f"💡 [green]{suggestion}[/green]")
raise typer.Exit(ExitCode.INVALID_ARGS)


def finalize_additional_context_dict(
context: Dict[str, Any],
*,
Expand Down Expand Up @@ -280,6 +321,7 @@ def finalize_additional_context_dict(
validate_additional_context_structure(context)
_normalize_docker_build_arg_values(context)
_validate_gpu_vendor_guest_after_defaults(context)
_validate_launcher_after_defaults(context)
return context


Expand Down
70 changes: 70 additions & 0 deletions src/madengine/deployment/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,76 @@ def __init__(self, config: DeploymentConfig):
self.config = config
self.manifest = self._load_manifest(config.manifest_file)
self.console = Console()
self._validate_launchers()

def _validate_launchers(self) -> None:
"""Validate every launcher this deployment will read, and canonicalize in place.

Deliberately in ``__init__`` rather than ``validate()``: ``execute()`` catches
bare ``Exception`` and returns a FAILED result without re-raising, so a
ConfigurationError raised any later never reaches the handler in
``cli/commands/run.py``. ``__init__`` runs under DeploymentFactory.create(),
which re-raises ConfigurationError, so the user gets INVALID_ARGS and a message
naming the correct spelling.

Raises:
ConfigurationError: If any configured launcher is not a valid launcher.
"""
# Imported here: common.py imports from core.errors, and a module-level import
# would make base.py part of that chain for every deployment consumer.
from madengine.core.errors import ConfigurationError, create_error_context

from .common import validate_launcher

context = self.config.additional_context or {}

distributed = context.get("distributed")
if isinstance(distributed, dict) and "launcher" in distributed:
distributed["launcher"] = validate_launcher(
distributed["launcher"], source="additional_context.distributed.launcher"
)

launcher_cfg = context.get("launcher")
if launcher_cfg is not None and not isinstance(launcher_cfg, dict):
# A bare string here is a natural mistake, since distributed.launcher *is*
# a string. Left alone it surfaces as AttributeError deep in the K8s
# template context, so name both valid shapes now.
raise ConfigurationError(
f"'launcher' in additional_context must be an object, got "
f"{type(launcher_cfg).__name__} ({launcher_cfg!r})",
context=create_error_context(
operation="validate_launchers",
component="deployment.base",
additional_info={"launcher": launcher_cfg},
),
suggestions=[
'Use {"launcher": {"type": "torchrun", "nnodes": 2}}',
'Or {"distributed": {"launcher": "torchrun", "nnodes": 2}}',
],
)
if isinstance(launcher_cfg, dict) and "type" in launcher_cfg:
launcher_cfg["type"] = validate_launcher(
launcher_cfg["type"], source="additional_context.launcher.type"
)

deployment_config = self.manifest.get("deployment_config")
if isinstance(deployment_config, dict):
manifest_distributed = deployment_config.get("distributed")
if isinstance(manifest_distributed, dict) and "launcher" in manifest_distributed:
manifest_distributed["launcher"] = validate_launcher(
manifest_distributed["launcher"],
source="build_manifest.json deployment_config.distributed.launcher",
)

for model_name, model_info in (self.manifest.get("built_models") or {}).items():
if not isinstance(model_info, dict):
continue
model_distributed = model_info.get("distributed")
if isinstance(model_distributed, dict) and "launcher" in model_distributed:
model_distributed["launcher"] = validate_launcher(
model_distributed["launcher"],
source=f"model '{model_name}' distributed.launcher",
)

def _load_manifest(self, manifest_file: str) -> Dict:
"""
Expand Down
Loading