[Feature] Pass env / secrets into agent host

README.md

@@ -80,6 +80,15 @@ Agents that need API keys read them from environment variables. Point `env_file`
 env_file: .env
 ```
 
+#### Step 1.1: Passing secrets to agents (optional)
+
+Agents that need API keys read them from environment variables. Point `env_file` at a `.env` file to have Ventis inject it into every agent container:
+
+```yaml
+# config/global_controller.yaml
+env_file: .env
+```
+
 #### Step 2: Build the project
 ```bash
 ventis build

ventis/cli.py

@@ -16,6 +16,8 @@
 import subprocess
 import sys
 
+from ventis.controller.utils.env_file import resolve_env_file
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("ventis")
 DEFAULT_DOCKER_PLATFORM = "linux/amd64"
@@ -446,7 +448,15 @@ def cmd_deploy(args):
     prefix = _artifact_prefix(project_dir)
     artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir
 
-    _ensure_grpc_stubs_importable(artifact_root)
+    # Fail here rather than after a fleet of containers is already up without
+    # the API keys they need.
+    try:
+        resolve_env_file(config, base_dir=project_dir)
+    except ValueError as e:
+        logger.error("%s", e)
+        sys.exit(1)
+
+    _ensure_grpc_stubs_importable(project_dir)
 
     if any(
         agent.get("provider", "local").upper() == "EC2"

ventis/controller/cloud_provider_logic/EC2/_runtime.py

@@ -23,6 +23,7 @@
 
 import boto3
 
+from ventis.controller.utils.env_file import env_file_args
 from ventis.controller.utils.redis_utils import _wait_for_redis
 from ventis.utils.redis_client import RedisClient
 
@@ -286,8 +287,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port,
             cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"])
         if project_id:
             cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"])
-    cmd.append(image)
-    result = _controller._run_cmd(cmd, host, user=ssh_user)
+
+    # User secrets from `env_file`. Explicit -e flags above still win over
+    # anything in the file.
+    with env_file_args(
+        _controller, host, ssh_user, container_name, is_local=False
+    ) as env_args:
+        cmd.extend(env_args)
+        cmd.append(image)
+        result = _controller._run_cmd(cmd, host, user=ssh_user)
     if result.returncode != 0:
         raise RuntimeError(
             f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}"

ventis/controller/cloud_provider_logic/Local/_runtime.py

@@ -8,6 +8,8 @@
 
 import logging
 
+from ventis.controller.utils.env_file import env_file_args
+
 logger = logging.getLogger(__name__)
 
 DEFAULT_HOST = "localhost"
@@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id):
         cmd.extend(["--memory", f"{resources['memory']}m"])
     if resources.get("gpu"):
         cmd.extend(["--gpus", str(resources["gpu"])])
-    cmd.append(image)
 
-    result = _require_controller()._run_cmd(cmd, host, user)
+    # User secrets from `env_file`. Explicit -e flags above still win, so a
+    # stray VENTIS_* line in someone's .env cannot break agent wiring.
+    with env_file_args(
+        _require_controller(), host, user, runtime_id, _is_local_host(host)
+    ) as env_args:
+        cmd.extend(env_args)
+        cmd.append(image)
+        result = _require_controller()._run_cmd(cmd, host, user)
     if result.returncode != 0:
         raise RuntimeError(f"Failed to launch {runtime_id}")
 

ventis/controller/global_controller.py

@@ -4,6 +4,7 @@
 
 import atexit
 import logging
+import shlex
 import signal
 import subprocess
 import threading
@@ -16,6 +17,7 @@
 import yaml
 from ventis.controller.instance_manager import InstanceManager
 from ventis.controller.utils.agent_specs import write_agent_specs
+from ventis.controller.utils.env_file import resolve_env_file
 from ventis.controller.utils.redis_utils import _wait_for_redis
 from ventis.controller.utils.telemetry_logging import (
     assign_project_id,
@@ -64,6 +66,9 @@ class GlobalController(object):
     def __init__(self, config_path):
         self.config_path = config_path
         self.config = self._load_config(config_path)
+        # Validate before launching anything: an agent that boots without its
+        # API keys fails deep inside a container, where it is expensive to debug.
+        self.env_file_path = resolve_env_file(self.config)
 
         redis_cfg = self.config.get("redis", {})
         self.redis = RedisClient(
@@ -174,6 +179,7 @@ def reload_config(self):
         """Reload the config file and rebuild the routing table."""
         logger.info("Reloading config from %s", self.config_path)
         self.config = self._load_config(self.config_path)
+        self.env_file_path = resolve_env_file(self.config)
         self.controllers = self.config.get("agents", [])
         self.poll_interval = self.config.get("poll_interval", 5)
         assign_project_id(self.config.get("project_id", 0))
@@ -642,6 +648,28 @@ def _send(instance):
     #  Runtime launching                                                  #
     # ------------------------------------------------------------------ #
 
+    def _ssh_args(self, host, user=None):
+        """Return the `ssh ... target` prefix used to reach a remote host."""
+        ssh_key_path = os.path.expanduser(
+            self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2")
+        )
+        return [
+            "ssh",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-o",
+            "IdentitiesOnly=yes",
+            "-o",
+            "ConnectTimeout=10",
+            "-o",
+            "ServerAliveInterval=10",
+            "-o",
+            "ServerAliveCountMax=3",
+            "-i",
+            ssh_key_path,
+            f"{user}@{host}" if user else host,
+        ]
+
     def _run_cmd(self, cmd, host, user=None):
         """
         Run a command locally or on a remote host via SSH.
@@ -656,41 +684,52 @@ def _run_cmd(self, cmd, host, user=None):
         """
         is_local = _is_local_host(host)
         if is_local:
-            return subprocess.run(
-                cmd, capture_output=True, text=True, timeout=180
-            )
-        else:
-            ssh_key_path = os.path.expanduser(
-                self.config.get("ec2", {}).get(
-                    "ssh_private_key_path", "~/.ssh/ventis_ec2"
-                )
-            )
-            ssh_target = f"{user}@{host}" if user else host
-            remote_cmd = " ".join(cmd)
-            if cmd and cmd[0] == "docker":
-                remote_cmd = f"sudo {remote_cmd}"
-            return subprocess.run(
-                [
-                    "ssh",
-                    "-o",
-                    "StrictHostKeyChecking=no",
-                    "-o",
-                    "IdentitiesOnly=yes",
-                    "-o",
-                    "ConnectTimeout=10",
-                    "-o",
-                    "ServerAliveInterval=10",
-                    "-o",
-                    "ServerAliveCountMax=3",
-                    "-i",
-                    ssh_key_path,
-                    ssh_target,
-                    remote_cmd,
-                ],
+            return subprocess.run(cmd, capture_output=True, text=True, timeout=180)
+
+        remote_cmd = " ".join(cmd)
+        if cmd and cmd[0] == "docker":
+            remote_cmd = f"sudo {remote_cmd}"
+        return subprocess.run(
+            self._ssh_args(host, user) + [remote_cmd],
+            capture_output=True,
+            text=True,
+            timeout=180,
+        )
+
+    def _push_file(self, local_path, remote_path, host, user=None):
+        """
+        Copy a local file to a remote host over SSH.
+
+        Streams the bytes through `cat` under `umask 077` rather than using
+        `scp`, so a secrets file is never briefly world-readable on the far
+        side.
+
+        Anything already sitting at the destination is removed first: `umask`
+        only governs files the shell creates, and `>` follows symlinks. Without
+        the `rm`, a local user on the remote host could pre-create the path
+        world-readable, or point it at a file of their own, and collect
+        whatever we write there.
+
+        Returns:
+            subprocess.CompletedProcess
+        """
+        quoted = shlex.quote(remote_path)
+        remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}"
+        with open(local_path, "rb") as f:
+            result = subprocess.run(
+                self._ssh_args(host, user) + [remote_cmd],
+                stdin=f,
                 capture_output=True,
                 text=True,
                 timeout=180,
+                check=False,
+            )
+        if result.returncode != 0:
+            raise RuntimeError(
+                f"Failed to copy {local_path} to {host}:{remote_path}: "
+                f"{(result.stderr or result.stdout or '').strip()}"
             )
+        return result
 
     def launch_docker_agents(self):
         """Launch all configured runtimes through InstanceManager."""