From 6f36d4f1c090def778f5275c8b25da45a9844864 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:22:28 -0700 Subject: [PATCH 1/6] Add the canyonos CLI package, porting skill, and .car examples The CLI layer on top of the ventis fixes: `canyonos` wraps the Global Controller in a container and drives it over HTTP, so a project goes from source to a running workflow with a local dashboard in one command. - cli/: the canyonos package -- deploy (which folds in init, sync, build and launch, then auto-starts the dashboard once the workflow reports up), serve, stop, logs, quit, clean, config, integrate, new-app. - cli/canyonos/dashboard.compose.yml: api/db/web stack. The api port is published so the GC container can POST OTLP spans to /v1/traces, which is also what renames ventis' `project_id` attribute to the `canyon.project.id` the dashboard queries filter on. serve replaces the api container every run, since it reads the controller's Redis identity only at startup and would otherwise keep serving a stale project. - cli/canyonos/constants.py: resolve the config path per call rather than at import, preferring .car/config over the flat layout. - .claude/skills/porting-to-canyonos-core/: the porting skill `canyonos integrate` installs, plus its validator. - examples/: joke_writer converted to the .car layout, epigenomics added, and workflow imports pointed at each agent's entrypoint path to match single-location stub placement. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 4 + README.md | 32 +- SESSION_NOTES.md | 79 ++ .../skills/porting-to-canyonos-core/SKILL.md | 240 ++++ .../references/ec2.md | 41 + .../references/llm-proxy.md | 64 + .../references/packaging.md | 81 ++ .../references/runtime-contract.md | 187 +++ .../references/troubleshooting.md | 60 + .../porting-to-canyonos-core/validate.py | 1257 +++++++++++++++++ cli/README.md | 21 + cli/canyonos/__init__.py | 0 cli/canyonos/clean.py | 28 + cli/canyonos/config.py | 345 +++++ cli/canyonos/constants.py | 9 + cli/canyonos/dashboard.compose.yml | 42 + cli/canyonos/dashboard_stack.py | 518 +++++++ cli/canyonos/deploy.py | 86 ++ cli/canyonos/init.py | 129 ++ cli/canyonos/integrate.py | 82 ++ cli/canyonos/logs.py | 37 + cli/canyonos/new_app.py | 21 + cli/canyonos/quit.py | 63 + cli/canyonos/serve.py | 18 + cli/canyonos/stop.py | 47 + cli/canyonos/sync.py | 40 + cli/canyonos/theme.py | 20 + cli/cli.py | 217 +++ cli/pyproject.toml | 27 + cli/tests/test_dashboard_stack.py | 432 ++++++ cli/utils/__init__.py | 0 cli/utils/tui.py | 113 ++ examples/epigenomics/README.md | 70 + examples/epigenomics/agents/dedup_agent.py | 44 + examples/epigenomics/agents/dedup_agent.yaml | 10 + examples/epigenomics/agents/filter_agent.py | 32 + examples/epigenomics/agents/filter_agent.yaml | 12 + examples/epigenomics/agents/index_agent.py | 30 + examples/epigenomics/agents/index_agent.yaml | 12 + examples/epigenomics/agents/map_agent.py | 33 + examples/epigenomics/agents/map_agent.yaml | 14 + examples/epigenomics/agents/sort_agent.py | 32 + examples/epigenomics/agents/sort_agent.yaml | 14 + examples/epigenomics/agents/split_agent.py | 27 + examples/epigenomics/agents/split_agent.yaml | 12 + .../epigenomics/config/global_controller.yaml | 76 + examples/epigenomics/config/policy.yaml | 20 + .../workflow/epigenomics_workflow.py | 97 ++ .../helloworld/config/global_controller.yaml | 6 +- examples/joke_writer/.car/app/.env.example | 20 + examples/joke_writer/.car/app/LICENSE | 21 + examples/joke_writer/.car/app/README.md | 177 +++ .../joke_writer/.car/app/joke_workflow.py | 39 + examples/joke_writer/.car/app/joke_writer.py | 164 +++ .../.car/config/global_controller.yaml | 52 + .../joke_writer/.car/config/joke_agent.yaml | 35 + examples/joke_writer/README.md | 8 +- examples/joke_writer/agents/joke_agent.py | 63 - examples/joke_writer/agents/joke_agent.yaml | 53 - .../joke_writer/config/global_controller.yaml | 82 -- examples/joke_writer/config/policy.yaml | 20 - .../joke_writer/workflow/joke_workflow.py | 59 - .../skills/porting-to-canyonos-core/SKILL.md | 240 ++++ .../references/ec2.md | 41 + .../references/llm-proxy.md | 64 + .../references/packaging.md | 81 ++ .../references/runtime-contract.md | 187 +++ .../references/troubleshooting.md | 60 + .../porting-to-canyonos-core/validate.py | 1257 +++++++++++++++++ examples/portfolio/agents/advisor_agent.py | 4 +- examples/portfolio/agents/intent_agent.py | 4 +- .../portfolio/config/global_controller.yaml | 32 +- .../portfolio/workflow/portfolio_workflow.py | 2 +- examples/text2sql/agents/vllm_agent.py | 4 +- 74 files changed, 7321 insertions(+), 329 deletions(-) create mode 100644 SESSION_NOTES.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/SKILL.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/ec2.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/packaging.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md create mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md create mode 100755 cli/.claude/skills/porting-to-canyonos-core/validate.py create mode 100644 cli/README.md create mode 100644 cli/canyonos/__init__.py create mode 100644 cli/canyonos/clean.py create mode 100644 cli/canyonos/config.py create mode 100644 cli/canyonos/constants.py create mode 100644 cli/canyonos/dashboard.compose.yml create mode 100644 cli/canyonos/dashboard_stack.py create mode 100644 cli/canyonos/deploy.py create mode 100644 cli/canyonos/init.py create mode 100644 cli/canyonos/integrate.py create mode 100644 cli/canyonos/logs.py create mode 100644 cli/canyonos/new_app.py create mode 100644 cli/canyonos/quit.py create mode 100644 cli/canyonos/serve.py create mode 100644 cli/canyonos/stop.py create mode 100644 cli/canyonos/sync.py create mode 100644 cli/canyonos/theme.py create mode 100644 cli/cli.py create mode 100644 cli/pyproject.toml create mode 100644 cli/tests/test_dashboard_stack.py create mode 100644 cli/utils/__init__.py create mode 100644 cli/utils/tui.py create mode 100644 examples/epigenomics/README.md create mode 100644 examples/epigenomics/agents/dedup_agent.py create mode 100644 examples/epigenomics/agents/dedup_agent.yaml create mode 100644 examples/epigenomics/agents/filter_agent.py create mode 100644 examples/epigenomics/agents/filter_agent.yaml create mode 100644 examples/epigenomics/agents/index_agent.py create mode 100644 examples/epigenomics/agents/index_agent.yaml create mode 100644 examples/epigenomics/agents/map_agent.py create mode 100644 examples/epigenomics/agents/map_agent.yaml create mode 100644 examples/epigenomics/agents/sort_agent.py create mode 100644 examples/epigenomics/agents/sort_agent.yaml create mode 100644 examples/epigenomics/agents/split_agent.py create mode 100644 examples/epigenomics/agents/split_agent.yaml create mode 100644 examples/epigenomics/config/global_controller.yaml create mode 100644 examples/epigenomics/config/policy.yaml create mode 100644 examples/epigenomics/workflow/epigenomics_workflow.py create mode 100644 examples/joke_writer/.car/app/.env.example create mode 100644 examples/joke_writer/.car/app/LICENSE create mode 100644 examples/joke_writer/.car/app/README.md create mode 100644 examples/joke_writer/.car/app/joke_workflow.py create mode 100644 examples/joke_writer/.car/app/joke_writer.py create mode 100644 examples/joke_writer/.car/config/global_controller.yaml create mode 100644 examples/joke_writer/.car/config/joke_agent.yaml delete mode 100644 examples/joke_writer/agents/joke_agent.py delete mode 100644 examples/joke_writer/agents/joke_agent.yaml delete mode 100644 examples/joke_writer/config/global_controller.yaml delete mode 100644 examples/joke_writer/config/policy.yaml delete mode 100644 examples/joke_writer/workflow/joke_workflow.py create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md create mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md create mode 100755 examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/.gitignore b/.gitignore index 085b4a3..1d7998b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ env/ Thumbs.db ._* +# Canyon artifacts. `.car` is generated from the application source by the +# porting skill and `ventis build`; it is never committed. +.car/ + # Generated stubs stubs/ grpc_stubs/ diff --git a/README.md b/README.md index 81d944f..1d61db8 100644 --- a/README.md +++ b/README.md @@ -39,26 +39,26 @@ cd my-app ``` This command creates a new directory `my-app` with the following structure: ``` -├── agents/ # Agent implementations and YAML definitions -│ ├── example_agent.py -│ └── example_agent.yaml -├── workflows/ # Workflow scripts (deployed as REST APIs) -│ └── example_workflow.py -├── config/ -│ ├── global_controller.yaml # Deployment configuration -│ └── policy.yaml # Access control rules -├── stubs/ # Generated agent stubs (auto-generated) -├── grpc_stubs/ # Generated gRPC stubs (auto-generated) -└── README.md # Readme for the project +├── .car/ +│ ├── app/ # Source copy used for builds +│ ├── config/ +│ │ ├── global_controller.yaml +│ │ ├── example_agent.yaml # Agent declaration +│ │ └── policy.yaml +│ ├── stubs/ +│ ├── grpc_stubs/ +│ └── docker_container/ +└── README.md ``` The Readme in the newly created project directory provides a quick overview of the project and how to use it. Including how to add new files etc. We provide some overview in next few steps. #### Step 2: Define Your Agents -Place your agent logic (`.py`) and definitions (`.yaml`) in the `agents/` directory. +Agent declarations live under `.car/config/`. The source used for builds is +copied to `.car/app/` by `canyonos integrate`. -- **`agents/my_agent.yaml`**: Defines methods and schemas. -- **`agents/my_agent.py`**: Contains the actual Python implementation. +- **`.car/config/my_agent.yaml`**: Defines methods and schemas. +- **`.car/app/path/to/my_agent.py`**: Contains the agent implementation. We have provided an example of a finance agent and a market research agent in the `examples/` directory. To run the example, copy files into your newly created project directory from within the your my-app directory with the command - @@ -69,14 +69,14 @@ cp -r ../examples/* ./ ## Deployment Guide #### Step 1: Configure the Global Controller -Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +Edit `.car/config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. #### Step 1.1: Passing secrets to agents (optional) Agents that need API keys read them from environment variables. Point `env_file` at a `.env` file to have Ventis inject it into every agent container: ```yaml -# config/global_controller.yaml +# .car/config/global_controller.yaml env_file: .env ``` diff --git a/SESSION_NOTES.md b/SESSION_NOTES.md new file mode 100644 index 0000000..360efcc --- /dev/null +++ b/SESSION_NOTES.md @@ -0,0 +1,79 @@ +# Session notes: canyonos serve, OTel pipeline, examples + +## Fixed (code changes, rebuilt+pushed `saakeths/canyonos:latest` where needed) + +1. **`ventis/stub_generator.py`**: stubs only ever got placed at ONE path + (nested-at-entrypoint, never flat), breaking any workflow that imports a + sibling agent directly (`from split_agent import SplitAgent`, e.g. + `examples/epigenomics`). Now placed at both. +2. **`otel_exporter.py`**: hardcoded Redis to `localhost`, but it runs inside + the GC container (bridge networking) while Redis is a sibling container — + crash-looped forever. Fixed to `host.docker.internal`. +3. **`ventis/OTLP_Exporter/db.py`**: `write_waiting_rows()` unconditionally + priced every future via a `aws_instance_pricing` table that only exists for + EC2 deployments — silently dropped **every** span for **every** + `provider: local` deployment, always. Wrapped cost lookups in try/except, + falls back to $0. +4. **`dashboard_stack.py`**: `web` port was hardcoded 8080 with no fallback. + Now searches for a free port (reuses an already-running dashboard's port + if one exists), same pattern as `init.py`'s GC port selection. +5. **`dashboard_stack.py`**: `database.url` was required; made optional + (dashboard boots fine with no DB configured in the project's own config). +6. **`dashboard.compose.yml`**: added `pull_policy: never` for the local-only + `canyonos-otel-receiver:local` image (compose was trying to pull it from a + registry that doesn't have it). Sequenced `otel-receiver` to start after + `api` (both raced to create `otel_spans`; `api`'s bare `CREATE TABLE` lost + the race and crashed). + +## Bundled (new, working) + +- `db` (plain Postgres) + `otel-receiver` (new `Dockerfile` for + `otlp_pg_receiver`, built locally as `canyonos-otel-receiver:local`) added + to `dashboard.compose.yml`. Verified real spans, sent via the actual OTLP + gRPC exporter, land in `otel_spans`. + +## Workarounds applied, NOT real fixes (will resurface) + +- **`canyonos quit` only tears down the GC container + volume**, never the + deployed agent/workflow/redis containers. Had to `docker rm -f` those by + exact name every time before a truly clean restart. +- **The named workspace volume is additive-only** (`docker cp`, never + clears) — files from a previous project leak into the next one's build + until you manually nuke the volume. +- **`otlp_pg_receiver` holds one Postgres connection with no reconnect + logic** — a DB restart silently kills every future write until the + receiver container itself is restarted. +- **`joke_writer/.car` layout** (`app/`, `config/`) doesn't match what + `ventis/cli.py`'s build step expects (`agents/`, flat entrypoints) — worked + around by manually copying files into the shape it wants. The real fix + (`nickhuo/car-artifact-layout`, already pushed) was not merged in. +- **joke_writer's LLM calls are stubbed to return `"animal"`** — for testing + only, real Bedrock creds needed to restore actual behavior (commented-out + code left in place). + +## Known, not touched + +- Pre-existing OrbStack local-provider startup race (first request right + after a container reports healthy can fail); a fix exists on an unrelated, + unmerged branch. +- Stale global `uv tool install` is a recurring trap — always + `uv tool install --reinstall .` after any `cli/` change. + +## What's still needed to actually see data in the UI + +The whole pipeline up to Postgres now genuinely works. **Nothing shows up in +the dashboard because `canyonos-api`/`canyonos-web` have zero code that reads +or displays `otel_spans`** — confirmed by inspecting their actual source +(they're a `cc-forge` rebrand: deploy/project management + a static +code-structure diagram, unrelated data model). To close the loop: + +1. New API route(s) in `canyon-code-forge/packages/api` that query + `otel_spans`. +2. New UI screen(s) in `canyon-code-forge/apps/web` to render it. +3. `web` currently has **no path to reach `api` at all** even once that + exists — no reverse proxy in its Caddyfile, and `api`'s port isn't + published to the host in `dashboard.compose.yml`. Needs one or the other + before the browser can fetch anything. + +All of the above is real feature work in a different repo, not a config or +wiring fix. diff --git a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md new file mode 100644 index 0000000..fe6dd49 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -0,0 +1,240 @@ +--- +name: porting-to-canyonos-core +description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. +compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. +--- + +# Port an agent project to CanyonOS Core + +CanyonOS Core is the product name. Its compatibility executable and Python +package remain `ventis`; environment variables and Docker resources retain the +`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding +strings. Do not rename them. + +## Load references only when needed + +- Read [references/packaging.md](references/packaging.md) when a source import + does not resolve from `/app`, the source is nested, or packaging metadata is + involved. +- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target + includes `llm_proxy`. +- Read [references/ec2.md](references/ec2.md) only when any config entry uses + `provider: EC2`. +- Read [references/troubleshooting.md](references/troubleshooting.md) after a + failed build, image probe, deploy, or request. +- Read [references/runtime-contract.md](references/runtime-contract.md) when a + validator finding needs explanation or the runtime mechanism is unclear. + +## Goal: thin scaffolding beside untouched source + +```text +agents/.yaml one callable surface per service +agents/.py one thin adapter per service, when needed +workflow/_workflow.py HTTP entry point; calls deploy() +config/global_controller.yaml deployment manifest +config/policy.yaml optional access restriction +pyproject.toml conditional nested-import scaffolding + unchanged +``` + +The file count follows the deployment. A multi-agent port has one yaml/adapter +pair per service that is worth deploying separately. If a source class already +satisfies the runtime contract, point its config entry at that file and do not +copy it into an adapter. + +Everything the source already owns—prompts, tools, schemas, parsing, retries, +model clients, and node bodies—is imported. The port re-expresses only the +CanyonOS Core boundary and framework-owned orchestration. + +The port root is the existing repository root and the directory from which +`ventis build` runs. Write scaffolding there beside existing directories. If the +repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, +and `config/` beside it. Never move or copy the repository into a new `src/` +directory, and never create an outer wrapper merely for the port. + +## 1. Survey before writing + +Identify: + +1. The source entry point and callable input/output. +2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, + `Send`, `Command`, interrupts). +3. Runtime-injected services nodes read: stores, context, memory, sessions, or + callback managers. +4. Sync versus async boundaries. +5. Imports and declared runtime dependencies. +6. Model provider, credential names, streaming use, and optional `llm_proxy`. +7. Whether independent work fans out and benefits from separate replicas. +8. Whether source imports resolve from the project root that becomes `/app`. + +Run the validator once now. Its header detects capabilities directly from the +importable runtime rather than external development metadata: + +```bash +python /validate.py . +``` + +If config or agent yaml is malformed, the validator defers to `ventis build`. +Capability-gated findings say which runtime behavior is available. + +## 2. Choose service boundaries + +Start with one service. Split only when it creates independent parallel work or +a distinct resource/replica profile. + +- Keep a ReAct loop together; every turn needs shared message history. +- Hoist supervisor task lists and `Send`-style fan-out into the workflow. +- Do not create a one-replica service with no distinct resource profile merely + to mirror every source graph node. + +Rewrite framework-owned edges as ordinary Python. Import the connected node +functions unchanged. Construct runtime-injected service objects from source +configuration; do not invent models, dimensions, stores, or defaults silently. +Report any choice the source does not specify. + +## 3. Write declarations and adapters + +### Agent yaml + +Use one yaml per deployed service. Argument types are bare builtins only: +`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is +required by the generated stub. `returns.type` is documentation; use `dict` or +`list` to signal that workflow callers must `json.loads` the returned string. + +### Adapter + +The entrypoint exposes a module-level class named exactly `agent.name`. It +constructs with no arguments and its declared methods are synchronous. Read +configuration from the environment in `__init__`. Bridge source coroutines +inside a synchronous method with `asyncio.run(...)`. Serialize framework objects +with their own JSON-safe serializer before returning. + +Do not duplicate source prompts, tools, schemas, or model calls. Keep the source +provider and SDK. + +### Workflow + +Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. +Import generated stubs by yaml basename and agent class name: + +```python +from deploy import deploy +from agents. import +``` + +The deployment platform sends `{query: string}` to `/main`. Pack richer input +inside `query`; any additional workflow parameter has a default. + +Dispatch every remote call before resolving any future: + +```python +futures = [agent.work(item=item) for item in items] +results = [json.loads(future.value()) for future in futures] +``` + +Do not fuse dispatch and `.value()` in one comprehension; that silently +serializes fan-out. Do not add an `if __name__ == "__main__":` block: the +workflow is executed with `__name__ == "__main__"` in production. + +### Config + +For each service, keep these names aligned: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a +list of distribution-name strings. Put `env_file` at config top level when the +runtime capability is available. Omit `policy.yaml` unless access must be +restricted; if present, give it a non-empty `rules` list. + +## Hard rules + +Capitalized **MUST** and **NEVER** are reserved for port-breaking or +source-integrity rules. The owner column states where each is decided. + +| ID | Rule | Owner | +|---|---|---| +| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | The class MUST construct with no arguments | V007 | +| M3 | yaml argument names MUST match Python parameter names | V008 | +| M4 | yaml argument types MUST be bare builtins | V010 | +| M5 | Declared adapter methods MUST be synchronous | V009 | +| M6 | Config names MUST match yaml agent names | build | +| M7 | Config names MUST not collide after lowercase normalization | build output | +| M8 | Local provider MUST be lowercase `local` | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements` MUST be a list of strings | build | +| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | +| M12 | Workflow MUST NEVER contain a main guard | V017 | +| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | +| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | +| M15 | Workflow MUST import stubs from `agents.` | V023 | +| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | +| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | +| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | +| M19 | NEVER hardcode or bake a real credential into an image | W003 | +| M20 | NEVER edit or vendor the source tree | `git status` | +| M21 | NEVER swap the source LLM provider | review | +| M22 | NEVER silently move, drop, or reclassify source dependencies | review | +| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | +| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | + +## 4. Validate, build, and probe + +Run static preflight, then let the build own build-time validation: + +```bash +python /validate.py . +ventis build -c config/global_controller.yaml +``` + +A green build never imports the adapter. Probe each agent image in this order: + +```bash +# Runtime startup path + docker run --rm ventis- \ + python -c "import local_controller" + +# Agent load path; include --env-file when configured + docker run --rm --env-file ventis- \ + python -c "import importlib.util,sys; \ +s=importlib.util.spec_from_file_location('m','.py'); \ +m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ +m.();print('ok')" +``` + +Also probe the workflow image with `python -c "import local_controller"`; it has +its own dependency resolve and generated-stub imports. + +Then deploy, send a representative request, and poll its status: + +```bash +ventis deploy -c config/global_controller.yaml +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query":""}' +curl http://localhost:8080/status/ +``` + +A successful outer request with a source-level failure still proves the port +reached and returned the source behavior. Record the distinction. + +## 5. Clean up + +After collecting evidence, stop foreground deploy with Ctrl+C and wait for +controller cleanup. Remove exact leftovers if startup crashed. Then remove build +products and exact images from this config: + +```bash +ventis clean +docker image rm ventis- \ + ventis- + +test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container +docker ps -a --format '{{.Names}}' +``` + +`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it +does not remove containers or images. Keep port scaffolding, untouched source, +and requested logs or reports. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md new file mode 100644 index 0000000..e06daa0 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md @@ -0,0 +1,41 @@ +# EC2 deployment + +Read this only when at least one config entry uses `provider: EC2`. + +## Configuration + +Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The +top-level `ec2` block supplies the runtime's required infrastructure and SSH +settings. Read the target checkout's deploy preflight and EC2 runtime before +writing the block; do not copy values from an example environment. + +Typical required categories are: + +- AMI and instance type +- region and subnet +- security groups +- SSH user and credentials accepted by the runtime + +`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof +that provisioning, SSH, image transfer, or remote container startup works. + +## Networking + +A remote container's `host.docker.internal` names its own EC2 Docker host. It +does not name the local controller machine. Databases, model proxies, and other +services must use addresses reachable from every selected host. + +The environment file may be copied temporarily to a remote host by runtimes that +expose the `env_file` capability. Confirm behavior from the capability probe and +target runtime rather than assuming local Docker semantics. + +## Probes and cleanup + +Run the same runtime and adapter probes against the exact image before remote +deployment. After deploy, verify the remote container logs; controller health +can be green even when agent loading failed. + +Stop foreground deploy normally so the controller can terminate recorded EC2 +instances. If provisioning or startup fails before an instance is recorded, +inspect the cloud provider directly and remove exact leaked resources. Never use +a broad cleanup command against unrelated instances. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md new file mode 100644 index 0000000..f726bd7 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md @@ -0,0 +1,64 @@ +# LLM proxy integration + +Read this only when the target checkout contains `llm_proxy` or the deployment +explicitly routes model SDKs through it. + +## Preserve provider protocols + +The proxy redirects provider endpoints; it does not convert providers. Keep the +source SDK, model ID, request body, and response parsing unchanged. + +Configure only the provider variables the source uses: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Some SDKs refuse to initialize without caller credentials. Give agent containers +non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS +credentials in the separate proxy process, not in the port's `env_file`. + +## Start locally + +The proxy defaults conflict with a typical deployment: host loopback is not +reachable from a container, and port 8080 is normally used by the workflow API. +Use a non-loopback bind and a different port: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers resolve `host.docker.internal` through their +Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the +machine running `ventis deploy`. Distributed deployments need a reachable proxy +address or one proxy on each host. + +## Supported call shape + +The implementation buffers complete requests and responses: + +- OpenAI and Anthropic non-streaming HTTP calls are forwarded. +- Bedrock `invoke` is reissued through the proxy's boto3 client. +- OpenAI/Anthropic streaming is unsupported. +- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are + unsupported. + +Survey the source before selecting the proxy. Do not silently disable streaming; +report the unsupported behavior and stop. + +## Credential behavior + +- The OpenAI adapter removes caller authorization and inserts the proxy key. +- The Anthropic adapter removes caller key headers and inserts the proxy key. +- Botocore still signs requests sent to a custom endpoint, so a caller may need + placeholder AWS credentials even though the proxy reissues upstream with its + own identity. +- `/healthz` proves provider registration and Flask availability, not upstream + credential validity. + +OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return +JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed +with the upstream status and are not byte-for-byte passthrough. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md new file mode 100644 index 0000000..8520c4a --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md @@ -0,0 +1,81 @@ +# Packaging and import roots + +Read this reference when an adapter imports nested source code, the source uses a +`src/` layout, or V031 reports an import-root problem. + +## What `/app` can import + +CanyonOS Core preserves project-relative paths in the image and starts Python at +`/app`. Without an editable install, Python resolves names rooted there: + +- `/app/tools.py` as `import tools` +- `/app/pkg/__init__.py` as `import pkg` +- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace + directories without `__init__.py` + +It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become +an import root first. + +## Detect support, do not infer it from release history + +Run: + +```bash +python /validate.py . +``` + +Read the `editable_install` capability. If it is unavailable and the original +import cannot resolve from `/app`, report a runtime capability blocker and stop. +Do not add a `sys.path` hack or relocate source files. + +## Root metadata is the trigger + +When editable install is supported, only packaging metadata at the **port root** +triggers `pip install -e .`: + +```text +port-root/pyproject.toml detected +port-root/source/pyproject.toml ignored as an install trigger +``` + +A nested source repository may remain untouched. Add minimal root scaffolding +that points package discovery at the existing source package: + +```toml +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "canyonos-port" +version = "0.0.0" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["source/src"] +include = ["pkg*"] +namespaces = true +``` + +Set `where` and `include` from the actual tree and original import spelling. Do +not reference a README or license from this wrapper metadata; file sweeps differ +by runtime capability and a missing referenced file makes the image build fail. + +## Dependencies in nested metadata + +A nested `pyproject.toml` is not installed merely because its Python files are +copied. Keep the source declaration unchanged and repeat its runtime +distributions in each relevant config entry's `requirements` list. This is +compatibility scaffolding, not permission to drop, move, or reclassify declared +dependencies. + +If source metadata is already at the port root, do not create a wrapper. Its +project dependencies participate in the same resolver as config requirements. +Report declared-but-unused toolchain dependencies and their image cost; let the +owner decide whether source metadata should change. + +## Validation boundary + +`ventis build` owns packaging syntax and installation errors. `validate.py` +checks only whether adapter imports appear to require a nested root that the +runtime will not expose. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md new file mode 100644 index 0000000..94f7401 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -0,0 +1,187 @@ +# CanyonOS Core runtime contract + +The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the +CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for +Docker resources. + +Read this reference when implementing an adapter or explaining a validator +finding. Runtime-dependent behavior is expressed as capabilities; run +`validate.py` against the target environment instead of inferring support from +release history. + +## Project root and discovery + +`ventis build` uses the current working directory as the project root. + +| Input | Discovery | +|---|---| +| `agents/*.yaml` | direct yaml glob under the project root | +| `config/global_controller.yaml` | default config, overridable with `-c` | +| workflow | `workflow_file` on a `type: workflow` entry | +| policy | `policy.yaml` beside the selected config file | +| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | + +The config name, yaml `agent.name`, and entrypoint class name form one binding: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +A missing match may skip an image while the command continues, so inspect build +output and generated image tags. + +## Agent yaml and generated stubs + +The consumed yaml shape is: + +```yaml +agent: + name: ExampleAgent + functions: + - name: work + arguments: + - name: query + type: str + returns: + type: dict +``` + +Argument annotations are generated from bare names without adding imports. Use +builtins. Generated methods have no defaults, so every declared argument is +required at the stub call site. `returns` does not control runtime conversion; +it documents whether workflow code should parse the returned string. + +Stub destinations differ by runtime capability and entrypoint layout. Workflow +code in this port convention imports the generated class from +`agents.`. The validator checks that import against declarations +before the workflow image starts. + +## Agent loading and execution + +The local controller effectively performs: + +```python +module = load(entrypoint) +agent_class = getattr(module, configured_name) +agent = agent_class() +result = getattr(agent, method_name)(**args) +``` + +Consequences: + +- The class is module-level and named exactly as configured. +- Construction takes no arguments. +- Declared methods accept yaml argument names as keyword arguments. +- Methods are synchronous; this path does not await a coroutine. +- Dicts and lists are JSON-encoded before entering Redis; other results become + strings. +- A remote Future's `.value()` returns text, not the original Python object. + +Agent import and construction exceptions are caught by the controller. A failed +agent may still advertise healthy because health is written independently of +successful agent loading. That is why image probes import both the runtime and +the entrypoint explicitly. + +## Workflow execution + +The workflow file is executed, not imported. Therefore: + +- module-level code runs at container startup; +- `__name__ == "__main__"`; +- `deploy()` blocks in the web server; +- the workflow function runs once per request; +- its function name determines the REST route exposed by the compatibility + runtime. + +The deployment platform additionally expects `/main` with a `{query: string}` +body. This platform constraint is stricter than the underlying transport. + +Each stub method returns a Future immediately. `.value()` blocks. Dispatching +and resolving inside one comprehension serializes work without raising an +error; dispatch all calls first, then resolve them. + +The workflow container also starts runtime controller code and has its own +package resolution. Probe it independently from agent images. + +## Build context and collisions + +The runtime copies project files while preserving relative paths, then writes +shared runtime modules, generated stubs, and entrypoints into the image. Later +writes can shadow project files. + +Avoid root project modules named like runtime files, including: + +```text +future.py +ventis_context.py +local_controller.py +local_controller_frontend.py +redis_client.py +grpc_options.py +bedrock.py +deploy.py +session_logging.py +workflow_launcher.py +``` + +Also avoid a yaml basename that shadows a different source module imported by an +adapter. The validator checks deterministic flat-name collisions. + +File sweep and editable-install behavior are runtime capabilities. For nested +imports, follow [packaging.md](packaging.md). + +## Dependencies and protobuf + +Agent and workflow images include a small runtime dependency set. Config +`requirements` adds source-specific distributions. A malformed requirements +value can be normalized away while image generation continues; missing imports +then surface only when the agent loads. + +The build compiles gRPC Python stubs on the host and copies them into images. +The image resolver does not necessarily know the generated-code version. A +source dependency that constrains protobuf below the host generator version can +produce a green image build that dies on: + +```text +import local_controller +``` + +Always run that probe before probing the entrypoint. Treat a generated-code / +runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter +source dependencies silently. + +## Credentials capability + +When `env_file` capability is available, the top-level config path is resolved +against the project root and passed at container start. Hidden env files are not +copied into images. Invalid paths are deploy-preflight errors. + +When the capability is unavailable, declaring `env_file` has no effect. If the +source needs credentials, report the capability blocker rather than hardcoding +or vendoring a secret. + +A source that constructs its client at import time works only when credentials +are already in the container environment. Image entrypoint probes therefore use +the same env file as deployment. + +For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). + +## Policy and provider behavior + +No policy file means unrestricted service access. If a policy exists, it needs a +non-empty rules list. Rules are evaluated by specificity and first match; +services excluded from the selected rule fail after request acceptance. + +Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior +and remote networking are covered in [ec2.md](ec2.md). + +## Cleanup boundary + +Stopping foreground deploy normally invokes controller cleanup for recorded +containers and Redis. Hard kills and failures before resource registration may +leave resources behind. + +`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and +`docker_container/`. It does not remove containers or images. Remove exact +leftovers explicitly and preserve source, port scaffolding, and requested +evidence. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md new file mode 100644 index 0000000..361acf9 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting + +Read this after a failed build, image probe, deploy, or request. For mechanisms, +read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, +read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). + +## Build or deploy stops early + +| Symptom | Likely cause | +|---|---| +| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | +| Two services produce one image | Config names collide after lowercase normalization | +| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | +| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | +| Replica conversion `TypeError` | `replicas` is not an integer | +| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | +| Port or container name already in use | A previous deployment did not complete cleanup | + +## Container exits or serves nothing + +| Symptom | Likely cause | +|---|---| +| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | +| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | +| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | +| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | +| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | +| Third-party module is missing | Distribution is absent from source metadata and config requirements | +| Stub import raises `NameError` | yaml argument type is not a bare builtin | +| Source module behaves like an empty stub | A generated stub basename shadowed the source module | +| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | + +## Request is accepted, then fails + +| Symptom | Likely cause | +|---|---| +| Unexpected keyword argument | yaml argument name differs from adapter parameter name | +| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | +| Unauthorized service | The first matching policy rule excludes that service | +| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | +| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | +| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | +| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | +| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | + +## Deployment platform endpoint + +| Symptom | Likely cause | +|---|---| +| 404 while workflow container is healthy | Workflow function is not named `main` | +| 400 before host receives request | Body is not the platform's `{query: string}` shape | +| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | + +## Cleanup + +| Symptom | Likely cause | +|---|---| +| `ventis clean` succeeds but containers remain | The command removes generated directories only | +| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | +| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/cli/.claude/skills/porting-to-canyonos-core/validate.py b/cli/.claude/skills/porting-to-canyonos-core/validate.py new file mode 100755 index 0000000..04baf37 --- /dev/null +++ b/cli/.claude/skills/porting-to-canyonos-core/validate.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Preflight the runtime traps that `ventis build` cannot see. + +This deliberately does not duplicate build-time validation such as malformed +YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those +checks. This script parses Python without importing it and catches failures that +otherwise stay hidden until a container loads an agent, starts a workflow, or +serves its first request. A replica is not evidence: the controller writes +`healthy` to Redis before `_load_agent` runs. + + python validate.py [project_dir] [-c config/global_controller.yaml] + [--json] [--strict] + +Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. + +Runtime capabilities vary across CanyonOS Core installations. This script probes +the importable `ventis` package directly. A capability-gated check reports +UNAVAILABLE when its behavior cannot be proven. +""" + +import argparse +import ast +import builtins +import json +import os +import re +import sys +from typing import ClassVar + +try: + import yaml +except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency + sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") + raise SystemExit(2) from None + + +DEFAULT_CONFIG_PATH = "config/global_controller.yaml" + +# Copied flat into every image over the swept project tree, so a project module +# landing flat under one of these names is overwritten. +# ventis/stub_generator.py generate_docker / generate_workflow_docker. +RUNTIME_FLAT_NAMES = frozenset( + { + "future.py", + "ventis_context.py", + "local_controller.py", + "local_controller_frontend.py", + "redis_client.py", + "grpc_options.py", + "gpu_metrics.py", + "bedrock.py", + "deploy.py", + "session_logging.py", + "workflow_launcher.py", + } +) + +# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. +BASE_AGENT_REQUIREMENTS = [ + "grpcio", + "grpcio-tools", + "redis", + "pyyaml", + "psutil", + "ipdb", + "ipython", + "boto3", +] +# Import name -> distribution name, for the handful where they differ and the +# mismatch would otherwise be reported as a missing requirement. +IMPORT_TO_DISTRIBUTION = { + "attr": "attrs", + "bs4": "beautifulsoup4", + "cv2": "opencv-python", + "dateutil": "python-dateutil", + "dotenv": "python-dotenv", + "grpc": "grpcio", + "grpc_tools": "grpcio-tools", + "jwt": "pyjwt", + "PIL": "pillow", + "psycopg": "psycopg", + "psycopg2": "psycopg2-binary", + "pydantic_settings": "pydantic-settings", + "sklearn": "scikit-learn", + "typing_extensions": "typing-extensions", + "yaml": "pyyaml", +} + +SECRET_PATTERNS = [ + (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), + (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), +] +SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) + +ERROR = "ERROR" +WARN = "WARN" +INFO = "INFO" + + +# ------------------------------------------------------------------ # +# Capabilities # +# ------------------------------------------------------------------ # +# +# Stable labels for behavior detected from the importable runtime. They contain +# no external development metadata. + +CAPABILITY_SOURCE = { + "env_file": "runtime env-file injection", + "editable_install": "editable project installation", + "sweeps_all_files": "full project-file sweep", + "stub_two_destinations": "flat and package stub destinations", +} + + +def probe_capabilities(): + """Ask the importable ventis package what it actually supports.""" + caps = dict.fromkeys(CAPABILITY_SOURCE, False) + caps["ventis"] = False + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a broken install must not crash the check + return caps + + caps["ventis"] = True + caps["editable_install"] = hasattr(stub_generator, "_install_step") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") + + import importlib + + for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): + try: + module = importlib.import_module(module_name) + except Exception: # noqa: BLE001,S112 - the other path is the live one + continue + if hasattr(module, "resolve_env_file"): + caps["env_file"] = True + break + return caps + + +# ------------------------------------------------------------------ # +# YAML with line numbers # +# ------------------------------------------------------------------ # + + +class LineDict(dict): + """A mapping that remembers where it and each of its keys were written.""" + + line = 0 + key_lines: ClassVar[dict] = {} + + +class LineLoader(yaml.SafeLoader): + pass + + +def _construct_mapping(loader, node): + data = LineDict() + yield data + data.update(loader.construct_mapping(node, deep=False)) + data.line = node.start_mark.line + 1 + data.key_lines = { + key.value: key.start_mark.line + 1 + for key, _ in node.value + if isinstance(key, yaml.ScalarNode) + } + + +LineLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping +) + + +def line_of(mapping, key=None): + """The source line of `key` inside `mapping`, or of the mapping itself.""" + if not isinstance(mapping, LineDict): + return 0 + if key is not None: + return mapping.key_lines.get(key, mapping.line) + return mapping.line + + +def load_yaml(path): + """Parse `path`, returning (data, error). Never raises.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return yaml.load(handle, Loader=LineLoader), None + except Exception as exc: # noqa: BLE001 - any parse failure is a finding + return None, str(exc) + + +# ------------------------------------------------------------------ # +# Findings # +# ------------------------------------------------------------------ # + + +class Report: + def __init__(self, project_dir, capabilities): + self.project_dir = project_dir + self.capabilities = capabilities + self.findings = [] + # A peer agent is imported by the name of its generated stub, which the + # build copies flat into every image. Those are not project modules and + # need no requirement. + self.stub_module_names = set() + + def add(self, check, level, path, line, summary, mechanism): + self.findings.append( + { + "check": check, + "level": level, + "path": self.rel(path) if path else "", + "line": line or 0, + "summary": summary, + "mechanism": mechanism, + } + ) + + def error(self, check, path, line, summary, mechanism): + self.add(check, ERROR, path, line, summary, mechanism) + + def warn(self, check, path, line, summary, mechanism): + self.add(check, WARN, path, line, summary, mechanism) + + def unavailable(self, check, summary): + self.add(check, INFO, "", 0, summary, "") + + def rel(self, path): + try: + return os.path.relpath(path, self.project_dir) + except ValueError: + return path + + def counts(self): + errors = sum(1 for f in self.findings if f["level"] == ERROR) + warnings = sum(1 for f in self.findings if f["level"] == WARN) + return errors, warnings + + +# ------------------------------------------------------------------ # +# Python source helpers # +# ------------------------------------------------------------------ # + + +def parse_python(path): + """AST for `path`, or (None, error). The port is never imported.""" + try: + with open(path, "r", encoding="utf-8") as handle: + source = handle.read() + except OSError as exc: + return None, str(exc) + try: + return ast.parse(source, filename=path), None + except SyntaxError as exc: + return None, f"{exc.msg} (line {exc.lineno})" + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def class_methods(class_node): + return { + node.name: node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def parameter_names(func_node): + """Every parameter a caller can pass by keyword, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + return positional + [a.arg for a in args.kwonlyargs] + + +def required_parameters(func_node): + """Parameters with no default, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + if args.defaults: + positional = positional[: len(positional) - len(args.defaults)] + kwonly = [ + arg.arg + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is None + ] + return positional + kwonly + + +def toplevel_import_names(tree): + """Top-level package name of every import in the module, with line numbers.""" + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.setdefault(alias.name.split(".")[0], node.lineno) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.setdefault(node.module.split(".")[0], node.lineno) + return names + + +# ------------------------------------------------------------------ # +# V006-V010 adapter failures hidden by _load_agent # +# ------------------------------------------------------------------ # + +BUILTIN_TYPE_NAMES = frozenset( + name + for name in dir(builtins) + if isinstance(getattr(builtins, name), type) and not name[0].isupper() +) + + +def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): + """V006 V007 V008 V009 V010.""" + name = agent_block["name"] + functions = agent_block.get("functions") or [] + check_argument_types(report, agent_yaml_path, functions) + + entrypoint = entry.get("entrypoint") + if not entrypoint: + return + entrypoint_path = os.path.join(project_dir, entrypoint) + if not os.path.isfile(entrypoint_path): + return + + tree, error = parse_python(entrypoint_path) + if tree is None: + report.error( + "V006", + entrypoint_path, + 0, + f"the entrypoint does not parse: {error}", + "_load_agent exec_module's it and swallows the exception; the first " + "request answers 'No agent loaded'.", + ) + return + + class_node = find_class(tree, name) + if class_node is None: + classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] + found = ", ".join(classes) if classes else "no classes at all" + report.error( + "V006", + entrypoint_path, + 1, + f"no class named `{name}` at module level (found: {found})", + "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " + "the AttributeError. The class name must equal agent.name exactly.", + ) + return + + methods = class_methods(class_node) + check_constructor(report, entrypoint_path, name, methods) + + for func in functions: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + continue + check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) + + +def check_argument_types(report, agent_yaml_path, functions): + """V010 -- the type string is pasted into an ast.Name, never checked.""" + for func in functions: + if not isinstance(func, dict): + continue + for arg in func.get("arguments") or []: + if not isinstance(arg, dict) or "type" not in arg: + continue + declared = arg.get("type") + if not isinstance(declared, str): + continue # stub generation reports malformed type values + if declared in BUILTIN_TYPE_NAMES: + continue + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared}` is not a builtin", + "stub_generator pastes it verbatim into the generated " + "annotation, and the stub module imports only Future and " + "inspect. Anything else raises NameError when the stub is " + "imported -- after a green build. Use str int float bool dict " + "list.", + ) + + +def check_constructor(report, entrypoint_path, name, methods): + """V007 -- _load_agent calls agent_class() with no arguments.""" + init = methods.get("__init__") + if init is None: + return + required = required_parameters(init) + if required: + report.error( + "V007", + entrypoint_path, + init.lineno, + f"`{name}.__init__` requires {', '.join(required)}", + "_load_agent calls agent_class() with no arguments; the TypeError is " + "swallowed and the first request answers 'No agent loaded'. Read " + "configuration from the environment inside __init__ instead.", + ) + + +def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): + """V008 V009.""" + func_name = func["name"] + method = methods.get(func_name) + if method is None: + report.error( + "V008", + entrypoint_path, + 0, + f"`{class_name}` has no method `{func_name}`", + "The yaml declares it, so callers get a stub for it; the controller " + f"then answers \"Agent {class_name} has no method '{func_name}'\".", + ) + return + + # V009 -- nothing on the execution path awaits. + if isinstance(method, ast.AsyncFunctionDef): + report.error( + "V009", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` is `async def`", + "The executor calls method(**args) with no await, so Redis receives " + "''. Keep the signature synchronous and call " + "asyncio.run(...) inside the body.", + ) + + # V008 -- the controller calls method(**args) with the yaml's names. + declared = [ + arg["name"] + for arg in func.get("arguments") or [] + if isinstance(arg, dict) and isinstance(arg.get("name"), str) + ] + actual = parameter_names(method) + required = required_parameters(method) + + missing = [arg for arg in declared if arg not in actual] + if missing: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` has no parameter " + f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", + "LocalController does method(**args) with the yaml's argument names. " + "A mismatch is TypeError: unexpected keyword argument, at request " + f"time. See {os.path.basename(agent_yaml_path)}.", + ) + + unfilled = [arg for arg in required if arg not in declared] + if unfilled: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " + "the yaml does not declare", + "Only declared arguments are ever sent, and the generated stub gives " + "none of them a default. Declare them in the yaml or default them " + "in the signature.", + ) + + + +# ------------------------------------------------------------------ # +# V016-V018 the workflow # +# ------------------------------------------------------------------ # + + +def check_stub_imports(report, workflow_path, tree, stub_classes): + """V023 -- the workflow must import a stub as `from agents. import `. + + The build copies each stub to exactly one path, and for the workflow image + that path is agents/.py. Two ways of writing this line fail, and + the project walks you into both: the flat form is what examples/ uses, and + the class name is the one `ventis build` prints, which is not the one it + writes. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + base = alias.name.split(".")[0] + if base in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`import {alias.name}` -- the stub is at " + f"agents/{base}.py, not flat", + "The build copies a stub to one path, and for the " + "workflow that path is under agents/. This is a " + "ModuleNotFoundError the moment the workflow runs. " + f"Write `from agents.{base} import {stub_classes[base]}`.", + ) + continue + + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + + module = node.module + if module in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`from {module} import ...` -- the stub is at " + f"agents/{module}.py, not flat", + "The build copies a stub to one path, and for the workflow " + "that path is under agents/. The flat form is what this " + "repository's own examples use and it raises " + "ModuleNotFoundError in the workflow image. Write " + f"`from agents.{module} import {stub_classes[module]}`.", + ) + continue + + if not module.startswith("agents."): + continue + base = module.split(".", 1)[1] + expected = stub_classes.get(base) + if expected is None: + continue + for alias in node.names: + if alias.name == expected: + continue + if alias.name == f"{expected}Stub": + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is the name the build prints, not the " + f"class it writes", + "generate_agent_stub sets class_name = agent_config['name'] " + "and then recomputes it with a 'Stub' suffix for the log " + "line only. The message names a class that does not exist; " + f"the class is `{expected}`.", + ) + else: + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is not a class the stub for {base} defines", + f"The stub's class carries the agent's own name: `{expected}`.", + ) + + +def check_workflow(report, workflow_path, stub_classes=None): + """V016 V017 V018 V023.""" + tree, error = parse_python(workflow_path) + if tree is None: + report.error("V016", workflow_path, 0, f"does not parse: {error}", "") + return + + main = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "main" + ): + main = node + break + + if main is None: + defined = [ + n.name + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + found = ", ".join(defined) if defined else "no top-level functions" + report.error( + "V016", + workflow_path, + 1, + f"no top-level function named `main` (found: {found})", + "CanyonOS Core serves POST /, but the deployment platform's " + "test endpoint posts to a hardcoded /main. A differently named " + "workflow builds, deploys and stays unreachable -- 404, container " + "healthy.", + ) + else: + check_main_signature(report, workflow_path, main) + + if stub_classes: + check_stub_imports(report, workflow_path, tree, stub_classes) + + # V016 -- deploy() is what starts Flask. + if not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "deploy" + for node in ast.walk(tree) + ): + report.error( + "V016", + workflow_path, + 1, + "the workflow never calls `deploy(...)`", + "workflow_launcher.py exec's this file and nothing else starts the " + "HTTP server; the container comes up serving nothing.", + ) + + check_main_guard(report, workflow_path, tree) + check_fused_fanout(report, workflow_path, tree) + + +def check_main_signature(report, workflow_path, main): + """V016 -- the platform sends exactly {"query": ...}.""" + if isinstance(main, ast.AsyncFunctionDef): + report.error( + "V016", + workflow_path, + main.lineno, + "`main` is `async def`", + "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " + "no await; the response body would be a coroutine repr.", + ) + params = parameter_names(main) + if not params: + report.error( + "V016", + workflow_path, + main.lineno, + "`main` takes no arguments", + 'The platform posts {"query": "..."} and deploy() splats the ' + "body in as kwargs -- TypeError on every request.", + ) + return + if params[0] != "query": + report.error( + "V016", + workflow_path, + main.lineno, + f"`main`'s first parameter is `{params[0]}`, not `query`", + "The platform's body schema is strictly validated as " + "{query: string}; any other key is rejected with 400 in the control " + "plane, before the request reaches the host.", + ) + extra = [p for p in required_parameters(main) if p != "query"] + if extra: + report.error( + "V016", + workflow_path, + main.lineno, + f"`main` requires {', '.join(extra)} beyond `query`", + "Only `query` is ever sent, so every other parameter needs a " + "default or the call raises on every request. Pack richer input " + "into `query`.", + ) + + +def check_main_guard(report, workflow_path, tree): + """V017 -- the workflow is exec'd, so __name__ == "__main__".""" + for node in tree.body: + if not isinstance(node, ast.If): + continue + test = node.test + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and any( + isinstance(c, ast.Constant) and c.value == "__main__" + for c in test.comparators + ) + ): + report.error( + "V017", + workflow_path, + node.lineno, + '`if __name__ == "__main__":` block in the workflow', + "workflow_launcher.py runs exec(open().read()), so " + "__name__ IS '__main__' here and this block executes in " + "production, at container start.", + ) + + +def check_fused_fanout(report, workflow_path, tree): + """V018 -- .value() blocks, so dispatching and resolving in one + comprehension runs the fan-out one call at a time.""" + comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) + for node in ast.walk(tree): + if not isinstance(node, comprehensions + (ast.DictComp,)): + continue + elements = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for element in elements: + for inner in ast.walk(element): + if ( + isinstance(inner, ast.Call) + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "value" + and isinstance(inner.func.value, ast.Call) + ): + report.error( + "V018", + workflow_path, + node.lineno, + "one comprehension both dispatches a call and resolves " + "it with .value()", + ".value() blocks, so each call completes before the " + "next is dispatched. It does not error -- the fan-out " + "is just silently serial, and with it the reason to be " + "on CanyonOS Core. Dispatch every call first, then resolve: " + "futures = [a.work(i) for i in items] then " + "[f.value() for f in futures].", + ) + return + + +# ------------------------------------------------------------------ # +# V019-V020 what the copy order overwrites # +# ------------------------------------------------------------------ # + + +def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): + """V019 V020 -- later copies land on earlier ones at the context root.""" + for entry in sorted(os.listdir(project_dir)): + path = os.path.join(project_dir, entry) + if not os.path.isfile(path) or not entry.endswith(".py"): + continue + + # V019 -- the shared runtime is copied flat, after the project sweep. + if entry in RUNTIME_FLAT_NAMES: + report.error( + "V019", + path, + 1, + f"a project module named `{entry}` sits at the project root", + "The shared CanyonOS Core runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by CanyonOS Core's own " + f"{entry}. Rename it or move it into a package directory.", + ) + + # V020 -- a stub is copied flat under its yaml's basename. + entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} + for yaml_path in yaml_paths: + stem = os.path.splitext(os.path.basename(yaml_path))[0] + module = f"{stem}.py" + if module in entrypoint_basenames: + continue # the entrypoint is copied last and wins its flat name back + candidate = os.path.join(project_dir, module) + if os.path.isfile(candidate): + report.error( + "V020", + candidate, + 1, + f"`{os.path.basename(yaml_path)}` generates a stub that lands on " + f"`{module}`", + "The yaml's basename names the stub, and the stub is copied flat " + "over the swept tree. Anything importing this module inside the " + "container gets the generated stub instead of the real code. " + "Rename the yaml to match its own entrypoint.", + ) + + +# ------------------------------------------------------------------ # +# V030-V031 capability-gated rules # +# ------------------------------------------------------------------ # + + +def check_env_file(report, config, config_path, project_dir): + """V030 -- gated on detected env-file injection support.""" + declared = config.get("env_file") + supported = report.capabilities.get("env_file") + + if not supported: + if declared: + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", + "No resolve_env_file in the importable ventis package, so the " + "key is silently dropped and the container answers a provider " + "credential error on the first request. This port requires the " + "`env_file` runtime capability.", + ) + else: + report.unavailable( + "V030", + "env_file is not supported by the importable `ventis` runtime. " + "Credentials have no declared path into a container on this tree.", + ) + return + + if not declared: + report.warn( + "V030", + config_path, + line_of(config), + "no `env_file:` in the config", + "Only runtime-managed VENTIS_* variables are guaranteed without it. " + "If the source reads credentials from the environment, the first " + "request fails on a provider error.", + ) + return + + # Path existence and readability are deploy-preflight checks. Do not + # duplicate them here. + + +def check_import_root(report, project_dir, entrypoint_paths): + """V031 -- gated on detected editable-install support.""" + supported = report.capabilities.get("editable_install") + has_metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + + non_flat = [] + for path in entrypoint_paths: + tree, _ = parse_python(path) + if tree is None: + continue + for name, lineno in toplevel_import_names(tree).items(): + if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: + continue + if _resolves_flat(project_dir, name): + continue + location = _resolves_nested(project_dir, name) + if location: + non_flat.append((path, lineno, name, location)) + + if not supported: + report.unavailable( + "V031", + "the editable install (`-e .`) is not supported by the importable " + "`ventis` runtime. Only names rooted at /app import inside a container.", + ) + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, which is not at the " + "project root", + "sys.path[0] is /app and this CanyonOS Core runs no editable install, " + "so only modules swept to the root import. The adapter raises " + "ModuleNotFoundError inside _load_agent and the first request " + "answers 'No agent loaded'.", + ) + return + + if non_flat and not has_metadata: + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, and the project root " + "has no packaging metadata", + "A pyproject.toml, setup.py or setup.cfg at the port root is " + "what adds `-e .`; metadata nested inside the untouched source " + "tree is ignored. Add minimal root metadata pointing at the " + "existing package directory. Without it the install is skipped " + "silently.", + ) + + +def _resolves_flat(project_dir, name): + """Whether Python can resolve `name` with /app as its import root. + + A directory does not need __init__.py: PEP 420 namespace packages resolve + from sys.path just like regular packages. + """ + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( + os.path.join(project_dir, name) + ) + + +def _resolves_nested(project_dir, name): + """Where below /app `name` lives but cannot resolve as a top-level name.""" + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] + if root == project_dir: + continue + if f"{name}.py" in files: + return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) + if name in dirs: + return os.path.relpath(os.path.join(root, name), project_dir) + return None + + +# ------------------------------------------------------------------ # +# W003, W006 secrets and imports a green build does not reject # +# ------------------------------------------------------------------ # + + +def check_secrets(report, port_paths): + """W003 -- env_file is the way in; nothing else is.""" + for path in port_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError: + continue + for number, line in enumerate(lines, start=1): + for pattern, description in SECRET_PATTERNS: + if pattern.search(line): + report.warn( + "W003", + path, + number, + f"this line looks like {description}", + "Never put a secret in the source tree or the build " + "context. The build sweeps the project into every " + "image.", + ) + break + + tree, _ = parse_python(path) + if tree is None: + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not ( + isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() + ): + continue + for target in node.targets: + if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): + report.warn( + "W003", + path, + node.lineno, + f"`{target.id}` is assigned a literal string", + "Read it from the environment instead; the build sweeps " + "this file into every image.", + ) + + +def _pyproject_dependencies(project_dir): + """What `-e .` installs alongside `requirements:`, or None if unreadable. + + None and the empty set mean different things here: empty means the project + declares no dependencies, None means we could not find out -- a setup.py, or + a tomllib this interpreter does not have. The caller must not treat the + second as the first, or it warns about imports the install would satisfy. + """ + path = os.path.join(project_dir, "pyproject.toml") + if not os.path.isfile(path): + return None + try: + import tomllib + except ImportError: # < 3.11 + return None + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except Exception: # noqa: BLE001 - malformed metadata is uv's error to give + return None + deps = (data.get("project") or {}).get("dependencies") + if not isinstance(deps, list): + return None + return {_normalize_distribution(d) for d in deps if isinstance(d, str)} + + +def check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path +): + """W006 -- an import the container cannot satisfy.""" + tree, _ = parse_python(entrypoint_path) + if tree is None: + return + + declared = { + _normalize_distribution(item) + for item in (entry.get("requirements") or []) + if isinstance(item, str) + } + + # Where the editable install exists, `-e .` resolves the project's own + # [project.dependencies] in the same pass as `requirements:`. Warning about + # those is a false positive, and a false warning about a dependency is worse + # than none: it teaches the reader to dismiss this check. + editable = report.capabilities.get("editable_install") + metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + unreadable_metadata = False + if editable and metadata: + project_deps = _pyproject_dependencies(project_dir) + if project_deps is None: + unreadable_metadata = True + else: + declared |= project_deps + base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} + stdlib = getattr(sys, "stdlib_module_names", frozenset()) + + for name, lineno in sorted(toplevel_import_names(tree).items()): + if name in stdlib or name == "ventis": + continue + # Provided by the image itself: the shared runtime is copied flat, and + # every agents/*.yaml generates a stub that is copied flat too. + if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: + continue + if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): + continue + distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) + if distribution in base or distribution in declared: + continue + if unreadable_metadata: + mechanism = ( + "The container installs the base list, `requirements:`, and -- " + "since this project declares packaging metadata -- whatever " + "`-e .` resolves from it. That metadata could not be read here, " + f"so if it already requires `{name}` this line is noise; " + "otherwise it is a ModuleNotFoundError inside _load_agent and " + "'No agent loaded' on the first request." + ) + else: + mechanism = ( + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside " + "_load_agent and 'No agent loaded' on the first request. If the " + f"distribution is named something other than `{name}`, declare " + f"that name in {report.rel(config_path)}." + ) + report.warn( + "W006", + entrypoint_path, + lineno, + f"`import {name}` is in neither the runtime's base list nor this " + "entry's `requirements:`", + mechanism, + ) + + +def _normalize_distribution(name): + return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( + "_", "-" + ) + + +# ------------------------------------------------------------------ # +# Driver # +# ------------------------------------------------------------------ # + + +def validate(project_dir, config_path, capabilities): + """Inspect only failures hidden behind a successful image build.""" + report = Report(project_dir, capabilities) + + # The build owns config/YAML syntax and shape validation. We read only enough + # valid structure to locate code for the deeper checks below. + config, error = load_yaml(config_path) + if error is not None or not isinstance(config, dict): + report.unavailable( + "BUILD", + "runtime preflight skipped because the config cannot be read; " + "ventis build owns and reports this error.", + ) + return report + + import glob + + yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) + report.stub_module_names = { + os.path.splitext(os.path.basename(path))[0] for path in yaml_paths + } + + agents_by_name = {} + stub_classes = {} + for path in yaml_paths: + data, yaml_error = load_yaml(path) + agent = data.get("agent") if isinstance(data, dict) else None + name = agent.get("name") if isinstance(agent, dict) else None + if yaml_error is not None or not isinstance(name, str): + continue # ventis build reports malformed agent declarations + agents_by_name[name] = (path, agent) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = name + + entries = config.get("agents") + if not isinstance(entries, list): + report.unavailable( + "BUILD", + "runtime preflight skipped because `agents:` is not a list; " + "ventis build owns and reports this error.", + ) + return report + + entrypoints = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": + continue + name = entry.get("name") + entrypoint = entry.get("entrypoint") + if isinstance(entrypoint, str): + entrypoints.append(entrypoint) + if name in agents_by_name: + yaml_path, agent_block = agents_by_name[name] + check_adapter(report, yaml_path, agent_block, entry, project_dir) + entrypoint_path = os.path.join(project_dir, entrypoint or "") + if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): + check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path + ) + + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": + continue + workflow_file = entry.get("workflow_file") + if not isinstance(workflow_file, str): + continue + workflow_path = os.path.join(project_dir, workflow_file) + if os.path.isfile(workflow_path): + check_workflow(report, workflow_path, stub_classes) + + # These survive a green build and otherwise surface only in a container or + # on its first request. + check_flat_collisions(report, project_dir, yaml_paths, entrypoints) + check_env_file(report, config, config_path, project_dir) + + entrypoint_paths = [ + os.path.join(project_dir, e) + for e in entrypoints + if os.path.isfile(os.path.join(project_dir, e)) + ] + check_import_root(report, project_dir, entrypoint_paths) + + port_paths = list(entrypoint_paths) + for entry in entries: + if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): + candidate = os.path.join(project_dir, entry["workflow_file"]) + if os.path.isfile(candidate): + port_paths.append(candidate) + + # Secret detection remains because a green image build would permanently + # bake the credential into every image. + check_secrets(report, port_paths) + return report + + + +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + +LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} + + +def _wrap(text, width, indent): + words = text.split() + lines = [] + current = "" + for word in words: + candidate = f"{current} {word}".strip() + if len(candidate) + len(indent) > width and current: + lines.append(indent + current) + current = word + else: + current = candidate + if current: + lines.append(indent + current) + return lines + + +def print_report(report, project_dir): + caps = report.capabilities + if not caps.get("ventis"): + print("ventis is not importable here -- capability-gated rules are") + print("reported UNAVAILABLE rather than checked.\n") + else: + print("CanyonOS Core capabilities detected:") + for key, source in CAPABILITY_SOURCE.items(): + mark = "yes" if caps.get(key) else "no " + print(f" {mark} {key:<22} {source}") + print() + + findings = sorted( + report.findings, + key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), + ) + for finding in findings: + where = finding["path"] + if where and finding["line"]: + where = f"{where}:{finding['line']}" + header = f"{finding['check']} {finding['level']:<5}" + print(f"{header} {where}" if where else header) + for line in _wrap(finding["summary"], 78, " "): + print(line) + if finding["mechanism"]: + for line in _wrap(finding["mechanism"], 78, " "): + print(line) + print() + + errors, warnings = report.counts() + if not findings: + print(f"{project_dir}: clean.") + return + print(f"{errors} error(s), {warnings} warning(s).") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Check a CanyonOS Core port against the rules in SKILL.md." + ) + parser.add_argument( + "project_dir", nargs="?", default=".", help="the port's project root" + ) + parser.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument("--json", action="store_true", help="emit findings as JSON") + parser.add_argument( + "--strict", action="store_true", help="fail on warnings as well as errors" + ) + args = parser.parse_args(argv) + + project_dir = os.path.abspath(args.project_dir) + config_path = ( + args.config + if os.path.isabs(args.config) + else os.path.join(project_dir, args.config) + ) + + capabilities = probe_capabilities() + report = validate(project_dir, config_path, capabilities) + errors, warnings = report.counts() + + if args.json: + print( + json.dumps( + { + "project_dir": project_dir, + "capabilities": capabilities, + "errors": errors, + "warnings": warnings, + "findings": report.findings, + }, + indent=2, + ) + ) + else: + print_report(report, report.rel(project_dir) or project_dir) + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..de66cd0 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,21 @@ +Lightweight CLI for CanyonOS + +Serves as a thin API layer, connecting to the global controller container. + +## Serve + +`canyonos serve -c config/global_controller.yaml` starts the local CanyonOS dashboard. It reads +`database.url` from the config and writes only `CANYONOS_`-prefixed settings to the current `.env`, +leaving other lines unchanged. + + +### To Republish to PyPi + +```Terminal +cd cli +# Go into, pyproject.toml, and increment version number +rm -rf dist/ # Removes the old distro, causes conflicts + +uv build +uv publish # Needs PyPi Auth Token, ask Saaketh +``` \ No newline at end of file diff --git a/cli/canyonos/__init__.py b/cli/canyonos/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py new file mode 100644 index 0000000..aabf105 --- /dev/null +++ b/cli/canyonos/clean.py @@ -0,0 +1,28 @@ +""" +Remove generated stubs, gRPC files, and Docker build contexts. + +Ported directly over from canyonos, moving the logic into here. +""" + +import os +import shutil + + +def run_clean(): + project_dir = os.getcwd() + + paths_to_clean = [ + os.path.join(project_dir, "stubs"), + os.path.join(project_dir, "grpc_stubs"), + os.path.join(project_dir, "docker_container"), + ] + + for path in paths_to_clean: + if os.path.exists(path): + print(f"Cleaning {path}...") + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + print("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py new file mode 100644 index 0000000..226312e --- /dev/null +++ b/cli/canyonos/config.py @@ -0,0 +1,345 @@ +""" +Logic for `canyonos config`: view or change project/deploy configuration. +""" + +import os + +import yaml +from rich.console import Console +from rich.table import Table +from ruamel.yaml import YAML + +from canyonos.constants import default_config_path +from canyonos.theme import GREEN, WHITE +from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu + +BACK = "__back__" + +OPTIONS = [ + ("view", "View"), + ("change", "Change"), +] + +BORDER = GREEN +HEADER = f"bold {GREEN}" + +# Rendered as their own tables (in this order); everything else scalar at the +# top level is collected into a single "General" table. +STRUCTURED_KEYS = ("agents", "otel") + + +def _fmt(value): + """Render a YAML value as a compact, single-cell string.""" + if value is None: + return "-" + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, list): + return ", ".join(_fmt(v) for v in value) if value else "-" + if isinstance(value, dict): + return ", ".join(f"{k}={_fmt(v)}" for k, v in value.items()) if value else "-" + return str(value) + + +def _agents_table(agents): + table = Table(title="Agents", border_style=BORDER, header_style=HEADER, title_style=HEADER) + for col in ("Name", "Type", "Replicas", "CPU", "Mem", "Provider", "Port", "Entrypoint"): + table.add_column(col) + + for agent in agents: + resources = agent.get("resources") or {} + # Workflows carry `workflow_file` + `api_port`; plain agents carry + # `entrypoint` + `redis_port`. + entry = agent.get("entrypoint") or agent.get("workflow_file") or "-" + port = agent.get("api_port") or agent.get("redis_port") + table.add_row( + _fmt(agent.get("name")), + agent.get("type", "agent"), + _fmt(agent.get("replicas")), + _fmt(resources.get("cpu")), + _fmt(resources.get("memory")), + _fmt(agent.get("provider")), + _fmt(port), + entry, + ) + return table + + +def _otel_table(otel): + destinations = (otel or {}).get("destinations") or [] + table = Table( + title="OTel Destinations", border_style=BORDER, header_style=HEADER, title_style=HEADER + ) + for col in ("Name", "Protocol", "Endpoint", "Insecure", "Headers"): + table.add_column(col) + + for dest in destinations: + headers = dest.get("headers") or {} + table.add_row( + _fmt(dest.get("name")), + _fmt(dest.get("protocol")), + _fmt(dest.get("endpoint")), + _fmt(dest.get("insecure", False)), + ", ".join(headers.keys()) if headers else "-", + ) + return table + + +def _kv_table(title, data): + """A two-column Setting/Value table from a flat-ish dict (or single value).""" + table = Table(title=title, border_style=BORDER, header_style=HEADER, title_style=HEADER) + table.add_column("Setting", style="bold") + table.add_column("Value") + + if isinstance(data, dict): + for key, value in data.items(): + table.add_row(str(key), _fmt(value)) + else: + table.add_row(title, _fmt(data)) + return table + + +def run_view_config(config_path=None): + config_path = config_path or default_config_path() + console = Console() + + if not os.path.isfile(config_path): + console.print(f"[red]Config file not found: {config_path}[/red]") + return + + with open(config_path) as f: + config = yaml.safe_load(f) or {} + + console.print(_agents_table(config.get("agents") or [])) + console.print() + + if config.get("otel"): + console.print(_otel_table(config["otel"])) + console.print() + + # Every other top-level key: dicts get their own table, bare scalars are + # gathered into a single "General" table. + general = {} + for key, value in config.items(): + if key in STRUCTURED_KEYS: + continue + if isinstance(value, dict): + console.print(_kv_table(key, value)) + console.print() + else: + general[key] = value + + if general: + console.print(_kv_table("General", general)) + + +def _is_leaf(value): + """A value the user edits directly: any scalar, or a list of only scalars. + + Lists of mappings (agents, otel.destinations) are containers to drill into; + lists of plain scalars (requirements, security_group_ids) are edited whole + via comma-separated input. + """ + if isinstance(value, dict): + return False + if isinstance(value, list): + return all(not isinstance(item, (dict, list)) for item in value) + return True + + +def _preview(value): + if isinstance(value, dict): + return f"{{{len(value)} keys}}" + if isinstance(value, list) and not _is_leaf(value): + return f"[{len(value)} items]" + return _fmt(value) + + +def _seq_label(index, item): + if isinstance(item, dict) and item.get("name"): + return str(item["name"]) + return f"[{index}]" + + +def _cast(raw, current): + """Coerce the typed string to the current value's type. Raises ValueError.""" + # bool must precede int: bool is a subclass of int. + if isinstance(current, bool): + low = raw.strip().lower() + if low in ("true", "yes", "y", "1"): + return True + if low in ("false", "no", "n", "0"): + return False + raise ValueError("expected yes/no") + if isinstance(current, int): + return int(raw) + if isinstance(current, float): + return float(raw) + if isinstance(current, list): + return [part.strip() for part in raw.split(",") if part.strip()] + return raw + + +class _Screen: + """Owns the alt-screen: clears and redraws a persistent breadcrumb header + (plus a transient status line) before each menu/prompt, so the change + session replaces the view in place instead of scrolling. + """ + + def __init__(self, console): + self.console = console + self.status = "" + + def render(self, breadcrumb): + self.console.clear() + path = " \u203a ".join(str(part) for part in breadcrumb) if breadcrumb else "config" + self.console.print(f"[bold {GREEN}]CanyonOS[/] [{WHITE}]config[/]") + self.console.print(f"[{WHITE}]{path}[/]") + if self.status: + self.console.print(f"[{GREEN}]{self.status}[/]") + self.console.print() + + +def _edit_leaf(screen, parent, key, breadcrumb): + """Prompt for and apply a new value for parent[key]. Returns True if changed.""" + screen.render(breadcrumb) + console = screen.console + current = parent[key] + label = key if not isinstance(key, int) else f"item {key}" + console.print(f"[bold]{label}[/bold] current: {_fmt(current)}") + if isinstance(current, list): + console.print("[dim]enter comma-separated values[/dim]") + + raw = input("New value (blank to cancel): ").strip() + if raw == "": + return False + + try: + parent[key] = _cast(raw, current) + except ValueError as exc: + screen.status = f"Invalid value: {exc}" + return False + + screen.status = f"Set {label} = {_fmt(parent[key])}" + return True + + +def _confirm_delete(screen, node, key, breadcrumb): + """Yes/No confirm menu for deleting node[key]. Returns True to delete.""" + screen.render(breadcrumb) + label = key if isinstance(node, dict) else _seq_label(key, node[key]) + options = [("yes", f"Yes, delete '{label}'"), ("no", "No, keep it")] + choice = select_menu( + options, + title=f"Delete '{label}' ({_preview(node[key])}) and everything inside?", + ) + return choice == "yes" + + +def _navigate(screen, node, breadcrumb): + """Drill into a mapping/sequence. Returns True if any value was changed or + deleted, None if the user backed out of this level, or QUIT_ACTION if the + user quit (which unwinds the whole session from any depth).""" + while True: + screen.render(breadcrumb) + if isinstance(node, dict): + options = [(k, f"{k}: {_preview(v)}") for k, v in node.items()] + else: # list + options = [(i, f"{_seq_label(i, item)}: {_preview(item)}") for i, item in enumerate(node)] + options.append((BACK, "\u2190 Back")) + + choice = select_menu( + options, title="Select a field (d to delete)", deletable=True, quittable=True + ) + if choice is None: + return None + # 'q' anywhere -> unwind the entire session, not just this level. + if choice is QUIT_ACTION: + return QUIT_ACTION + + # 'd' over an item -> (DELETE_ACTION, hovered_value). + if isinstance(choice, tuple) and choice[0] is DELETE_ACTION: + target = choice[1] + if target == BACK: + continue # the Back entry isn't deletable + if _confirm_delete(screen, node, target, breadcrumb): + label = target if isinstance(node, dict) else _seq_label(target, node[target]) + del node[target] + screen.status = f"Deleted '{label}'" + return True + continue # delete cancelled: stay on this menu + + if choice == BACK: + return None + + child = node[choice] + label = choice if isinstance(node, dict) else _seq_label(choice, child) + if _is_leaf(child): + if _edit_leaf(screen, node, choice, breadcrumb + [str(label)]): + return True + # cancelled/invalid: stay on this menu + else: + result = _navigate(screen, child, breadcrumb + [str(label)]) + if result is QUIT_ACTION: + return QUIT_ACTION + if result: + return True + # backed out of the child: stay on this menu + + +def run_change_config(config_path=None): + config_path = config_path or default_config_path() + console = Console() + + if not os.path.isfile(config_path): + console.print(f"[red]Config file not found: {config_path}[/red]") + return + + # Round-trip loader preserves comments, key order, quoting and ${ENV} refs. + yaml_rt = YAML() + yaml_rt.preserve_quotes = True + # Match the project's YAML style so edits don't reflow list indentation: + # block sequences indented under their key (` - item`). + yaml_rt.indent(mapping=2, sequence=4, offset=2) + with open(config_path) as f: + data = yaml_rt.load(f) + + if not data: + console.print("[yellow]Config is empty; nothing to change.[/yellow]") + return + + screen = _Screen(console) + saves = 0 + # Alternate screen: the whole session replaces the view, and the terminal + # scrollback is restored untouched on exit. + console.set_alt_screen(True) + try: + while True: + changed = _navigate(screen, data, ["config"]) + # None = backed out at root, QUIT_ACTION = quit from any depth. + if changed is None or changed is QUIT_ACTION: + break + with open(config_path, "w") as f: + yaml_rt.dump(data, f) + saves += 1 + screen.status = f"Saved to {config_path}" + finally: + console.set_alt_screen(False) + + if saves: + console.print(f"[{GREEN}]Saved {saves} change(s) to {config_path}[/]") + else: + console.print("No changes made.") + + +def run_config(): + console = Console() + choice = select_menu(OPTIONS, title="What do you want to do?") + if choice is None: + console.print("Cancelled.") + return + + if choice == "view": + run_view_config() + elif choice == "change": + run_change_config() diff --git a/cli/canyonos/constants.py b/cli/canyonos/constants.py new file mode 100644 index 0000000..d34316e --- /dev/null +++ b/cli/canyonos/constants.py @@ -0,0 +1,9 @@ +"""Shared constants for the canyonos CLI.""" + +import os + + +def default_config_path(): + """Global controller config for the current directory, preferring the .car artifact layout.""" + car = os.path.join(".car", "config", "global_controller.yaml") + return car if os.path.isfile(car) else os.path.join("config", "global_controller.yaml") diff --git a/cli/canyonos/dashboard.compose.yml b/cli/canyonos/dashboard.compose.yml new file mode 100644 index 0000000..db775aa --- /dev/null +++ b/cli/canyonos/dashboard.compose.yml @@ -0,0 +1,42 @@ +services: + # Fast-path bundled DB, hardcoded creds -- fine for local dev, not for anything real. + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: canyonos + POSTGRES_PASSWORD: canyonos + POSTGRES_DB: canyonos + healthcheck: + test: ["CMD-SHELL", "pg_isready -U canyonos"] + interval: 2s + timeout: 3s + retries: 20 + ports: + - "127.0.0.1:5432:5432" + + api: + image: ${CANYONOS_API_IMAGE} + depends_on: + db: + condition: service_healthy + # Published so a GC container can POST OTLP spans to /v1/traces via + # host.docker.internal; that route also renames ventis' `project_id` + # attribute to the `canyon.project.id` every dashboard query filters on. + ports: + - "127.0.0.1:3000:3000" + environment: + DATABASE_URL: postgresql://canyonos:canyonos@db:5432/canyonos + JWT_SECRET: ${CANYONOS_JWT_SECRET} + CANYONOS_DISABLE_AUTH: "true" + CANYONOS_REDIS_HOST: ${CANYONOS_REDIS_HOST} + CANYONOS_REDIS_PORT: ${CANYONOS_REDIS_PORT} + LOG_LEVEL: info + extra_hosts: + - host.docker.internal:host-gateway + web: + image: ${CANYONOS_WEB_IMAGE} + depends_on: + api: + condition: service_healthy + ports: + - "127.0.0.1:${CANYONOS_WEB_PORT}:8080" diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py new file mode 100644 index 0000000..6abcfb1 --- /dev/null +++ b/cli/canyonos/dashboard_stack.py @@ -0,0 +1,518 @@ +"""Manage the local CanyonOS dashboard stack.""" + +from __future__ import annotations + +import importlib.resources +import json +import os +import re +import secrets +import shutil +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable +from urllib.parse import urlsplit, urlunsplit + +import yaml + +from canyonos.constants import default_config_path + +COMPOSE_PROJECT = "canyonos-dashboard" +STACK_VERSION = "v0.1.0-rc.2" +API_IMAGE = f"ghcr.io/canyoncodecoreai/canyonos-api:{STACK_VERSION}" +WEB_IMAGE = f"ghcr.io/canyoncodecoreai/canyonos-web:{STACK_VERSION}" +HOST_GATEWAY = "host.docker.internal" +REDIS_HOST = HOST_GATEWAY +REDIS_PORT = "6379" +ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +@dataclass(frozen=True) +class ServeResult: + ok: bool + phase: str + message: str + url: str | None = None + log_path: str | None = None + + +class PhaseFailure(Exception): + def __init__(self, phase: str, message: str, *, had_containers: bool | None = None): + super().__init__(message) + self.phase = phase + self.message = message + self.had_containers = had_containers + + +@dataclass(frozen=True) +class DashboardStack: + database_url: str | None + state_dir: Path + project_dir: Path + web_port: int = 8080 + + @property + def env_path(self) -> Path: + return self.project_dir / ".env" + + +def _run(argv: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(argv, capture_output=True, check=False, text=True) + + +def _state_dir() -> Path: + return Path.home() / ".canyonos" / "dashboard" + + +def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: + # Absolute, not "./.env": `canyonos serve` may cd into .car/ before + # running, so a cwd-relative path would miss the project root .env that + # `prepare()` actually writes to (stack.env_path). + return [ + "docker", + "compose", + "-p", + COMPOSE_PROJECT, + "--env-file", + str(stack.env_path), + "-f", + str(manifest), + ] + + +def _managed_database_url(database_url: str) -> tuple[str, str | None]: + parsed = urlsplit(database_url) + if parsed.hostname not in {"localhost", "127.0.0.1"}: + return database_url, None + + hostname = parsed.hostname + credentials = "" + if parsed.username is not None: + credentials = parsed.username + if parsed.password is not None: + credentials = f"{credentials}:{parsed.password}" + credentials = f"{credentials}@" + port = f":{parsed.port}" if parsed.port is not None else "" + rewritten = urlunsplit( + (parsed.scheme, f"{credentials}{HOST_GATEWAY}{port}", parsed.path, parsed.query, parsed.fragment) + ) + return rewritten, hostname + + +def _existing_dashboard_port() -> int | None: + """The host port an already-running dashboard `web` container owns, if any + -- so re-running `canyonos serve` reconnects to the same stack instead of + picking a new port out from under it.""" + try: + result = _run(["docker", "container", "inspect", "canyonos-dashboard-web-1"]) + except OSError: + return None + if result.returncode != 0: + return None + + try: + containers = json.loads(result.stdout) + bindings = containers[0]["NetworkSettings"]["Ports"].get("8080/tcp") or [] + except (IndexError, KeyError, TypeError, json.JSONDecodeError): + return None + + for binding in bindings: + if binding.get("HostIp") in {"127.0.0.1", "0.0.0.0", "::"}: + try: + return int(binding["HostPort"]) + except (KeyError, TypeError, ValueError): + continue + return None + + +def _port_is_free(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + try: + probe.bind(("127.0.0.1", port)) + except OSError: + return False + return True + + +def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: + """First free port at or after `start` -- same retry-on-conflict shape as + init.py's GC port selection, so an unrelated process/container squatting + on 8080 (e.g. a deployed Workflow's own api_port) doesn't hard-block serve. + """ + for port in range(start, start + max_attempts): + if _port_is_free(port): + return port + raise PhaseFailure( + "validate", f"no free port found for the dashboard after {max_attempts} attempts starting at {start}" + ) + + +def _load_project_config(config_path: str) -> tuple[object, Path]: + project_root = Path(os.path.abspath(os.path.join(os.path.dirname(config_path), ".."))) + # `canyonos serve` cds into .car/ before calling here, so the naive + # parent-of-parent lands on .car itself -- go up one more level to reach + # the actual project root, where .env lives. + if project_root.name == ".car": + project_root = project_root.parent + dotenv_path = project_root / ".env" + if dotenv_path.is_file(): + with dotenv_path.open(encoding="utf-8") as dotenv: + for line in dotenv: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = _env_value(value) + if key and key not in os.environ: + os.environ[key] = value + + with open(config_path, encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) + return _expand_env_value(config), project_root + + +def _expand_env_value(value: object) -> object: + if isinstance(value, str): + return ENV_REFERENCE.sub(lambda match: os.environ.get(match.group(1), match.group(0)), value) + if isinstance(value, dict): + return {key: _expand_env_value(item) for key, item in value.items()} + if isinstance(value, list): + return [_expand_env_value(item) for item in value] + return value + + +def validate(config_path: str) -> DashboardStack: + if shutil.which("docker") is None: + raise PhaseFailure("validate", "docker is not on PATH") + + try: + if _run(["docker", "info"]).returncode != 0: + raise PhaseFailure("validate", "docker daemon or socket is unavailable") + if _run(["docker", "compose", "version"]).returncode != 0: + raise PhaseFailure("validate", "docker compose is unavailable") + except OSError: + raise PhaseFailure("validate", "docker daemon or socket is unavailable") + + try: + # Keep interpolation consistent with GlobalController._load_config in ventis/controller/global_controller.py. + config, project_root = _load_project_config(config_path) + except (OSError, yaml.YAMLError): + raise PhaseFailure("validate", f"config file is not readable: {config_path}") + + # database.url is optional -- the dashboard works without a database configured + # (e.g. OTLP-only setups); if present, it still needs to actually be usable. + database = config.get("database") if isinstance(config, dict) else None + database_url = database.get("url") if isinstance(database, dict) else None + if database_url is not None: + if not isinstance(database_url, str) or not database_url.strip(): + raise PhaseFailure("validate", "database.url must be a non-empty string") + unresolved = ENV_REFERENCE.search(database_url) + if unresolved: + name = unresolved.group(1) + raise PhaseFailure( + "validate", f"database.url needs ${{{name}}}, which is not set in the project .env" + ) + database_url = database_url.strip() + + state_dir = _state_dir() + try: + state_dir.mkdir(parents=True, exist_ok=True) + probe_path = state_dir / ".write-probe" + with open(probe_path, "w", encoding="utf-8") as probe: + probe.write("") + probe_path.unlink() + except OSError: + raise PhaseFailure("validate", "dashboard state directory is not writable") + + web_port = _existing_dashboard_port() or _find_web_port() + + return DashboardStack(database_url, state_dir, project_root, web_port) + + +def _env_value(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value + + +def _read_existing_secret(env_path: Path) -> str | None: + try: + lines = env_path.read_text(encoding="utf-8").splitlines() + except OSError: + return None + + for line in lines: + key, separator, value = line.partition("=") + if separator and key == "CANYONOS_JWT_SECRET" and _env_value(value): + return _env_value(value) + return None + + +def _write_private_file(path: Path, contents: str) -> None: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(contents) + + +def _env_line(key: str, value: str) -> str: + if " " in value or "#" in value: + return f'{key}="{value}"\n' + return f"{key}={value}\n" + + +def _write_project_env(env_path: Path, managed_env: dict[str, str]) -> None: + try: + lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) + except FileNotFoundError: + lines = [] + + managed_keys = set(managed_env) + replaced: set[str] = set() + updated_lines: list[str] = [] + for line in lines: + key, separator, _ = line.partition("=") + if separator and key in managed_keys: + if key not in replaced: + updated_lines.append(_env_line(key, managed_env[key])) + replaced.add(key) + continue + updated_lines.append(line) + + for key, value in managed_env.items(): + if key not in replaced: + updated_lines.append(_env_line(key, value)) + + descriptor, temporary_name = tempfile.mkstemp(prefix=".env.", dir=env_path.parent) + temporary_path = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + os.fchmod(output.fileno(), 0o600) + output.writelines(updated_lines) + os.replace(temporary_path, env_path) + except Exception: + try: + temporary_path.unlink() + except FileNotFoundError: + pass + raise + + +def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: + try: + stack.state_dir.mkdir(parents=True, exist_ok=True) + os.chmod(stack.state_dir, 0o700) + managed_env = { + "CANYONOS_JWT_SECRET": _read_existing_secret(stack.env_path) or secrets.token_urlsafe(32), + "CANYONOS_REDIS_HOST": REDIS_HOST, + "CANYONOS_REDIS_PORT": REDIS_PORT, + "CANYONOS_API_IMAGE": API_IMAGE, + "CANYONOS_WEB_IMAGE": WEB_IMAGE, + "CANYONOS_WEB_PORT": str(stack.web_port), + } + rewritten_host = None + if stack.database_url is not None: + managed_database_url, rewritten_host = _managed_database_url(stack.database_url) + managed_env["CANYONOS_DATABASE_URL"] = managed_database_url + _write_project_env(stack.env_path, managed_env) + (stack.state_dir / "stack.json").write_text( + json.dumps( + { + "schema_version": 1, + "stack_version": STACK_VERSION, + "compose_project": COMPOSE_PROJECT, + } + ) + + "\n", + encoding="utf-8", + ) + except (OSError, ValueError): + raise PhaseFailure("prepare", "could not prepare the dashboard state directory") + + message = "dashboard state prepared" + if rewritten_host: + message = ( + f"database host {rewritten_host} is reachable from the stack as host.docker.internal" + ) + return managed_env, message + + +def _redact_stack_text(text: str, stack: DashboardStack, managed_env: dict[str, str]) -> str: + secret = managed_env["CANYONOS_JWT_SECRET"] + redacted = redact_logs(text, secret) + if stack.database_url is not None: + redacted = redact_logs(redacted.replace(stack.database_url, "[redacted]"), secret) + return redacted + + +def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: + return next((line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None) + + +def _command_failure_message( + message: str, + result: subprocess.CompletedProcess[str], + stack: DashboardStack, + managed_env: dict[str, str], +) -> str: + detail = _last_stderr_line(result) + if detail is None: + return message + return f"{message}: {_redact_stack_text(detail, stack, managed_env)}" + + +def pull( + stack: DashboardStack, + manifest: Path, + managed_env: dict[str, str], + had_containers: bool, +) -> str: + try: + result = _run([*_compose_argv(stack, manifest), "pull"]) + except OSError: + raise PhaseFailure("pull", "could not run docker compose pull", had_containers=had_containers) + if result.returncode != 0: + raise PhaseFailure( + "pull", + _command_failure_message("docker compose pull failed", result, stack, managed_env), + had_containers=had_containers, + ) + return "dashboard images pulled" + + +def _project_has_running_containers(stack: DashboardStack, manifest: Path) -> bool: + try: + result = _run([*_compose_argv(stack, manifest), "ps", "-q"]) + except OSError: + return False + return result.returncode == 0 and bool(result.stdout.strip()) + + +def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> bool: + had_containers = _project_has_running_containers(stack, manifest) + # The api reads the controller's Redis identity once at startup to create + # its project row, so a surviving container keeps serving whichever project + # was deployed before it. Replace it every serve rather than reuse it. + _run([*_compose_argv(stack, manifest), "rm", "-sf", "api"]) + try: + result = _run( + [*_compose_argv(stack, manifest), "up", "-d", "--wait", "--wait-timeout", "180"] + ) + except OSError: + raise PhaseFailure("start", "could not run docker compose up", had_containers=had_containers) + if result.returncode != 0: + raise PhaseFailure( + "start", + _command_failure_message("docker compose up failed", result, stack, managed_env), + had_containers=had_containers, + ) + return had_containers + + +def verify(port: int) -> str: + dashboard_url = f"http://127.0.0.1:{port}" + deadline = time.monotonic() + 30 + endpoints = (f"{dashboard_url}/healthz", f"{dashboard_url}/api/healthz") + while time.monotonic() < deadline: + healthy = True + for endpoint in endpoints: + try: + response = urllib.request.urlopen(endpoint, timeout=5) + try: + status = response.status + finally: + response.close() + except (OSError, urllib.error.URLError): + healthy = False + break + if status != 200: + healthy = False + break + if healthy: + return dashboard_url + if time.monotonic() < deadline: + time.sleep(1) + raise PhaseFailure("verify", "dashboard health checks did not return 200 within 30 seconds") + + +def redact_logs(logs: str, jwt_secret: str) -> str: + redacted = logs.replace(jwt_secret, "[redacted]") + return re.sub(r"://[^/\s@]+@", "://[redacted]@", redacted) + + +def _capture_failure_logs(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> Path: + try: + result = _run([*_compose_argv(stack, manifest), "logs", "--no-color", "--tail", "200"]) + logs = f"{result.stdout}\n{result.stderr}" + except OSError: + logs = "Unable to collect docker compose logs." + + log_dir = stack.state_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + log_path = log_dir / f"serve-{timestamp}.log" + _write_private_file( + log_path, + _redact_stack_text(logs, stack, managed_env), + ) + return log_path + + +def _cleanup(stack: DashboardStack, manifest: Path) -> None: + try: + _run([*_compose_argv(stack, manifest), "down"]) + except OSError: + return + + +def run_dashboard( + config_path: str | None = None, + phase_reporter: Callable[[str, str], None] | None = None, +) -> ServeResult: + config_path = config_path or default_config_path() + def report(result: ServeResult) -> None: + if phase_reporter is not None: + phase_reporter(result.phase, result.message) + + stack: DashboardStack | None = None + managed_env: dict[str, str] | None = None + manifest: Path | None = None + had_containers = False + with ExitStack() as resources: + try: + stack = validate(config_path) + report(ServeResult(True, "validate", "dashboard prerequisites validated")) + + managed_env, prepare_message = prepare(stack) + report(ServeResult(True, "prepare", prepare_message)) + + manifest_resource = importlib.resources.files("canyonos").joinpath("dashboard.compose.yml") + manifest = resources.enter_context(importlib.resources.as_file(manifest_resource)) + had_containers_before_pull = _project_has_running_containers(stack, manifest) + pull_message = pull(stack, manifest, managed_env, had_containers_before_pull) + report(ServeResult(True, "pull", pull_message)) + + had_containers = start(stack, manifest, managed_env) + report(ServeResult(True, "start", "dashboard stack started")) + + url = verify(stack.web_port) + report(ServeResult(True, "verify", "dashboard health checks passed", url)) + return ServeResult(True, "verify", "dashboard health checks passed", url) + except PhaseFailure as failure: + log_path = None + if failure.phase in {"pull", "start", "verify"} and stack and managed_env and manifest: + log_path = _capture_failure_logs(stack, manifest, managed_env) + if not (failure.had_containers if failure.had_containers is not None else had_containers): + _cleanup(stack, manifest) + return ServeResult( + False, failure.phase, failure.message, None, str(log_path) if log_path else None + ) diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py new file mode 100644 index 0000000..a2239a2 --- /dev/null +++ b/cli/canyonos/deploy.py @@ -0,0 +1,86 @@ +""" +Logic for `canyonos deploy`: copy the project into the container's /workspace +volume (via `canyonos sync`), then tell the Global Controller container to +build and deploy it. The container's `ventis deploy` handles both the build +(stubs, protos, Docker images) and the launch -- the CLI just ships files, +triggers it, and streams the logs. + +Once the deploy's logs report the workflow is actually up, `canyonos serve` +is kicked off automatically so the local dashboard is ready without an extra +manual step. +""" + +import json +import subprocess +import urllib.error +import urllib.request + +from canyonos.constants import default_config_path +from canyonos.init import load_state, run_init +from canyonos.serve import run_serve +from canyonos.sync import run_sync + +# Logged exactly once by GlobalController.run(), right after `_wait_for_healthy()` +# returns -- the signal that the workflow finished coming up and entered its +# steady-state polling loop. +_WORKFLOW_UP_MARKER = "Global controller started, polling every" + + +def run_deploy(config_path=None, serve=True): + config_path = config_path or default_config_path() + run_init() + + # Copy the current project into the container before building/deploying. + if not run_sync(): + return + + state = load_state() + + url = f"http://127.0.0.1:{state['port']}/deploy" + body = json.dumps({"config_path": config_path}).encode() + req = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"}, method="POST" + ) + + try: + with urllib.request.urlopen(req) as resp: + json.loads(resp.read()) + _stream_logs_and_autoserve(state["container_id"], serve=serve) + except urllib.error.HTTPError as e: + data = json.loads(e.read()) + print(f"Deploy failed: {data.get('error')}") + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") + + +def _stream_logs_and_autoserve(container_id, serve=True): + """Tail the GC container's logs (same as before), and -- unless disabled + via `serve=False` -- launch `canyonos serve` the moment they show the + workflow is up, so the dashboard is ready alongside it. Log tailing + continues afterwards exactly as before. + """ + process = subprocess.Popen( + ["docker", "logs", "-f", container_id], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + served = not serve + try: + for line in process.stdout: + print(line, end="") + if not served and _WORKFLOW_UP_MARKER in line: + served = True + print("\nWorkflow is up -- starting the local dashboard (canyonos serve)...") + try: + run_serve() + except Exception as e: + print(f"Could not start the dashboard automatically: {e}") + print("Run `canyonos serve` manually to view it.") + except KeyboardInterrupt: + print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + print("To resubscribe to log stream run `canyonos logs`.") + finally: + if process.poll() is None: + process.terminate() diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py new file mode 100644 index 0000000..3d9cee9 --- /dev/null +++ b/cli/canyonos/init.py @@ -0,0 +1,129 @@ +""" +Logic for `canyonos init`, does the following: +1. Pull the Global Controller image +2. Start a container from it +3. Record where it's listening so cli knows where to send requests. +""" + +import json +import os +import subprocess +import urllib.error +import urllib.request + +# Formatting +from pyfiglet import figlet_format +from rich.console import Console + +from canyonos.theme import GRADIENT + + + +# Image Name, need to switch to CanyonCore Organization Namespace later +GC_IMAGE = "saakeths/canyonos:latest" +GC_CONTAINER_PORT = 8000 + +# Named docker volume mounted at /workspace inside the container. Unlike a bind +# mount, this lives in the container's docker volume (not the host filesystem): +# it persists across `canyonos quit` (docker rm leaves named volumes intact) and +# is unaffected by host-side changes. Files are copied in via `canyonos sync` +# (docker cp), not mounted live. +GC_WORKSPACE_VOLUME = "canyonos-workspace" +GC_WORKSPACE_PATH = "/workspace" + +STATE_DIR = os.path.expanduser("~/.canyonos") +STATE_PATH = os.path.join(STATE_DIR, "state.json") + + +def pull_image(image=GC_IMAGE): + # Capture output so the rich status spinner isn't clobbered by docker's own + # layer-progress printing. + subprocess.run(["docker", "pull", image], check=True, capture_output=True) + + +def _port_reachable(port, attempts=10, delay=0.5): + """ + A successful `docker run` only means Docker accepted the port binding -- + not that traffic actually flows. OrbStack's own port-forwarding proxy for + a given port can get stuck (heavy churn on the same port is enough to + trigger it), which looks fine at the Docker level but resets every real + connection. Confirm the container is actually reachable before trusting it. + """ + import time + + url = f"http://127.0.0.1:{port}/status" + for _ in range(attempts): + try: + urllib.request.urlopen(url, timeout=1) + return True + except (urllib.error.URLError, OSError): + time.sleep(delay) + return False + + +def run_container(image=GC_IMAGE, max_attempts=50): + port = GC_CONTAINER_PORT + for _ in range(max_attempts): + result = subprocess.run( + [ + "docker", + "run", + "-d", + "-p", + f"127.0.0.1:{port}:{GC_CONTAINER_PORT}", + # Docker-outside-of-Docker: GC shells out to `docker` to launch + # Redis/agent containers, so it needs the host's real daemon, + # not a nested one. + "-v", + "/var/run/docker.sock:/var/run/docker.sock", + "-v", + f"{GC_WORKSPACE_VOLUME}:{GC_WORKSPACE_PATH}", + "--add-host=host.docker.internal:host-gateway", + "-e", + "VENTIS_REDIS_HOST=host.docker.internal", + image, + ], + capture_output=True, + text=True, + ) + if result.returncode == 0: + container_id = result.stdout.strip() + if _port_reachable(port): + return container_id, port + # Port bound fine but never actually became reachable -- treat + # like a conflict, since that's effectively what it is. + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True) + port += 1 + continue + if "port is already allocated" in result.stderr: + port += 1 + continue + raise RuntimeError(result.stderr) + raise RuntimeError(f"no free port found after {max_attempts} attempts starting at {GC_CONTAINER_PORT}") + + +def save_state(container_id, port): + os.makedirs(STATE_DIR, exist_ok=True) + with open(STATE_PATH, "w") as f: + json.dump({"container_id": container_id, "port": port}, f) + + +def load_state(): + with open(STATE_PATH) as f: + return json.load(f) + + +def run_init(): + console = Console() + banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) + + for line, color in zip(banner.splitlines(), GRADIENT): + console.print(line, style=color) + + + with console.status("Pulling Global Controller image..."): + pull_image() + with console.status("Starting Global Controller container..."): + container_id, port = run_container() + save_state(container_id, port) + print(f"Global Controller running in container {container_id[:12]} on port {port}") diff --git a/cli/canyonos/integrate.py b/cli/canyonos/integrate.py new file mode 100644 index 0000000..23e61b1 --- /dev/null +++ b/cli/canyonos/integrate.py @@ -0,0 +1,82 @@ +""" +Logic for `canyonos integrate`: install the CanyonOS skill on a coding agent, +then launch that agent with a prompt to apply it to the current project. +""" + +import os +import shutil +import subprocess + +from rich.console import Console + +from utils.tui import select_menu + +# Points at the skill's folder, so SKILL.md and references/ both come along. +SKILL_SOURCE_URL = "https://github.com/CanyonCodeCoreAI/canyoncodecore/tree/nickhuo/porting-skill-car-layout/.claude/skills/porting-to-canyonos" + +# The porting skill emits no otel config; without this the dashboard stays empty. +OTEL_BLOCK = """otel: + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {}""" + +INTEGRATE_PROMPT = ( + "Use the CanyonOS porting-to-canyonos-core skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." + "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," + " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" + " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK +) + +AGENTS = { + "claude": { + "label": "Claude Code", + "cli": "claude", + # Claude Code auto-loads project-local skills from here. + "skill_dir": ".claude/skills/porting-to-canyonos-core", + }, + "codex": { + "label": "Codex", + "cli": "codex", + # Codex only auto-loads skills from the user's home directory, not per-project. + "skill_dir": os.path.expanduser("~/.codex/skills/porting-to-canyonos-core"), + }, +} + + +def prompt_agent(): + options = [(key, spec["label"]) for key, spec in AGENTS.items()] + return select_menu(options, title="Which coding agent do you want to integrate with?") + + +def install_skill(agent): + spec = AGENTS[agent] + # -f overwrites an existing skill dir; without it gitpick exits 1 when the + # target already exists and is non-empty (e.g. re-running `integrate`). + subprocess.run( + ["npx", "-y", "gitpick", "-f", SKILL_SOURCE_URL, spec["skill_dir"]], + check=True, + ) + + +def launch_agent(agent, prompt): + spec = AGENTS[agent] + if not shutil.which(spec["cli"]): + print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + return + subprocess.run([spec["cli"], prompt], check=True) + + +def run_integrate(): + console = Console() + agent = prompt_agent() + if agent is None: + console.print("Cancelled.") + return + + console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") + install_skill(agent) + + console.print(f"Launching {AGENTS[agent]['label']}...") + launch_agent(agent, INTEGRATE_PROMPT) diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py new file mode 100644 index 0000000..9b2b1d8 --- /dev/null +++ b/cli/canyonos/logs.py @@ -0,0 +1,37 @@ +""" +Logic for `canyonos logs`: re-subscribe to the running deploy's log stream. +""" + +import json +import subprocess +import urllib.error +import urllib.request + +from canyonos.init import load_state + + +def run_logs(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return + + url = f"http://127.0.0.1:{state['port']}/status" + req = urllib.request.Request(url, method="GET") + + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") + return + + if not data.get("running"): + print("No deploy running, run `canyonos deploy` to deploy project.") + return + + try: + subprocess.run(["docker", "logs", "-f", state["container_id"]]) + except KeyboardInterrupt: + print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") diff --git a/cli/canyonos/new_app.py b/cli/canyonos/new_app.py new file mode 100644 index 0000000..30e93b0 --- /dev/null +++ b/cli/canyonos/new_app.py @@ -0,0 +1,21 @@ +""" +Logic for `canyonos new-app`: scaffold a new project in the current +directory. Runs locally, no container involved. +""" + +import os + + +def run_new_app(): + if os.listdir("."): + print("Directory is not empty. Run `canyonos new-app` in an empty directory.") + return + + for folder in ("agents", "config", "workflow"): + os.makedirs(folder) + open(".env", "w").close() + + for filename in ("global_controller.yaml", "policy.yaml"): + open(os.path.join("config", filename), "w").close() + + print("Created new CanyonOS project.") diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py new file mode 100644 index 0000000..15aff2c --- /dev/null +++ b/cli/canyonos/quit.py @@ -0,0 +1,63 @@ +""" +Logic for `canyonos quit`: full teardown. Stops and removes the Global +Controller container AND deletes the /workspace named volume, so the project +files copied into it are discarded too. (Use `canyonos stop` to only halt a +running deploy while keeping the container and files around.) +""" + +import os +import subprocess + +from rich.console import Console + +from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH, load_state +from canyonos.stop import _post_clean + + +def _container_exists(container_id): + result = subprocess.run( + ["docker", "inspect", container_id], capture_output=True + ) + return result.returncode == 0 + + +def run_quit(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running.") + return + + container_id = state["container_id"] + console = Console() + with console.status("Tearing down..."): + # Stop any running deploy first, so the local controller and Redis + # containers it spawned via docker-outside-of-docker get torn down + # too. Removing the GC container itself doesn't touch them -- they're + # sibling containers on the host, not nested inside it. + try: + _post_clean(state["port"]) + except OSError: + # Covers urllib.error.HTTPError/URLError (both subclass OSError) + # plus raw connection errors -- nothing was running, or the GC is + # already unreachable/gone. + pass + + # state.json can go stale (daemon restarted, container removed by + # hand, a previous `quit` died partway through) -- don't let a + # missing container turn `quit` into a crash instead of a cleanup. + already_gone = not _container_exists(container_id) + if not already_gone: + subprocess.run(["docker", "stop", container_id], check=False, capture_output=True) + subprocess.run(["docker", "rm", container_id], check=False, capture_output=True) + + # Remove the workspace volume only after the container is gone (docker + # refuses to remove a volume still in use). check=False so a missing + # volume doesn't turn teardown into an error. + subprocess.run(["docker", "volume", "rm", GC_WORKSPACE_VOLUME], check=False, capture_output=True) + os.remove(STATE_PATH) + + if already_gone: + print(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") + else: + print(f"Global Controller container {container_id[:12]} torn down (volume removed)") diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py new file mode 100644 index 0000000..ddfd7d6 --- /dev/null +++ b/cli/canyonos/serve.py @@ -0,0 +1,18 @@ +"""CLI output for the local dashboard stack.""" + +from .dashboard_stack import run_dashboard + + +def run_serve(config_path: str | None = None) -> int: + def report(phase: str, message: str) -> None: + print(f"[serve] {phase}: {message}") + + result = run_dashboard(config_path, report) + if result.ok: + print(f"Dashboard: {result.url}") + return 0 + + print(f"serve failed in {result.phase}: {result.message}") + if result.log_path: + print(f"log: {result.log_path}") + return 1 diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py new file mode 100644 index 0000000..2f6ad40 --- /dev/null +++ b/cli/canyonos/stop.py @@ -0,0 +1,47 @@ +""" +Logic for `canyonos stop`: stop the running deploy inside the Global +Controller container (SIGTERM, same teardown as Ctrl+C would trigger). +""" + +import json +import urllib.error +import urllib.request + +from rich.console import Console + +from canyonos.init import load_state + + +def _post_clean(port): + """POST /clean to the Global Controller container. + + This is what actually tears down the local controller and Redis + containers a deploy spawned via docker-outside-of-docker: it sends + SIGTERM to the in-container `ventis deploy` process, whose handler calls + `GlobalController.stop()` and blocks until it returns. Shared with + `canyonos quit`, which needs the same teardown before removing the GC + container itself. + """ + url = f"http://127.0.0.1:{port}/clean" + req = urllib.request.Request(url, method="POST") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def run_stop(): + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return + + console = Console() + try: + with console.status("Stopping deploy..."): + _post_clean(state["port"]) + print("Deploy stopped.") + except urllib.error.HTTPError as e: + data = json.loads(e.read()) + print(f"Stop failed: {data.get('error')}") + except urllib.error.URLError as e: + print(f"Could not reach Global Controller container: {e}") diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py new file mode 100644 index 0000000..f350a1c --- /dev/null +++ b/cli/canyonos/sync.py @@ -0,0 +1,40 @@ +""" +Logic for `canyonos sync`: copy the current project directory into the Global +Controller container's /workspace volume via `docker cp`. + +Files live inside the container's named volume (see `init.py`), not on a live +bind mount -- so they persist across `canyonos quit` and survive host-side +changes. `docker cp` is additive: it overwrites/adds files but never deletes, +so build outputs generated inside the container (stubs/, grpc_stubs/, +docker_container/) survive a re-sync of the host source. +""" + +import os +import subprocess + +from canyonos.init import GC_WORKSPACE_PATH, load_state + + +def run_sync(): + """Copy the current directory into the container. Returns True on success.""" + try: + state = load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos init` first.") + return False + + container_id = state["container_id"] + # Trailing "/." copies the *contents* of the current directory into + # /workspace, rather than nesting it under /workspace/. + src = os.path.join(os.getcwd(), ".") + print(f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH} ...") + + result = subprocess.run( + ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"] + ) + if result.returncode != 0: + print("Sync failed.") + return False + + print("Sync complete.") + return True diff --git a/cli/canyonos/theme.py b/cli/canyonos/theme.py new file mode 100644 index 0000000..e74a069 --- /dev/null +++ b/cli/canyonos/theme.py @@ -0,0 +1,20 @@ +""" +CanyonOS standard color palette. + +The green->white gradient introduced by the `canyonos init` banner, reused +across the CLI so everything shares one look. `GREEN` is the primary brand +color; `WHITE` the secondary; `GRADIENT` the full ramp for multi-line output. +""" + +GREEN = "#2BD17E" +WHITE = "#FFFFFF" + +# Primary -> secondary ramp (used for the init banner, top to bottom). +GRADIENT = [ + "#2BD17E", + "#55DA98", + "#80E3B2", + "#AAEDCB", + "#D5F6E5", + "#FFFFFF", +] diff --git a/cli/cli.py b/cli/cli.py new file mode 100644 index 0000000..4c78043 --- /dev/null +++ b/cli/cli.py @@ -0,0 +1,217 @@ +""" +Most of the commands will be executed by code in the canyonos container. +Anything executing in this CLI pertains to file/folder modification +""" + +import argparse +import sys + +from canyonos.clean import run_clean +from canyonos.constants import default_config_path +from canyonos.config import run_config +from canyonos.deploy import run_deploy +from canyonos.integrate import run_integrate +from canyonos.logs import run_logs +from canyonos.new_app import run_new_app +from canyonos.quit import run_quit +from canyonos.serve import run_serve +from canyonos.stop import run_stop +from canyonos.sync import run_sync + +try: + from rich.console import Console + from rich.panel import Panel + from rich.text import Text + from rich.table import Table + RICH_AVAILABLE = True +except ImportError: + RICH_AVAILABLE = False + +def cmd_connect(args): + pass + +def cmd_quit(args): + run_quit() + +def cmd_new_app(args): + run_new_app() + +# Executed in canyonos: syncs files, then builds + deploys +def cmd_deploy(args): + run_deploy(args.config, serve=args.serve) + +def cmd_clean(args): + run_clean() + +def cmd_stop(args): + run_stop() + +def cmd_logs(args): + run_logs() + +def cmd_sync(args): + run_sync() + +def cmd_config(args): + run_config() + +def cmd_integrate(args): + run_integrate() + +def cmd_doctor(args): + pass + +def cmd_serve(args): + sys.exit(run_serve(args.config)) + +# Executed in canyonos +def cmd_test(args): + pass + +# Executed in canyonos +def cmd_mega_build(args): + pass + + +def cmd_version(args): + pass + + +def print_custom_help(): + """Print a custom, visually appealing help screen.""" + if RICH_AVAILABLE: + console = Console() + + # Header + title = Text("CanyonOS CLI", style="bold cyan") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") + + console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + + # Core commands + console.print("\n[bold yellow]Core Commands[/bold yellow]") + core_table = Table(show_header=False, border_style="dim", padding=(0, 2)) + core_table.add_column(style="cyan", width=20) + core_table.add_column(style="white") + core_table.add_row("integrate", "Sync source files to .car/app/") + core_table.add_row("deploy", "Build and deploy agents to configured hosts") + core_table.add_row("config", "Configure project settings") + console.print(core_table) + + # Utils commands + console.print("\n[bold yellow]Utils[/bold yellow]") + utils_table = Table(show_header=False, border_style="dim", padding=(0, 2)) + utils_table.add_column(style="cyan", width=20) + utils_table.add_column(style="white") + utils_table.add_row("new-app", "Create a new CanyonOS project") + utils_table.add_row("serve", "Start local CanyonOS dashboard") + utils_table.add_row("sync", "Sync files with container") + utils_table.add_row("stop", "Stop running containers") + utils_table.add_row("clean", "Remove generated files") + utils_table.add_row("logs", "View container logs") + utils_table.add_row("doctor", "Check system health") + utils_table.add_row("connect", "Connect to remote host") + utils_table.add_row("quit", "Shut down CanyonOS services") + console.print(utils_table) + + # Quick start + console.print("\n[bold green]Quick Start:[/bold green]") + console.print(" [dim]1.[/dim] canyonos new-app [cyan]my-app[/cyan]") + console.print(" [dim]2.[/dim] cd [cyan]my-app[/cyan]") + console.print(" [dim]3.[/dim] canyonos integrate") + console.print(" [dim]4.[/dim] canyonos deploy") + console.print(" [dim]5.[/dim] canyonos serve\n") + + console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") + else: + # Fallback to simple text if rich is not available + print("\n" + "="*60) + print(" " * 20 + "CanyonOS CLI") + print(" " * 10 + "Build, deploy, and manage agentic workflows") + print("="*60 + "\n") + + print("CORE COMMANDS:") + print(" integrate Sync source files to .car/app/") + print(" deploy Build and deploy agents to configured hosts") + print(" config Configure project settings\n") + + print("UTILS:") + print(" new-app Create a new CanyonOS project") + print(" serve Start local CanyonOS dashboard") + print(" sync Sync files with container") + print(" stop Stop running containers") + print(" clean Remove generated files") + print(" logs View container logs") + print(" doctor Check system health") + print(" connect Connect to remote host") + print(" quit Shut down CanyonOS services\n") + + print("QUICK START:") + print(" 1. canyonos new-app my-app") + print(" 2. cd my-app") + print(" 3. canyonos integrate") + print(" 4. canyonos deploy") + print(" 5. canyonos serve\n") + + print("For command-specific help: canyonos --help\n") + + +def _parse_bool(value): + if value.lower() in ("true", "1", "yes"): + return True + if value.lower() in ("false", "0", "no"): + return False + raise argparse.ArgumentTypeError(f"expected true/false, got: {value!r}") + + +def main(): + parser = argparse.ArgumentParser(prog="canyonos") + subparsers = parser.add_subparsers(dest="command") + config_default = default_config_path() + + subparsers.add_parser("new-app").set_defaults(func=cmd_new_app) + deploy = subparsers.add_parser("deploy") + deploy.add_argument( + "-c", + "--config", + default=config_default, + help=f"Path to global controller config (default: {config_default})", + ) + deploy.add_argument( + "--serve", + type=_parse_bool, + default=True, + metavar="true|false", + help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", + ) + deploy.set_defaults(func=cmd_deploy) + subparsers.add_parser("clean").set_defaults(func=cmd_clean) + subparsers.add_parser("stop").set_defaults(func=cmd_stop) + subparsers.add_parser("logs").set_defaults(func=cmd_logs) + subparsers.add_parser("quit").set_defaults(func=cmd_quit) + subparsers.add_parser("connect").set_defaults(func=cmd_connect) + subparsers.add_parser("sync").set_defaults(func=cmd_sync) + subparsers.add_parser("config").set_defaults(func=cmd_config) + subparsers.add_parser("integrate").set_defaults(func=cmd_integrate) + subparsers.add_parser("doctor").set_defaults(func=cmd_doctor) + serve = subparsers.add_parser("serve") + serve.add_argument( + "-c", + "--config", + default=config_default, + help=f"Path to global controller config (default: {config_default})", + ) + serve.set_defaults(func=cmd_serve) + subparsers.add_parser("test").set_defaults(func=cmd_test) + subparsers.add_parser("mega-build").set_defaults(func=cmd_mega_build) + + args = parser.parse_args() + if not getattr(args, "command", None): + print_custom_help() + return + + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/cli/pyproject.toml b/cli/pyproject.toml new file mode 100644 index 0000000..e85c825 --- /dev/null +++ b/cli/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "canyonos" +version = "0.1.4" +description = "CanyonOS CLI" +requires-python = ">=3.10" +dependencies = [ + "pyfiglet", + "pyyaml", + "rich", + "ruamel.yaml", +] + +[project.scripts] +canyonos = "cli:main" + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["canyonos*", "utils*"] + +[tool.setuptools] +py-modules = ["cli"] + +[tool.setuptools.package-data] +canyonos = ["dashboard.compose.yml"] diff --git a/cli/tests/test_dashboard_stack.py b/cli/tests/test_dashboard_stack.py new file mode 100644 index 0000000..2e62480 --- /dev/null +++ b/cli/tests/test_dashboard_stack.py @@ -0,0 +1,432 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from canyonos import dashboard_stack + + +def completed(argv, returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess(argv, returncode, stdout, stderr) + + +@pytest.fixture +def project(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.chdir(tmp_path) + for key in ( + "DATABASE_URL", + "JWT_SECRET", + "CANYONOS_DATABASE_URL", + "CANYONOS_JWT_SECRET", + "CANYONOS_REDIS_HOST", + "CANYONOS_REDIS_PORT", + "CANYONOS_API_IMAGE", + "CANYONOS_WEB_IMAGE", + ): + monkeypatch.delenv(key, raising=False) + config_dir = tmp_path / "config" + config_dir.mkdir() + config = config_dir / "global_controller.yaml" + config.write_text("database:\n url: postgres://user:password@db.example/canyonos\n") + monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: "/usr/bin/docker") + monkeypatch.setattr(dashboard_stack, "_port_is_free", lambda _port: True) + return config + + +def install_docker(monkeypatch, calls, responses=None): + responses = responses or {} + + def fake_run(argv, **_): + calls.append(argv) + if callable(responses): + return responses(argv) + for marker, result in responses.items(): + if argv[-len(marker) :] == list(marker): + return result(argv) + if argv[:3] == ["docker", "container", "inspect"]: + return completed(argv, returncode=1) + return completed(argv) + + monkeypatch.setattr(dashboard_stack.subprocess, "run", fake_run) + + +@pytest.mark.parametrize( + ("prepare", "message"), + [ + (lambda monkeypatch, _: monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: None), "docker is not on PATH"), + ( + lambda _, responses: responses.update( + {("info",): lambda argv: completed(argv, returncode=1)} + ), + "docker daemon or socket is unavailable", + ), + ( + lambda _, responses: responses.update( + {("compose", "version"): lambda argv: completed(argv, returncode=1)} + ), + "docker compose is unavailable", + ), + ], +) +def test_docker_validation_failures_do_not_pull(monkeypatch, project, prepare, message): + calls = [] + responses = {} + prepare(monkeypatch, responses) + install_docker(monkeypatch, calls, responses) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult(False, "validate", message) + assert all(command[-1] != "pull" for command in calls) + + +def test_empty_database_url_fails_validation_but_absent_one_does_not(monkeypatch, project): + project.write_text("database:\n url: ''\n") + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + False, "validate", "database.url must be a non-empty string" + ) + assert all(command[-1] != "pull" for command in calls) + + +def test_missing_database_section_is_not_a_validation_failure(monkeypatch, project): + project.write_text("") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + + assert stack.database_url is None + + +def test_prepare_omits_database_env_when_not_configured(project): + project.write_text("") + stack = dashboard_stack.DashboardStack( + None, dashboard_stack._state_dir(), Path.cwd() + ) + + managed_env, message = dashboard_stack.prepare(stack) + + assert message == "dashboard state prepared" + assert "CANYONOS_DATABASE_URL" not in managed_env + assert "CANYONOS_DATABASE_URL" not in stack.env_path.read_text() + + +def test_config_substitutes_quoted_dotenv_value_without_replacing_source(monkeypatch, project): + source_line = 'DATABASE_URL="postgres://user:password@db.example/canyonos"\n' + Path.cwd().joinpath(".env").write_text(source_line) + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + managed_env, _ = dashboard_stack.prepare(stack) + + assert stack.database_url == "postgres://user:password@db.example/canyonos" + assert managed_env["CANYONOS_DATABASE_URL"] == stack.database_url + assert stack.env_path.read_text().startswith(source_line) + + +def test_missing_database_url_variable_fails_without_pulling(monkeypatch, project): + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + False, + "validate", + "database.url needs ${DATABASE_URL}, which is not set in the project .env", + ) + assert all(command[-1] != "pull" for command in calls) + + +def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): + source_line = "JWT_SECRET=user-value\n" + Path.cwd().joinpath(".env").write_text(source_line) + stack = dashboard_stack.DashboardStack( + "postgres://user:password@db.example/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + + first_env, _ = dashboard_stack.prepare(stack) + second_env, _ = dashboard_stack.prepare(stack) + + env_contents = stack.env_path.read_text() + assert env_contents.startswith(source_line) + assert first_env["CANYONOS_JWT_SECRET"] != "user-value" + assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] + + +def test_process_environment_database_url_wins_over_project_dotenv(monkeypatch, project): + Path.cwd().joinpath(".env").write_text("DATABASE_URL=postgres://from-file/canyonos\n") + monkeypatch.setenv("DATABASE_URL", "postgres://from-process/canyonos") + project.write_text("database:\n url: ${DATABASE_URL}\n") + calls = [] + install_docker(monkeypatch, calls) + + stack = dashboard_stack.validate(str(project)) + + assert stack.database_url == "postgres://from-process/canyonos" + + +def test_unreadable_config_does_not_pull(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + + result = dashboard_stack.run_dashboard(str(project.with_name("missing.yaml"))) + + assert result.message == f"config file is not readable: {project.with_name('missing.yaml')}" + assert all(command[-1] != "pull" for command in calls) + + +def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, project, tmp_path): + calls = [] + install_docker(monkeypatch, calls) + blocked_state_dir = tmp_path / "blocked" + blocked_state_dir.write_text("not a directory") + monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: blocked_state_dir) + + state_result = dashboard_stack.run_dashboard(str(project)) + + assert state_result.message == "dashboard state directory is not writable" + assert all(command[-1] != "pull" for command in calls) + + calls.clear() + monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: tmp_path / "state") + monkeypatch.setattr(dashboard_stack, "_find_web_port", lambda start=8080, max_attempts=50: (_ for _ in ()).throw( + dashboard_stack.PhaseFailure("validate", "no free port found for the dashboard after 50 attempts starting at 8080") + )) + port_result = dashboard_stack.run_dashboard(str(project)) + + assert port_result.message == "no free port found for the dashboard after 50 attempts starting at 8080" + assert all(command[-1] != "pull" for command in calls) + + +def test_prepare_preserves_unrelated_env_lines_and_mode(project): + Path.cwd().joinpath(".env").write_text( + "OTHER=one\n# preserved\nJWT_SECRET=kept-secret\nLAST=two\n" + ) + stack = dashboard_stack.DashboardStack( + "postgres://user:password@localhost:5432/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + + managed_env, message = dashboard_stack.prepare(stack) + + assert message == "database host localhost is reachable from the stack as host.docker.internal" + env_lines = stack.env_path.read_text().splitlines() + assert env_lines[:2] == ["OTHER=one", "# preserved"] + assert env_lines[2] == "JWT_SECRET=kept-secret" + assert env_lines[3] == "LAST=two" + assert {line.split("=", 1)[0] for line in env_lines if "=" in line} == { + "OTHER", + "JWT_SECRET", + "LAST", + "CANYONOS_DATABASE_URL", + "CANYONOS_JWT_SECRET", + "CANYONOS_REDIS_HOST", + "CANYONOS_REDIS_PORT", + "CANYONOS_API_IMAGE", + "CANYONOS_WEB_IMAGE", + "CANYONOS_WEB_PORT", + } + assert "CANYONOS_DATABASE_URL=postgres://user:password@host.docker.internal:5432/canyonos" in env_lines + assert stack.env_path.stat().st_mode & 0o777 == 0o600 + assert stack.state_dir.stat().st_mode & 0o777 == 0o700 + assert sorted(path.name for path in stack.state_dir.iterdir()) == ["stack.json"] + + +def test_prepare_reuses_secret_and_rewrites_only_local_hosts(project): + stack = dashboard_stack.DashboardStack( + "postgres://user:password@localhost/canyonos", + dashboard_stack._state_dir(), + Path.cwd(), + ) + first_env, first_message = dashboard_stack.prepare(stack) + second_env, second_message = dashboard_stack.prepare(stack) + + assert first_message.startswith("database host localhost") + assert second_message.startswith("database host localhost") + assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] + assert ( + first_env["CANYONOS_DATABASE_URL"] + == "postgres://user:password@host.docker.internal/canyonos" + ) + + remote_stack = dashboard_stack.DashboardStack( + "postgres://db.example/canyonos", dashboard_stack._state_dir(), Path.cwd() + ) + remote_env, _ = dashboard_stack.prepare(remote_stack) + assert remote_env["CANYONOS_DATABASE_URL"] == "postgres://db.example/canyonos" + + +def test_redaction_removes_urls_secrets_and_credentials(): + database_url = "postgres://user:password@db.example/canyonos" + secret = "secret-value" + logs = f"{database_url}\n{secret}\nredis://other:credential@cache:6379/0" + + redacted = dashboard_stack.redact_logs(logs, secret) + + assert database_url not in redacted # credentials portion is stripped by the generic regex + assert secret not in redacted + assert "user:password@" not in redacted + assert "other:credential@" not in redacted + + +@pytest.mark.parametrize("had_containers", [False, True]) +def test_start_failure_saves_log_and_cleans_up_only_new_stack(monkeypatch, project, had_containers): + calls = [] + + def response(argv): + if argv[-2:] == ["ps", "-q"]: + return completed(argv, stdout="existing\n" if had_containers else "") + if argv[-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"]: + return completed(argv, returncode=1) + if argv[-4:] == ["logs", "--no-color", "--tail", "200"]: + return completed(argv, stdout="postgres://user:password@db.example/canyonos") + return completed(argv) + + install_docker(monkeypatch, calls, response) + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok is False + assert result.phase == "start" + assert result.log_path is not None + assert Path(result.log_path).stat().st_mode & 0o777 == 0o600 + assert "postgres://user:password@db.example/canyonos" not in Path(result.log_path).read_text() + assert any("logs" in command for command in calls) + assert any(command[-1] == "down" for command in calls) is (not had_containers) + + +def test_pull_failure_includes_redacted_stderr(monkeypatch, project): + calls = [] + secret = None + + def response(argv): + nonlocal secret + if argv[-1] == "pull": + secret = next( + line.split("=", 1)[1] + for line in Path.cwd().joinpath(".env").read_text().splitlines() + if line.startswith("CANYONOS_JWT_SECRET=") + ) + return completed( + argv, + returncode=1, + stderr=f"first line\npull unauthorized postgres://user:password@db.example/canyonos {secret}\n", + ) + return completed(argv) + + install_docker(monkeypatch, calls, response) + result = dashboard_stack.run_dashboard(str(project)) + + assert result.phase == "pull" + assert "pull unauthorized" in result.message + assert "postgres://user:password@db.example/canyonos" not in result.message + assert "user:password@" not in result.message + assert secret not in result.message + + +def test_verify_failure_saves_a_log(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + probes = [] + + def urlopen(*_args, **_kwargs): + probes.append(True) + raise dashboard_stack.urllib.error.URLError("down") + + monkeypatch.setattr( + dashboard_stack.urllib.request, + "urlopen", + urlopen, + ) + clock = iter([0, 0, 0, 31]) + monkeypatch.setattr(dashboard_stack.time, "monotonic", lambda: next(clock)) + monkeypatch.setattr(dashboard_stack.time, "sleep", lambda _: None) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok is False + assert result.phase == "verify" + assert result.log_path is not None + assert Path(result.log_path).is_file() + assert probes == [True] + assert any("logs" in command for command in calls) + assert any(command[-1] == "down" for command in calls) + + +def test_success_pulls_starts_and_verifies(monkeypatch, project): + calls = [] + install_docker(monkeypatch, calls) + + class Response: + status = 200 + + def close(self): + pass + + endpoints = [] + + def urlopen(endpoint, timeout): + endpoints.append((endpoint, timeout)) + return Response() + + monkeypatch.setattr(dashboard_stack.urllib.request, "urlopen", urlopen) + result = dashboard_stack.run_dashboard(str(project)) + + assert result == dashboard_stack.ServeResult( + True, "verify", "dashboard health checks passed", "http://127.0.0.1:8080" + ) + pull_index = next(index for index, command in enumerate(calls) if command[-1] == "pull") + up_index = next(index for index, command in enumerate(calls) if "up" in command) + assert pull_index < up_index + assert calls[pull_index][4:6] == ["--env-file", str(project.parent.parent / ".env")] + assert calls[up_index][-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"] + assert endpoints == [ + ("http://127.0.0.1:8080/healthz", 5), + ("http://127.0.0.1:8080/api/healthz", 5), + ] + + +def test_existing_dashboard_container_skips_port_check(monkeypatch, project): + calls = [] + + def response(argv): + if argv[:3] == ["docker", "container", "inspect"]: + return completed( + argv, + stdout=json.dumps( + [{"NetworkSettings": {"Ports": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "8080"}]}}}] + ), + ) + if argv[-2:] == ["ps", "-q"]: + return completed(argv, stdout="existing\n") + return completed(argv) + + install_docker(monkeypatch, calls, response) + monkeypatch.setattr( + dashboard_stack, + "_find_web_port", + lambda *a, **k: pytest.fail("the existing dashboard owns port 8080, should not search for a new one"), + ) + monkeypatch.setattr( + dashboard_stack.urllib.request, + "urlopen", + lambda *_args, **_kwargs: type("Response", (), {"status": 200, "close": lambda self: None})(), + ) + + result = dashboard_stack.run_dashboard(str(project)) + + assert result.ok + assert result.url == "http://127.0.0.1:8080" diff --git a/cli/utils/__init__.py b/cli/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/utils/tui.py b/cli/utils/tui.py new file mode 100644 index 0000000..3fcdad5 --- /dev/null +++ b/cli/utils/tui.py @@ -0,0 +1,113 @@ +""" +Minimal arrow-key select menu, no dependency beyond the standard library. +""" + +import os +import select as select_syscall +import sys +import termios +import tty + +UP_KEYS = ("\x1b[A", "\x1bOA", "k") +DOWN_KEYS = ("\x1b[B", "\x1bOB", "j") +CANCEL_KEYS = ("\x03", "\x1b") +DELETE_KEYS = ("d", "D") +QUIT_KEYS = ("q", "Q") + +# Sentinel returned (paired with the hovered value) when the delete key is +# pressed and `deletable=True`. Callers check `result[0] is DELETE_ACTION`. +DELETE_ACTION = object() + +# Sentinel returned when the quit key is pressed and `quittable=True`. Distinct +# from None (which callers use for a single-level cancel/back) so a caller can +# unwind an entire nested session. Callers check `result is QUIT_ACTION`. +QUIT_ACTION = object() + + +def _read_key(fd): + # Reads straight off the fd (not sys.stdin) so this stays in sync with + # the select() call below -- stdin's own buffering can silently swallow + # an arrow key's trailing bytes before select() ever sees them queued. + ch = os.read(fd, 1).decode() + if ch == "\x1b": + # An arrow key arrives as a multi-byte escape sequence; a bare Esc + # press has nothing queued right behind it. + if select_syscall.select([fd], [], [], 0.01)[0]: + ch += os.read(fd, 1).decode() + if ch[-1] in ("[", "O"): + ch += os.read(fd, 1).decode() + return ch + + +def select_menu(options, title, deletable=False, quittable=False): + """Arrow-key single-select over `options` (a list of (value, label) pairs). + + Returns the chosen value, or None if there's nothing to choose from or + the user cancelled (Esc/Ctrl-C). + + If `deletable` is True, pressing the delete key ('d') over an item returns + the tuple `(DELETE_ACTION, hovered_value)` so the caller can act on the + currently-hovered item instead of selecting it. + + If `quittable` is True, pressing the quit key ('q') returns the sentinel + `QUIT_ACTION` -- distinct from None -- so the caller can unwind an entire + nested session rather than just this one menu. + """ + if len(options) == 1: + return options[0][0] + if not options or not sys.stdin.isatty(): + return None + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + out = sys.stderr + idx = 0 + n = len(options) + + def frame(): + lines = [f"\x1b[1m{title}\x1b[0m", ""] + for i, (_, label) in enumerate(options): + lines.append(f"\x1b[36m❯ {label}\x1b[0m" if i == idx else f" {label}") + hint = "↑/↓ move · 1-9 jump · enter select" + if deletable: + hint += " · d delete" + if quittable: + hint += " · q quit" + hint += " · esc cancel" + lines.append(f"\x1b[2m{hint}\x1b[0m") + return "\r\n".join(lines) + + prev_frame = None + try: + tty.setraw(fd) + out.write("\x1b[?25l") + while True: + text = frame() + if prev_frame is not None: + # How far back up to move is read off the frame we actually + # wrote last time, not recomputed separately -- it can't drift + # out of sync with what's really on screen. + out.write(f"\r\x1b[{prev_frame.count(chr(10))}A\x1b[J") + out.write(text) + out.flush() + prev_frame = text + + key = _read_key(fd) + if key in ("\r", "\n"): + return options[idx][0] + if key in CANCEL_KEYS: + return None + if key in UP_KEYS: + idx = (idx - 1) % n + elif key in DOWN_KEYS: + idx = (idx + 1) % n + elif deletable and key in DELETE_KEYS: + return (DELETE_ACTION, options[idx][0]) + elif quittable and key in QUIT_KEYS: + return QUIT_ACTION + elif key.isdigit() and key != "0" and int(key) <= n: + return options[int(key) - 1][0] + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + out.write("\x1b[?25h\r\n") + out.flush() diff --git a/examples/epigenomics/README.md b/examples/epigenomics/README.md new file mode 100644 index 0000000..19fb748 --- /dev/null +++ b/examples/epigenomics/README.md @@ -0,0 +1,70 @@ +# Epigenomics Example + +A synthetic, LLM-free workflow modeled on the +[WfCommons Epigenomics recipe](https://docs.wfcommons.org/en/latest/generating_workflows.html): +a split → fan-out (filter → align → sort) → fan-in (dedup) → index pipeline. +Every stage does deterministic SHA-256 work sized off chunk byte counts, so +results are reproducible and the fan-out width scales with `num_chunks` -- +useful for exercising scheduling/replica behavior locally without any real +model calls. + +## Pipeline + +``` +SplitAgent --> FilterAgent --> MapAgent --> SortAgent --\ + (1 call) (fan-out) (fan-out) (fan-out) --> DedupAgent --> IndexAgent + (fan-in barrier) (1 call) +``` + +- **SplitAgent** — splits `input_size` bytes into `num_chunks` equal chunks. +- **FilterAgent** — per-chunk contaminant filter (light cost). +- **MapAgent** — per-chunk alignment, the heaviest stage. +- **SortAgent** — per-chunk sort (moderate cost). +- **DedupAgent** — merges every sorted chunk into one digest (the barrier). +- **IndexAgent** — builds the final index from the merged digest. + +## Quick Start + +```bash +# Build stubs and Docker images +ventis build + +# Launch all agents +ventis deploy + +# Test with curl +curl -X POST http://:8080/main \ + -H 'Content-Type: application/json' \ + -d '{"input_size": 65536, "num_chunks": 4}' + +# Check result +curl http://:8080/status/ +``` + +## Project Structure + +``` +├── agents/ # Agent implementations and YAML definitions +│ ├── split_agent.py/.yaml +│ ├── filter_agent.py/.yaml +│ ├── map_agent.py/.yaml +│ ├── sort_agent.py/.yaml +│ ├── dedup_agent.py/.yaml +│ └── index_agent.py/.yaml +├── workflow/ # Workflow script (deployed as a REST API) +│ └── epigenomics_workflow.py +└── config/ + ├── global_controller.yaml # Deployment configuration (provider: local) + └── policy.yaml # Access control rules +``` + +## Policy Rules + +Edit `config/policy.yaml` to control which callers can access which agents. +Pass `_context` in your curl request to set the caller identity: + +```bash +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' \ + -d '{"input_size": 65536, "num_chunks": 4, "_context": {"origin": "admin"}}' +``` diff --git a/examples/epigenomics/agents/dedup_agent.py b/examples/epigenomics/agents/dedup_agent.py new file mode 100644 index 0000000..d883854 --- /dev/null +++ b/examples/epigenomics/agents/dedup_agent.py @@ -0,0 +1,44 @@ +# Dedup Agent +# +# Fan-in barrier (mirrors Epigenomics' mark-duplicates/merge stage): needs +# every sorted chunk before it can run. Combines all chunk digests into one +# merged digest, with cost scaling off the total merged data volume. +# +# Resource profile: moderate CPU, single call per request (the barrier). + +import hashlib + + +class DedupAgent(object): + def __init__(self): + self.tools = [self.merge_dedup] + + def merge_dedup(self, chunks: list) -> dict: + """Merge and deduplicate every sorted chunk into one combined digest.""" + total_size = sum(c["size"] for c in chunks) + seed = "".join(c["digest"] for c in sorted(chunks, key=lambda c: c["chunk_id"])) + merged_digest = self._cpu_work(seed, total_size) + return { + "merged_digest": merged_digest, + "total_size": total_size, + "n_chunks": len(chunks), + } + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = DedupAgent() + print( + agent.merge_dedup( + [ + {"chunk_id": "chunk-0", "size": 16384, "digest": "aa"}, + {"chunk_id": "chunk-1", "size": 16384, "digest": "bb"}, + ] + ) + ) diff --git a/examples/epigenomics/agents/dedup_agent.yaml b/examples/epigenomics/agents/dedup_agent.yaml new file mode 100644 index 0000000..1c577cd --- /dev/null +++ b/examples/epigenomics/agents/dedup_agent.yaml @@ -0,0 +1,10 @@ +agent: + name: DedupAgent + functions: + - name: merge_dedup + description: Merge and deduplicate every sorted chunk into one combined digest. + arguments: + - name: chunks + type: list + returns: + type: dict diff --git a/examples/epigenomics/agents/filter_agent.py b/examples/epigenomics/agents/filter_agent.py new file mode 100644 index 0000000..24dfc48 --- /dev/null +++ b/examples/epigenomics/agents/filter_agent.py @@ -0,0 +1,32 @@ +# Filter Agent +# +# First per-chunk stage in the fan-out (mirrors Epigenomics' filter_contams +# stage): scrubs one chunk and hands back a content digest the later stages +# build on. The "work" is a deterministic SHA-256 chain sized off the +# chunk's declared byte size, standing in for the real stage's per-byte cost. +# +# Resource profile: light CPU, high fan-out (one call per chunk). + +import hashlib + + +class FilterAgent(object): + def __init__(self): + self.tools = [self.filter_contams] + + def filter_contams(self, chunk_id: str, size: int) -> dict: + """Filter contaminants out of one chunk, returning its content digest.""" + digest = self._cpu_work(chunk_id, size) + return {"chunk_id": chunk_id, "size": size, "digest": digest} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = FilterAgent() + print(agent.filter_contams("chunk-0", 16384)) diff --git a/examples/epigenomics/agents/filter_agent.yaml b/examples/epigenomics/agents/filter_agent.yaml new file mode 100644 index 0000000..9f10d29 --- /dev/null +++ b/examples/epigenomics/agents/filter_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: FilterAgent + functions: + - name: filter_contams + description: Filter contaminants out of one chunk, returning its content digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + returns: + type: dict diff --git a/examples/epigenomics/agents/index_agent.py b/examples/epigenomics/agents/index_agent.py new file mode 100644 index 0000000..6faa984 --- /dev/null +++ b/examples/epigenomics/agents/index_agent.py @@ -0,0 +1,30 @@ +# Index Agent +# +# Final stage (mirrors Epigenomics' index-build stage): produces the +# workflow's terminal artifact from the merged, deduplicated digest. +# +# Resource profile: light CPU, single call per request. + +import hashlib + + +class IndexAgent(object): + def __init__(self): + self.tools = [self.build_index] + + def build_index(self, merged_digest: str, total_size: int) -> dict: + """Build the final index from the merged digest.""" + index_digest = self._cpu_work(merged_digest, max(1, total_size // 4)) + return {"index_digest": index_digest, "total_size": total_size} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = IndexAgent() + print(agent.build_index("deadbeef", 65536)) diff --git a/examples/epigenomics/agents/index_agent.yaml b/examples/epigenomics/agents/index_agent.yaml new file mode 100644 index 0000000..3424c44 --- /dev/null +++ b/examples/epigenomics/agents/index_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: IndexAgent + functions: + - name: build_index + description: Build the final index from the merged digest. + arguments: + - name: merged_digest + type: str + - name: total_size + type: int + returns: + type: dict diff --git a/examples/epigenomics/agents/map_agent.py b/examples/epigenomics/agents/map_agent.py new file mode 100644 index 0000000..b9db3ef --- /dev/null +++ b/examples/epigenomics/agents/map_agent.py @@ -0,0 +1,33 @@ +# Map Agent +# +# Sequence-alignment stand-in (Epigenomics' map stage) -- by far the most +# CPU-expensive stage in the real workflow, so its per-byte cost multiplier +# here is set well above the other stages to match that shape. +# +# Resource profile: heavy CPU, high fan-out (one call per chunk). + +import hashlib + +COST_MULTIPLIER = 8 + + +class MapAgent(object): + def __init__(self): + self.tools = [self.align] + + def align(self, chunk_id: str, size: int, digest: str) -> dict: + """Align one filtered chunk, returning its post-alignment digest.""" + aligned = self._cpu_work(digest, size * COST_MULTIPLIER) + return {"chunk_id": chunk_id, "size": size, "digest": aligned} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = MapAgent() + print(agent.align("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/map_agent.yaml b/examples/epigenomics/agents/map_agent.yaml new file mode 100644 index 0000000..66c1210 --- /dev/null +++ b/examples/epigenomics/agents/map_agent.yaml @@ -0,0 +1,14 @@ +agent: + name: MapAgent + functions: + - name: align + description: Align one filtered chunk, returning its post-alignment digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + - name: digest + type: str + returns: + type: dict diff --git a/examples/epigenomics/agents/sort_agent.py b/examples/epigenomics/agents/sort_agent.py new file mode 100644 index 0000000..2aa2df7 --- /dev/null +++ b/examples/epigenomics/agents/sort_agent.py @@ -0,0 +1,32 @@ +# Sort Agent +# +# Third per-chunk stage in the fan-out (mirrors Epigenomics' sort stage): +# orders one aligned chunk, returning an updated digest for the fan-in below. +# +# Resource profile: moderate CPU, high fan-out (one call per chunk). + +import hashlib + +COST_MULTIPLIER = 2 + + +class SortAgent(object): + def __init__(self): + self.tools = [self.sort] + + def sort(self, chunk_id: str, size: int, digest: str) -> dict: + """Sort one aligned chunk, returning its post-sort digest.""" + sorted_digest = self._cpu_work(digest, size * COST_MULTIPLIER) + return {"chunk_id": chunk_id, "size": size, "digest": sorted_digest} + + def _cpu_work(self, seed: str, iterations: int) -> str: + """Deterministic CPU-bound stand-in for the stage's real processing cost.""" + digest = seed.encode() + for _ in range(max(1, iterations)): + digest = hashlib.sha256(digest).digest() + return digest.hex() + + +if __name__ == "__main__": + agent = SortAgent() + print(agent.sort("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/sort_agent.yaml b/examples/epigenomics/agents/sort_agent.yaml new file mode 100644 index 0000000..464dc85 --- /dev/null +++ b/examples/epigenomics/agents/sort_agent.yaml @@ -0,0 +1,14 @@ +agent: + name: SortAgent + functions: + - name: sort + description: Sort one aligned chunk, returning its post-sort digest. + arguments: + - name: chunk_id + type: str + - name: size + type: int + - name: digest + type: str + returns: + type: dict diff --git a/examples/epigenomics/agents/split_agent.py b/examples/epigenomics/agents/split_agent.py new file mode 100644 index 0000000..1b1a6be --- /dev/null +++ b/examples/epigenomics/agents/split_agent.py @@ -0,0 +1,27 @@ +# Split Agent +# +# Entry stage of the pipeline (mirrors WfCommons' Epigenomics fastq-split +# stage): splits one logical input into num_chunks equal-sized chunks for the +# downstream fan-out. There's no real sequence file here -- each chunk's +# "size" just stands in for its data volume, which is what every downstream +# stage prices its synthetic CPU work off of. +# +# Resource profile: cheap CPU, single call per request. + + +class SplitAgent(object): + def __init__(self): + self.tools = [self.split] + + def split(self, input_size: int, num_chunks: int) -> dict: + """Split input_size bytes of data into num_chunks equal chunks.""" + chunk_size = max(1, input_size // num_chunks) + chunks = [ + {"chunk_id": f"chunk-{i}", "size": chunk_size} for i in range(num_chunks) + ] + return {"chunks": chunks} + + +if __name__ == "__main__": + agent = SplitAgent() + print(agent.split(65536, 4)) diff --git a/examples/epigenomics/agents/split_agent.yaml b/examples/epigenomics/agents/split_agent.yaml new file mode 100644 index 0000000..cc64e8e --- /dev/null +++ b/examples/epigenomics/agents/split_agent.yaml @@ -0,0 +1,12 @@ +agent: + name: SplitAgent + functions: + - name: split + description: Split input_size bytes of data into num_chunks equal chunks. + arguments: + - name: input_size + type: int + - name: num_chunks + type: int + returns: + type: dict diff --git a/examples/epigenomics/config/global_controller.yaml b/examples/epigenomics/config/global_controller.yaml new file mode 100644 index 0000000..6e8b4b0 --- /dev/null +++ b/examples/epigenomics/config/global_controller.yaml @@ -0,0 +1,76 @@ +# Global Controller Configuration — synthetic Epigenomics DAG, local provider +# Lists all agents and the workflow that Ventis manages. +# +# FilterAgent/MapAgent/SortAgent get 2 replicas each since the workflow fans +# out one call per chunk to them -- exercises multi-replica scheduling on a +# purely local, LLM-free run. + +agents: + - name: SplitAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/split_agent.py + provider: local + + - name: FilterAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/filter_agent.py + provider: local + + - name: MapAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/map_agent.py + provider: local + + - name: SortAgent + replicas: 2 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/sort_agent.py + provider: local + + - name: DedupAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/dedup_agent.py + provider: local + + - name: IndexAgent + replicas: 1 + redis_port: 6379 + resources: + cpu: 1 + memory: 256 + entrypoint: agents/index_agent.py + provider: local + + - name: Workflow + replicas: 1 + type: workflow + redis_port: 6379 + api_port: 8080 + workflow_file: workflow/epigenomics_workflow.py + provider: local + +poll_interval: 5 + +redis: + host: localhost + port: 6379 + db: 0 diff --git a/examples/epigenomics/config/policy.yaml b/examples/epigenomics/config/policy.yaml new file mode 100644 index 0000000..2c91415 --- /dev/null +++ b/examples/epigenomics/config/policy.yaml @@ -0,0 +1,20 @@ +# Policy-Based Routing Rules — synthetic Epigenomics DAG +# Each rule defines a match condition (key-value pairs to check against +# request context) and an access list of allowed services. +# Rules are evaluated most-specific-first (most matching keys wins). +# An empty match ({}) acts as a default fallback. + +rules: + - match: + origin: admin + access: all + + - match: {} + access: + - Workflow + - SplitAgent + - FilterAgent + - MapAgent + - SortAgent + - DedupAgent + - IndexAgent diff --git a/examples/epigenomics/workflow/epigenomics_workflow.py b/examples/epigenomics/workflow/epigenomics_workflow.py new file mode 100644 index 0000000..00bcc2b --- /dev/null +++ b/examples/epigenomics/workflow/epigenomics_workflow.py @@ -0,0 +1,97 @@ +# Epigenomics Workflow +# +# WfCommons-style synthetic Epigenomics DAG for local, LLM-free testing: +# 0. SplitAgent - split input_size bytes into num_chunks chunks (single call) +# 1. FilterAgent - per-chunk contaminant filter (fan-out) +# 2. MapAgent - per-chunk alignment, the heaviest stage (fan-out) +# 3. SortAgent - per-chunk sort (fan-out) +# 4. DedupAgent - merge + dedup every sorted chunk (fan-in barrier) +# 5. IndexAgent - build the final index from the merged digest (single call) +# +# Every stage does deterministic SHA-256 work sized off chunk byte counts -- +# no LLM calls, no external services -- so results are reproducible and the +# fan-out width scales with num_chunks. +# +# After running `ventis build` and `ventis deploy`: +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' \ +# -d '{"input_size": 65536, "num_chunks": 4}' +# curl http://localhost:8080/status/ + +import sys +import os + +# These path inserts are needed when running inside a Docker container +# where all files are copied flat into /app/. +sys.path.insert(0, os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) + +import json + +from deploy import deploy +from agents.split_agent import SplitAgent +from agents.filter_agent import FilterAgent +from agents.map_agent import MapAgent +from agents.sort_agent import SortAgent +from agents.dedup_agent import DedupAgent +from agents.index_agent import IndexAgent + + +def main(input_size: int = 65536, num_chunks: int = 4): + split_agent = SplitAgent() + filter_agent = FilterAgent() + map_agent = MapAgent() + sort_agent = SortAgent() + dedup_agent = DedupAgent() + index_agent = IndexAgent() + + # Stage 0: single call, produces the chunk list the fan-out below runs over. + split = json.loads( + split_agent.split(input_size=input_size, num_chunks=num_chunks).value() + ) + chunks = split["chunks"] + + # Stage 1: fan out one filter call per chunk -- every call returns a Future + # immediately, so all chunks are dispatched before we block on any of them. + filter_futures = { + c["chunk_id"]: filter_agent.filter_contams(chunk_id=c["chunk_id"], size=c["size"]) + for c in chunks + } + filtered = {cid: json.loads(f.value()) for cid, f in filter_futures.items()} + + # Stage 2: fan out alignment -- the heaviest stage -- one call per chunk. + map_futures = { + cid: map_agent.align(chunk_id=cid, size=r["size"], digest=r["digest"]) + for cid, r in filtered.items() + } + mapped = {cid: json.loads(f.value()) for cid, f in map_futures.items()} + + # Stage 3: fan out sort, one call per chunk. + sort_futures = { + cid: sort_agent.sort(chunk_id=cid, size=r["size"], digest=r["digest"]) + for cid, r in mapped.items() + } + sorted_chunks = {cid: json.loads(f.value()) for cid, f in sort_futures.items()} + + # Stage 4: fan-in barrier -- dedup needs every sorted chunk before it can run. + merged = json.loads( + dedup_agent.merge_dedup(chunks=list(sorted_chunks.values())).value() + ) + + # Stage 5: build the final index from the merged digest. + index = json.loads( + index_agent.build_index( + merged_digest=merged["merged_digest"], total_size=merged["total_size"] + ).value() + ) + + return { + "input_size": input_size, + "num_chunks": num_chunks, + "merged_digest": merged["merged_digest"], + "n_chunks": merged["n_chunks"], + "index_digest": index["index_digest"], + } + + +deploy(main, port=8080) diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 5f6c0cc..0b9c194 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -10,7 +10,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/example_agent.py - provider: local + provider: EC2 - name: VllmAgent replicas: 1 @@ -19,7 +19,7 @@ agents: cpu: 2 memory: 2048 entrypoint: agents/vllm_agent.py - provider: local + provider: EC2 instance_type: t3.micro - name: Workflow @@ -28,7 +28,7 @@ agents: redis_port: 6379 api_port: 8080 # Only needed for workflows, defaults to 8080 if not filled workflow_file: workflow/example_workflow.py - provider: local + provider: EC2 instance_type: t3.micro poll_interval: 5 diff --git a/examples/joke_writer/.car/app/.env.example b/examples/joke_writer/.car/app/.env.example new file mode 100644 index 0000000..b846149 --- /dev/null +++ b/examples/joke_writer/.car/app/.env.example @@ -0,0 +1,20 @@ +# Copy this to `.env` and fill in the token. `config/global_controller.yaml` +# points `env_file:` at that copy, and it reaches every container as +# `docker run --env-file`. +# +# Keep the real token out of THIS file. `.env.example` is the one exception to +# the build context's exclusion of `.env*`, so whatever is written here is baked +# into the image; `.env` itself never enters the build and never leaves the host. + +# A Bedrock API key -- the long-term kind generated in the console, or a +# short-term one. botocore matches this exact name against bedrock-runtime's +# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by +# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions +# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and +# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. +AWS_BEARER_TOKEN_BEDROCK= + +# Neither is a secret, and both have defaults in joke_writer.py -- they are here +# to name what the source reads. +BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 +AWS_REGION=us-east-1 diff --git a/examples/joke_writer/.car/app/LICENSE b/examples/joke_writer/.car/app/LICENSE new file mode 100644 index 0000000..5600729 --- /dev/null +++ b/examples/joke_writer/.car/app/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 LangChain, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/joke_writer/.car/app/README.md b/examples/joke_writer/.car/app/README.md new file mode 100644 index 0000000..930a410 --- /dev/null +++ b/examples/joke_writer/.car/app/README.md @@ -0,0 +1,177 @@ +# Joke Writer + +A LangGraph map-reduce, ported to Ventis. Derived from +[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) +at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). + +Unlike the other targets in `examples/`, **the source here is not unmodified**. +`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port +an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked +at the credential wall until the model call was rewritten onto Bedrock. That wall +is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed +anyway, and [What the port cost](#what-the-port-cost) is honest about what that +means. + +## Overview + +Given a topic, the graph splits it into sub-topics, writes one joke per +sub-topic in parallel, then picks the best of them. + +1. `generate_topics` — one LLM call, turns the topic into three sub-topics, + validated into `Subjects`. +2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a + `Send` per subject, so this node runs N times per request with no shared + state between the runs. `jokes` is an `Annotated[list, operator.add]`, which + is how the N results merge back into one state. +3. `best_joke` — one LLM call over every joke, returns the winner by index. + +``` + START + | + generate_topics 1 call + | + continue_to_jokes Send x N + / | \ + joke joke joke N calls, no shared state + \ | / + best_joke 1 call + | + END +``` + +### Why this one + +It is the smallest project in reach whose control flow does something a single +process cannot: `Send` fans out to N independent calls per request. Everything +else about it is deliberately boring — four packages, no tools, no external +service, one API key. + +## The port + +| File | What it holds | +| --- | --- | +| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | +| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | +| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | +| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | +| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | +| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | + +Two decisions worth naming: + +**One agent, not three.** `generate_topics` and `best_joke` run once per request +and have no resource profile of their own. Splitting them out would buy two more +images and two more Redis round trips. What is hoisted is the fan-out, and that +is a workflow concern. + +**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, +operator.add]` reducer are control flow owned by the LangGraph runtime, and +Ventis has no runtime to execute them. The workflow dispatches N +`generate_joke` calls across the three replicas and concatenates the results +itself. Every call is dispatched before any is resolved — `.value()` blocks, so +fusing the two lines into one comprehension would silently serialize the fan-out +and remove the reason to be on Ventis at all. + +## What the port cost + +This is no longer upstream's model stack. `ChatOpenAI` and +`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw +converse API, so each node asks for JSON in its prompt and validates the reply +through the same pydantic schema upstream used. `_extract_json` exists only +because `with_structured_output` used to do that work. + +That rewrite is not something the `porting-to-ventis` skill should do on a +user's project — it is the credential wall, and the skill's instruction is to +report it. It was done here deliberately, so that this example is one that +actually deploys. + +**It would not be necessary today.** The rewrite bought one thing: boto3 builds +no client at import, so the agent could be *loaded* with no secret in the +container, back when `_launch_locally` passed five `-e` flags and all five were +`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches +a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module +scope would import fine. What the rewrite still buys is narrower: a module-scope +client turns a missing key into `"No agent loaded"`, while a per-call one turns +it into a real error on `/status`. Worth knowing, not worth a rewrite. + +The example stays on Bedrock because it is the model call that has been end-to-end +verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call +token telemetry onto the future. + +## Running it + +Copy `.env.example` to `.env` and put a Bedrock API key in it: + +```shell +cp .env.example .env +$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... +``` + +`config/global_controller.yaml` points `env_file:` at that file, and every +container gets it as `docker run --env-file`. Nothing in this project reads the +variable: botocore matches the name against `bedrock-runtime`'s signingName and +switches the client from SigV4 to bearer auth on its own, so +`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. +An IAM access key instead of the bearer token works the same way. + +`.env` is gitignored and excluded from the build context — the key is in the +container's environment and not in the image. Deploy checks the path before it +launches anything, so a missing `.env` is one error line rather than three +replicas that come up and fail every request. + +```shell +ventis build +ventis deploy +``` + +```shell +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' +curl http://localhost:8080/status/ +``` + +```json +{"request_id": "cb6cb62d...", "status": "done", "result": { + "topic": "animals", + "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], + "jokes": ["...", "...", "..."], + "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" +}} +``` + +`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in +`joke_writer.py`; neither is a secret. The region has to match the one the key +was issued for. + +### Running the source outside Ventis + +`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat +`bedrock` copy an agent image gets, so the compiled graph still runs on its own +from a checkout of this repo: + +```shell +pip install -e ../.. # the ventis package +pip install langgraph pydantic typing_extensions boto3 +``` + +```python +from joke_writer import graph + +graph.invoke({"topic": "animals"}) +``` + +## Provenance + +Taken from `module-4/studio/`, which holds four unrelated graphs sharing one +directory. Only `map_reduce.py` and its license are here. + +| Left behind | Why | +| --- | --- | +| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | +| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | +| The module-4 notebooks | Teaching material for the same code. | +| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | + +Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` +or `requirements.txt`, exactly as upstream has none for module-4. That is why +`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/.car/app/joke_workflow.py b/examples/joke_writer/.car/app/joke_workflow.py new file mode 100644 index 0000000..76e4027 --- /dev/null +++ b/examples/joke_writer/.car/app/joke_workflow.py @@ -0,0 +1,39 @@ +r"""CanyonOS Core workflow for the map-reduce joke writer. + +This file is where the graph went. `generate_topics -> continue_to_jokes -> +generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the +three statements below, and the `Send` fan-out is N calls dispatched across +JokeAgent's replicas. + + curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query": "animals"}' + curl http://localhost:8080/status/ +""" + +import json + +from deploy import deploy +from joke_writer import JokeAgent + + +def main(query): + """Route: POST /main {"query": ""}""" + agent = JokeAgent() + + subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] + + futures = [agent.generate_joke(subject=s) for s in subjects] + written = [json.loads(f.value()) for f in futures] + written_jokes = [joke for result in written for joke in result["jokes"]] + + best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) + + return { + "topic": query, + "subjects": subjects, + "jokes": written_jokes, + "best_selected_joke": best["best_selected_joke"], + } + + +deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/joke_writer.py b/examples/joke_writer/.car/app/joke_writer.py new file mode 100644 index 0000000..9f96eb5 --- /dev/null +++ b/examples/joke_writer/.car/app/joke_writer.py @@ -0,0 +1,164 @@ +"""Map-reduce joke writer. + +Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` +(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are +upstream's. The model call is not: upstream builds a `ChatOpenAI` at module +scope, and when this was ported nothing could carry an OPENAI_API_KEY into an +agent container. Bedrock reaches the model through boto3, which builds no client +at import, so the same code loaded with no secret injected. + +`env_file` has since removed that constraint -- the key now travels to the +container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The +rewrite stayed regardless; README.md says what that costs. + +`with_structured_output` went with it. `call_bedrock` is the raw converse API, so +each node asks for JSON in the prompt and validates the reply through the same +pydantic schema upstream used. +""" + +import json +import operator +import os +import re +from typing import Annotated + +from typing_extensions import TypedDict + +from pydantic import BaseModel, ValidationError + +from langgraph.constants import Send +from langgraph.graph import END, StateGraph, START + +# Ventis copies bedrock.py flat into every agent image; the package path is for +# running this module outside a container. +try: + from ventis.llm.bedrock import call_bedrock +except ImportError: + from bedrock import call_bedrock + +# Prompts we will use. Upstream's, plus the JSON instruction that +# `with_structured_output` used to add on our behalf. +subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. +Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" +joke_prompt = """Generate a joke about {subject}. +Respond with JSON only, no prose: {{"joke": "..."}}""" +best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} +Respond with JSON only, no prose: {{"id": 0}}""" + +# LLM. Both are read once at import; the container gets them from its +# environment, and neither is a secret. +MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") +REGION = os.environ.get("AWS_REGION", "us-east-1") + + +def _extract_json(text): + """Pull the first JSON object out of a model reply. + + Even told to answer with JSON only, a model wraps it in a ```json fence or + prefaces it with a sentence. Upstream never needed this because + `with_structured_output` handled it; the converse API does not. + """ + text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) + try: + return json.loads(text) + except json.JSONDecodeError: + pass + # Fall back to the outermost braced span. + match = re.search(r"\{.*\}", text, flags=re.DOTALL) + if not match: + raise ValueError(f"joke_writer: no JSON in model output: {text!r}") + return json.loads(match.group(0)) + + +def _ask(prompt, schema, max_tokens): + """One converse() call, validated into `schema`. + + Raising on a bad reply is deliberate. A node that returned a default would + put a plausible-looking wrong answer into the state, and the reduce step + downstream indexes into the jokes list by an id the model chose -- a silent + default there picks the wrong joke instead of failing. + """ + response = call_bedrock( + model_id=MODEL_ID, + messages=[{"role": "user", "content": [{"text": prompt}]}], + inference_config={"maxTokens": max_tokens, "temperature": 0.0}, + region=REGION, + ) + text = response["output"]["message"]["content"][0]["text"] + if not text: + raise ValueError("joke_writer: LLM returned no output.") + try: + return schema(**_extract_json(text)) + except (ValidationError, TypeError) as exc: + raise ValueError( + f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" + ) from exc + + +# Define the state +class Subjects(BaseModel): + subjects: list[str] + +class BestJoke(BaseModel): + id: int + +class OverallState(TypedDict): + topic: str + subjects: list + jokes: Annotated[list, operator.add] + best_selected_joke: str + +def generate_topics(state: OverallState): + prompt = subjects_prompt.format(topic=state["topic"]) + response = _ask(prompt, Subjects, max_tokens=300) + return {"subjects": response.subjects} + +class JokeState(TypedDict): + subject: str + +class Joke(BaseModel): + joke: str + +def generate_joke(state: JokeState): + prompt = joke_prompt.format(subject=state["subject"]) + response = _ask(prompt, Joke, max_tokens=300) + return {"jokes": [response.joke]} + +def best_joke(state: OverallState): + jokes = "\n\n".join(state["jokes"]) + prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) + response = _ask(prompt, BestJoke, max_tokens=100) + if not 0 <= response.id < len(state["jokes"]): + raise ValueError( + f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." + ) + return {"best_selected_joke": state["jokes"][response.id]} + +def continue_to_jokes(state: OverallState): + return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] + +# Construct the graph: here we put everything together to construct our graph +graph_builder = StateGraph(OverallState) +graph_builder.add_node("generate_topics", generate_topics) +graph_builder.add_node("generate_joke", generate_joke) +graph_builder.add_node("best_joke", best_joke) +graph_builder.add_edge(START, "generate_topics") +graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) +graph_builder.add_edge("generate_joke", "best_joke") +graph_builder.add_edge("best_joke", END) + +# Compile the graph +graph = graph_builder.compile() + + +class JokeAgent(object): + """The graph's nodes, exposed under the class name `agent.name` declares.""" + + def generate_topics(self, topic: str) -> dict: + return generate_topics({"topic": topic}) + + def generate_joke(self, subject: str) -> dict: + return generate_joke({"subject": subject}) + + def best_joke(self, topic: str, jokes: list) -> dict: + return best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/.car/config/global_controller.yaml b/examples/joke_writer/.car/config/global_controller.yaml new file mode 100644 index 0000000..8ed54ab --- /dev/null +++ b/examples/joke_writer/.car/config/global_controller.yaml @@ -0,0 +1,52 @@ +# Deployment manifest for the map-reduce joke writer. +# +# `entrypoint` is the copied source itself: the adapter is appended to the +# bottom of joke_writer.py, so the module the agent needs is the one the class +# already lives in. + +agents: + - name: JokeAgent + # The fan-out. `generate_joke` is stateless, so the controller picks a + # replica at random per call and the workflow's N dispatched calls spread + # across these three. + entrypoint: joke_writer.py + provider: local + replicas: 3 + redis_port: 6379 + resources: + cpu: 1 + memory: 1024 + requirements: + - langgraph + - pydantic + - typing_extensions + + - name: Workflow + type: workflow + workflow_file: joke_workflow.py + api_port: 8080 + provider: local + replicas: 1 + redis_port: 6379 + +poll_interval: 5 + +redis: + host: localhost + port: 6379 + db: 0 + +otel: + # The dashboard api's own OTLP ingest. Must be the full url including the + # path: the http exporter uses an explicitly-passed endpoint verbatim and + # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {} + +# Relative to the application root (the directory `ventis` runs from), not +# `.car`. .env is gitignored and excluded from the build context; +# .env.example names what belongs in it. +env_file: .env diff --git a/examples/joke_writer/.car/config/joke_agent.yaml b/examples/joke_writer/.car/config/joke_agent.yaml new file mode 100644 index 0000000..a5bc13f --- /dev/null +++ b/examples/joke_writer/.car/config/joke_agent.yaml @@ -0,0 +1,35 @@ +# The graph's three nodes, exposed as three methods on one agent. +# +# One agent, not three. `generate_topics` and `best_joke` run once per request +# and have no resource profile of their own; splitting them out would add two +# images, two dependency trees and a Redis round trip to buy nothing. What is +# hoisted is the `Send` fan-out, and that is a workflow concern. + +agent: + name: JokeAgent + functions: + - name: generate_topics + description: Split a topic into three related sub-topics. + arguments: + - name: topic + type: str + returns: + type: dict + + - name: generate_joke + description: Write one joke about one subject. + arguments: + - name: subject + type: str + returns: + type: dict + + - name: best_joke + description: Pick the best joke out of the ones written for a topic. + arguments: + - name: topic + type: str + - name: jokes + type: list + returns: + type: dict diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md index 3ba7930..930a410 100644 --- a/examples/joke_writer/README.md +++ b/examples/joke_writer/README.md @@ -80,7 +80,7 @@ converse API, so each node asks for JSON in its prompt and validates the reply through the same pydantic schema upstream used. `_extract_json` exists only because `with_structured_output` used to do that work. -That rewrite is not something the `porting-to-canyonos-core` skill should do on a +That rewrite is not something the `porting-to-ventis` skill should do on a user's project — it is the credential wall, and the skill's instruction is to report it. It was done here deliberately, so that this example is one that actually deploys. @@ -107,12 +107,6 @@ cp .env.example .env $EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... ``` -> **`env_file:` needs PR #53** (`jiajunh/can-232-...`), still open against main. -> Until it merges nothing in `ventis/` reads the key, so the steps below leave -> the container without a credential and every request answers a Bedrock -> credential error. `python ../../.claude/skills/porting-to-canyonos-core/validate.py .` -> reports this as V030 and stops reporting it the day the PR lands. - `config/global_controller.yaml` points `env_file:` at that file, and every container gets it as `docker run --env-file`. Nothing in this project reads the variable: botocore matches the name against `bedrock-runtime`'s signingName and diff --git a/examples/joke_writer/agents/joke_agent.py b/examples/joke_writer/agents/joke_agent.py deleted file mode 100644 index 8fa3b93..0000000 --- a/examples/joke_writer/agents/joke_agent.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Ventis entrypoint for the map-reduce joke writer. - -Nothing here restates the project. The three prompts, the two schemas and the -Bedrock binding all live in `joke_writer.py` and are reached with an import -- -the whole project tree is in the image. - -What could not be reused is the graph itself. `StateGraph`, the `Send` in -`continue_to_jokes` and the `Annotated[list, operator.add]` reducer are control -flow owned by the LangGraph runtime, and Ventis has no runtime to execute them. -That wiring is re-expressed as ordinary Python in workflow/joke_workflow.py, -where the fan-out becomes N dispatched calls across this agent's replicas. The -nodes those edges connected are imported, unchanged. - -The module is imported whole rather than by name so that -`joke_writer.generate_joke` inside a method named `generate_joke` reads as what -it is: the source's node. -""" - -# The source tree. Importing it reads BEDROCK_MODEL_ID and AWS_REGION, imports -# bedrock.py (which builds a RedisClient at module scope) and compiles the graph -# -- but it constructs no API client, so the import needs no credential. -# -# The credential arrives by a different road: `env_file` in -# config/global_controller.yaml hands the container a .env holding -# AWS_BEARER_TOKEN_BEDROCK, and botocore picks that name up by itself. Nothing -# here or in joke_writer.py names it. -# -# Constructing no client at import is no longer what makes this agent loadable -- -# env_file would carry a key to a module-scope client too. It only changes the -# failure: a missing key is an error on /status rather than "No agent loaded". -import joke_writer - - -class JokeAgent(object): - """The graph's nodes, exposed under the class name `agent.name` declares.""" - - # No constructor arguments -- LocalController does `JokeAgent()`. The model - # id and region are the source's own module-level constants, read from the - # environment there; there is nothing to configure here. - - def generate_topics(self, topic: str) -> dict: - """Split a topic into sub-topics. Returns {"subjects": [...]}. - - Synchronous by signature -- the executor calls this with no `await`, and - returning a coroutine would put `` into Redis. - """ - # The node's own state dict goes in, the node's own return comes out. - # Both hold nothing but str and list, so the executor's json.dumps is - # happy without a serializer -- unlike a graph that hands back messages. - return joke_writer.generate_topics({"topic": topic}) - - def generate_joke(self, subject: str) -> dict: - """Write one joke about one subject. Returns {"jokes": ["..."]}. - - The single-element list is the node's own shape: it is what - `Annotated[list, operator.add]` merged N of. The workflow does that - concatenation now. - """ - return joke_writer.generate_joke({"subject": subject}) - - def best_joke(self, topic: str, jokes: list) -> dict: - """Pick the winner. Returns {"best_selected_joke": "..."}.""" - return joke_writer.best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/agents/joke_agent.yaml b/examples/joke_writer/agents/joke_agent.yaml deleted file mode 100644 index fee8607..0000000 --- a/examples/joke_writer/agents/joke_agent.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# The graph's three nodes, exposed as three methods on one agent. -# -# One agent, not three. `generate_topics` and `best_joke` run once per request -# and have no resource profile of their own, so splitting them out would add -# two images, two dependency trees and a Redis round trip to buy nothing. What -# is hoisted is the `Send` fan-out, and that is a workflow concern, not a -# second agent: the workflow dispatches N `generate_joke` calls and the -# routing table spreads them across this agent's replicas. -# -# This file's basename names the generated stub, not the agent. Sharing it with -# joke_agent.py is why both land at /app/joke_agent.py -- the entrypoint is -# copied last and wins it, so the agent container loads the real class while the -# stub keeps /app/agents/joke_agent.py for callers. What the basename must not -# match is a source module: a `joke_writer.yaml` would put a stub at -# /app/joke_writer.py, on top of the module the adapter imports. - -agent: - name: JokeAgent - functions: - # Node 1 of the graph. One LLM call, structured output into `Subjects`. - - name: generate_topics - description: Split a topic into three related sub-topics. - arguments: - # Must equal the Python parameter name character for character -- - # LocalController calls method(**args). - - name: topic - type: str - # dict -> the workflow must json.loads what .value() hands back - returns: - type: dict - - # Node 2. The fan-out: one call per sub-topic, no shared state between - # them. This is the only reason this project is on Ventis. - - name: generate_joke - description: Write one joke about one subject. - arguments: - - name: subject - type: str - returns: - type: dict - - # Node 3. The reduce: one call over every joke the fan-out produced. - - name: best_joke - description: Pick the best joke out of the ones written for a topic. - arguments: - - name: topic - type: str - # `list` is a builtin, so the stub's annotation resolves. `list[str]` - # would be pasted into the AST verbatim and NameError on import. - - name: jokes - type: list - returns: - type: dict diff --git a/examples/joke_writer/config/global_controller.yaml b/examples/joke_writer/config/global_controller.yaml deleted file mode 100644 index eb26090..0000000 --- a/examples/joke_writer/config/global_controller.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Deployment manifest for the map-reduce joke writer. -# -# `entrypoint` is the adapter, which imports the untouched-in-shape source tree. -# -# The source has no pyproject.toml, setup.py or setup.cfg, so the Dockerfile's -# `-e .` is skipped -- silently. It does not matter here: `joke_writer.py` sits -# at the project root, so it lands flat at /app, which is sys.path[0]. A source -# laid out under src/ would need its own packaging metadata to import at all. - -agents: - - name: JokeAgent - # The fan-out. `generate_joke` is stateless, so LocalController picks a - # replica at random per call and the workflow's N dispatched calls spread - # across these three. N is whatever the model returns (the prompt asks for - # three sub-topics); replicas bound how many run at once, not how many run. - replicas: 3 - redis_port: 6379 - resources: - cpu: 1 - memory: 1024 - entrypoint: agents/joke_agent.py - provider: local - # What the source imports beyond the runtime's own list, which the generator - # prepends. boto3 is already in it, which is the whole reason the Bedrock - # call needs nothing declared here. The graph is never executed in this - # container, but `joke_writer.py` imports langgraph at module scope, so it - # still has to be installed. - requirements: - - langgraph - - pydantic - - typing_extensions - - - name: Workflow - type: workflow - replicas: 1 - redis_port: 6379 - api_port: 8080 - workflow_file: workflow/joke_workflow.py - provider: local - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 - -# `provider` must be lowercase. InstanceManager.launch_all tests -# `provider == "local"` to decide whether to reserve a host port; `Local` fails -# that test, reserved_port stays None, and Local/_runtime.py raises -# `int() argument must be ... not 'NoneType'` before any container starts. -# -# The credential. `_launch_locally` passes exactly five `-e` flags, all VENTIS_*, -# and .env is excluded from the build context, so for a while the only model call -# that could work here was one that needed no secret in the container: boto3 -# resolving an instance role per call. `env_file` is what changed. It points at a -# local .env, unresolved paths relative to this project root, and every container -# gets it as `docker run --env-file` -- so the key is in the environment without -# ever entering the image. -# -# What lands there is AWS_BEARER_TOKEN_BEDROCK. Nothing in this project reads it: -# botocore matches the name against bedrock-runtime's signingName and switches -# the client to bearer auth on its own, so `ventis/llm/bedrock.py` still builds a -# plain `boto3.client("bedrock-runtime")`. -# -# Deploy fails here rather than in a container: resolve_env_file checks the path -# before InstanceManager launches anything, so a missing .env is one error line -# instead of three replicas that come up and then answer -# {"status": "error", "error": "Unable to locate credentials"} on every request. -# -# What it costs: this is no longer upstream's model stack. See README.md. - -# Relative to this project root, same as `entrypoint` and `workflow_file`. -# .env is gitignored and excluded from the build context; .env.example names -# what belongs in it. -# -# NOTE: this key needs PR #53 (jiajunh/can-232-...), which is still open against -# main. On main nothing reads it -- `grep -rn env_file ventis/` finds no hits -- -# so the key is inert, no credential reaches the container, and every request -# answers a Bedrock credential error. `validate.py` reports that as V030 until -# the PR lands. -env_file: .env diff --git a/examples/joke_writer/config/policy.yaml b/examples/joke_writer/config/policy.yaml deleted file mode 100644 index 2cb9cb3..0000000 --- a/examples/joke_writer/config/policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Policy-Based Routing Rules — map-reduce joke writer -# Each rule defines a match condition (key-value pairs checked against the -# request context) and an access list of allowed services. -# Rules are evaluated most-specific-first (most matching keys wins). -# An empty match ({}) acts as a default fallback. -# -# This file IS optional -- `_load_policy_rules` logs "No policy file found" and -# returns [], and `_check_policy` allows everything when the rule list is empty. -# What is not safe is a half-written one: past the isfile() guard the read is -# unguarded, so an empty file (`.get("rules")` on None) or a null `rules:` -# (`None.sort()`) raises inside GlobalController.__init__ and `ventis deploy` -# dies before any container starts. Delete it or fill it; do not leave it empty. - -rules: - # Default fallback: the workflow and the one agent behind it. A service left - # out of this list answers "Unauthorized: Policy denied access to service". - - match: {} - access: - - Workflow - - JokeAgent diff --git a/examples/joke_writer/workflow/joke_workflow.py b/examples/joke_writer/workflow/joke_workflow.py deleted file mode 100644 index 9fdf3cc..0000000 --- a/examples/joke_writer/workflow/joke_workflow.py +++ /dev/null @@ -1,59 +0,0 @@ -r"""Ventis workflow for the map-reduce joke writer. - -This file is where the graph went. `generate_topics -> continue_to_jokes -> -generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the -three statements below, and the `Send` fan-out is N calls dispatched across -JokeAgent's replicas. - -The function is `main` and its one argument is `query` because the deployment -platform's test endpoint posts to a hardcoded /main with a strictly validated -{query: string} body. Ventis would serve any name and any kwargs -- the route is -the function's __name__ and the body is splatted in -- so nothing here fails if -you rename it; it just stops being reachable through the platform. - - curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' - curl http://localhost:8080/status/ -""" - -import json - -from deploy import deploy -from agents.joke_agent import JokeAgent - - -def main(query): - """Route: POST /main {"query": ""}""" - agent = JokeAgent() - - # Node 1: one call, and the fan-out width comes out of it. The agent's own - # parameter is still `topic` -- that name is bound by joke_agent.yaml and the - # source's node, and only the workflow's entry point is pinned to `query`. - subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] - - # `continue_to_jokes`, re-expressed. Every call is dispatched before any - # of them is resolved -- .value() blocks, so fusing these two lines into one - # comprehension would run the jokes one after another. It would not error; - # the fan-out would just be gone, and with it the reason to be on Ventis. - futures = [agent.generate_joke(subject=s) for s in subjects] - written = [json.loads(f.value()) for f in futures] - - # `Annotated[list, operator.add]`, re-expressed: the reducer that merged N - # single-joke lists back into one list was part of the graph, not of a node. - written_jokes = [joke for result in written for joke in result["jokes"]] - - # Node 3: the reduce. `list` in the yaml is what lets this argument through. - best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) - - return { - "topic": query, - "subjects": subjects, - "jokes": written_jokes, - "best_selected_joke": best["best_selected_joke"], - } - - -# This file is exec'd, not imported, so __name__ == "__main__" here and any -# `if __name__ == "__main__":` block would run in production. deploy() blocks -# on app.run(); nothing after it executes. -deploy(main, port=8080) diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md new file mode 100644 index 0000000..fe6dd49 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md @@ -0,0 +1,240 @@ +--- +name: porting-to-canyonos-core +description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. +compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. +--- + +# Port an agent project to CanyonOS Core + +CanyonOS Core is the product name. Its compatibility executable and Python +package remain `ventis`; environment variables and Docker resources retain the +`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding +strings. Do not rename them. + +## Load references only when needed + +- Read [references/packaging.md](references/packaging.md) when a source import + does not resolve from `/app`, the source is nested, or packaging metadata is + involved. +- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target + includes `llm_proxy`. +- Read [references/ec2.md](references/ec2.md) only when any config entry uses + `provider: EC2`. +- Read [references/troubleshooting.md](references/troubleshooting.md) after a + failed build, image probe, deploy, or request. +- Read [references/runtime-contract.md](references/runtime-contract.md) when a + validator finding needs explanation or the runtime mechanism is unclear. + +## Goal: thin scaffolding beside untouched source + +```text +agents/.yaml one callable surface per service +agents/.py one thin adapter per service, when needed +workflow/_workflow.py HTTP entry point; calls deploy() +config/global_controller.yaml deployment manifest +config/policy.yaml optional access restriction +pyproject.toml conditional nested-import scaffolding + unchanged +``` + +The file count follows the deployment. A multi-agent port has one yaml/adapter +pair per service that is worth deploying separately. If a source class already +satisfies the runtime contract, point its config entry at that file and do not +copy it into an adapter. + +Everything the source already owns—prompts, tools, schemas, parsing, retries, +model clients, and node bodies—is imported. The port re-expresses only the +CanyonOS Core boundary and framework-owned orchestration. + +The port root is the existing repository root and the directory from which +`ventis build` runs. Write scaffolding there beside existing directories. If the +repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, +and `config/` beside it. Never move or copy the repository into a new `src/` +directory, and never create an outer wrapper merely for the port. + +## 1. Survey before writing + +Identify: + +1. The source entry point and callable input/output. +2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, + `Send`, `Command`, interrupts). +3. Runtime-injected services nodes read: stores, context, memory, sessions, or + callback managers. +4. Sync versus async boundaries. +5. Imports and declared runtime dependencies. +6. Model provider, credential names, streaming use, and optional `llm_proxy`. +7. Whether independent work fans out and benefits from separate replicas. +8. Whether source imports resolve from the project root that becomes `/app`. + +Run the validator once now. Its header detects capabilities directly from the +importable runtime rather than external development metadata: + +```bash +python /validate.py . +``` + +If config or agent yaml is malformed, the validator defers to `ventis build`. +Capability-gated findings say which runtime behavior is available. + +## 2. Choose service boundaries + +Start with one service. Split only when it creates independent parallel work or +a distinct resource/replica profile. + +- Keep a ReAct loop together; every turn needs shared message history. +- Hoist supervisor task lists and `Send`-style fan-out into the workflow. +- Do not create a one-replica service with no distinct resource profile merely + to mirror every source graph node. + +Rewrite framework-owned edges as ordinary Python. Import the connected node +functions unchanged. Construct runtime-injected service objects from source +configuration; do not invent models, dimensions, stores, or defaults silently. +Report any choice the source does not specify. + +## 3. Write declarations and adapters + +### Agent yaml + +Use one yaml per deployed service. Argument types are bare builtins only: +`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is +required by the generated stub. `returns.type` is documentation; use `dict` or +`list` to signal that workflow callers must `json.loads` the returned string. + +### Adapter + +The entrypoint exposes a module-level class named exactly `agent.name`. It +constructs with no arguments and its declared methods are synchronous. Read +configuration from the environment in `__init__`. Bridge source coroutines +inside a synchronous method with `asyncio.run(...)`. Serialize framework objects +with their own JSON-safe serializer before returning. + +Do not duplicate source prompts, tools, schemas, or model calls. Keep the source +provider and SDK. + +### Workflow + +Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. +Import generated stubs by yaml basename and agent class name: + +```python +from deploy import deploy +from agents. import +``` + +The deployment platform sends `{query: string}` to `/main`. Pack richer input +inside `query`; any additional workflow parameter has a default. + +Dispatch every remote call before resolving any future: + +```python +futures = [agent.work(item=item) for item in items] +results = [json.loads(future.value()) for future in futures] +``` + +Do not fuse dispatch and `.value()` in one comprehension; that silently +serializes fan-out. Do not add an `if __name__ == "__main__":` block: the +workflow is executed with `__name__ == "__main__"` in production. + +### Config + +For each service, keep these names aligned: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a +list of distribution-name strings. Put `env_file` at config top level when the +runtime capability is available. Omit `policy.yaml` unless access must be +restricted; if present, give it a non-empty `rules` list. + +## Hard rules + +Capitalized **MUST** and **NEVER** are reserved for port-breaking or +source-integrity rules. The owner column states where each is decided. + +| ID | Rule | Owner | +|---|---|---| +| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | +| M2 | The class MUST construct with no arguments | V007 | +| M3 | yaml argument names MUST match Python parameter names | V008 | +| M4 | yaml argument types MUST be bare builtins | V010 | +| M5 | Declared adapter methods MUST be synchronous | V009 | +| M6 | Config names MUST match yaml agent names | build | +| M7 | Config names MUST not collide after lowercase normalization | build output | +| M8 | Local provider MUST be lowercase `local` | deploy preflight | +| M9 | `replicas` MUST be an integer | deploy preflight | +| M10 | `requirements` MUST be a list of strings | build | +| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | +| M12 | Workflow MUST NEVER contain a main guard | V017 | +| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | +| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | +| M15 | Workflow MUST import stubs from `agents.` | V023 | +| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | +| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | +| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | +| M19 | NEVER hardcode or bake a real credential into an image | W003 | +| M20 | NEVER edit or vendor the source tree | `git status` | +| M21 | NEVER swap the source LLM provider | review | +| M22 | NEVER silently move, drop, or reclassify source dependencies | review | +| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | +| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | + +## 4. Validate, build, and probe + +Run static preflight, then let the build own build-time validation: + +```bash +python /validate.py . +ventis build -c config/global_controller.yaml +``` + +A green build never imports the adapter. Probe each agent image in this order: + +```bash +# Runtime startup path + docker run --rm ventis- \ + python -c "import local_controller" + +# Agent load path; include --env-file when configured + docker run --rm --env-file ventis- \ + python -c "import importlib.util,sys; \ +s=importlib.util.spec_from_file_location('m','.py'); \ +m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ +m.();print('ok')" +``` + +Also probe the workflow image with `python -c "import local_controller"`; it has +its own dependency resolve and generated-stub imports. + +Then deploy, send a representative request, and poll its status: + +```bash +ventis deploy -c config/global_controller.yaml +curl -X POST http://localhost:8080/main \ + -H 'Content-Type: application/json' -d '{"query":""}' +curl http://localhost:8080/status/ +``` + +A successful outer request with a source-level failure still proves the port +reached and returned the source behavior. Record the distinction. + +## 5. Clean up + +After collecting evidence, stop foreground deploy with Ctrl+C and wait for +controller cleanup. Remove exact leftovers if startup crashed. Then remove build +products and exact images from this config: + +```bash +ventis clean +docker image rm ventis- \ + ventis- + +test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container +docker ps -a --format '{{.Names}}' +``` + +`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it +does not remove containers or images. Keep port scaffolding, untouched source, +and requested logs or reports. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md new file mode 100644 index 0000000..e06daa0 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md @@ -0,0 +1,41 @@ +# EC2 deployment + +Read this only when at least one config entry uses `provider: EC2`. + +## Configuration + +Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The +top-level `ec2` block supplies the runtime's required infrastructure and SSH +settings. Read the target checkout's deploy preflight and EC2 runtime before +writing the block; do not copy values from an example environment. + +Typical required categories are: + +- AMI and instance type +- region and subnet +- security groups +- SSH user and credentials accepted by the runtime + +`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof +that provisioning, SSH, image transfer, or remote container startup works. + +## Networking + +A remote container's `host.docker.internal` names its own EC2 Docker host. It +does not name the local controller machine. Databases, model proxies, and other +services must use addresses reachable from every selected host. + +The environment file may be copied temporarily to a remote host by runtimes that +expose the `env_file` capability. Confirm behavior from the capability probe and +target runtime rather than assuming local Docker semantics. + +## Probes and cleanup + +Run the same runtime and adapter probes against the exact image before remote +deployment. After deploy, verify the remote container logs; controller health +can be green even when agent loading failed. + +Stop foreground deploy normally so the controller can terminate recorded EC2 +instances. If provisioning or startup fails before an instance is recorded, +inspect the cloud provider directly and remove exact leaked resources. Never use +a broad cleanup command against unrelated instances. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md new file mode 100644 index 0000000..f726bd7 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md @@ -0,0 +1,64 @@ +# LLM proxy integration + +Read this only when the target checkout contains `llm_proxy` or the deployment +explicitly routes model SDKs through it. + +## Preserve provider protocols + +The proxy redirects provider endpoints; it does not convert providers. Keep the +source SDK, model ID, request body, and response parsing unchanged. + +Configure only the provider variables the source uses: + +```dotenv +OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 +ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic +AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +``` + +Some SDKs refuse to initialize without caller credentials. Give agent containers +non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS +credentials in the separate proxy process, not in the port's `env_file`. + +## Start locally + +The proxy defaults conflict with a typical deployment: host loopback is not +reachable from a container, and port 8080 is normally used by the workflow API. +Use a non-loopback bind and a different port: + +```bash +PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy +curl http://127.0.0.1:8081/healthz +``` + +Local CanyonOS Core containers resolve `host.docker.internal` through their +Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the +machine running `ventis deploy`. Distributed deployments need a reachable proxy +address or one proxy on each host. + +## Supported call shape + +The implementation buffers complete requests and responses: + +- OpenAI and Anthropic non-streaming HTTP calls are forwarded. +- Bedrock `invoke` is reissued through the proxy's boto3 client. +- OpenAI/Anthropic streaming is unsupported. +- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are + unsupported. + +Survey the source before selecting the proxy. Do not silently disable streaming; +report the unsupported behavior and stop. + +## Credential behavior + +- The OpenAI adapter removes caller authorization and inserts the proxy key. +- The Anthropic adapter removes caller key headers and inserts the proxy key. +- Botocore still signs requests sent to a custom endpoint, so a caller may need + placeholder AWS credentials even though the proxy reissues upstream with its + own identity. +- `/healthz` proves provider registration and Flask availability, not upstream + credential validity. + +OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return +JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed +with the upstream status and are not byte-for-byte passthrough. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md new file mode 100644 index 0000000..8520c4a --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md @@ -0,0 +1,81 @@ +# Packaging and import roots + +Read this reference when an adapter imports nested source code, the source uses a +`src/` layout, or V031 reports an import-root problem. + +## What `/app` can import + +CanyonOS Core preserves project-relative paths in the image and starts Python at +`/app`. Without an editable install, Python resolves names rooted there: + +- `/app/tools.py` as `import tools` +- `/app/pkg/__init__.py` as `import pkg` +- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace + directories without `__init__.py` + +It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become +an import root first. + +## Detect support, do not infer it from release history + +Run: + +```bash +python /validate.py . +``` + +Read the `editable_install` capability. If it is unavailable and the original +import cannot resolve from `/app`, report a runtime capability blocker and stop. +Do not add a `sys.path` hack or relocate source files. + +## Root metadata is the trigger + +When editable install is supported, only packaging metadata at the **port root** +triggers `pip install -e .`: + +```text +port-root/pyproject.toml detected +port-root/source/pyproject.toml ignored as an install trigger +``` + +A nested source repository may remain untouched. Add minimal root scaffolding +that points package discovery at the existing source package: + +```toml +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "canyonos-port" +version = "0.0.0" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["source/src"] +include = ["pkg*"] +namespaces = true +``` + +Set `where` and `include` from the actual tree and original import spelling. Do +not reference a README or license from this wrapper metadata; file sweeps differ +by runtime capability and a missing referenced file makes the image build fail. + +## Dependencies in nested metadata + +A nested `pyproject.toml` is not installed merely because its Python files are +copied. Keep the source declaration unchanged and repeat its runtime +distributions in each relevant config entry's `requirements` list. This is +compatibility scaffolding, not permission to drop, move, or reclassify declared +dependencies. + +If source metadata is already at the port root, do not create a wrapper. Its +project dependencies participate in the same resolver as config requirements. +Report declared-but-unused toolchain dependencies and their image cost; let the +owner decide whether source metadata should change. + +## Validation boundary + +`ventis build` owns packaging syntax and installation errors. `validate.py` +checks only whether adapter imports appear to require a nested root that the +runtime will not expose. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md new file mode 100644 index 0000000..94f7401 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -0,0 +1,187 @@ +# CanyonOS Core runtime contract + +The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the +CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for +Docker resources. + +Read this reference when implementing an adapter or explaining a validator +finding. Runtime-dependent behavior is expressed as capabilities; run +`validate.py` against the target environment instead of inferring support from +release history. + +## Project root and discovery + +`ventis build` uses the current working directory as the project root. + +| Input | Discovery | +|---|---| +| `agents/*.yaml` | direct yaml glob under the project root | +| `config/global_controller.yaml` | default config, overridable with `-c` | +| workflow | `workflow_file` on a `type: workflow` entry | +| policy | `policy.yaml` beside the selected config file | +| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | + +The config name, yaml `agent.name`, and entrypoint class name form one binding: + +```text +config entry name == yaml agent.name == entrypoint class name +``` + +A missing match may skip an image while the command continues, so inspect build +output and generated image tags. + +## Agent yaml and generated stubs + +The consumed yaml shape is: + +```yaml +agent: + name: ExampleAgent + functions: + - name: work + arguments: + - name: query + type: str + returns: + type: dict +``` + +Argument annotations are generated from bare names without adding imports. Use +builtins. Generated methods have no defaults, so every declared argument is +required at the stub call site. `returns` does not control runtime conversion; +it documents whether workflow code should parse the returned string. + +Stub destinations differ by runtime capability and entrypoint layout. Workflow +code in this port convention imports the generated class from +`agents.`. The validator checks that import against declarations +before the workflow image starts. + +## Agent loading and execution + +The local controller effectively performs: + +```python +module = load(entrypoint) +agent_class = getattr(module, configured_name) +agent = agent_class() +result = getattr(agent, method_name)(**args) +``` + +Consequences: + +- The class is module-level and named exactly as configured. +- Construction takes no arguments. +- Declared methods accept yaml argument names as keyword arguments. +- Methods are synchronous; this path does not await a coroutine. +- Dicts and lists are JSON-encoded before entering Redis; other results become + strings. +- A remote Future's `.value()` returns text, not the original Python object. + +Agent import and construction exceptions are caught by the controller. A failed +agent may still advertise healthy because health is written independently of +successful agent loading. That is why image probes import both the runtime and +the entrypoint explicitly. + +## Workflow execution + +The workflow file is executed, not imported. Therefore: + +- module-level code runs at container startup; +- `__name__ == "__main__"`; +- `deploy()` blocks in the web server; +- the workflow function runs once per request; +- its function name determines the REST route exposed by the compatibility + runtime. + +The deployment platform additionally expects `/main` with a `{query: string}` +body. This platform constraint is stricter than the underlying transport. + +Each stub method returns a Future immediately. `.value()` blocks. Dispatching +and resolving inside one comprehension serializes work without raising an +error; dispatch all calls first, then resolve them. + +The workflow container also starts runtime controller code and has its own +package resolution. Probe it independently from agent images. + +## Build context and collisions + +The runtime copies project files while preserving relative paths, then writes +shared runtime modules, generated stubs, and entrypoints into the image. Later +writes can shadow project files. + +Avoid root project modules named like runtime files, including: + +```text +future.py +ventis_context.py +local_controller.py +local_controller_frontend.py +redis_client.py +grpc_options.py +bedrock.py +deploy.py +session_logging.py +workflow_launcher.py +``` + +Also avoid a yaml basename that shadows a different source module imported by an +adapter. The validator checks deterministic flat-name collisions. + +File sweep and editable-install behavior are runtime capabilities. For nested +imports, follow [packaging.md](packaging.md). + +## Dependencies and protobuf + +Agent and workflow images include a small runtime dependency set. Config +`requirements` adds source-specific distributions. A malformed requirements +value can be normalized away while image generation continues; missing imports +then surface only when the agent loads. + +The build compiles gRPC Python stubs on the host and copies them into images. +The image resolver does not necessarily know the generated-code version. A +source dependency that constrains protobuf below the host generator version can +produce a green image build that dies on: + +```text +import local_controller +``` + +Always run that probe before probing the entrypoint. Treat a generated-code / +runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter +source dependencies silently. + +## Credentials capability + +When `env_file` capability is available, the top-level config path is resolved +against the project root and passed at container start. Hidden env files are not +copied into images. Invalid paths are deploy-preflight errors. + +When the capability is unavailable, declaring `env_file` has no effect. If the +source needs credentials, report the capability blocker rather than hardcoding +or vendoring a secret. + +A source that constructs its client at import time works only when credentials +are already in the container environment. Image entrypoint probes therefore use +the same env file as deployment. + +For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). + +## Policy and provider behavior + +No policy file means unrestricted service access. If a policy exists, it needs a +non-empty rules list. Rules are evaluated by specificity and first match; +services excluded from the selected rule fail after request acceptance. + +Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior +and remote networking are covered in [ec2.md](ec2.md). + +## Cleanup boundary + +Stopping foreground deploy normally invokes controller cleanup for recorded +containers and Redis. Hard kills and failures before resource registration may +leave resources behind. + +`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and +`docker_container/`. It does not remove containers or images. Remove exact +leftovers explicitly and preserve source, port scaffolding, and requested +evidence. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md new file mode 100644 index 0000000..361acf9 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md @@ -0,0 +1,60 @@ +# Troubleshooting + +Read this after a failed build, image probe, deploy, or request. For mechanisms, +read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, +read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). + +## Build or deploy stops early + +| Symptom | Likely cause | +|---|---| +| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | +| Two services produce one image | Config names collide after lowercase normalization | +| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | +| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | +| Replica conversion `TypeError` | `replicas` is not an integer | +| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | +| Port or container name already in use | A previous deployment did not complete cleanup | + +## Container exits or serves nothing + +| Symptom | Likely cause | +|---|---| +| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | +| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | +| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | +| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | +| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | +| Third-party module is missing | Distribution is absent from source metadata and config requirements | +| Stub import raises `NameError` | yaml argument type is not a bare builtin | +| Source module behaves like an empty stub | A generated stub basename shadowed the source module | +| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | + +## Request is accepted, then fails + +| Symptom | Likely cause | +|---|---| +| Unexpected keyword argument | yaml argument name differs from adapter parameter name | +| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | +| Unauthorized service | The first matching policy rule excludes that service | +| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | +| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | +| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | +| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | +| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | + +## Deployment platform endpoint + +| Symptom | Likely cause | +|---|---| +| 404 while workflow container is healthy | Workflow function is not named `main` | +| 400 before host receives request | Body is not the platform's `{query: string}` shape | +| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | + +## Cleanup + +| Symptom | Likely cause | +|---|---| +| `ventis clean` succeeds but containers remain | The command removes generated directories only | +| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | +| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py new file mode 100755 index 0000000..04baf37 --- /dev/null +++ b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python3 +"""Preflight the runtime traps that `ventis build` cannot see. + +This deliberately does not duplicate build-time validation such as malformed +YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those +checks. This script parses Python without importing it and catches failures that +otherwise stay hidden until a container loads an agent, starts a workflow, or +serves its first request. A replica is not evidence: the controller writes +`healthy` to Redis before `_load_agent` runs. + + python validate.py [project_dir] [-c config/global_controller.yaml] + [--json] [--strict] + +Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. + +Runtime capabilities vary across CanyonOS Core installations. This script probes +the importable `ventis` package directly. A capability-gated check reports +UNAVAILABLE when its behavior cannot be proven. +""" + +import argparse +import ast +import builtins +import json +import os +import re +import sys +from typing import ClassVar + +try: + import yaml +except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency + sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") + raise SystemExit(2) from None + + +DEFAULT_CONFIG_PATH = "config/global_controller.yaml" + +# Copied flat into every image over the swept project tree, so a project module +# landing flat under one of these names is overwritten. +# ventis/stub_generator.py generate_docker / generate_workflow_docker. +RUNTIME_FLAT_NAMES = frozenset( + { + "future.py", + "ventis_context.py", + "local_controller.py", + "local_controller_frontend.py", + "redis_client.py", + "grpc_options.py", + "gpu_metrics.py", + "bedrock.py", + "deploy.py", + "session_logging.py", + "workflow_launcher.py", + } +) + +# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. +BASE_AGENT_REQUIREMENTS = [ + "grpcio", + "grpcio-tools", + "redis", + "pyyaml", + "psutil", + "ipdb", + "ipython", + "boto3", +] +# Import name -> distribution name, for the handful where they differ and the +# mismatch would otherwise be reported as a missing requirement. +IMPORT_TO_DISTRIBUTION = { + "attr": "attrs", + "bs4": "beautifulsoup4", + "cv2": "opencv-python", + "dateutil": "python-dateutil", + "dotenv": "python-dotenv", + "grpc": "grpcio", + "grpc_tools": "grpcio-tools", + "jwt": "pyjwt", + "PIL": "pillow", + "psycopg": "psycopg", + "psycopg2": "psycopg2-binary", + "pydantic_settings": "pydantic-settings", + "sklearn": "scikit-learn", + "typing_extensions": "typing-extensions", + "yaml": "pyyaml", +} + +SECRET_PATTERNS = [ + (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), + (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), + (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), + (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), +] +SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) + +ERROR = "ERROR" +WARN = "WARN" +INFO = "INFO" + + +# ------------------------------------------------------------------ # +# Capabilities # +# ------------------------------------------------------------------ # +# +# Stable labels for behavior detected from the importable runtime. They contain +# no external development metadata. + +CAPABILITY_SOURCE = { + "env_file": "runtime env-file injection", + "editable_install": "editable project installation", + "sweeps_all_files": "full project-file sweep", + "stub_two_destinations": "flat and package stub destinations", +} + + +def probe_capabilities(): + """Ask the importable ventis package what it actually supports.""" + caps = dict.fromkeys(CAPABILITY_SOURCE, False) + caps["ventis"] = False + try: + from ventis import stub_generator + except Exception: # noqa: BLE001 - a broken install must not crash the check + return caps + + caps["ventis"] = True + caps["editable_install"] = hasattr(stub_generator, "_install_step") + caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") + caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") + + import importlib + + for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): + try: + module = importlib.import_module(module_name) + except Exception: # noqa: BLE001,S112 - the other path is the live one + continue + if hasattr(module, "resolve_env_file"): + caps["env_file"] = True + break + return caps + + +# ------------------------------------------------------------------ # +# YAML with line numbers # +# ------------------------------------------------------------------ # + + +class LineDict(dict): + """A mapping that remembers where it and each of its keys were written.""" + + line = 0 + key_lines: ClassVar[dict] = {} + + +class LineLoader(yaml.SafeLoader): + pass + + +def _construct_mapping(loader, node): + data = LineDict() + yield data + data.update(loader.construct_mapping(node, deep=False)) + data.line = node.start_mark.line + 1 + data.key_lines = { + key.value: key.start_mark.line + 1 + for key, _ in node.value + if isinstance(key, yaml.ScalarNode) + } + + +LineLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping +) + + +def line_of(mapping, key=None): + """The source line of `key` inside `mapping`, or of the mapping itself.""" + if not isinstance(mapping, LineDict): + return 0 + if key is not None: + return mapping.key_lines.get(key, mapping.line) + return mapping.line + + +def load_yaml(path): + """Parse `path`, returning (data, error). Never raises.""" + try: + with open(path, "r", encoding="utf-8") as handle: + return yaml.load(handle, Loader=LineLoader), None + except Exception as exc: # noqa: BLE001 - any parse failure is a finding + return None, str(exc) + + +# ------------------------------------------------------------------ # +# Findings # +# ------------------------------------------------------------------ # + + +class Report: + def __init__(self, project_dir, capabilities): + self.project_dir = project_dir + self.capabilities = capabilities + self.findings = [] + # A peer agent is imported by the name of its generated stub, which the + # build copies flat into every image. Those are not project modules and + # need no requirement. + self.stub_module_names = set() + + def add(self, check, level, path, line, summary, mechanism): + self.findings.append( + { + "check": check, + "level": level, + "path": self.rel(path) if path else "", + "line": line or 0, + "summary": summary, + "mechanism": mechanism, + } + ) + + def error(self, check, path, line, summary, mechanism): + self.add(check, ERROR, path, line, summary, mechanism) + + def warn(self, check, path, line, summary, mechanism): + self.add(check, WARN, path, line, summary, mechanism) + + def unavailable(self, check, summary): + self.add(check, INFO, "", 0, summary, "") + + def rel(self, path): + try: + return os.path.relpath(path, self.project_dir) + except ValueError: + return path + + def counts(self): + errors = sum(1 for f in self.findings if f["level"] == ERROR) + warnings = sum(1 for f in self.findings if f["level"] == WARN) + return errors, warnings + + +# ------------------------------------------------------------------ # +# Python source helpers # +# ------------------------------------------------------------------ # + + +def parse_python(path): + """AST for `path`, or (None, error). The port is never imported.""" + try: + with open(path, "r", encoding="utf-8") as handle: + source = handle.read() + except OSError as exc: + return None, str(exc) + try: + return ast.parse(source, filename=path), None + except SyntaxError as exc: + return None, f"{exc.msg} (line {exc.lineno})" + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + return None + + +def class_methods(class_node): + return { + node.name: node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def parameter_names(func_node): + """Every parameter a caller can pass by keyword, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + return positional + [a.arg for a in args.kwonlyargs] + + +def required_parameters(func_node): + """Parameters with no default, minus self.""" + args = func_node.args + positional = [a.arg for a in args.posonlyargs + args.args] + if positional and positional[0] in ("self", "cls"): + positional = positional[1:] + if args.defaults: + positional = positional[: len(positional) - len(args.defaults)] + kwonly = [ + arg.arg + for arg, default in zip(args.kwonlyargs, args.kw_defaults) + if default is None + ] + return positional + kwonly + + +def toplevel_import_names(tree): + """Top-level package name of every import in the module, with line numbers.""" + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.setdefault(alias.name.split(".")[0], node.lineno) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.setdefault(node.module.split(".")[0], node.lineno) + return names + + +# ------------------------------------------------------------------ # +# V006-V010 adapter failures hidden by _load_agent # +# ------------------------------------------------------------------ # + +BUILTIN_TYPE_NAMES = frozenset( + name + for name in dir(builtins) + if isinstance(getattr(builtins, name), type) and not name[0].isupper() +) + + +def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): + """V006 V007 V008 V009 V010.""" + name = agent_block["name"] + functions = agent_block.get("functions") or [] + check_argument_types(report, agent_yaml_path, functions) + + entrypoint = entry.get("entrypoint") + if not entrypoint: + return + entrypoint_path = os.path.join(project_dir, entrypoint) + if not os.path.isfile(entrypoint_path): + return + + tree, error = parse_python(entrypoint_path) + if tree is None: + report.error( + "V006", + entrypoint_path, + 0, + f"the entrypoint does not parse: {error}", + "_load_agent exec_module's it and swallows the exception; the first " + "request answers 'No agent loaded'.", + ) + return + + class_node = find_class(tree, name) + if class_node is None: + classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] + found = ", ".join(classes) if classes else "no classes at all" + report.error( + "V006", + entrypoint_path, + 1, + f"no class named `{name}` at module level (found: {found})", + "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " + "the AttributeError. The class name must equal agent.name exactly.", + ) + return + + methods = class_methods(class_node) + check_constructor(report, entrypoint_path, name, methods) + + for func in functions: + if not isinstance(func, dict) or not isinstance(func.get("name"), str): + continue + check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) + + +def check_argument_types(report, agent_yaml_path, functions): + """V010 -- the type string is pasted into an ast.Name, never checked.""" + for func in functions: + if not isinstance(func, dict): + continue + for arg in func.get("arguments") or []: + if not isinstance(arg, dict) or "type" not in arg: + continue + declared = arg.get("type") + if not isinstance(declared, str): + continue # stub generation reports malformed type values + if declared in BUILTIN_TYPE_NAMES: + continue + report.error( + "V010", + agent_yaml_path, + line_of(arg, "type"), + f"`type: {declared}` is not a builtin", + "stub_generator pastes it verbatim into the generated " + "annotation, and the stub module imports only Future and " + "inspect. Anything else raises NameError when the stub is " + "imported -- after a green build. Use str int float bool dict " + "list.", + ) + + +def check_constructor(report, entrypoint_path, name, methods): + """V007 -- _load_agent calls agent_class() with no arguments.""" + init = methods.get("__init__") + if init is None: + return + required = required_parameters(init) + if required: + report.error( + "V007", + entrypoint_path, + init.lineno, + f"`{name}.__init__` requires {', '.join(required)}", + "_load_agent calls agent_class() with no arguments; the TypeError is " + "swallowed and the first request answers 'No agent loaded'. Read " + "configuration from the environment inside __init__ instead.", + ) + + +def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): + """V008 V009.""" + func_name = func["name"] + method = methods.get(func_name) + if method is None: + report.error( + "V008", + entrypoint_path, + 0, + f"`{class_name}` has no method `{func_name}`", + "The yaml declares it, so callers get a stub for it; the controller " + f"then answers \"Agent {class_name} has no method '{func_name}'\".", + ) + return + + # V009 -- nothing on the execution path awaits. + if isinstance(method, ast.AsyncFunctionDef): + report.error( + "V009", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` is `async def`", + "The executor calls method(**args) with no await, so Redis receives " + "''. Keep the signature synchronous and call " + "asyncio.run(...) inside the body.", + ) + + # V008 -- the controller calls method(**args) with the yaml's names. + declared = [ + arg["name"] + for arg in func.get("arguments") or [] + if isinstance(arg, dict) and isinstance(arg.get("name"), str) + ] + actual = parameter_names(method) + required = required_parameters(method) + + missing = [arg for arg in declared if arg not in actual] + if missing: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` has no parameter " + f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", + "LocalController does method(**args) with the yaml's argument names. " + "A mismatch is TypeError: unexpected keyword argument, at request " + f"time. See {os.path.basename(agent_yaml_path)}.", + ) + + unfilled = [arg for arg in required if arg not in declared] + if unfilled: + report.error( + "V008", + entrypoint_path, + method.lineno, + f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " + "the yaml does not declare", + "Only declared arguments are ever sent, and the generated stub gives " + "none of them a default. Declare them in the yaml or default them " + "in the signature.", + ) + + + +# ------------------------------------------------------------------ # +# V016-V018 the workflow # +# ------------------------------------------------------------------ # + + +def check_stub_imports(report, workflow_path, tree, stub_classes): + """V023 -- the workflow must import a stub as `from agents. import `. + + The build copies each stub to exactly one path, and for the workflow image + that path is agents/.py. Two ways of writing this line fail, and + the project walks you into both: the flat form is what examples/ uses, and + the class name is the one `ventis build` prints, which is not the one it + writes. + """ + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + base = alias.name.split(".")[0] + if base in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`import {alias.name}` -- the stub is at " + f"agents/{base}.py, not flat", + "The build copies a stub to one path, and for the " + "workflow that path is under agents/. This is a " + "ModuleNotFoundError the moment the workflow runs. " + f"Write `from agents.{base} import {stub_classes[base]}`.", + ) + continue + + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + + module = node.module + if module in stub_classes: + report.error( + "V023", workflow_path, node.lineno, + f"`from {module} import ...` -- the stub is at " + f"agents/{module}.py, not flat", + "The build copies a stub to one path, and for the workflow " + "that path is under agents/. The flat form is what this " + "repository's own examples use and it raises " + "ModuleNotFoundError in the workflow image. Write " + f"`from agents.{module} import {stub_classes[module]}`.", + ) + continue + + if not module.startswith("agents."): + continue + base = module.split(".", 1)[1] + expected = stub_classes.get(base) + if expected is None: + continue + for alias in node.names: + if alias.name == expected: + continue + if alias.name == f"{expected}Stub": + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is the name the build prints, not the " + f"class it writes", + "generate_agent_stub sets class_name = agent_config['name'] " + "and then recomputes it with a 'Stub' suffix for the log " + "line only. The message names a class that does not exist; " + f"the class is `{expected}`.", + ) + else: + report.error( + "V023", workflow_path, node.lineno, + f"`{alias.name}` is not a class the stub for {base} defines", + f"The stub's class carries the agent's own name: `{expected}`.", + ) + + +def check_workflow(report, workflow_path, stub_classes=None): + """V016 V017 V018 V023.""" + tree, error = parse_python(workflow_path) + if tree is None: + report.error("V016", workflow_path, 0, f"does not parse: {error}", "") + return + + main = None + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( + node.name == "main" + ): + main = node + break + + if main is None: + defined = [ + n.name + for n in tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) + ] + found = ", ".join(defined) if defined else "no top-level functions" + report.error( + "V016", + workflow_path, + 1, + f"no top-level function named `main` (found: {found})", + "CanyonOS Core serves POST /, but the deployment platform's " + "test endpoint posts to a hardcoded /main. A differently named " + "workflow builds, deploys and stays unreachable -- 404, container " + "healthy.", + ) + else: + check_main_signature(report, workflow_path, main) + + if stub_classes: + check_stub_imports(report, workflow_path, tree, stub_classes) + + # V016 -- deploy() is what starts Flask. + if not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "deploy" + for node in ast.walk(tree) + ): + report.error( + "V016", + workflow_path, + 1, + "the workflow never calls `deploy(...)`", + "workflow_launcher.py exec's this file and nothing else starts the " + "HTTP server; the container comes up serving nothing.", + ) + + check_main_guard(report, workflow_path, tree) + check_fused_fanout(report, workflow_path, tree) + + +def check_main_signature(report, workflow_path, main): + """V016 -- the platform sends exactly {"query": ...}.""" + if isinstance(main, ast.AsyncFunctionDef): + report.error( + "V016", + workflow_path, + main.lineno, + "`main` is `async def`", + "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " + "no await; the response body would be a coroutine repr.", + ) + params = parameter_names(main) + if not params: + report.error( + "V016", + workflow_path, + main.lineno, + "`main` takes no arguments", + 'The platform posts {"query": "..."} and deploy() splats the ' + "body in as kwargs -- TypeError on every request.", + ) + return + if params[0] != "query": + report.error( + "V016", + workflow_path, + main.lineno, + f"`main`'s first parameter is `{params[0]}`, not `query`", + "The platform's body schema is strictly validated as " + "{query: string}; any other key is rejected with 400 in the control " + "plane, before the request reaches the host.", + ) + extra = [p for p in required_parameters(main) if p != "query"] + if extra: + report.error( + "V016", + workflow_path, + main.lineno, + f"`main` requires {', '.join(extra)} beyond `query`", + "Only `query` is ever sent, so every other parameter needs a " + "default or the call raises on every request. Pack richer input " + "into `query`.", + ) + + +def check_main_guard(report, workflow_path, tree): + """V017 -- the workflow is exec'd, so __name__ == "__main__".""" + for node in tree.body: + if not isinstance(node, ast.If): + continue + test = node.test + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and test.left.id == "__name__" + and any( + isinstance(c, ast.Constant) and c.value == "__main__" + for c in test.comparators + ) + ): + report.error( + "V017", + workflow_path, + node.lineno, + '`if __name__ == "__main__":` block in the workflow', + "workflow_launcher.py runs exec(open().read()), so " + "__name__ IS '__main__' here and this block executes in " + "production, at container start.", + ) + + +def check_fused_fanout(report, workflow_path, tree): + """V018 -- .value() blocks, so dispatching and resolving in one + comprehension runs the fan-out one call at a time.""" + comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) + for node in ast.walk(tree): + if not isinstance(node, comprehensions + (ast.DictComp,)): + continue + elements = ( + [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] + ) + for element in elements: + for inner in ast.walk(element): + if ( + isinstance(inner, ast.Call) + and isinstance(inner.func, ast.Attribute) + and inner.func.attr == "value" + and isinstance(inner.func.value, ast.Call) + ): + report.error( + "V018", + workflow_path, + node.lineno, + "one comprehension both dispatches a call and resolves " + "it with .value()", + ".value() blocks, so each call completes before the " + "next is dispatched. It does not error -- the fan-out " + "is just silently serial, and with it the reason to be " + "on CanyonOS Core. Dispatch every call first, then resolve: " + "futures = [a.work(i) for i in items] then " + "[f.value() for f in futures].", + ) + return + + +# ------------------------------------------------------------------ # +# V019-V020 what the copy order overwrites # +# ------------------------------------------------------------------ # + + +def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): + """V019 V020 -- later copies land on earlier ones at the context root.""" + for entry in sorted(os.listdir(project_dir)): + path = os.path.join(project_dir, entry) + if not os.path.isfile(path) or not entry.endswith(".py"): + continue + + # V019 -- the shared runtime is copied flat, after the project sweep. + if entry in RUNTIME_FLAT_NAMES: + report.error( + "V019", + path, + 1, + f"a project module named `{entry}` sits at the project root", + "The shared CanyonOS Core runtime is copied flat into the image after " + "the project sweep, so this file is overwritten by CanyonOS Core's own " + f"{entry}. Rename it or move it into a package directory.", + ) + + # V020 -- a stub is copied flat under its yaml's basename. + entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} + for yaml_path in yaml_paths: + stem = os.path.splitext(os.path.basename(yaml_path))[0] + module = f"{stem}.py" + if module in entrypoint_basenames: + continue # the entrypoint is copied last and wins its flat name back + candidate = os.path.join(project_dir, module) + if os.path.isfile(candidate): + report.error( + "V020", + candidate, + 1, + f"`{os.path.basename(yaml_path)}` generates a stub that lands on " + f"`{module}`", + "The yaml's basename names the stub, and the stub is copied flat " + "over the swept tree. Anything importing this module inside the " + "container gets the generated stub instead of the real code. " + "Rename the yaml to match its own entrypoint.", + ) + + +# ------------------------------------------------------------------ # +# V030-V031 capability-gated rules # +# ------------------------------------------------------------------ # + + +def check_env_file(report, config, config_path, project_dir): + """V030 -- gated on detected env-file injection support.""" + declared = config.get("env_file") + supported = report.capabilities.get("env_file") + + if not supported: + if declared: + report.error( + "V030", + config_path, + line_of(config, "env_file"), + f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", + "No resolve_env_file in the importable ventis package, so the " + "key is silently dropped and the container answers a provider " + "credential error on the first request. This port requires the " + "`env_file` runtime capability.", + ) + else: + report.unavailable( + "V030", + "env_file is not supported by the importable `ventis` runtime. " + "Credentials have no declared path into a container on this tree.", + ) + return + + if not declared: + report.warn( + "V030", + config_path, + line_of(config), + "no `env_file:` in the config", + "Only runtime-managed VENTIS_* variables are guaranteed without it. " + "If the source reads credentials from the environment, the first " + "request fails on a provider error.", + ) + return + + # Path existence and readability are deploy-preflight checks. Do not + # duplicate them here. + + +def check_import_root(report, project_dir, entrypoint_paths): + """V031 -- gated on detected editable-install support.""" + supported = report.capabilities.get("editable_install") + has_metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + + non_flat = [] + for path in entrypoint_paths: + tree, _ = parse_python(path) + if tree is None: + continue + for name, lineno in toplevel_import_names(tree).items(): + if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: + continue + if _resolves_flat(project_dir, name): + continue + location = _resolves_nested(project_dir, name) + if location: + non_flat.append((path, lineno, name, location)) + + if not supported: + report.unavailable( + "V031", + "the editable install (`-e .`) is not supported by the importable " + "`ventis` runtime. Only names rooted at /app import inside a container.", + ) + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, which is not at the " + "project root", + "sys.path[0] is /app and this CanyonOS Core runs no editable install, " + "so only modules swept to the root import. The adapter raises " + "ModuleNotFoundError inside _load_agent and the first request " + "answers 'No agent loaded'.", + ) + return + + if non_flat and not has_metadata: + for path, lineno, name, location in non_flat: + report.error( + "V031", + path, + lineno, + f"`import {name}` resolves to {location}, and the project root " + "has no packaging metadata", + "A pyproject.toml, setup.py or setup.cfg at the port root is " + "what adds `-e .`; metadata nested inside the untouched source " + "tree is ignored. Add minimal root metadata pointing at the " + "existing package directory. Without it the install is skipped " + "silently.", + ) + + +def _resolves_flat(project_dir, name): + """Whether Python can resolve `name` with /app as its import root. + + A directory does not need __init__.py: PEP 420 namespace packages resolve + from sys.path just like regular packages. + """ + return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( + os.path.join(project_dir, name) + ) + + +def _resolves_nested(project_dir, name): + """Where below /app `name` lives but cannot resolve as a top-level name.""" + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] + if root == project_dir: + continue + if f"{name}.py" in files: + return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) + if name in dirs: + return os.path.relpath(os.path.join(root, name), project_dir) + return None + + +# ------------------------------------------------------------------ # +# W003, W006 secrets and imports a green build does not reject # +# ------------------------------------------------------------------ # + + +def check_secrets(report, port_paths): + """W003 -- env_file is the way in; nothing else is.""" + for path in port_paths: + try: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError: + continue + for number, line in enumerate(lines, start=1): + for pattern, description in SECRET_PATTERNS: + if pattern.search(line): + report.warn( + "W003", + path, + number, + f"this line looks like {description}", + "Never put a secret in the source tree or the build " + "context. The build sweeps the project into every " + "image.", + ) + break + + tree, _ = parse_python(path) + if tree is None: + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if not ( + isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and node.value.value.strip() + ): + continue + for target in node.targets: + if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): + report.warn( + "W003", + path, + node.lineno, + f"`{target.id}` is assigned a literal string", + "Read it from the environment instead; the build sweeps " + "this file into every image.", + ) + + +def _pyproject_dependencies(project_dir): + """What `-e .` installs alongside `requirements:`, or None if unreadable. + + None and the empty set mean different things here: empty means the project + declares no dependencies, None means we could not find out -- a setup.py, or + a tomllib this interpreter does not have. The caller must not treat the + second as the first, or it warns about imports the install would satisfy. + """ + path = os.path.join(project_dir, "pyproject.toml") + if not os.path.isfile(path): + return None + try: + import tomllib + except ImportError: # < 3.11 + return None + try: + with open(path, "rb") as handle: + data = tomllib.load(handle) + except Exception: # noqa: BLE001 - malformed metadata is uv's error to give + return None + deps = (data.get("project") or {}).get("dependencies") + if not isinstance(deps, list): + return None + return {_normalize_distribution(d) for d in deps if isinstance(d, str)} + + +def check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path +): + """W006 -- an import the container cannot satisfy.""" + tree, _ = parse_python(entrypoint_path) + if tree is None: + return + + declared = { + _normalize_distribution(item) + for item in (entry.get("requirements") or []) + if isinstance(item, str) + } + + # Where the editable install exists, `-e .` resolves the project's own + # [project.dependencies] in the same pass as `requirements:`. Warning about + # those is a false positive, and a false warning about a dependency is worse + # than none: it teaches the reader to dismiss this check. + editable = report.capabilities.get("editable_install") + metadata = any( + os.path.isfile(os.path.join(project_dir, name)) + for name in ("pyproject.toml", "setup.py", "setup.cfg") + ) + unreadable_metadata = False + if editable and metadata: + project_deps = _pyproject_dependencies(project_dir) + if project_deps is None: + unreadable_metadata = True + else: + declared |= project_deps + base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} + stdlib = getattr(sys, "stdlib_module_names", frozenset()) + + for name, lineno in sorted(toplevel_import_names(tree).items()): + if name in stdlib or name == "ventis": + continue + # Provided by the image itself: the shared runtime is copied flat, and + # every agents/*.yaml generates a stub that is copied flat too. + if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: + continue + if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): + continue + distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) + if distribution in base or distribution in declared: + continue + if unreadable_metadata: + mechanism = ( + "The container installs the base list, `requirements:`, and -- " + "since this project declares packaging metadata -- whatever " + "`-e .` resolves from it. That metadata could not be read here, " + f"so if it already requires `{name}` this line is noise; " + "otherwise it is a ModuleNotFoundError inside _load_agent and " + "'No agent loaded' on the first request." + ) + else: + mechanism = ( + "The container installs the base list plus `requirements:` and " + "nothing else, so this is a ModuleNotFoundError inside " + "_load_agent and 'No agent loaded' on the first request. If the " + f"distribution is named something other than `{name}`, declare " + f"that name in {report.rel(config_path)}." + ) + report.warn( + "W006", + entrypoint_path, + lineno, + f"`import {name}` is in neither the runtime's base list nor this " + "entry's `requirements:`", + mechanism, + ) + + +def _normalize_distribution(name): + return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( + "_", "-" + ) + + +# ------------------------------------------------------------------ # +# Driver # +# ------------------------------------------------------------------ # + + +def validate(project_dir, config_path, capabilities): + """Inspect only failures hidden behind a successful image build.""" + report = Report(project_dir, capabilities) + + # The build owns config/YAML syntax and shape validation. We read only enough + # valid structure to locate code for the deeper checks below. + config, error = load_yaml(config_path) + if error is not None or not isinstance(config, dict): + report.unavailable( + "BUILD", + "runtime preflight skipped because the config cannot be read; " + "ventis build owns and reports this error.", + ) + return report + + import glob + + yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) + report.stub_module_names = { + os.path.splitext(os.path.basename(path))[0] for path in yaml_paths + } + + agents_by_name = {} + stub_classes = {} + for path in yaml_paths: + data, yaml_error = load_yaml(path) + agent = data.get("agent") if isinstance(data, dict) else None + name = agent.get("name") if isinstance(agent, dict) else None + if yaml_error is not None or not isinstance(name, str): + continue # ventis build reports malformed agent declarations + agents_by_name[name] = (path, agent) + stub_classes[os.path.splitext(os.path.basename(path))[0]] = name + + entries = config.get("agents") + if not isinstance(entries, list): + report.unavailable( + "BUILD", + "runtime preflight skipped because `agents:` is not a list; " + "ventis build owns and reports this error.", + ) + return report + + entrypoints = [] + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": + continue + name = entry.get("name") + entrypoint = entry.get("entrypoint") + if isinstance(entrypoint, str): + entrypoints.append(entrypoint) + if name in agents_by_name: + yaml_path, agent_block = agents_by_name[name] + check_adapter(report, yaml_path, agent_block, entry, project_dir) + entrypoint_path = os.path.join(project_dir, entrypoint or "") + if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): + check_requirements_coverage( + report, project_dir, entry, entrypoint_path, config_path + ) + + for entry in entries: + if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": + continue + workflow_file = entry.get("workflow_file") + if not isinstance(workflow_file, str): + continue + workflow_path = os.path.join(project_dir, workflow_file) + if os.path.isfile(workflow_path): + check_workflow(report, workflow_path, stub_classes) + + # These survive a green build and otherwise surface only in a container or + # on its first request. + check_flat_collisions(report, project_dir, yaml_paths, entrypoints) + check_env_file(report, config, config_path, project_dir) + + entrypoint_paths = [ + os.path.join(project_dir, e) + for e in entrypoints + if os.path.isfile(os.path.join(project_dir, e)) + ] + check_import_root(report, project_dir, entrypoint_paths) + + port_paths = list(entrypoint_paths) + for entry in entries: + if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): + candidate = os.path.join(project_dir, entry["workflow_file"]) + if os.path.isfile(candidate): + port_paths.append(candidate) + + # Secret detection remains because a green image build would permanently + # bake the credential into every image. + check_secrets(report, port_paths) + return report + + + +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + +LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} + + +def _wrap(text, width, indent): + words = text.split() + lines = [] + current = "" + for word in words: + candidate = f"{current} {word}".strip() + if len(candidate) + len(indent) > width and current: + lines.append(indent + current) + current = word + else: + current = candidate + if current: + lines.append(indent + current) + return lines + + +def print_report(report, project_dir): + caps = report.capabilities + if not caps.get("ventis"): + print("ventis is not importable here -- capability-gated rules are") + print("reported UNAVAILABLE rather than checked.\n") + else: + print("CanyonOS Core capabilities detected:") + for key, source in CAPABILITY_SOURCE.items(): + mark = "yes" if caps.get(key) else "no " + print(f" {mark} {key:<22} {source}") + print() + + findings = sorted( + report.findings, + key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), + ) + for finding in findings: + where = finding["path"] + if where and finding["line"]: + where = f"{where}:{finding['line']}" + header = f"{finding['check']} {finding['level']:<5}" + print(f"{header} {where}" if where else header) + for line in _wrap(finding["summary"], 78, " "): + print(line) + if finding["mechanism"]: + for line in _wrap(finding["mechanism"], 78, " "): + print(line) + print() + + errors, warnings = report.counts() + if not findings: + print(f"{project_dir}: clean.") + return + print(f"{errors} error(s), {warnings} warning(s).") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Check a CanyonOS Core port against the rules in SKILL.md." + ) + parser.add_argument( + "project_dir", nargs="?", default=".", help="the port's project root" + ) + parser.add_argument( + "-c", + "--config", + default=DEFAULT_CONFIG_PATH, + help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", + ) + parser.add_argument("--json", action="store_true", help="emit findings as JSON") + parser.add_argument( + "--strict", action="store_true", help="fail on warnings as well as errors" + ) + args = parser.parse_args(argv) + + project_dir = os.path.abspath(args.project_dir) + config_path = ( + args.config + if os.path.isabs(args.config) + else os.path.join(project_dir, args.config) + ) + + capabilities = probe_capabilities() + report = validate(project_dir, config_path, capabilities) + errors, warnings = report.counts() + + if args.json: + print( + json.dumps( + { + "project_dir": project_dir, + "capabilities": capabilities, + "errors": errors, + "warnings": warnings, + "findings": report.findings, + }, + indent=2, + ) + ) + else: + print_report(report, report.rel(project_dir) or project_dir) + + if errors or (args.strict and warnings): + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/portfolio/agents/advisor_agent.py b/examples/portfolio/agents/advisor_agent.py index 0915eea..82d8936 100644 --- a/examples/portfolio/agents/advisor_agent.py +++ b/examples/portfolio/agents/advisor_agent.py @@ -2,7 +2,7 @@ # # Final stage. Turns the computed portfolio metrics and risk figures into a # short, plain-English briefing using a small, cheap model on AWS Bedrock -# (Converse API), called via ventis.llm.bedrock so token/cost telemetry gets +# (Converse API), called via ventis.controller.bedrock so token/cost telemetry gets # recorded onto this execution's future: hash. Configure # with env vars: # BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) @@ -16,7 +16,7 @@ import os try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock diff --git a/examples/portfolio/agents/intent_agent.py b/examples/portfolio/agents/intent_agent.py index d74b27b..04124cf 100644 --- a/examples/portfolio/agents/intent_agent.py +++ b/examples/portfolio/agents/intent_agent.py @@ -7,7 +7,7 @@ # -> {"holdings": {"AAPL": 0.4, "MSFT": 0.35, "NVDA": 0.25}, # "lookback_days": 180} # -# Calls AWS Bedrock (Converse API) via ventis.llm.bedrock -- same pattern as +# Calls AWS Bedrock (Converse API) via ventis.controller.bedrock -- same pattern as # AdvisorAgent -- so token/cost telemetry gets recorded onto this execution's # future: hash. Configure with env vars: # BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) @@ -25,7 +25,7 @@ import json try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 96f371c..dbff7bb 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -16,7 +16,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 0b: price history fetch. Network/IO-bound, cheap CPU. Called by @@ -28,7 +28,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/price_agent.py - provider: EC2 + provider: local instance_type: t3.micro requirements: [yfinance] @@ -41,7 +41,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/metrics_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 2: portfolio-level risk aggregation. Single call per request; needs @@ -53,7 +53,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/risk_agent.py - provider: EC2 + provider: local instance_type: t3.micro # Stage 3: LLM briefing via Bedrock. On the critical path, one call per @@ -65,7 +65,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/advisor_agent.py - provider: EC2 + provider: local instance_type: t3.micro # The workflow, exposed as a REST API. @@ -75,27 +75,23 @@ agents: redis_port: 6379 replicas: 1 workflow_file: workflow/portfolio_workflow.py - provider: EC2 + provider: local instance_type: t3.micro otel: + # The dashboard api's own OTLP ingest. Must be the full url including the + # path: the http exporter uses an explicitly-passed endpoint verbatim and + # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. destinations: - - name: railway - protocol: grpc - endpoint: ${RAILWAY_OTLP_ENDPOINT} - insecure: true - headers: {} - - name: grafana + - name: local protocol: http - endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces - headers: - Authorization: Basic ${GRAFANA_OTLP_HEADERS} + endpoint: http://host.docker.internal:3000/v1/traces + headers: {} # Polling interval in seconds -poll_interval: 5 +poll_interval: 4 -# Redis connection redis: host: localhost port: 6379 @@ -111,5 +107,3 @@ ec2: ssh_user: ${EC2_SSH_USER} ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} -database: - url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 619c4bd..8b0a76a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,7 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query) + intent = json.loads(intent_agent.parse(query=query).value()) holdings = intent["holdings"] lookback_days = intent["lookback_days"] diff --git a/examples/text2sql/agents/vllm_agent.py b/examples/text2sql/agents/vllm_agent.py index 4a7a245..ea23616 100644 --- a/examples/text2sql/agents/vllm_agent.py +++ b/examples/text2sql/agents/vllm_agent.py @@ -1,7 +1,7 @@ # VLLM Agent # # LLM backend for SQL candidate generation, called remotely by -# SQLGeneratorAgent. Calls AWS Bedrock (Converse API) via ventis.llm.bedrock +# SQLGeneratorAgent. Calls AWS Bedrock (Converse API) via ventis.controller.bedrock # so token/cost telemetry gets recorded onto this execution's # future: hash — same pattern as # examples/portfolio/agents/advisor_agent.py. @@ -14,7 +14,7 @@ import os try: - from ventis.llm.bedrock import call_bedrock + from ventis.controller.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock From 7cbd976185527e46019532766cc53deca51c7e79 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:24:43 -0700 Subject: [PATCH 2/6] removed some useless code --- SESSION_NOTES.md | 79 -- examples/epigenomics/README.md | 70 - examples/epigenomics/agents/dedup_agent.py | 44 - examples/epigenomics/agents/dedup_agent.yaml | 10 - examples/epigenomics/agents/filter_agent.py | 32 - examples/epigenomics/agents/filter_agent.yaml | 12 - examples/epigenomics/agents/index_agent.py | 30 - examples/epigenomics/agents/index_agent.yaml | 12 - examples/epigenomics/agents/map_agent.py | 33 - examples/epigenomics/agents/map_agent.yaml | 14 - examples/epigenomics/agents/sort_agent.py | 32 - examples/epigenomics/agents/sort_agent.yaml | 14 - examples/epigenomics/agents/split_agent.py | 27 - examples/epigenomics/agents/split_agent.yaml | 12 - .../epigenomics/config/global_controller.yaml | 76 - examples/epigenomics/config/policy.yaml | 20 - .../workflow/epigenomics_workflow.py | 97 -- examples/joke_writer/.car/app/.env.example | 20 - examples/joke_writer/.car/app/LICENSE | 21 - examples/joke_writer/.car/app/README.md | 177 --- .../joke_writer/.car/app/joke_workflow.py | 39 - examples/joke_writer/.car/app/joke_writer.py | 164 --- .../.car/config/global_controller.yaml | 52 - .../joke_writer/.car/config/joke_agent.yaml | 35 - examples/joke_writer/.env.example | 20 - examples/joke_writer/LICENSE | 21 - examples/joke_writer/README.md | 177 --- examples/joke_writer/joke_writer.py | 151 -- .../skills/porting-to-canyonos-core/SKILL.md | 240 ---- .../references/ec2.md | 41 - .../references/llm-proxy.md | 64 - .../references/packaging.md | 81 -- .../references/runtime-contract.md | 187 --- .../references/troubleshooting.md | 60 - .../porting-to-canyonos-core/validate.py | 1257 ----------------- 35 files changed, 3421 deletions(-) delete mode 100644 SESSION_NOTES.md delete mode 100644 examples/epigenomics/README.md delete mode 100644 examples/epigenomics/agents/dedup_agent.py delete mode 100644 examples/epigenomics/agents/dedup_agent.yaml delete mode 100644 examples/epigenomics/agents/filter_agent.py delete mode 100644 examples/epigenomics/agents/filter_agent.yaml delete mode 100644 examples/epigenomics/agents/index_agent.py delete mode 100644 examples/epigenomics/agents/index_agent.yaml delete mode 100644 examples/epigenomics/agents/map_agent.py delete mode 100644 examples/epigenomics/agents/map_agent.yaml delete mode 100644 examples/epigenomics/agents/sort_agent.py delete mode 100644 examples/epigenomics/agents/sort_agent.yaml delete mode 100644 examples/epigenomics/agents/split_agent.py delete mode 100644 examples/epigenomics/agents/split_agent.yaml delete mode 100644 examples/epigenomics/config/global_controller.yaml delete mode 100644 examples/epigenomics/config/policy.yaml delete mode 100644 examples/epigenomics/workflow/epigenomics_workflow.py delete mode 100644 examples/joke_writer/.car/app/.env.example delete mode 100644 examples/joke_writer/.car/app/LICENSE delete mode 100644 examples/joke_writer/.car/app/README.md delete mode 100644 examples/joke_writer/.car/app/joke_workflow.py delete mode 100644 examples/joke_writer/.car/app/joke_writer.py delete mode 100644 examples/joke_writer/.car/config/global_controller.yaml delete mode 100644 examples/joke_writer/.car/config/joke_agent.yaml delete mode 100644 examples/joke_writer/.env.example delete mode 100644 examples/joke_writer/LICENSE delete mode 100644 examples/joke_writer/README.md delete mode 100644 examples/joke_writer/joke_writer.py delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md delete mode 100644 examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md delete mode 100755 examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/SESSION_NOTES.md b/SESSION_NOTES.md deleted file mode 100644 index 360efcc..0000000 --- a/SESSION_NOTES.md +++ /dev/null @@ -1,79 +0,0 @@ -# Session notes: canyonos serve, OTel pipeline, examples - -## Fixed (code changes, rebuilt+pushed `saakeths/canyonos:latest` where needed) - -1. **`ventis/stub_generator.py`**: stubs only ever got placed at ONE path - (nested-at-entrypoint, never flat), breaking any workflow that imports a - sibling agent directly (`from split_agent import SplitAgent`, e.g. - `examples/epigenomics`). Now placed at both. -2. **`otel_exporter.py`**: hardcoded Redis to `localhost`, but it runs inside - the GC container (bridge networking) while Redis is a sibling container — - crash-looped forever. Fixed to `host.docker.internal`. -3. **`ventis/OTLP_Exporter/db.py`**: `write_waiting_rows()` unconditionally - priced every future via a `aws_instance_pricing` table that only exists for - EC2 deployments — silently dropped **every** span for **every** - `provider: local` deployment, always. Wrapped cost lookups in try/except, - falls back to $0. -4. **`dashboard_stack.py`**: `web` port was hardcoded 8080 with no fallback. - Now searches for a free port (reuses an already-running dashboard's port - if one exists), same pattern as `init.py`'s GC port selection. -5. **`dashboard_stack.py`**: `database.url` was required; made optional - (dashboard boots fine with no DB configured in the project's own config). -6. **`dashboard.compose.yml`**: added `pull_policy: never` for the local-only - `canyonos-otel-receiver:local` image (compose was trying to pull it from a - registry that doesn't have it). Sequenced `otel-receiver` to start after - `api` (both raced to create `otel_spans`; `api`'s bare `CREATE TABLE` lost - the race and crashed). - -## Bundled (new, working) - -- `db` (plain Postgres) + `otel-receiver` (new `Dockerfile` for - `otlp_pg_receiver`, built locally as `canyonos-otel-receiver:local`) added - to `dashboard.compose.yml`. Verified real spans, sent via the actual OTLP - gRPC exporter, land in `otel_spans`. - -## Workarounds applied, NOT real fixes (will resurface) - -- **`canyonos quit` only tears down the GC container + volume**, never the - deployed agent/workflow/redis containers. Had to `docker rm -f` those by - exact name every time before a truly clean restart. -- **The named workspace volume is additive-only** (`docker cp`, never - clears) — files from a previous project leak into the next one's build - until you manually nuke the volume. -- **`otlp_pg_receiver` holds one Postgres connection with no reconnect - logic** — a DB restart silently kills every future write until the - receiver container itself is restarted. -- **`joke_writer/.car` layout** (`app/`, `config/`) doesn't match what - `ventis/cli.py`'s build step expects (`agents/`, flat entrypoints) — worked - around by manually copying files into the shape it wants. The real fix - (`nickhuo/car-artifact-layout`, already pushed) was not merged in. -- **joke_writer's LLM calls are stubbed to return `"animal"`** — for testing - only, real Bedrock creds needed to restore actual behavior (commented-out - code left in place). - -## Known, not touched - -- Pre-existing OrbStack local-provider startup race (first request right - after a container reports healthy can fail); a fix exists on an unrelated, - unmerged branch. -- Stale global `uv tool install` is a recurring trap — always - `uv tool install --reinstall .` after any `cli/` change. - -## What's still needed to actually see data in the UI - -The whole pipeline up to Postgres now genuinely works. **Nothing shows up in -the dashboard because `canyonos-api`/`canyonos-web` have zero code that reads -or displays `otel_spans`** — confirmed by inspecting their actual source -(they're a `cc-forge` rebrand: deploy/project management + a static -code-structure diagram, unrelated data model). To close the loop: - -1. New API route(s) in `canyon-code-forge/packages/api` that query - `otel_spans`. -2. New UI screen(s) in `canyon-code-forge/apps/web` to render it. -3. `web` currently has **no path to reach `api` at all** even once that - exists — no reverse proxy in its Caddyfile, and `api`'s port isn't - published to the host in `dashboard.compose.yml`. Needs one or the other - before the browser can fetch anything. - -All of the above is real feature work in a different repo, not a config or -wiring fix. diff --git a/examples/epigenomics/README.md b/examples/epigenomics/README.md deleted file mode 100644 index 19fb748..0000000 --- a/examples/epigenomics/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Epigenomics Example - -A synthetic, LLM-free workflow modeled on the -[WfCommons Epigenomics recipe](https://docs.wfcommons.org/en/latest/generating_workflows.html): -a split → fan-out (filter → align → sort) → fan-in (dedup) → index pipeline. -Every stage does deterministic SHA-256 work sized off chunk byte counts, so -results are reproducible and the fan-out width scales with `num_chunks` -- -useful for exercising scheduling/replica behavior locally without any real -model calls. - -## Pipeline - -``` -SplitAgent --> FilterAgent --> MapAgent --> SortAgent --\ - (1 call) (fan-out) (fan-out) (fan-out) --> DedupAgent --> IndexAgent - (fan-in barrier) (1 call) -``` - -- **SplitAgent** — splits `input_size` bytes into `num_chunks` equal chunks. -- **FilterAgent** — per-chunk contaminant filter (light cost). -- **MapAgent** — per-chunk alignment, the heaviest stage. -- **SortAgent** — per-chunk sort (moderate cost). -- **DedupAgent** — merges every sorted chunk into one digest (the barrier). -- **IndexAgent** — builds the final index from the merged digest. - -## Quick Start - -```bash -# Build stubs and Docker images -ventis build - -# Launch all agents -ventis deploy - -# Test with curl -curl -X POST http://:8080/main \ - -H 'Content-Type: application/json' \ - -d '{"input_size": 65536, "num_chunks": 4}' - -# Check result -curl http://:8080/status/ -``` - -## Project Structure - -``` -├── agents/ # Agent implementations and YAML definitions -│ ├── split_agent.py/.yaml -│ ├── filter_agent.py/.yaml -│ ├── map_agent.py/.yaml -│ ├── sort_agent.py/.yaml -│ ├── dedup_agent.py/.yaml -│ └── index_agent.py/.yaml -├── workflow/ # Workflow script (deployed as a REST API) -│ └── epigenomics_workflow.py -└── config/ - ├── global_controller.yaml # Deployment configuration (provider: local) - └── policy.yaml # Access control rules -``` - -## Policy Rules - -Edit `config/policy.yaml` to control which callers can access which agents. -Pass `_context` in your curl request to set the caller identity: - -```bash -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' \ - -d '{"input_size": 65536, "num_chunks": 4, "_context": {"origin": "admin"}}' -``` diff --git a/examples/epigenomics/agents/dedup_agent.py b/examples/epigenomics/agents/dedup_agent.py deleted file mode 100644 index d883854..0000000 --- a/examples/epigenomics/agents/dedup_agent.py +++ /dev/null @@ -1,44 +0,0 @@ -# Dedup Agent -# -# Fan-in barrier (mirrors Epigenomics' mark-duplicates/merge stage): needs -# every sorted chunk before it can run. Combines all chunk digests into one -# merged digest, with cost scaling off the total merged data volume. -# -# Resource profile: moderate CPU, single call per request (the barrier). - -import hashlib - - -class DedupAgent(object): - def __init__(self): - self.tools = [self.merge_dedup] - - def merge_dedup(self, chunks: list) -> dict: - """Merge and deduplicate every sorted chunk into one combined digest.""" - total_size = sum(c["size"] for c in chunks) - seed = "".join(c["digest"] for c in sorted(chunks, key=lambda c: c["chunk_id"])) - merged_digest = self._cpu_work(seed, total_size) - return { - "merged_digest": merged_digest, - "total_size": total_size, - "n_chunks": len(chunks), - } - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = DedupAgent() - print( - agent.merge_dedup( - [ - {"chunk_id": "chunk-0", "size": 16384, "digest": "aa"}, - {"chunk_id": "chunk-1", "size": 16384, "digest": "bb"}, - ] - ) - ) diff --git a/examples/epigenomics/agents/dedup_agent.yaml b/examples/epigenomics/agents/dedup_agent.yaml deleted file mode 100644 index 1c577cd..0000000 --- a/examples/epigenomics/agents/dedup_agent.yaml +++ /dev/null @@ -1,10 +0,0 @@ -agent: - name: DedupAgent - functions: - - name: merge_dedup - description: Merge and deduplicate every sorted chunk into one combined digest. - arguments: - - name: chunks - type: list - returns: - type: dict diff --git a/examples/epigenomics/agents/filter_agent.py b/examples/epigenomics/agents/filter_agent.py deleted file mode 100644 index 24dfc48..0000000 --- a/examples/epigenomics/agents/filter_agent.py +++ /dev/null @@ -1,32 +0,0 @@ -# Filter Agent -# -# First per-chunk stage in the fan-out (mirrors Epigenomics' filter_contams -# stage): scrubs one chunk and hands back a content digest the later stages -# build on. The "work" is a deterministic SHA-256 chain sized off the -# chunk's declared byte size, standing in for the real stage's per-byte cost. -# -# Resource profile: light CPU, high fan-out (one call per chunk). - -import hashlib - - -class FilterAgent(object): - def __init__(self): - self.tools = [self.filter_contams] - - def filter_contams(self, chunk_id: str, size: int) -> dict: - """Filter contaminants out of one chunk, returning its content digest.""" - digest = self._cpu_work(chunk_id, size) - return {"chunk_id": chunk_id, "size": size, "digest": digest} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = FilterAgent() - print(agent.filter_contams("chunk-0", 16384)) diff --git a/examples/epigenomics/agents/filter_agent.yaml b/examples/epigenomics/agents/filter_agent.yaml deleted file mode 100644 index 9f10d29..0000000 --- a/examples/epigenomics/agents/filter_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: FilterAgent - functions: - - name: filter_contams - description: Filter contaminants out of one chunk, returning its content digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - returns: - type: dict diff --git a/examples/epigenomics/agents/index_agent.py b/examples/epigenomics/agents/index_agent.py deleted file mode 100644 index 6faa984..0000000 --- a/examples/epigenomics/agents/index_agent.py +++ /dev/null @@ -1,30 +0,0 @@ -# Index Agent -# -# Final stage (mirrors Epigenomics' index-build stage): produces the -# workflow's terminal artifact from the merged, deduplicated digest. -# -# Resource profile: light CPU, single call per request. - -import hashlib - - -class IndexAgent(object): - def __init__(self): - self.tools = [self.build_index] - - def build_index(self, merged_digest: str, total_size: int) -> dict: - """Build the final index from the merged digest.""" - index_digest = self._cpu_work(merged_digest, max(1, total_size // 4)) - return {"index_digest": index_digest, "total_size": total_size} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = IndexAgent() - print(agent.build_index("deadbeef", 65536)) diff --git a/examples/epigenomics/agents/index_agent.yaml b/examples/epigenomics/agents/index_agent.yaml deleted file mode 100644 index 3424c44..0000000 --- a/examples/epigenomics/agents/index_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: IndexAgent - functions: - - name: build_index - description: Build the final index from the merged digest. - arguments: - - name: merged_digest - type: str - - name: total_size - type: int - returns: - type: dict diff --git a/examples/epigenomics/agents/map_agent.py b/examples/epigenomics/agents/map_agent.py deleted file mode 100644 index b9db3ef..0000000 --- a/examples/epigenomics/agents/map_agent.py +++ /dev/null @@ -1,33 +0,0 @@ -# Map Agent -# -# Sequence-alignment stand-in (Epigenomics' map stage) -- by far the most -# CPU-expensive stage in the real workflow, so its per-byte cost multiplier -# here is set well above the other stages to match that shape. -# -# Resource profile: heavy CPU, high fan-out (one call per chunk). - -import hashlib - -COST_MULTIPLIER = 8 - - -class MapAgent(object): - def __init__(self): - self.tools = [self.align] - - def align(self, chunk_id: str, size: int, digest: str) -> dict: - """Align one filtered chunk, returning its post-alignment digest.""" - aligned = self._cpu_work(digest, size * COST_MULTIPLIER) - return {"chunk_id": chunk_id, "size": size, "digest": aligned} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = MapAgent() - print(agent.align("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/map_agent.yaml b/examples/epigenomics/agents/map_agent.yaml deleted file mode 100644 index 66c1210..0000000 --- a/examples/epigenomics/agents/map_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: MapAgent - functions: - - name: align - description: Align one filtered chunk, returning its post-alignment digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - - name: digest - type: str - returns: - type: dict diff --git a/examples/epigenomics/agents/sort_agent.py b/examples/epigenomics/agents/sort_agent.py deleted file mode 100644 index 2aa2df7..0000000 --- a/examples/epigenomics/agents/sort_agent.py +++ /dev/null @@ -1,32 +0,0 @@ -# Sort Agent -# -# Third per-chunk stage in the fan-out (mirrors Epigenomics' sort stage): -# orders one aligned chunk, returning an updated digest for the fan-in below. -# -# Resource profile: moderate CPU, high fan-out (one call per chunk). - -import hashlib - -COST_MULTIPLIER = 2 - - -class SortAgent(object): - def __init__(self): - self.tools = [self.sort] - - def sort(self, chunk_id: str, size: int, digest: str) -> dict: - """Sort one aligned chunk, returning its post-sort digest.""" - sorted_digest = self._cpu_work(digest, size * COST_MULTIPLIER) - return {"chunk_id": chunk_id, "size": size, "digest": sorted_digest} - - def _cpu_work(self, seed: str, iterations: int) -> str: - """Deterministic CPU-bound stand-in for the stage's real processing cost.""" - digest = seed.encode() - for _ in range(max(1, iterations)): - digest = hashlib.sha256(digest).digest() - return digest.hex() - - -if __name__ == "__main__": - agent = SortAgent() - print(agent.sort("chunk-0", 16384, "deadbeef")) diff --git a/examples/epigenomics/agents/sort_agent.yaml b/examples/epigenomics/agents/sort_agent.yaml deleted file mode 100644 index 464dc85..0000000 --- a/examples/epigenomics/agents/sort_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: SortAgent - functions: - - name: sort - description: Sort one aligned chunk, returning its post-sort digest. - arguments: - - name: chunk_id - type: str - - name: size - type: int - - name: digest - type: str - returns: - type: dict diff --git a/examples/epigenomics/agents/split_agent.py b/examples/epigenomics/agents/split_agent.py deleted file mode 100644 index 1b1a6be..0000000 --- a/examples/epigenomics/agents/split_agent.py +++ /dev/null @@ -1,27 +0,0 @@ -# Split Agent -# -# Entry stage of the pipeline (mirrors WfCommons' Epigenomics fastq-split -# stage): splits one logical input into num_chunks equal-sized chunks for the -# downstream fan-out. There's no real sequence file here -- each chunk's -# "size" just stands in for its data volume, which is what every downstream -# stage prices its synthetic CPU work off of. -# -# Resource profile: cheap CPU, single call per request. - - -class SplitAgent(object): - def __init__(self): - self.tools = [self.split] - - def split(self, input_size: int, num_chunks: int) -> dict: - """Split input_size bytes of data into num_chunks equal chunks.""" - chunk_size = max(1, input_size // num_chunks) - chunks = [ - {"chunk_id": f"chunk-{i}", "size": chunk_size} for i in range(num_chunks) - ] - return {"chunks": chunks} - - -if __name__ == "__main__": - agent = SplitAgent() - print(agent.split(65536, 4)) diff --git a/examples/epigenomics/agents/split_agent.yaml b/examples/epigenomics/agents/split_agent.yaml deleted file mode 100644 index cc64e8e..0000000 --- a/examples/epigenomics/agents/split_agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -agent: - name: SplitAgent - functions: - - name: split - description: Split input_size bytes of data into num_chunks equal chunks. - arguments: - - name: input_size - type: int - - name: num_chunks - type: int - returns: - type: dict diff --git a/examples/epigenomics/config/global_controller.yaml b/examples/epigenomics/config/global_controller.yaml deleted file mode 100644 index 6e8b4b0..0000000 --- a/examples/epigenomics/config/global_controller.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Global Controller Configuration — synthetic Epigenomics DAG, local provider -# Lists all agents and the workflow that Ventis manages. -# -# FilterAgent/MapAgent/SortAgent get 2 replicas each since the workflow fans -# out one call per chunk to them -- exercises multi-replica scheduling on a -# purely local, LLM-free run. - -agents: - - name: SplitAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/split_agent.py - provider: local - - - name: FilterAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/filter_agent.py - provider: local - - - name: MapAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/map_agent.py - provider: local - - - name: SortAgent - replicas: 2 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/sort_agent.py - provider: local - - - name: DedupAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/dedup_agent.py - provider: local - - - name: IndexAgent - replicas: 1 - redis_port: 6379 - resources: - cpu: 1 - memory: 256 - entrypoint: agents/index_agent.py - provider: local - - - name: Workflow - replicas: 1 - type: workflow - redis_port: 6379 - api_port: 8080 - workflow_file: workflow/epigenomics_workflow.py - provider: local - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 diff --git a/examples/epigenomics/config/policy.yaml b/examples/epigenomics/config/policy.yaml deleted file mode 100644 index 2c91415..0000000 --- a/examples/epigenomics/config/policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Policy-Based Routing Rules — synthetic Epigenomics DAG -# Each rule defines a match condition (key-value pairs to check against -# request context) and an access list of allowed services. -# Rules are evaluated most-specific-first (most matching keys wins). -# An empty match ({}) acts as a default fallback. - -rules: - - match: - origin: admin - access: all - - - match: {} - access: - - Workflow - - SplitAgent - - FilterAgent - - MapAgent - - SortAgent - - DedupAgent - - IndexAgent diff --git a/examples/epigenomics/workflow/epigenomics_workflow.py b/examples/epigenomics/workflow/epigenomics_workflow.py deleted file mode 100644 index 00bcc2b..0000000 --- a/examples/epigenomics/workflow/epigenomics_workflow.py +++ /dev/null @@ -1,97 +0,0 @@ -# Epigenomics Workflow -# -# WfCommons-style synthetic Epigenomics DAG for local, LLM-free testing: -# 0. SplitAgent - split input_size bytes into num_chunks chunks (single call) -# 1. FilterAgent - per-chunk contaminant filter (fan-out) -# 2. MapAgent - per-chunk alignment, the heaviest stage (fan-out) -# 3. SortAgent - per-chunk sort (fan-out) -# 4. DedupAgent - merge + dedup every sorted chunk (fan-in barrier) -# 5. IndexAgent - build the final index from the merged digest (single call) -# -# Every stage does deterministic SHA-256 work sized off chunk byte counts -- -# no LLM calls, no external services -- so results are reproducible and the -# fan-out width scales with num_chunks. -# -# After running `ventis build` and `ventis deploy`: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' \ -# -d '{"input_size": 65536, "num_chunks": 4}' -# curl http://localhost:8080/status/ - -import sys -import os - -# These path inserts are needed when running inside a Docker container -# where all files are copied flat into /app/. -sys.path.insert(0, os.path.dirname(__file__)) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) - -import json - -from deploy import deploy -from agents.split_agent import SplitAgent -from agents.filter_agent import FilterAgent -from agents.map_agent import MapAgent -from agents.sort_agent import SortAgent -from agents.dedup_agent import DedupAgent -from agents.index_agent import IndexAgent - - -def main(input_size: int = 65536, num_chunks: int = 4): - split_agent = SplitAgent() - filter_agent = FilterAgent() - map_agent = MapAgent() - sort_agent = SortAgent() - dedup_agent = DedupAgent() - index_agent = IndexAgent() - - # Stage 0: single call, produces the chunk list the fan-out below runs over. - split = json.loads( - split_agent.split(input_size=input_size, num_chunks=num_chunks).value() - ) - chunks = split["chunks"] - - # Stage 1: fan out one filter call per chunk -- every call returns a Future - # immediately, so all chunks are dispatched before we block on any of them. - filter_futures = { - c["chunk_id"]: filter_agent.filter_contams(chunk_id=c["chunk_id"], size=c["size"]) - for c in chunks - } - filtered = {cid: json.loads(f.value()) for cid, f in filter_futures.items()} - - # Stage 2: fan out alignment -- the heaviest stage -- one call per chunk. - map_futures = { - cid: map_agent.align(chunk_id=cid, size=r["size"], digest=r["digest"]) - for cid, r in filtered.items() - } - mapped = {cid: json.loads(f.value()) for cid, f in map_futures.items()} - - # Stage 3: fan out sort, one call per chunk. - sort_futures = { - cid: sort_agent.sort(chunk_id=cid, size=r["size"], digest=r["digest"]) - for cid, r in mapped.items() - } - sorted_chunks = {cid: json.loads(f.value()) for cid, f in sort_futures.items()} - - # Stage 4: fan-in barrier -- dedup needs every sorted chunk before it can run. - merged = json.loads( - dedup_agent.merge_dedup(chunks=list(sorted_chunks.values())).value() - ) - - # Stage 5: build the final index from the merged digest. - index = json.loads( - index_agent.build_index( - merged_digest=merged["merged_digest"], total_size=merged["total_size"] - ).value() - ) - - return { - "input_size": input_size, - "num_chunks": num_chunks, - "merged_digest": merged["merged_digest"], - "n_chunks": merged["n_chunks"], - "index_digest": index["index_digest"], - } - - -deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/.env.example b/examples/joke_writer/.car/app/.env.example deleted file mode 100644 index b846149..0000000 --- a/examples/joke_writer/.car/app/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Copy this to `.env` and fill in the token. `config/global_controller.yaml` -# points `env_file:` at that copy, and it reaches every container as -# `docker run --env-file`. -# -# Keep the real token out of THIS file. `.env.example` is the one exception to -# the build context's exclusion of `.env*`, so whatever is written here is baked -# into the image; `.env` itself never enters the build and never leaves the host. - -# A Bedrock API key -- the long-term kind generated in the console, or a -# short-term one. botocore matches this exact name against bedrock-runtime's -# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by -# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions -# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and -# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. -AWS_BEARER_TOKEN_BEDROCK= - -# Neither is a secret, and both have defaults in joke_writer.py -- they are here -# to name what the source reads. -BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 -AWS_REGION=us-east-1 diff --git a/examples/joke_writer/.car/app/LICENSE b/examples/joke_writer/.car/app/LICENSE deleted file mode 100644 index 5600729..0000000 --- a/examples/joke_writer/.car/app/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 LangChain, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/examples/joke_writer/.car/app/README.md b/examples/joke_writer/.car/app/README.md deleted file mode 100644 index 930a410..0000000 --- a/examples/joke_writer/.car/app/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Joke Writer - -A LangGraph map-reduce, ported to Ventis. Derived from -[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) -at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). - -Unlike the other targets in `examples/`, **the source here is not unmodified**. -`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port -an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked -at the credential wall until the model call was rewritten onto Bedrock. That wall -is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed -anyway, and [What the port cost](#what-the-port-cost) is honest about what that -means. - -## Overview - -Given a topic, the graph splits it into sub-topics, writes one joke per -sub-topic in parallel, then picks the best of them. - -1. `generate_topics` — one LLM call, turns the topic into three sub-topics, - validated into `Subjects`. -2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a - `Send` per subject, so this node runs N times per request with no shared - state between the runs. `jokes` is an `Annotated[list, operator.add]`, which - is how the N results merge back into one state. -3. `best_joke` — one LLM call over every joke, returns the winner by index. - -``` - START - | - generate_topics 1 call - | - continue_to_jokes Send x N - / | \ - joke joke joke N calls, no shared state - \ | / - best_joke 1 call - | - END -``` - -### Why this one - -It is the smallest project in reach whose control flow does something a single -process cannot: `Send` fans out to N independent calls per request. Everything -else about it is deliberately boring — four packages, no tools, no external -service, one API key. - -## The port - -| File | What it holds | -| --- | --- | -| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | -| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | -| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | -| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | -| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | -| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | - -Two decisions worth naming: - -**One agent, not three.** `generate_topics` and `best_joke` run once per request -and have no resource profile of their own. Splitting them out would buy two more -images and two more Redis round trips. What is hoisted is the fan-out, and that -is a workflow concern. - -**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, -operator.add]` reducer are control flow owned by the LangGraph runtime, and -Ventis has no runtime to execute them. The workflow dispatches N -`generate_joke` calls across the three replicas and concatenates the results -itself. Every call is dispatched before any is resolved — `.value()` blocks, so -fusing the two lines into one comprehension would silently serialize the fan-out -and remove the reason to be on Ventis at all. - -## What the port cost - -This is no longer upstream's model stack. `ChatOpenAI` and -`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw -converse API, so each node asks for JSON in its prompt and validates the reply -through the same pydantic schema upstream used. `_extract_json` exists only -because `with_structured_output` used to do that work. - -That rewrite is not something the `porting-to-ventis` skill should do on a -user's project — it is the credential wall, and the skill's instruction is to -report it. It was done here deliberately, so that this example is one that -actually deploys. - -**It would not be necessary today.** The rewrite bought one thing: boto3 builds -no client at import, so the agent could be *loaded* with no secret in the -container, back when `_launch_locally` passed five `-e` flags and all five were -`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches -a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module -scope would import fine. What the rewrite still buys is narrower: a module-scope -client turns a missing key into `"No agent loaded"`, while a per-call one turns -it into a real error on `/status`. Worth knowing, not worth a rewrite. - -The example stays on Bedrock because it is the model call that has been end-to-end -verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call -token telemetry onto the future. - -## Running it - -Copy `.env.example` to `.env` and put a Bedrock API key in it: - -```shell -cp .env.example .env -$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... -``` - -`config/global_controller.yaml` points `env_file:` at that file, and every -container gets it as `docker run --env-file`. Nothing in this project reads the -variable: botocore matches the name against `bedrock-runtime`'s signingName and -switches the client from SigV4 to bearer auth on its own, so -`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. -An IAM access key instead of the bearer token works the same way. - -`.env` is gitignored and excluded from the build context — the key is in the -container's environment and not in the image. Deploy checks the path before it -launches anything, so a missing `.env` is one error line rather than three -replicas that come up and fail every request. - -```shell -ventis build -ventis deploy -``` - -```shell -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' -curl http://localhost:8080/status/ -``` - -```json -{"request_id": "cb6cb62d...", "status": "done", "result": { - "topic": "animals", - "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], - "jokes": ["...", "...", "..."], - "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" -}} -``` - -`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in -`joke_writer.py`; neither is a secret. The region has to match the one the key -was issued for. - -### Running the source outside Ventis - -`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat -`bedrock` copy an agent image gets, so the compiled graph still runs on its own -from a checkout of this repo: - -```shell -pip install -e ../.. # the ventis package -pip install langgraph pydantic typing_extensions boto3 -``` - -```python -from joke_writer import graph - -graph.invoke({"topic": "animals"}) -``` - -## Provenance - -Taken from `module-4/studio/`, which holds four unrelated graphs sharing one -directory. Only `map_reduce.py` and its license are here. - -| Left behind | Why | -| --- | --- | -| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | -| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | -| The module-4 notebooks | Teaching material for the same code. | -| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | - -Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` -or `requirements.txt`, exactly as upstream has none for module-4. That is why -`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/.car/app/joke_workflow.py b/examples/joke_writer/.car/app/joke_workflow.py deleted file mode 100644 index 76e4027..0000000 --- a/examples/joke_writer/.car/app/joke_workflow.py +++ /dev/null @@ -1,39 +0,0 @@ -r"""CanyonOS Core workflow for the map-reduce joke writer. - -This file is where the graph went. `generate_topics -> continue_to_jokes -> -generate_joke x N -> best_joke` is not a compiled StateGraph any more; it is the -three statements below, and the `Send` fan-out is N calls dispatched across -JokeAgent's replicas. - - curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' - curl http://localhost:8080/status/ -""" - -import json - -from deploy import deploy -from joke_writer import JokeAgent - - -def main(query): - """Route: POST /main {"query": ""}""" - agent = JokeAgent() - - subjects = json.loads(agent.generate_topics(topic=query).value())["subjects"] - - futures = [agent.generate_joke(subject=s) for s in subjects] - written = [json.loads(f.value()) for f in futures] - written_jokes = [joke for result in written for joke in result["jokes"]] - - best = json.loads(agent.best_joke(topic=query, jokes=written_jokes).value()) - - return { - "topic": query, - "subjects": subjects, - "jokes": written_jokes, - "best_selected_joke": best["best_selected_joke"], - } - - -deploy(main, port=8080) diff --git a/examples/joke_writer/.car/app/joke_writer.py b/examples/joke_writer/.car/app/joke_writer.py deleted file mode 100644 index 9f96eb5..0000000 --- a/examples/joke_writer/.car/app/joke_writer.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Map-reduce joke writer. - -Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` -(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are -upstream's. The model call is not: upstream builds a `ChatOpenAI` at module -scope, and when this was ported nothing could carry an OPENAI_API_KEY into an -agent container. Bedrock reaches the model through boto3, which builds no client -at import, so the same code loaded with no secret injected. - -`env_file` has since removed that constraint -- the key now travels to the -container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The -rewrite stayed regardless; README.md says what that costs. - -`with_structured_output` went with it. `call_bedrock` is the raw converse API, so -each node asks for JSON in the prompt and validates the reply through the same -pydantic schema upstream used. -""" - -import json -import operator -import os -import re -from typing import Annotated - -from typing_extensions import TypedDict - -from pydantic import BaseModel, ValidationError - -from langgraph.constants import Send -from langgraph.graph import END, StateGraph, START - -# Ventis copies bedrock.py flat into every agent image; the package path is for -# running this module outside a container. -try: - from ventis.llm.bedrock import call_bedrock -except ImportError: - from bedrock import call_bedrock - -# Prompts we will use. Upstream's, plus the JSON instruction that -# `with_structured_output` used to add on our behalf. -subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. -Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" -joke_prompt = """Generate a joke about {subject}. -Respond with JSON only, no prose: {{"joke": "..."}}""" -best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} -Respond with JSON only, no prose: {{"id": 0}}""" - -# LLM. Both are read once at import; the container gets them from its -# environment, and neither is a secret. -MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") -REGION = os.environ.get("AWS_REGION", "us-east-1") - - -def _extract_json(text): - """Pull the first JSON object out of a model reply. - - Even told to answer with JSON only, a model wraps it in a ```json fence or - prefaces it with a sentence. Upstream never needed this because - `with_structured_output` handled it; the converse API does not. - """ - text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) - try: - return json.loads(text) - except json.JSONDecodeError: - pass - # Fall back to the outermost braced span. - match = re.search(r"\{.*\}", text, flags=re.DOTALL) - if not match: - raise ValueError(f"joke_writer: no JSON in model output: {text!r}") - return json.loads(match.group(0)) - - -def _ask(prompt, schema, max_tokens): - """One converse() call, validated into `schema`. - - Raising on a bad reply is deliberate. A node that returned a default would - put a plausible-looking wrong answer into the state, and the reduce step - downstream indexes into the jokes list by an id the model chose -- a silent - default there picks the wrong joke instead of failing. - """ - response = call_bedrock( - model_id=MODEL_ID, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inference_config={"maxTokens": max_tokens, "temperature": 0.0}, - region=REGION, - ) - text = response["output"]["message"]["content"][0]["text"] - if not text: - raise ValueError("joke_writer: LLM returned no output.") - try: - return schema(**_extract_json(text)) - except (ValidationError, TypeError) as exc: - raise ValueError( - f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" - ) from exc - - -# Define the state -class Subjects(BaseModel): - subjects: list[str] - -class BestJoke(BaseModel): - id: int - -class OverallState(TypedDict): - topic: str - subjects: list - jokes: Annotated[list, operator.add] - best_selected_joke: str - -def generate_topics(state: OverallState): - prompt = subjects_prompt.format(topic=state["topic"]) - response = _ask(prompt, Subjects, max_tokens=300) - return {"subjects": response.subjects} - -class JokeState(TypedDict): - subject: str - -class Joke(BaseModel): - joke: str - -def generate_joke(state: JokeState): - prompt = joke_prompt.format(subject=state["subject"]) - response = _ask(prompt, Joke, max_tokens=300) - return {"jokes": [response.joke]} - -def best_joke(state: OverallState): - jokes = "\n\n".join(state["jokes"]) - prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) - response = _ask(prompt, BestJoke, max_tokens=100) - if not 0 <= response.id < len(state["jokes"]): - raise ValueError( - f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." - ) - return {"best_selected_joke": state["jokes"][response.id]} - -def continue_to_jokes(state: OverallState): - return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] - -# Construct the graph: here we put everything together to construct our graph -graph_builder = StateGraph(OverallState) -graph_builder.add_node("generate_topics", generate_topics) -graph_builder.add_node("generate_joke", generate_joke) -graph_builder.add_node("best_joke", best_joke) -graph_builder.add_edge(START, "generate_topics") -graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) -graph_builder.add_edge("generate_joke", "best_joke") -graph_builder.add_edge("best_joke", END) - -# Compile the graph -graph = graph_builder.compile() - - -class JokeAgent(object): - """The graph's nodes, exposed under the class name `agent.name` declares.""" - - def generate_topics(self, topic: str) -> dict: - return generate_topics({"topic": topic}) - - def generate_joke(self, subject: str) -> dict: - return generate_joke({"subject": subject}) - - def best_joke(self, topic: str, jokes: list) -> dict: - return best_joke({"topic": topic, "jokes": jokes}) diff --git a/examples/joke_writer/.car/config/global_controller.yaml b/examples/joke_writer/.car/config/global_controller.yaml deleted file mode 100644 index 8ed54ab..0000000 --- a/examples/joke_writer/.car/config/global_controller.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Deployment manifest for the map-reduce joke writer. -# -# `entrypoint` is the copied source itself: the adapter is appended to the -# bottom of joke_writer.py, so the module the agent needs is the one the class -# already lives in. - -agents: - - name: JokeAgent - # The fan-out. `generate_joke` is stateless, so the controller picks a - # replica at random per call and the workflow's N dispatched calls spread - # across these three. - entrypoint: joke_writer.py - provider: local - replicas: 3 - redis_port: 6379 - resources: - cpu: 1 - memory: 1024 - requirements: - - langgraph - - pydantic - - typing_extensions - - - name: Workflow - type: workflow - workflow_file: joke_workflow.py - api_port: 8080 - provider: local - replicas: 1 - redis_port: 6379 - -poll_interval: 5 - -redis: - host: localhost - port: 6379 - db: 0 - -otel: - # The dashboard api's own OTLP ingest. Must be the full url including the - # path: the http exporter uses an explicitly-passed endpoint verbatim and - # only appends /v1/traces when reading OTEL_EXPORTER_OTLP_ENDPOINT. - destinations: - - name: local - protocol: http - endpoint: http://host.docker.internal:3000/v1/traces - headers: {} - -# Relative to the application root (the directory `ventis` runs from), not -# `.car`. .env is gitignored and excluded from the build context; -# .env.example names what belongs in it. -env_file: .env diff --git a/examples/joke_writer/.car/config/joke_agent.yaml b/examples/joke_writer/.car/config/joke_agent.yaml deleted file mode 100644 index a5bc13f..0000000 --- a/examples/joke_writer/.car/config/joke_agent.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# The graph's three nodes, exposed as three methods on one agent. -# -# One agent, not three. `generate_topics` and `best_joke` run once per request -# and have no resource profile of their own; splitting them out would add two -# images, two dependency trees and a Redis round trip to buy nothing. What is -# hoisted is the `Send` fan-out, and that is a workflow concern. - -agent: - name: JokeAgent - functions: - - name: generate_topics - description: Split a topic into three related sub-topics. - arguments: - - name: topic - type: str - returns: - type: dict - - - name: generate_joke - description: Write one joke about one subject. - arguments: - - name: subject - type: str - returns: - type: dict - - - name: best_joke - description: Pick the best joke out of the ones written for a topic. - arguments: - - name: topic - type: str - - name: jokes - type: list - returns: - type: dict diff --git a/examples/joke_writer/.env.example b/examples/joke_writer/.env.example deleted file mode 100644 index b846149..0000000 --- a/examples/joke_writer/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -# Copy this to `.env` and fill in the token. `config/global_controller.yaml` -# points `env_file:` at that copy, and it reaches every container as -# `docker run --env-file`. -# -# Keep the real token out of THIS file. `.env.example` is the one exception to -# the build context's exclusion of `.env*`, so whatever is written here is baked -# into the image; `.env` itself never enters the build and never leaves the host. - -# A Bedrock API key -- the long-term kind generated in the console, or a -# short-term one. botocore matches this exact name against bedrock-runtime's -# signingName (`bedrock`) and switches the client from SigV4 to bearer auth by -# itself, which is why neither joke_writer.py nor ventis/llm/bedrock.py mentions -# it. An IAM access key works too: drop AWS_ACCESS_KEY_ID and -# AWS_SECRET_ACCESS_KEY in instead and the same client signs with SigV4. -AWS_BEARER_TOKEN_BEDROCK= - -# Neither is a secret, and both have defaults in joke_writer.py -- they are here -# to name what the source reads. -BEDROCK_MODEL_ID=meta.llama3-8b-instruct-v1:0 -AWS_REGION=us-east-1 diff --git a/examples/joke_writer/LICENSE b/examples/joke_writer/LICENSE deleted file mode 100644 index 5600729..0000000 --- a/examples/joke_writer/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 LangChain, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/examples/joke_writer/README.md b/examples/joke_writer/README.md deleted file mode 100644 index 930a410..0000000 --- a/examples/joke_writer/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Joke Writer - -A LangGraph map-reduce, ported to Ventis. Derived from -[langchain-ai/langchain-academy](https://github.com/langchain-ai/langchain-academy) -at `fa15bec` (`module-4/studio/map_reduce.py`, MIT — see `LICENSE`). - -Unlike the other targets in `examples/`, **the source here is not unmodified**. -`map_reduce.py` built a `ChatOpenAI` at module scope, and at the time of the port -an agent container had no way to carry an `OPENAI_API_KEY` — the port was blocked -at the credential wall until the model call was rewritten onto Bedrock. That wall -is gone now: `env_file` puts any key in the container. The Bedrock rewrite stayed -anyway, and [What the port cost](#what-the-port-cost) is honest about what that -means. - -## Overview - -Given a topic, the graph splits it into sub-topics, writes one joke per -sub-topic in parallel, then picks the best of them. - -1. `generate_topics` — one LLM call, turns the topic into three sub-topics, - validated into `Subjects`. -2. `generate_joke` — one LLM call per sub-topic. `continue_to_jokes` emits a - `Send` per subject, so this node runs N times per request with no shared - state between the runs. `jokes` is an `Annotated[list, operator.add]`, which - is how the N results merge back into one state. -3. `best_joke` — one LLM call over every joke, returns the winner by index. - -``` - START - | - generate_topics 1 call - | - continue_to_jokes Send x N - / | \ - joke joke joke N calls, no shared state - \ | / - best_joke 1 call - | - END -``` - -### Why this one - -It is the smallest project in reach whose control flow does something a single -process cannot: `Send` fans out to N independent calls per request. Everything -else about it is deliberately boring — four packages, no tools, no external -service, one API key. - -## The port - -| File | What it holds | -| --- | --- | -| `joke_writer.py` | The source. Three prompts, two schemas, three nodes, and the graph — still compiled, never executed under Ventis. | -| `agents/joke_agent.py` | `JokeAgent`. Three methods, each calling the source's node with the node's own state dict. Imports `joke_writer`; restates nothing. | -| `agents/joke_agent.yaml` | The three nodes declared as three functions on one agent. | -| `workflow/joke_workflow.py` | Where the graph went — the edges, the `Send` fan-out and the `operator.add` reducer, re-expressed as ordinary Python. | -| `config/global_controller.yaml` | `JokeAgent` at `replicas: 3`, plus the workflow. | -| `config/policy.yaml` | Default-allow for the two services. Not optional — a missing file kills `ventis deploy`. | - -Two decisions worth naming: - -**One agent, not three.** `generate_topics` and `best_joke` run once per request -and have no resource profile of their own. Splitting them out would buy two more -images and two more Redis round trips. What is hoisted is the fan-out, and that -is a workflow concern. - -**The graph is not the port.** `StateGraph`, `Send` and the `Annotated[list, -operator.add]` reducer are control flow owned by the LangGraph runtime, and -Ventis has no runtime to execute them. The workflow dispatches N -`generate_joke` calls across the three replicas and concatenates the results -itself. Every call is dispatched before any is resolved — `.value()` blocks, so -fusing the two lines into one comprehension would silently serialize the fan-out -and remove the reason to be on Ventis at all. - -## What the port cost - -This is no longer upstream's model stack. `ChatOpenAI` and -`with_structured_output` are gone; `ventis.llm.bedrock.call_bedrock` is the raw -converse API, so each node asks for JSON in its prompt and validates the reply -through the same pydantic schema upstream used. `_extract_json` exists only -because `with_structured_output` used to do that work. - -That rewrite is not something the `porting-to-ventis` skill should do on a -user's project — it is the credential wall, and the skill's instruction is to -report it. It was done here deliberately, so that this example is one that -actually deploys. - -**It would not be necessary today.** The rewrite bought one thing: boto3 builds -no client at import, so the agent could be *loaded* with no secret in the -container, back when `_launch_locally` passed five `-e` flags and all five were -`VENTIS_*`. `env_file` removes that constraint — an `OPENAI_API_KEY` now reaches -a container as readily as a Bedrock one, and upstream's `ChatOpenAI` at module -scope would import fine. What the rewrite still buys is narrower: a module-scope -client turns a missing key into `"No agent loaded"`, while a per-call one turns -it into a real error on `/status`. Worth knowing, not worth a rewrite. - -The example stays on Bedrock because it is the model call that has been end-to-end -verified here, and because `ventis/llm/bedrock.py` is where Ventis writes per-call -token telemetry onto the future. - -## Running it - -Copy `.env.example` to `.env` and put a Bedrock API key in it: - -```shell -cp .env.example .env -$EDITOR .env # AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-... -``` - -`config/global_controller.yaml` points `env_file:` at that file, and every -container gets it as `docker run --env-file`. Nothing in this project reads the -variable: botocore matches the name against `bedrock-runtime`'s signingName and -switches the client from SigV4 to bearer auth on its own, so -`ventis/llm/bedrock.py` still builds a plain `boto3.client("bedrock-runtime")`. -An IAM access key instead of the bearer token works the same way. - -`.env` is gitignored and excluded from the build context — the key is in the -container's environment and not in the image. Deploy checks the path before it -launches anything, so a missing `.env` is one error line rather than three -replicas that come up and fail every request. - -```shell -ventis build -ventis deploy -``` - -```shell -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query": "animals"}' -curl http://localhost:8080/status/ -``` - -```json -{"request_id": "cb6cb62d...", "status": "done", "result": { - "topic": "animals", - "subjects": ["Wildlife Conservation", "Animal Behavior", "Endangered Species"], - "jokes": ["...", "...", "..."], - "best_selected_joke": "Why did the chimpanzee go to the doctor? Because it was going bananas!" -}} -``` - -`BEDROCK_MODEL_ID` and `AWS_REGION` (see `.env.example`) have defaults in -`joke_writer.py`; neither is a secret. The region has to match the one the key -was issued for. - -### Running the source outside Ventis - -`joke_writer.py` imports `ventis.llm.bedrock` first and falls back to the flat -`bedrock` copy an agent image gets, so the compiled graph still runs on its own -from a checkout of this repo: - -```shell -pip install -e ../.. # the ventis package -pip install langgraph pydantic typing_extensions boto3 -``` - -```python -from joke_writer import graph - -graph.invoke({"topic": "animals"}) -``` - -## Provenance - -Taken from `module-4/studio/`, which holds four unrelated graphs sharing one -directory. Only `map_reduce.py` and its license are here. - -| Left behind | Why | -| --- | --- | -| `parallelization.py`, `research_assistant.py`, `sub_graphs.py` | Other graphs in the same studio directory. The first two also need a Tavily key and Wikipedia. | -| `langgraph.json` | Registers all four graphs and points at `./.env`; a trimmed copy would only be useful for LangGraph Studio. | -| The module-4 notebooks | Teaching material for the same code. | -| `OPENAI_API_KEY`, `TAVILY_API_KEY` in `.env.example` | The first belongs to a model call that is no longer here; the second to the two graphs that are not here. | - -Nothing was added at the project root: there is no `pyproject.toml`, `setup.py` -or `requirements.txt`, exactly as upstream has none for module-4. That is why -`config/global_controller.yaml` has to declare `requirements:` by hand. diff --git a/examples/joke_writer/joke_writer.py b/examples/joke_writer/joke_writer.py deleted file mode 100644 index 3ad49d5..0000000 --- a/examples/joke_writer/joke_writer.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Map-reduce joke writer. - -Derived from langchain-ai/langchain-academy `module-4/studio/map_reduce.py` -(MIT, see LICENSE). The graph shape, the three prompts and the two schemas are -upstream's. The model call is not: upstream builds a `ChatOpenAI` at module -scope, and when this was ported nothing could carry an OPENAI_API_KEY into an -agent container. Bedrock reaches the model through boto3, which builds no client -at import, so the same code loaded with no secret injected. - -`env_file` has since removed that constraint -- the key now travels to the -container in a .env and botocore reads AWS_BEARER_TOKEN_BEDROCK on its own. The -rewrite stayed regardless; README.md says what that costs. - -`with_structured_output` went with it. `call_bedrock` is the raw converse API, so -each node asks for JSON in the prompt and validates the reply through the same -pydantic schema upstream used. -""" - -import json -import operator -import os -import re -from typing import Annotated - -from typing_extensions import TypedDict - -from pydantic import BaseModel, ValidationError - -from langgraph.constants import Send -from langgraph.graph import END, StateGraph, START - -# Ventis copies bedrock.py flat into every agent image; the package path is for -# running this module outside a container. -try: - from ventis.llm.bedrock import call_bedrock -except ImportError: - from bedrock import call_bedrock - -# Prompts we will use. Upstream's, plus the JSON instruction that -# `with_structured_output` used to add on our behalf. -subjects_prompt = """Generate a list of 3 sub-topics that are all related to this overall topic: {topic}. -Respond with JSON only, no prose: {{"subjects": ["...", "...", "..."]}}""" -joke_prompt = """Generate a joke about {subject}. -Respond with JSON only, no prose: {{"joke": "..."}}""" -best_joke_prompt = """Below are a bunch of jokes about {topic}. Select the best one! Return the ID of the best one, starting 0 as the ID for the first joke. Jokes: \n\n {jokes} -Respond with JSON only, no prose: {{"id": 0}}""" - -# LLM. Both are read once at import; the container gets them from its -# environment, and neither is a secret. -MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0") -REGION = os.environ.get("AWS_REGION", "us-east-1") - - -def _extract_json(text): - """Pull the first JSON object out of a model reply. - - Even told to answer with JSON only, a model wraps it in a ```json fence or - prefaces it with a sentence. Upstream never needed this because - `with_structured_output` handled it; the converse API does not. - """ - text = re.sub(r"^\s*```(?:json)?|```\s*$", "", text.strip(), flags=re.MULTILINE) - try: - return json.loads(text) - except json.JSONDecodeError: - pass - # Fall back to the outermost braced span. - match = re.search(r"\{.*\}", text, flags=re.DOTALL) - if not match: - raise ValueError(f"joke_writer: no JSON in model output: {text!r}") - return json.loads(match.group(0)) - - -def _ask(prompt, schema, max_tokens): - """One converse() call, validated into `schema`. - - Raising on a bad reply is deliberate. A node that returned a default would - put a plausible-looking wrong answer into the state, and the reduce step - downstream indexes into the jokes list by an id the model chose -- a silent - default there picks the wrong joke instead of failing. - """ - response = call_bedrock( - model_id=MODEL_ID, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inference_config={"maxTokens": max_tokens, "temperature": 0.0}, - region=REGION, - ) - text = response["output"]["message"]["content"][0]["text"] - if not text: - raise ValueError("joke_writer: LLM returned no output.") - try: - return schema(**_extract_json(text)) - except (ValidationError, TypeError) as exc: - raise ValueError( - f"joke_writer: {schema.__name__} not satisfied by model output: {text!r}" - ) from exc - - -# Define the state -class Subjects(BaseModel): - subjects: list[str] - -class BestJoke(BaseModel): - id: int - -class OverallState(TypedDict): - topic: str - subjects: list - jokes: Annotated[list, operator.add] - best_selected_joke: str - -def generate_topics(state: OverallState): - prompt = subjects_prompt.format(topic=state["topic"]) - response = _ask(prompt, Subjects, max_tokens=300) - return {"subjects": response.subjects} - -class JokeState(TypedDict): - subject: str - -class Joke(BaseModel): - joke: str - -def generate_joke(state: JokeState): - prompt = joke_prompt.format(subject=state["subject"]) - response = _ask(prompt, Joke, max_tokens=300) - return {"jokes": [response.joke]} - -def best_joke(state: OverallState): - jokes = "\n\n".join(state["jokes"]) - prompt = best_joke_prompt.format(topic=state["topic"], jokes=jokes) - response = _ask(prompt, BestJoke, max_tokens=100) - if not 0 <= response.id < len(state["jokes"]): - raise ValueError( - f"joke_writer: model chose joke {response.id} of {len(state['jokes'])}." - ) - return {"best_selected_joke": state["jokes"][response.id]} - -def continue_to_jokes(state: OverallState): - return [Send("generate_joke", {"subject": s}) for s in state["subjects"]] - -# Construct the graph: here we put everything together to construct our graph -graph_builder = StateGraph(OverallState) -graph_builder.add_node("generate_topics", generate_topics) -graph_builder.add_node("generate_joke", generate_joke) -graph_builder.add_node("best_joke", best_joke) -graph_builder.add_edge(START, "generate_topics") -graph_builder.add_conditional_edges("generate_topics", continue_to_jokes, ["generate_joke"]) -graph_builder.add_edge("generate_joke", "best_joke") -graph_builder.add_edge("best_joke", END) - -# Compile the graph -graph = graph_builder.compile() diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md deleted file mode 100644 index fe6dd49..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/SKILL.md +++ /dev/null @@ -1,240 +0,0 @@ ---- -name: porting-to-canyonos-core -description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. -compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. ---- - -# Port an agent project to CanyonOS Core - -CanyonOS Core is the product name. Its compatibility executable and Python -package remain `ventis`; environment variables and Docker resources retain the -`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding -strings. Do not rename them. - -## Load references only when needed - -- Read [references/packaging.md](references/packaging.md) when a source import - does not resolve from `/app`, the source is nested, or packaging metadata is - involved. -- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target - includes `llm_proxy`. -- Read [references/ec2.md](references/ec2.md) only when any config entry uses - `provider: EC2`. -- Read [references/troubleshooting.md](references/troubleshooting.md) after a - failed build, image probe, deploy, or request. -- Read [references/runtime-contract.md](references/runtime-contract.md) when a - validator finding needs explanation or the runtime mechanism is unclear. - -## Goal: thin scaffolding beside untouched source - -```text -agents/.yaml one callable surface per service -agents/.py one thin adapter per service, when needed -workflow/_workflow.py HTTP entry point; calls deploy() -config/global_controller.yaml deployment manifest -config/policy.yaml optional access restriction -pyproject.toml conditional nested-import scaffolding - unchanged -``` - -The file count follows the deployment. A multi-agent port has one yaml/adapter -pair per service that is worth deploying separately. If a source class already -satisfies the runtime contract, point its config entry at that file and do not -copy it into an adapter. - -Everything the source already owns—prompts, tools, schemas, parsing, retries, -model clients, and node bodies—is imported. The port re-expresses only the -CanyonOS Core boundary and framework-owned orchestration. - -The port root is the existing repository root and the directory from which -`ventis build` runs. Write scaffolding there beside existing directories. If the -repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, -and `config/` beside it. Never move or copy the repository into a new `src/` -directory, and never create an outer wrapper merely for the port. - -## 1. Survey before writing - -Identify: - -1. The source entry point and callable input/output. -2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, - `Send`, `Command`, interrupts). -3. Runtime-injected services nodes read: stores, context, memory, sessions, or - callback managers. -4. Sync versus async boundaries. -5. Imports and declared runtime dependencies. -6. Model provider, credential names, streaming use, and optional `llm_proxy`. -7. Whether independent work fans out and benefits from separate replicas. -8. Whether source imports resolve from the project root that becomes `/app`. - -Run the validator once now. Its header detects capabilities directly from the -importable runtime rather than external development metadata: - -```bash -python /validate.py . -``` - -If config or agent yaml is malformed, the validator defers to `ventis build`. -Capability-gated findings say which runtime behavior is available. - -## 2. Choose service boundaries - -Start with one service. Split only when it creates independent parallel work or -a distinct resource/replica profile. - -- Keep a ReAct loop together; every turn needs shared message history. -- Hoist supervisor task lists and `Send`-style fan-out into the workflow. -- Do not create a one-replica service with no distinct resource profile merely - to mirror every source graph node. - -Rewrite framework-owned edges as ordinary Python. Import the connected node -functions unchanged. Construct runtime-injected service objects from source -configuration; do not invent models, dimensions, stores, or defaults silently. -Report any choice the source does not specify. - -## 3. Write declarations and adapters - -### Agent yaml - -Use one yaml per deployed service. Argument types are bare builtins only: -`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is -required by the generated stub. `returns.type` is documentation; use `dict` or -`list` to signal that workflow callers must `json.loads` the returned string. - -### Adapter - -The entrypoint exposes a module-level class named exactly `agent.name`. It -constructs with no arguments and its declared methods are synchronous. Read -configuration from the environment in `__init__`. Bridge source coroutines -inside a synchronous method with `asyncio.run(...)`. Serialize framework objects -with their own JSON-safe serializer before returning. - -Do not duplicate source prompts, tools, schemas, or model calls. Keep the source -provider and SDK. - -### Workflow - -Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. -Import generated stubs by yaml basename and agent class name: - -```python -from deploy import deploy -from agents. import -``` - -The deployment platform sends `{query: string}` to `/main`. Pack richer input -inside `query`; any additional workflow parameter has a default. - -Dispatch every remote call before resolving any future: - -```python -futures = [agent.work(item=item) for item in items] -results = [json.loads(future.value()) for future in futures] -``` - -Do not fuse dispatch and `.value()` in one comprehension; that silently -serializes fan-out. Do not add an `if __name__ == "__main__":` block: the -workflow is executed with `__name__ == "__main__"` in production. - -### Config - -For each service, keep these names aligned: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a -list of distribution-name strings. Put `env_file` at config top level when the -runtime capability is available. Omit `policy.yaml` unless access must be -restricted; if present, give it a non-empty `rules` list. - -## Hard rules - -Capitalized **MUST** and **NEVER** are reserved for port-breaking or -source-integrity rules. The owner column states where each is decided. - -| ID | Rule | Owner | -|---|---|---| -| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | -| M2 | The class MUST construct with no arguments | V007 | -| M3 | yaml argument names MUST match Python parameter names | V008 | -| M4 | yaml argument types MUST be bare builtins | V010 | -| M5 | Declared adapter methods MUST be synchronous | V009 | -| M6 | Config names MUST match yaml agent names | build | -| M7 | Config names MUST not collide after lowercase normalization | build output | -| M8 | Local provider MUST be lowercase `local` | deploy preflight | -| M9 | `replicas` MUST be an integer | deploy preflight | -| M10 | `requirements` MUST be a list of strings | build | -| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | -| M12 | Workflow MUST NEVER contain a main guard | V017 | -| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | -| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | -| M15 | Workflow MUST import stubs from `agents.` | V023 | -| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | -| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | -| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | -| M19 | NEVER hardcode or bake a real credential into an image | W003 | -| M20 | NEVER edit or vendor the source tree | `git status` | -| M21 | NEVER swap the source LLM provider | review | -| M22 | NEVER silently move, drop, or reclassify source dependencies | review | -| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | -| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | - -## 4. Validate, build, and probe - -Run static preflight, then let the build own build-time validation: - -```bash -python /validate.py . -ventis build -c config/global_controller.yaml -``` - -A green build never imports the adapter. Probe each agent image in this order: - -```bash -# Runtime startup path - docker run --rm ventis- \ - python -c "import local_controller" - -# Agent load path; include --env-file when configured - docker run --rm --env-file ventis- \ - python -c "import importlib.util,sys; \ -s=importlib.util.spec_from_file_location('m','.py'); \ -m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ -m.();print('ok')" -``` - -Also probe the workflow image with `python -c "import local_controller"`; it has -its own dependency resolve and generated-stub imports. - -Then deploy, send a representative request, and poll its status: - -```bash -ventis deploy -c config/global_controller.yaml -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query":""}' -curl http://localhost:8080/status/ -``` - -A successful outer request with a source-level failure still proves the port -reached and returned the source behavior. Record the distinction. - -## 5. Clean up - -After collecting evidence, stop foreground deploy with Ctrl+C and wait for -controller cleanup. Remove exact leftovers if startup crashed. Then remove build -products and exact images from this config: - -```bash -ventis clean -docker image rm ventis- \ - ventis- - -test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container -docker ps -a --format '{{.Names}}' -``` - -`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it -does not remove containers or images. Keep port scaffolding, untouched source, -and requested logs or reports. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md deleted file mode 100644 index e06daa0..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/ec2.md +++ /dev/null @@ -1,41 +0,0 @@ -# EC2 deployment - -Read this only when at least one config entry uses `provider: EC2`. - -## Configuration - -Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The -top-level `ec2` block supplies the runtime's required infrastructure and SSH -settings. Read the target checkout's deploy preflight and EC2 runtime before -writing the block; do not copy values from an example environment. - -Typical required categories are: - -- AMI and instance type -- region and subnet -- security groups -- SSH user and credentials accepted by the runtime - -`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof -that provisioning, SSH, image transfer, or remote container startup works. - -## Networking - -A remote container's `host.docker.internal` names its own EC2 Docker host. It -does not name the local controller machine. Databases, model proxies, and other -services must use addresses reachable from every selected host. - -The environment file may be copied temporarily to a remote host by runtimes that -expose the `env_file` capability. Confirm behavior from the capability probe and -target runtime rather than assuming local Docker semantics. - -## Probes and cleanup - -Run the same runtime and adapter probes against the exact image before remote -deployment. After deploy, verify the remote container logs; controller health -can be green even when agent loading failed. - -Stop foreground deploy normally so the controller can terminate recorded EC2 -instances. If provisioning or startup fails before an instance is recorded, -inspect the cloud provider directly and remove exact leaked resources. Never use -a broad cleanup command against unrelated instances. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md deleted file mode 100644 index f726bd7..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md +++ /dev/null @@ -1,64 +0,0 @@ -# LLM proxy integration - -Read this only when the target checkout contains `llm_proxy` or the deployment -explicitly routes model SDKs through it. - -## Preserve provider protocols - -The proxy redirects provider endpoints; it does not convert providers. Keep the -source SDK, model ID, request body, and response parsing unchanged. - -Configure only the provider variables the source uses: - -```dotenv -OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 -ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic -AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock -``` - -Some SDKs refuse to initialize without caller credentials. Give agent containers -non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS -credentials in the separate proxy process, not in the port's `env_file`. - -## Start locally - -The proxy defaults conflict with a typical deployment: host loopback is not -reachable from a container, and port 8080 is normally used by the workflow API. -Use a non-loopback bind and a different port: - -```bash -PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy -curl http://127.0.0.1:8081/healthz -``` - -Local CanyonOS Core containers resolve `host.docker.internal` through their -Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the -machine running `ventis deploy`. Distributed deployments need a reachable proxy -address or one proxy on each host. - -## Supported call shape - -The implementation buffers complete requests and responses: - -- OpenAI and Anthropic non-streaming HTTP calls are forwarded. -- Bedrock `invoke` is reissued through the proxy's boto3 client. -- OpenAI/Anthropic streaming is unsupported. -- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are - unsupported. - -Survey the source before selecting the proxy. Do not silently disable streaming; -report the unsupported behavior and stop. - -## Credential behavior - -- The OpenAI adapter removes caller authorization and inserts the proxy key. -- The Anthropic adapter removes caller key headers and inserts the proxy key. -- Botocore still signs requests sent to a custom endpoint, so a caller may need - placeholder AWS credentials even though the proxy reissues upstream with its - own identity. -- `/healthz` proves provider registration and Flask availability, not upstream - credential validity. - -OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return -JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed -with the upstream status and are not byte-for-byte passthrough. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md deleted file mode 100644 index 8520c4a..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/packaging.md +++ /dev/null @@ -1,81 +0,0 @@ -# Packaging and import roots - -Read this reference when an adapter imports nested source code, the source uses a -`src/` layout, or V031 reports an import-root problem. - -## What `/app` can import - -CanyonOS Core preserves project-relative paths in the image and starts Python at -`/app`. Without an editable install, Python resolves names rooted there: - -- `/app/tools.py` as `import tools` -- `/app/pkg/__init__.py` as `import pkg` -- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace - directories without `__init__.py` - -It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become -an import root first. - -## Detect support, do not infer it from release history - -Run: - -```bash -python /validate.py . -``` - -Read the `editable_install` capability. If it is unavailable and the original -import cannot resolve from `/app`, report a runtime capability blocker and stop. -Do not add a `sys.path` hack or relocate source files. - -## Root metadata is the trigger - -When editable install is supported, only packaging metadata at the **port root** -triggers `pip install -e .`: - -```text -port-root/pyproject.toml detected -port-root/source/pyproject.toml ignored as an install trigger -``` - -A nested source repository may remain untouched. Add minimal root scaffolding -that points package discovery at the existing source package: - -```toml -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "canyonos-port" -version = "0.0.0" -dependencies = [] - -[tool.setuptools.packages.find] -where = ["source/src"] -include = ["pkg*"] -namespaces = true -``` - -Set `where` and `include` from the actual tree and original import spelling. Do -not reference a README or license from this wrapper metadata; file sweeps differ -by runtime capability and a missing referenced file makes the image build fail. - -## Dependencies in nested metadata - -A nested `pyproject.toml` is not installed merely because its Python files are -copied. Keep the source declaration unchanged and repeat its runtime -distributions in each relevant config entry's `requirements` list. This is -compatibility scaffolding, not permission to drop, move, or reclassify declared -dependencies. - -If source metadata is already at the port root, do not create a wrapper. Its -project dependencies participate in the same resolver as config requirements. -Report declared-but-unused toolchain dependencies and their image cost; let the -owner decide whether source metadata should change. - -## Validation boundary - -`ventis build` owns packaging syntax and installation errors. `validate.py` -checks only whether adapter imports appear to require a nested root that the -runtime will not expose. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md deleted file mode 100644 index 94f7401..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md +++ /dev/null @@ -1,187 +0,0 @@ -# CanyonOS Core runtime contract - -The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the -CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for -Docker resources. - -Read this reference when implementing an adapter or explaining a validator -finding. Runtime-dependent behavior is expressed as capabilities; run -`validate.py` against the target environment instead of inferring support from -release history. - -## Project root and discovery - -`ventis build` uses the current working directory as the project root. - -| Input | Discovery | -|---|---| -| `agents/*.yaml` | direct yaml glob under the project root | -| `config/global_controller.yaml` | default config, overridable with `-c` | -| workflow | `workflow_file` on a `type: workflow` entry | -| policy | `policy.yaml` beside the selected config file | -| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | - -The config name, yaml `agent.name`, and entrypoint class name form one binding: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -A missing match may skip an image while the command continues, so inspect build -output and generated image tags. - -## Agent yaml and generated stubs - -The consumed yaml shape is: - -```yaml -agent: - name: ExampleAgent - functions: - - name: work - arguments: - - name: query - type: str - returns: - type: dict -``` - -Argument annotations are generated from bare names without adding imports. Use -builtins. Generated methods have no defaults, so every declared argument is -required at the stub call site. `returns` does not control runtime conversion; -it documents whether workflow code should parse the returned string. - -Stub destinations differ by runtime capability and entrypoint layout. Workflow -code in this port convention imports the generated class from -`agents.`. The validator checks that import against declarations -before the workflow image starts. - -## Agent loading and execution - -The local controller effectively performs: - -```python -module = load(entrypoint) -agent_class = getattr(module, configured_name) -agent = agent_class() -result = getattr(agent, method_name)(**args) -``` - -Consequences: - -- The class is module-level and named exactly as configured. -- Construction takes no arguments. -- Declared methods accept yaml argument names as keyword arguments. -- Methods are synchronous; this path does not await a coroutine. -- Dicts and lists are JSON-encoded before entering Redis; other results become - strings. -- A remote Future's `.value()` returns text, not the original Python object. - -Agent import and construction exceptions are caught by the controller. A failed -agent may still advertise healthy because health is written independently of -successful agent loading. That is why image probes import both the runtime and -the entrypoint explicitly. - -## Workflow execution - -The workflow file is executed, not imported. Therefore: - -- module-level code runs at container startup; -- `__name__ == "__main__"`; -- `deploy()` blocks in the web server; -- the workflow function runs once per request; -- its function name determines the REST route exposed by the compatibility - runtime. - -The deployment platform additionally expects `/main` with a `{query: string}` -body. This platform constraint is stricter than the underlying transport. - -Each stub method returns a Future immediately. `.value()` blocks. Dispatching -and resolving inside one comprehension serializes work without raising an -error; dispatch all calls first, then resolve them. - -The workflow container also starts runtime controller code and has its own -package resolution. Probe it independently from agent images. - -## Build context and collisions - -The runtime copies project files while preserving relative paths, then writes -shared runtime modules, generated stubs, and entrypoints into the image. Later -writes can shadow project files. - -Avoid root project modules named like runtime files, including: - -```text -future.py -ventis_context.py -local_controller.py -local_controller_frontend.py -redis_client.py -grpc_options.py -bedrock.py -deploy.py -session_logging.py -workflow_launcher.py -``` - -Also avoid a yaml basename that shadows a different source module imported by an -adapter. The validator checks deterministic flat-name collisions. - -File sweep and editable-install behavior are runtime capabilities. For nested -imports, follow [packaging.md](packaging.md). - -## Dependencies and protobuf - -Agent and workflow images include a small runtime dependency set. Config -`requirements` adds source-specific distributions. A malformed requirements -value can be normalized away while image generation continues; missing imports -then surface only when the agent loads. - -The build compiles gRPC Python stubs on the host and copies them into images. -The image resolver does not necessarily know the generated-code version. A -source dependency that constrains protobuf below the host generator version can -produce a green image build that dies on: - -```text -import local_controller -``` - -Always run that probe before probing the entrypoint. Treat a generated-code / -runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter -source dependencies silently. - -## Credentials capability - -When `env_file` capability is available, the top-level config path is resolved -against the project root and passed at container start. Hidden env files are not -copied into images. Invalid paths are deploy-preflight errors. - -When the capability is unavailable, declaring `env_file` has no effect. If the -source needs credentials, report the capability blocker rather than hardcoding -or vendoring a secret. - -A source that constructs its client at import time works only when credentials -are already in the container environment. Image entrypoint probes therefore use -the same env file as deployment. - -For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). - -## Policy and provider behavior - -No policy file means unrestricted service access. If a policy exists, it needs a -non-empty rules list. Rules are evaluated by specificity and first match; -services excluded from the selected rule fail after request acceptance. - -Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior -and remote networking are covered in [ec2.md](ec2.md). - -## Cleanup boundary - -Stopping foreground deploy normally invokes controller cleanup for recorded -containers and Redis. Hard kills and failures before resource registration may -leave resources behind. - -`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and -`docker_container/`. It does not remove containers or images. Remove exact -leftovers explicitly and preserve source, port scaffolding, and requested -evidence. diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md deleted file mode 100644 index 361acf9..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md +++ /dev/null @@ -1,60 +0,0 @@ -# Troubleshooting - -Read this after a failed build, image probe, deploy, or request. For mechanisms, -read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, -read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). - -## Build or deploy stops early - -| Symptom | Likely cause | -|---|---| -| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | -| Two services produce one image | Config names collide after lowercase normalization | -| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | -| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | -| Replica conversion `TypeError` | `replicas` is not an integer | -| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | -| Port or container name already in use | A previous deployment did not complete cleanup | - -## Container exits or serves nothing - -| Symptom | Likely cause | -|---|---| -| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | -| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | -| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | -| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | -| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | -| Third-party module is missing | Distribution is absent from source metadata and config requirements | -| Stub import raises `NameError` | yaml argument type is not a bare builtin | -| Source module behaves like an empty stub | A generated stub basename shadowed the source module | -| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | - -## Request is accepted, then fails - -| Symptom | Likely cause | -|---|---| -| Unexpected keyword argument | yaml argument name differs from adapter parameter name | -| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | -| Unauthorized service | The first matching policy rule excludes that service | -| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | -| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | -| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | -| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | -| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | - -## Deployment platform endpoint - -| Symptom | Likely cause | -|---|---| -| 404 while workflow container is healthy | Workflow function is not named `main` | -| 400 before host receives request | Body is not the platform's `{query: string}` shape | -| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | - -## Cleanup - -| Symptom | Likely cause | -|---|---| -| `ventis clean` succeeds but containers remain | The command removes generated directories only | -| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | -| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py b/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py deleted file mode 100755 index 04baf37..0000000 --- a/examples/portfolio/.claude/skills/porting-to-canyonos-core/validate.py +++ /dev/null @@ -1,1257 +0,0 @@ -#!/usr/bin/env python3 -"""Preflight the runtime traps that `ventis build` cannot see. - -This deliberately does not duplicate build-time validation such as malformed -YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those -checks. This script parses Python without importing it and catches failures that -otherwise stay hidden until a container loads an agent, starts a workflow, or -serves its first request. A replica is not evidence: the controller writes -`healthy` to Redis before `_load_agent` runs. - - python validate.py [project_dir] [-c config/global_controller.yaml] - [--json] [--strict] - -Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. - -Runtime capabilities vary across CanyonOS Core installations. This script probes -the importable `ventis` package directly. A capability-gated check reports -UNAVAILABLE when its behavior cannot be proven. -""" - -import argparse -import ast -import builtins -import json -import os -import re -import sys -from typing import ClassVar - -try: - import yaml -except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency - sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") - raise SystemExit(2) from None - - -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" - -# Copied flat into every image over the swept project tree, so a project module -# landing flat under one of these names is overwritten. -# ventis/stub_generator.py generate_docker / generate_workflow_docker. -RUNTIME_FLAT_NAMES = frozenset( - { - "future.py", - "ventis_context.py", - "local_controller.py", - "local_controller_frontend.py", - "redis_client.py", - "grpc_options.py", - "gpu_metrics.py", - "bedrock.py", - "deploy.py", - "session_logging.py", - "workflow_launcher.py", - } -) - -# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. -BASE_AGENT_REQUIREMENTS = [ - "grpcio", - "grpcio-tools", - "redis", - "pyyaml", - "psutil", - "ipdb", - "ipython", - "boto3", -] -# Import name -> distribution name, for the handful where they differ and the -# mismatch would otherwise be reported as a missing requirement. -IMPORT_TO_DISTRIBUTION = { - "attr": "attrs", - "bs4": "beautifulsoup4", - "cv2": "opencv-python", - "dateutil": "python-dateutil", - "dotenv": "python-dotenv", - "grpc": "grpcio", - "grpc_tools": "grpcio-tools", - "jwt": "pyjwt", - "PIL": "pillow", - "psycopg": "psycopg", - "psycopg2": "psycopg2-binary", - "pydantic_settings": "pydantic-settings", - "sklearn": "scikit-learn", - "typing_extensions": "typing-extensions", - "yaml": "pyyaml", -} - -SECRET_PATTERNS = [ - (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), - (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), - (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), - (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), -] -SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) - -ERROR = "ERROR" -WARN = "WARN" -INFO = "INFO" - - -# ------------------------------------------------------------------ # -# Capabilities # -# ------------------------------------------------------------------ # -# -# Stable labels for behavior detected from the importable runtime. They contain -# no external development metadata. - -CAPABILITY_SOURCE = { - "env_file": "runtime env-file injection", - "editable_install": "editable project installation", - "sweeps_all_files": "full project-file sweep", - "stub_two_destinations": "flat and package stub destinations", -} - - -def probe_capabilities(): - """Ask the importable ventis package what it actually supports.""" - caps = dict.fromkeys(CAPABILITY_SOURCE, False) - caps["ventis"] = False - try: - from ventis import stub_generator - except Exception: # noqa: BLE001 - a broken install must not crash the check - return caps - - caps["ventis"] = True - caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") - caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") - - import importlib - - for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): - try: - module = importlib.import_module(module_name) - except Exception: # noqa: BLE001,S112 - the other path is the live one - continue - if hasattr(module, "resolve_env_file"): - caps["env_file"] = True - break - return caps - - -# ------------------------------------------------------------------ # -# YAML with line numbers # -# ------------------------------------------------------------------ # - - -class LineDict(dict): - """A mapping that remembers where it and each of its keys were written.""" - - line = 0 - key_lines: ClassVar[dict] = {} - - -class LineLoader(yaml.SafeLoader): - pass - - -def _construct_mapping(loader, node): - data = LineDict() - yield data - data.update(loader.construct_mapping(node, deep=False)) - data.line = node.start_mark.line + 1 - data.key_lines = { - key.value: key.start_mark.line + 1 - for key, _ in node.value - if isinstance(key, yaml.ScalarNode) - } - - -LineLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping -) - - -def line_of(mapping, key=None): - """The source line of `key` inside `mapping`, or of the mapping itself.""" - if not isinstance(mapping, LineDict): - return 0 - if key is not None: - return mapping.key_lines.get(key, mapping.line) - return mapping.line - - -def load_yaml(path): - """Parse `path`, returning (data, error). Never raises.""" - try: - with open(path, "r", encoding="utf-8") as handle: - return yaml.load(handle, Loader=LineLoader), None - except Exception as exc: # noqa: BLE001 - any parse failure is a finding - return None, str(exc) - - -# ------------------------------------------------------------------ # -# Findings # -# ------------------------------------------------------------------ # - - -class Report: - def __init__(self, project_dir, capabilities): - self.project_dir = project_dir - self.capabilities = capabilities - self.findings = [] - # A peer agent is imported by the name of its generated stub, which the - # build copies flat into every image. Those are not project modules and - # need no requirement. - self.stub_module_names = set() - - def add(self, check, level, path, line, summary, mechanism): - self.findings.append( - { - "check": check, - "level": level, - "path": self.rel(path) if path else "", - "line": line or 0, - "summary": summary, - "mechanism": mechanism, - } - ) - - def error(self, check, path, line, summary, mechanism): - self.add(check, ERROR, path, line, summary, mechanism) - - def warn(self, check, path, line, summary, mechanism): - self.add(check, WARN, path, line, summary, mechanism) - - def unavailable(self, check, summary): - self.add(check, INFO, "", 0, summary, "") - - def rel(self, path): - try: - return os.path.relpath(path, self.project_dir) - except ValueError: - return path - - def counts(self): - errors = sum(1 for f in self.findings if f["level"] == ERROR) - warnings = sum(1 for f in self.findings if f["level"] == WARN) - return errors, warnings - - -# ------------------------------------------------------------------ # -# Python source helpers # -# ------------------------------------------------------------------ # - - -def parse_python(path): - """AST for `path`, or (None, error). The port is never imported.""" - try: - with open(path, "r", encoding="utf-8") as handle: - source = handle.read() - except OSError as exc: - return None, str(exc) - try: - return ast.parse(source, filename=path), None - except SyntaxError as exc: - return None, f"{exc.msg} (line {exc.lineno})" - - -def find_class(tree, name): - for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == name: - return node - return None - - -def class_methods(class_node): - return { - node.name: node - for node in class_node.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - - -def parameter_names(func_node): - """Every parameter a caller can pass by keyword, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - return positional + [a.arg for a in args.kwonlyargs] - - -def required_parameters(func_node): - """Parameters with no default, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - if args.defaults: - positional = positional[: len(positional) - len(args.defaults)] - kwonly = [ - arg.arg - for arg, default in zip(args.kwonlyargs, args.kw_defaults) - if default is None - ] - return positional + kwonly - - -def toplevel_import_names(tree): - """Top-level package name of every import in the module, with line numbers.""" - names = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - names.setdefault(alias.name.split(".")[0], node.lineno) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - names.setdefault(node.module.split(".")[0], node.lineno) - return names - - -# ------------------------------------------------------------------ # -# V006-V010 adapter failures hidden by _load_agent # -# ------------------------------------------------------------------ # - -BUILTIN_TYPE_NAMES = frozenset( - name - for name in dir(builtins) - if isinstance(getattr(builtins, name), type) and not name[0].isupper() -) - - -def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): - """V006 V007 V008 V009 V010.""" - name = agent_block["name"] - functions = agent_block.get("functions") or [] - check_argument_types(report, agent_yaml_path, functions) - - entrypoint = entry.get("entrypoint") - if not entrypoint: - return - entrypoint_path = os.path.join(project_dir, entrypoint) - if not os.path.isfile(entrypoint_path): - return - - tree, error = parse_python(entrypoint_path) - if tree is None: - report.error( - "V006", - entrypoint_path, - 0, - f"the entrypoint does not parse: {error}", - "_load_agent exec_module's it and swallows the exception; the first " - "request answers 'No agent loaded'.", - ) - return - - class_node = find_class(tree, name) - if class_node is None: - classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] - found = ", ".join(classes) if classes else "no classes at all" - report.error( - "V006", - entrypoint_path, - 1, - f"no class named `{name}` at module level (found: {found})", - "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " - "the AttributeError. The class name must equal agent.name exactly.", - ) - return - - methods = class_methods(class_node) - check_constructor(report, entrypoint_path, name, methods) - - for func in functions: - if not isinstance(func, dict) or not isinstance(func.get("name"), str): - continue - check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) - - -def check_argument_types(report, agent_yaml_path, functions): - """V010 -- the type string is pasted into an ast.Name, never checked.""" - for func in functions: - if not isinstance(func, dict): - continue - for arg in func.get("arguments") or []: - if not isinstance(arg, dict) or "type" not in arg: - continue - declared = arg.get("type") - if not isinstance(declared, str): - continue # stub generation reports malformed type values - if declared in BUILTIN_TYPE_NAMES: - continue - report.error( - "V010", - agent_yaml_path, - line_of(arg, "type"), - f"`type: {declared}` is not a builtin", - "stub_generator pastes it verbatim into the generated " - "annotation, and the stub module imports only Future and " - "inspect. Anything else raises NameError when the stub is " - "imported -- after a green build. Use str int float bool dict " - "list.", - ) - - -def check_constructor(report, entrypoint_path, name, methods): - """V007 -- _load_agent calls agent_class() with no arguments.""" - init = methods.get("__init__") - if init is None: - return - required = required_parameters(init) - if required: - report.error( - "V007", - entrypoint_path, - init.lineno, - f"`{name}.__init__` requires {', '.join(required)}", - "_load_agent calls agent_class() with no arguments; the TypeError is " - "swallowed and the first request answers 'No agent loaded'. Read " - "configuration from the environment inside __init__ instead.", - ) - - -def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): - """V008 V009.""" - func_name = func["name"] - method = methods.get(func_name) - if method is None: - report.error( - "V008", - entrypoint_path, - 0, - f"`{class_name}` has no method `{func_name}`", - "The yaml declares it, so callers get a stub for it; the controller " - f"then answers \"Agent {class_name} has no method '{func_name}'\".", - ) - return - - # V009 -- nothing on the execution path awaits. - if isinstance(method, ast.AsyncFunctionDef): - report.error( - "V009", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` is `async def`", - "The executor calls method(**args) with no await, so Redis receives " - "''. Keep the signature synchronous and call " - "asyncio.run(...) inside the body.", - ) - - # V008 -- the controller calls method(**args) with the yaml's names. - declared = [ - arg["name"] - for arg in func.get("arguments") or [] - if isinstance(arg, dict) and isinstance(arg.get("name"), str) - ] - actual = parameter_names(method) - required = required_parameters(method) - - missing = [arg for arg in declared if arg not in actual] - if missing: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` has no parameter " - f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", - "LocalController does method(**args) with the yaml's argument names. " - "A mismatch is TypeError: unexpected keyword argument, at request " - f"time. See {os.path.basename(agent_yaml_path)}.", - ) - - unfilled = [arg for arg in required if arg not in declared] - if unfilled: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " - "the yaml does not declare", - "Only declared arguments are ever sent, and the generated stub gives " - "none of them a default. Declare them in the yaml or default them " - "in the signature.", - ) - - - -# ------------------------------------------------------------------ # -# V016-V018 the workflow # -# ------------------------------------------------------------------ # - - -def check_stub_imports(report, workflow_path, tree, stub_classes): - """V023 -- the workflow must import a stub as `from agents. import `. - - The build copies each stub to exactly one path, and for the workflow image - that path is agents/.py. Two ways of writing this line fail, and - the project walks you into both: the flat form is what examples/ uses, and - the class name is the one `ventis build` prints, which is not the one it - writes. - """ - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - base = alias.name.split(".")[0] - if base in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`import {alias.name}` -- the stub is at " - f"agents/{base}.py, not flat", - "The build copies a stub to one path, and for the " - "workflow that path is under agents/. This is a " - "ModuleNotFoundError the moment the workflow runs. " - f"Write `from agents.{base} import {stub_classes[base]}`.", - ) - continue - - if not isinstance(node, ast.ImportFrom) or not node.module: - continue - - module = node.module - if module in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`from {module} import ...` -- the stub is at " - f"agents/{module}.py, not flat", - "The build copies a stub to one path, and for the workflow " - "that path is under agents/. The flat form is what this " - "repository's own examples use and it raises " - "ModuleNotFoundError in the workflow image. Write " - f"`from agents.{module} import {stub_classes[module]}`.", - ) - continue - - if not module.startswith("agents."): - continue - base = module.split(".", 1)[1] - expected = stub_classes.get(base) - if expected is None: - continue - for alias in node.names: - if alias.name == expected: - continue - if alias.name == f"{expected}Stub": - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is the name the build prints, not the " - f"class it writes", - "generate_agent_stub sets class_name = agent_config['name'] " - "and then recomputes it with a 'Stub' suffix for the log " - "line only. The message names a class that does not exist; " - f"the class is `{expected}`.", - ) - else: - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is not a class the stub for {base} defines", - f"The stub's class carries the agent's own name: `{expected}`.", - ) - - -def check_workflow(report, workflow_path, stub_classes=None): - """V016 V017 V018 V023.""" - tree, error = parse_python(workflow_path) - if tree is None: - report.error("V016", workflow_path, 0, f"does not parse: {error}", "") - return - - main = None - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( - node.name == "main" - ): - main = node - break - - if main is None: - defined = [ - n.name - for n in tree.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - found = ", ".join(defined) if defined else "no top-level functions" - report.error( - "V016", - workflow_path, - 1, - f"no top-level function named `main` (found: {found})", - "CanyonOS Core serves POST /, but the deployment platform's " - "test endpoint posts to a hardcoded /main. A differently named " - "workflow builds, deploys and stays unreachable -- 404, container " - "healthy.", - ) - else: - check_main_signature(report, workflow_path, main) - - if stub_classes: - check_stub_imports(report, workflow_path, tree, stub_classes) - - # V016 -- deploy() is what starts Flask. - if not any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "deploy" - for node in ast.walk(tree) - ): - report.error( - "V016", - workflow_path, - 1, - "the workflow never calls `deploy(...)`", - "workflow_launcher.py exec's this file and nothing else starts the " - "HTTP server; the container comes up serving nothing.", - ) - - check_main_guard(report, workflow_path, tree) - check_fused_fanout(report, workflow_path, tree) - - -def check_main_signature(report, workflow_path, main): - """V016 -- the platform sends exactly {"query": ...}.""" - if isinstance(main, ast.AsyncFunctionDef): - report.error( - "V016", - workflow_path, - main.lineno, - "`main` is `async def`", - "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " - "no await; the response body would be a coroutine repr.", - ) - params = parameter_names(main) - if not params: - report.error( - "V016", - workflow_path, - main.lineno, - "`main` takes no arguments", - 'The platform posts {"query": "..."} and deploy() splats the ' - "body in as kwargs -- TypeError on every request.", - ) - return - if params[0] != "query": - report.error( - "V016", - workflow_path, - main.lineno, - f"`main`'s first parameter is `{params[0]}`, not `query`", - "The platform's body schema is strictly validated as " - "{query: string}; any other key is rejected with 400 in the control " - "plane, before the request reaches the host.", - ) - extra = [p for p in required_parameters(main) if p != "query"] - if extra: - report.error( - "V016", - workflow_path, - main.lineno, - f"`main` requires {', '.join(extra)} beyond `query`", - "Only `query` is ever sent, so every other parameter needs a " - "default or the call raises on every request. Pack richer input " - "into `query`.", - ) - - -def check_main_guard(report, workflow_path, tree): - """V017 -- the workflow is exec'd, so __name__ == "__main__".""" - for node in tree.body: - if not isinstance(node, ast.If): - continue - test = node.test - if ( - isinstance(test, ast.Compare) - and isinstance(test.left, ast.Name) - and test.left.id == "__name__" - and any( - isinstance(c, ast.Constant) and c.value == "__main__" - for c in test.comparators - ) - ): - report.error( - "V017", - workflow_path, - node.lineno, - '`if __name__ == "__main__":` block in the workflow', - "workflow_launcher.py runs exec(open().read()), so " - "__name__ IS '__main__' here and this block executes in " - "production, at container start.", - ) - - -def check_fused_fanout(report, workflow_path, tree): - """V018 -- .value() blocks, so dispatching and resolving in one - comprehension runs the fan-out one call at a time.""" - comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) - for node in ast.walk(tree): - if not isinstance(node, comprehensions + (ast.DictComp,)): - continue - elements = ( - [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] - ) - for element in elements: - for inner in ast.walk(element): - if ( - isinstance(inner, ast.Call) - and isinstance(inner.func, ast.Attribute) - and inner.func.attr == "value" - and isinstance(inner.func.value, ast.Call) - ): - report.error( - "V018", - workflow_path, - node.lineno, - "one comprehension both dispatches a call and resolves " - "it with .value()", - ".value() blocks, so each call completes before the " - "next is dispatched. It does not error -- the fan-out " - "is just silently serial, and with it the reason to be " - "on CanyonOS Core. Dispatch every call first, then resolve: " - "futures = [a.work(i) for i in items] then " - "[f.value() for f in futures].", - ) - return - - -# ------------------------------------------------------------------ # -# V019-V020 what the copy order overwrites # -# ------------------------------------------------------------------ # - - -def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): - """V019 V020 -- later copies land on earlier ones at the context root.""" - for entry in sorted(os.listdir(project_dir)): - path = os.path.join(project_dir, entry) - if not os.path.isfile(path) or not entry.endswith(".py"): - continue - - # V019 -- the shared runtime is copied flat, after the project sweep. - if entry in RUNTIME_FLAT_NAMES: - report.error( - "V019", - path, - 1, - f"a project module named `{entry}` sits at the project root", - "The shared CanyonOS Core runtime is copied flat into the image after " - "the project sweep, so this file is overwritten by CanyonOS Core's own " - f"{entry}. Rename it or move it into a package directory.", - ) - - # V020 -- a stub is copied flat under its yaml's basename. - entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} - for yaml_path in yaml_paths: - stem = os.path.splitext(os.path.basename(yaml_path))[0] - module = f"{stem}.py" - if module in entrypoint_basenames: - continue # the entrypoint is copied last and wins its flat name back - candidate = os.path.join(project_dir, module) - if os.path.isfile(candidate): - report.error( - "V020", - candidate, - 1, - f"`{os.path.basename(yaml_path)}` generates a stub that lands on " - f"`{module}`", - "The yaml's basename names the stub, and the stub is copied flat " - "over the swept tree. Anything importing this module inside the " - "container gets the generated stub instead of the real code. " - "Rename the yaml to match its own entrypoint.", - ) - - -# ------------------------------------------------------------------ # -# V030-V031 capability-gated rules # -# ------------------------------------------------------------------ # - - -def check_env_file(report, config, config_path, project_dir): - """V030 -- gated on detected env-file injection support.""" - declared = config.get("env_file") - supported = report.capabilities.get("env_file") - - if not supported: - if declared: - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", - "No resolve_env_file in the importable ventis package, so the " - "key is silently dropped and the container answers a provider " - "credential error on the first request. This port requires the " - "`env_file` runtime capability.", - ) - else: - report.unavailable( - "V030", - "env_file is not supported by the importable `ventis` runtime. " - "Credentials have no declared path into a container on this tree.", - ) - return - - if not declared: - report.warn( - "V030", - config_path, - line_of(config), - "no `env_file:` in the config", - "Only runtime-managed VENTIS_* variables are guaranteed without it. " - "If the source reads credentials from the environment, the first " - "request fails on a provider error.", - ) - return - - # Path existence and readability are deploy-preflight checks. Do not - # duplicate them here. - - -def check_import_root(report, project_dir, entrypoint_paths): - """V031 -- gated on detected editable-install support.""" - supported = report.capabilities.get("editable_install") - has_metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - - non_flat = [] - for path in entrypoint_paths: - tree, _ = parse_python(path) - if tree is None: - continue - for name, lineno in toplevel_import_names(tree).items(): - if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: - continue - if _resolves_flat(project_dir, name): - continue - location = _resolves_nested(project_dir, name) - if location: - non_flat.append((path, lineno, name, location)) - - if not supported: - report.unavailable( - "V031", - "the editable install (`-e .`) is not supported by the importable " - "`ventis` runtime. Only names rooted at /app import inside a container.", - ) - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, which is not at the " - "project root", - "sys.path[0] is /app and this CanyonOS Core runs no editable install, " - "so only modules swept to the root import. The adapter raises " - "ModuleNotFoundError inside _load_agent and the first request " - "answers 'No agent loaded'.", - ) - return - - if non_flat and not has_metadata: - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, and the project root " - "has no packaging metadata", - "A pyproject.toml, setup.py or setup.cfg at the port root is " - "what adds `-e .`; metadata nested inside the untouched source " - "tree is ignored. Add minimal root metadata pointing at the " - "existing package directory. Without it the install is skipped " - "silently.", - ) - - -def _resolves_flat(project_dir, name): - """Whether Python can resolve `name` with /app as its import root. - - A directory does not need __init__.py: PEP 420 namespace packages resolve - from sys.path just like regular packages. - """ - return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( - os.path.join(project_dir, name) - ) - - -def _resolves_nested(project_dir, name): - """Where below /app `name` lives but cannot resolve as a top-level name.""" - for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] - if root == project_dir: - continue - if f"{name}.py" in files: - return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) - if name in dirs: - return os.path.relpath(os.path.join(root, name), project_dir) - return None - - -# ------------------------------------------------------------------ # -# W003, W006 secrets and imports a green build does not reject # -# ------------------------------------------------------------------ # - - -def check_secrets(report, port_paths): - """W003 -- env_file is the way in; nothing else is.""" - for path in port_paths: - try: - with open(path, "r", encoding="utf-8") as handle: - lines = handle.read().splitlines() - except OSError: - continue - for number, line in enumerate(lines, start=1): - for pattern, description in SECRET_PATTERNS: - if pattern.search(line): - report.warn( - "W003", - path, - number, - f"this line looks like {description}", - "Never put a secret in the source tree or the build " - "context. The build sweeps the project into every " - "image.", - ) - break - - tree, _ = parse_python(path) - if tree is None: - continue - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - if not ( - isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and node.value.value.strip() - ): - continue - for target in node.targets: - if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): - report.warn( - "W003", - path, - node.lineno, - f"`{target.id}` is assigned a literal string", - "Read it from the environment instead; the build sweeps " - "this file into every image.", - ) - - -def _pyproject_dependencies(project_dir): - """What `-e .` installs alongside `requirements:`, or None if unreadable. - - None and the empty set mean different things here: empty means the project - declares no dependencies, None means we could not find out -- a setup.py, or - a tomllib this interpreter does not have. The caller must not treat the - second as the first, or it warns about imports the install would satisfy. - """ - path = os.path.join(project_dir, "pyproject.toml") - if not os.path.isfile(path): - return None - try: - import tomllib - except ImportError: # < 3.11 - return None - try: - with open(path, "rb") as handle: - data = tomllib.load(handle) - except Exception: # noqa: BLE001 - malformed metadata is uv's error to give - return None - deps = (data.get("project") or {}).get("dependencies") - if not isinstance(deps, list): - return None - return {_normalize_distribution(d) for d in deps if isinstance(d, str)} - - -def check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path -): - """W006 -- an import the container cannot satisfy.""" - tree, _ = parse_python(entrypoint_path) - if tree is None: - return - - declared = { - _normalize_distribution(item) - for item in (entry.get("requirements") or []) - if isinstance(item, str) - } - - # Where the editable install exists, `-e .` resolves the project's own - # [project.dependencies] in the same pass as `requirements:`. Warning about - # those is a false positive, and a false warning about a dependency is worse - # than none: it teaches the reader to dismiss this check. - editable = report.capabilities.get("editable_install") - metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - unreadable_metadata = False - if editable and metadata: - project_deps = _pyproject_dependencies(project_dir) - if project_deps is None: - unreadable_metadata = True - else: - declared |= project_deps - base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} - stdlib = getattr(sys, "stdlib_module_names", frozenset()) - - for name, lineno in sorted(toplevel_import_names(tree).items()): - if name in stdlib or name == "ventis": - continue - # Provided by the image itself: the shared runtime is copied flat, and - # every agents/*.yaml generates a stub that is copied flat too. - if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: - continue - if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): - continue - distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) - if distribution in base or distribution in declared: - continue - if unreadable_metadata: - mechanism = ( - "The container installs the base list, `requirements:`, and -- " - "since this project declares packaging metadata -- whatever " - "`-e .` resolves from it. That metadata could not be read here, " - f"so if it already requires `{name}` this line is noise; " - "otherwise it is a ModuleNotFoundError inside _load_agent and " - "'No agent loaded' on the first request." - ) - else: - mechanism = ( - "The container installs the base list plus `requirements:` and " - "nothing else, so this is a ModuleNotFoundError inside " - "_load_agent and 'No agent loaded' on the first request. If the " - f"distribution is named something other than `{name}`, declare " - f"that name in {report.rel(config_path)}." - ) - report.warn( - "W006", - entrypoint_path, - lineno, - f"`import {name}` is in neither the runtime's base list nor this " - "entry's `requirements:`", - mechanism, - ) - - -def _normalize_distribution(name): - return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( - "_", "-" - ) - - -# ------------------------------------------------------------------ # -# Driver # -# ------------------------------------------------------------------ # - - -def validate(project_dir, config_path, capabilities): - """Inspect only failures hidden behind a successful image build.""" - report = Report(project_dir, capabilities) - - # The build owns config/YAML syntax and shape validation. We read only enough - # valid structure to locate code for the deeper checks below. - config, error = load_yaml(config_path) - if error is not None or not isinstance(config, dict): - report.unavailable( - "BUILD", - "runtime preflight skipped because the config cannot be read; " - "ventis build owns and reports this error.", - ) - return report - - import glob - - yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) - report.stub_module_names = { - os.path.splitext(os.path.basename(path))[0] for path in yaml_paths - } - - agents_by_name = {} - stub_classes = {} - for path in yaml_paths: - data, yaml_error = load_yaml(path) - agent = data.get("agent") if isinstance(data, dict) else None - name = agent.get("name") if isinstance(agent, dict) else None - if yaml_error is not None or not isinstance(name, str): - continue # ventis build reports malformed agent declarations - agents_by_name[name] = (path, agent) - stub_classes[os.path.splitext(os.path.basename(path))[0]] = name - - entries = config.get("agents") - if not isinstance(entries, list): - report.unavailable( - "BUILD", - "runtime preflight skipped because `agents:` is not a list; " - "ventis build owns and reports this error.", - ) - return report - - entrypoints = [] - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": - continue - name = entry.get("name") - entrypoint = entry.get("entrypoint") - if isinstance(entrypoint, str): - entrypoints.append(entrypoint) - if name in agents_by_name: - yaml_path, agent_block = agents_by_name[name] - check_adapter(report, yaml_path, agent_block, entry, project_dir) - entrypoint_path = os.path.join(project_dir, entrypoint or "") - if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): - check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path - ) - - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": - continue - workflow_file = entry.get("workflow_file") - if not isinstance(workflow_file, str): - continue - workflow_path = os.path.join(project_dir, workflow_file) - if os.path.isfile(workflow_path): - check_workflow(report, workflow_path, stub_classes) - - # These survive a green build and otherwise surface only in a container or - # on its first request. - check_flat_collisions(report, project_dir, yaml_paths, entrypoints) - check_env_file(report, config, config_path, project_dir) - - entrypoint_paths = [ - os.path.join(project_dir, e) - for e in entrypoints - if os.path.isfile(os.path.join(project_dir, e)) - ] - check_import_root(report, project_dir, entrypoint_paths) - - port_paths = list(entrypoint_paths) - for entry in entries: - if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): - candidate = os.path.join(project_dir, entry["workflow_file"]) - if os.path.isfile(candidate): - port_paths.append(candidate) - - # Secret detection remains because a green image build would permanently - # bake the credential into every image. - check_secrets(report, port_paths) - return report - - - -# ------------------------------------------------------------------ # -# Output # -# ------------------------------------------------------------------ # - -LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} - - -def _wrap(text, width, indent): - words = text.split() - lines = [] - current = "" - for word in words: - candidate = f"{current} {word}".strip() - if len(candidate) + len(indent) > width and current: - lines.append(indent + current) - current = word - else: - current = candidate - if current: - lines.append(indent + current) - return lines - - -def print_report(report, project_dir): - caps = report.capabilities - if not caps.get("ventis"): - print("ventis is not importable here -- capability-gated rules are") - print("reported UNAVAILABLE rather than checked.\n") - else: - print("CanyonOS Core capabilities detected:") - for key, source in CAPABILITY_SOURCE.items(): - mark = "yes" if caps.get(key) else "no " - print(f" {mark} {key:<22} {source}") - print() - - findings = sorted( - report.findings, - key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), - ) - for finding in findings: - where = finding["path"] - if where and finding["line"]: - where = f"{where}:{finding['line']}" - header = f"{finding['check']} {finding['level']:<5}" - print(f"{header} {where}" if where else header) - for line in _wrap(finding["summary"], 78, " "): - print(line) - if finding["mechanism"]: - for line in _wrap(finding["mechanism"], 78, " "): - print(line) - print() - - errors, warnings = report.counts() - if not findings: - print(f"{project_dir}: clean.") - return - print(f"{errors} error(s), {warnings} warning(s).") - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="Check a CanyonOS Core port against the rules in SKILL.md." - ) - parser.add_argument( - "project_dir", nargs="?", default=".", help="the port's project root" - ) - parser.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", - ) - parser.add_argument("--json", action="store_true", help="emit findings as JSON") - parser.add_argument( - "--strict", action="store_true", help="fail on warnings as well as errors" - ) - args = parser.parse_args(argv) - - project_dir = os.path.abspath(args.project_dir) - config_path = ( - args.config - if os.path.isabs(args.config) - else os.path.join(project_dir, args.config) - ) - - capabilities = probe_capabilities() - report = validate(project_dir, config_path, capabilities) - errors, warnings = report.counts() - - if args.json: - print( - json.dumps( - { - "project_dir": project_dir, - "capabilities": capabilities, - "errors": errors, - "warnings": warnings, - "findings": report.findings, - }, - indent=2, - ) - ) - else: - print_report(report, report.rel(project_dir) or project_dir) - - if errors or (args.strict and warnings): - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From c12c5c5b023c5c7c4a38d5273feac98b5665aa81 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 12:31:18 -0700 Subject: [PATCH 3/6] removed more useless code --- .../skills/porting-to-canyonos-core/SKILL.md | 240 ---- .../references/ec2.md | 41 - .../references/llm-proxy.md | 64 - .../references/packaging.md | 81 -- .../references/runtime-contract.md | 187 --- .../references/troubleshooting.md | 60 - .../porting-to-canyonos-core/validate.py | 1257 ----------------- 7 files changed, 1930 deletions(-) delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/SKILL.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/ec2.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/packaging.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md delete mode 100644 cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md delete mode 100755 cli/.claude/skills/porting-to-canyonos-core/validate.py diff --git a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md b/cli/.claude/skills/porting-to-canyonos-core/SKILL.md deleted file mode 100644 index fe6dd49..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/SKILL.md +++ /dev/null @@ -1,240 +0,0 @@ ---- -name: porting-to-canyonos-core -description: Ports existing LangChain, LangGraph, CrewAI, AutoGen, and hand-rolled Python agent projects to CanyonOS Core. Use when converting, migrating, adapting, packaging, building, or deploying an existing agent or multi-agent project onto CanyonOS Core. -compatibility: Requires Python, Docker, and the `ventis` compatibility CLI. Runtime identifiers remain `ventis`, `VENTIS_*`, and `ventis-*`. ---- - -# Port an agent project to CanyonOS Core - -CanyonOS Core is the product name. Its compatibility executable and Python -package remain `ventis`; environment variables and Docker resources retain the -`VENTIS_*` and `ventis-*` prefixes. These are protocol identifiers, not branding -strings. Do not rename them. - -## Load references only when needed - -- Read [references/packaging.md](references/packaging.md) when a source import - does not resolve from `/app`, the source is nested, or packaging metadata is - involved. -- Read [references/llm-proxy.md](references/llm-proxy.md) only when the target - includes `llm_proxy`. -- Read [references/ec2.md](references/ec2.md) only when any config entry uses - `provider: EC2`. -- Read [references/troubleshooting.md](references/troubleshooting.md) after a - failed build, image probe, deploy, or request. -- Read [references/runtime-contract.md](references/runtime-contract.md) when a - validator finding needs explanation or the runtime mechanism is unclear. - -## Goal: thin scaffolding beside untouched source - -```text -agents/.yaml one callable surface per service -agents/.py one thin adapter per service, when needed -workflow/_workflow.py HTTP entry point; calls deploy() -config/global_controller.yaml deployment manifest -config/policy.yaml optional access restriction -pyproject.toml conditional nested-import scaffolding - unchanged -``` - -The file count follows the deployment. A multi-agent port has one yaml/adapter -pair per service that is worth deploying separately. If a source class already -satisfies the runtime contract, point its config entry at that file and do not -copy it into an adapter. - -Everything the source already owns—prompts, tools, schemas, parsing, retries, -model clients, and node bodies—is imported. The port re-expresses only the -CanyonOS Core boundary and framework-owned orchestration. - -The port root is the existing repository root and the directory from which -`ventis build` runs. Write scaffolding there beside existing directories. If the -repository already uses `src/`, leave it in place and put `agents/`, `workflow/`, -and `config/` beside it. Never move or copy the repository into a new `src/` -directory, and never create an outer wrapper merely for the port. - -## 1. Survey before writing - -Identify: - -1. The source entry point and callable input/output. -2. Framework-owned control flow (`StateGraph`, `Crew`, `GroupChat`, routing, - `Send`, `Command`, interrupts). -3. Runtime-injected services nodes read: stores, context, memory, sessions, or - callback managers. -4. Sync versus async boundaries. -5. Imports and declared runtime dependencies. -6. Model provider, credential names, streaming use, and optional `llm_proxy`. -7. Whether independent work fans out and benefits from separate replicas. -8. Whether source imports resolve from the project root that becomes `/app`. - -Run the validator once now. Its header detects capabilities directly from the -importable runtime rather than external development metadata: - -```bash -python /validate.py . -``` - -If config or agent yaml is malformed, the validator defers to `ventis build`. -Capability-gated findings say which runtime behavior is available. - -## 2. Choose service boundaries - -Start with one service. Split only when it creates independent parallel work or -a distinct resource/replica profile. - -- Keep a ReAct loop together; every turn needs shared message history. -- Hoist supervisor task lists and `Send`-style fan-out into the workflow. -- Do not create a one-replica service with no distinct resource profile merely - to mirror every source graph node. - -Rewrite framework-owned edges as ordinary Python. Import the connected node -functions unchanged. Construct runtime-injected service objects from source -configuration; do not invent models, dimensions, stores, or defaults silently. -Report any choice the source does not specify. - -## 3. Write declarations and adapters - -### Agent yaml - -Use one yaml per deployed service. Argument types are bare builtins only: -`str`, `int`, `float`, `bool`, `dict`, or `list`. Every declared argument is -required by the generated stub. `returns.type` is documentation; use `dict` or -`list` to signal that workflow callers must `json.loads` the returned string. - -### Adapter - -The entrypoint exposes a module-level class named exactly `agent.name`. It -constructs with no arguments and its declared methods are synchronous. Read -configuration from the environment in `__init__`. Bridge source coroutines -inside a synchronous method with `asyncio.run(...)`. Serialize framework objects -with their own JSON-safe serializer before returning. - -Do not duplicate source prompts, tools, schemas, or model calls. Keep the source -provider and SDK. - -### Workflow - -Expose `main(query: str)` and call `deploy(main, port=...)` at module scope. -Import generated stubs by yaml basename and agent class name: - -```python -from deploy import deploy -from agents. import -``` - -The deployment platform sends `{query: string}` to `/main`. Pack richer input -inside `query`; any additional workflow parameter has a default. - -Dispatch every remote call before resolving any future: - -```python -futures = [agent.work(item=item) for item in items] -results = [json.loads(future.value()) for future in futures] -``` - -Do not fuse dispatch and `.value()` in one comprehension; that silently -serializes fan-out. Do not add an `if __name__ == "__main__":` block: the -workflow is executed with `__name__ == "__main__"` in production. - -### Config - -For each service, keep these names aligned: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -Use lowercase `provider: local`; `replicas` is an integer; `requirements` is a -list of distribution-name strings. Put `env_file` at config top level when the -runtime capability is available. Omit `policy.yaml` unless access must be -restricted; if present, give it a non-empty `rules` list. - -## Hard rules - -Capitalized **MUST** and **NEVER** are reserved for port-breaking or -source-integrity rules. The owner column states where each is decided. - -| ID | Rule | Owner | -|---|---|---| -| M1 | Entrypoint MUST define a class named exactly `agent.name` | V006 | -| M2 | The class MUST construct with no arguments | V007 | -| M3 | yaml argument names MUST match Python parameter names | V008 | -| M4 | yaml argument types MUST be bare builtins | V010 | -| M5 | Declared adapter methods MUST be synchronous | V009 | -| M6 | Config names MUST match yaml agent names | build | -| M7 | Config names MUST not collide after lowercase normalization | build output | -| M8 | Local provider MUST be lowercase `local` | deploy preflight | -| M9 | `replicas` MUST be an integer | deploy preflight | -| M10 | `requirements` MUST be a list of strings | build | -| M11 | Workflow MUST expose `main(query)`; extra parameters MUST default | V016 | -| M12 | Workflow MUST NEVER contain a main guard | V017 | -| M13 | Fan-out MUST dispatch all calls before resolving any | V018 | -| M14 | Project modules MUST not take runtime or generated-stub flat names | V019/V020 | -| M15 | Workflow MUST import stubs from `agents.` | V023 | -| M16 | Policy MUST be absent or contain a non-empty `rules` list | deploy preflight | -| M17 | EC2 entries MUST satisfy the EC2 deployment contract | deploy preflight | -| M18 | NEVER copy source prompts, tools, schemas, or model calls | review | -| M19 | NEVER hardcode or bake a real credential into an image | W003 | -| M20 | NEVER edit or vendor the source tree | `git status` | -| M21 | NEVER swap the source LLM provider | review | -| M22 | NEVER silently move, drop, or reclassify source dependencies | review | -| M23 | Framework control flow MUST be rewritten; source node logic MUST be imported | review | -| M24 | A non-resolving source import MUST have usable root packaging metadata when editable install is supported | V031 | - -## 4. Validate, build, and probe - -Run static preflight, then let the build own build-time validation: - -```bash -python /validate.py . -ventis build -c config/global_controller.yaml -``` - -A green build never imports the adapter. Probe each agent image in this order: - -```bash -# Runtime startup path - docker run --rm ventis- \ - python -c "import local_controller" - -# Agent load path; include --env-file when configured - docker run --rm --env-file ventis- \ - python -c "import importlib.util,sys; \ -s=importlib.util.spec_from_file_location('m','.py'); \ -m=importlib.util.module_from_spec(s);sys.modules['m']=m;s.loader.exec_module(m); \ -m.();print('ok')" -``` - -Also probe the workflow image with `python -c "import local_controller"`; it has -its own dependency resolve and generated-stub imports. - -Then deploy, send a representative request, and poll its status: - -```bash -ventis deploy -c config/global_controller.yaml -curl -X POST http://localhost:8080/main \ - -H 'Content-Type: application/json' -d '{"query":""}' -curl http://localhost:8080/status/ -``` - -A successful outer request with a source-level failure still proves the port -reached and returned the source behavior. Record the distinction. - -## 5. Clean up - -After collecting evidence, stop foreground deploy with Ctrl+C and wait for -controller cleanup. Remove exact leftovers if startup crashed. Then remove build -products and exact images from this config: - -```bash -ventis clean -docker image rm ventis- \ - ventis- - -test ! -e stubs && test ! -e grpc_stubs && test ! -e docker_container -docker ps -a --format '{{.Names}}' -``` - -`ventis clean` removes only `stubs/`, `grpc_stubs/`, and `docker_container/`; it -does not remove containers or images. Keep port scaffolding, untouched source, -and requested logs or reports. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md b/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md deleted file mode 100644 index e06daa0..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/ec2.md +++ /dev/null @@ -1,41 +0,0 @@ -# EC2 deployment - -Read this only when at least one config entry uses `provider: EC2`. - -## Configuration - -Use `provider: EC2` and declare `instance_type` on every EC2 service entry. The -top-level `ec2` block supplies the runtime's required infrastructure and SSH -settings. Read the target checkout's deploy preflight and EC2 runtime before -writing the block; do not copy values from an example environment. - -Typical required categories are: - -- AMI and instance type -- region and subnet -- security groups -- SSH user and credentials accepted by the runtime - -`ventis deploy` owns basic EC2 config validation. A preflight pass is not proof -that provisioning, SSH, image transfer, or remote container startup works. - -## Networking - -A remote container's `host.docker.internal` names its own EC2 Docker host. It -does not name the local controller machine. Databases, model proxies, and other -services must use addresses reachable from every selected host. - -The environment file may be copied temporarily to a remote host by runtimes that -expose the `env_file` capability. Confirm behavior from the capability probe and -target runtime rather than assuming local Docker semantics. - -## Probes and cleanup - -Run the same runtime and adapter probes against the exact image before remote -deployment. After deploy, verify the remote container logs; controller health -can be green even when agent loading failed. - -Stop foreground deploy normally so the controller can terminate recorded EC2 -instances. If provisioning or startup fails before an instance is recorded, -inspect the cloud provider directly and remove exact leaked resources. Never use -a broad cleanup command against unrelated instances. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md b/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md deleted file mode 100644 index f726bd7..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/llm-proxy.md +++ /dev/null @@ -1,64 +0,0 @@ -# LLM proxy integration - -Read this only when the target checkout contains `llm_proxy` or the deployment -explicitly routes model SDKs through it. - -## Preserve provider protocols - -The proxy redirects provider endpoints; it does not convert providers. Keep the -source SDK, model ID, request body, and response parsing unchanged. - -Configure only the provider variables the source uses: - -```dotenv -OPENAI_BASE_URL=http://host.docker.internal:8081/openai/v1 -ANTHROPIC_BASE_URL=http://host.docker.internal:8081/anthropic -AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock -``` - -Some SDKs refuse to initialize without caller credentials. Give agent containers -non-secret placeholders only when required. Keep real OpenAI, Anthropic, or AWS -credentials in the separate proxy process, not in the port's `env_file`. - -## Start locally - -The proxy defaults conflict with a typical deployment: host loopback is not -reachable from a container, and port 8080 is normally used by the workflow API. -Use a non-loopback bind and a different port: - -```bash -PROXY_HOST=0.0.0.0 PROXY_PORT=8081 python -m llm_proxy -curl http://127.0.0.1:8081/healthz -``` - -Local CanyonOS Core containers resolve `host.docker.internal` through their -Docker host mapping. On EC2 that name resolves to each EC2 Docker host, not the -machine running `ventis deploy`. Distributed deployments need a reachable proxy -address or one proxy on each host. - -## Supported call shape - -The implementation buffers complete requests and responses: - -- OpenAI and Anthropic non-streaming HTTP calls are forwarded. -- Bedrock `invoke` is reissued through the proxy's boto3 client. -- OpenAI/Anthropic streaming is unsupported. -- Bedrock `invoke-with-response-stream`, `converse`, and `converse-stream` are - unsupported. - -Survey the source before selecting the proxy. Do not silently disable streaming; -report the unsupported behavior and stop. - -## Credential behavior - -- The OpenAI adapter removes caller authorization and inserts the proxy key. -- The Anthropic adapter removes caller key headers and inserts the proxy key. -- Botocore still signs requests sent to a custom endpoint, so a caller may need - placeholder AWS credentials even though the proxy reissues upstream with its - own identity. -- `/healthz` proves provider registration and Flask availability, not upstream - credential validity. - -OpenAI and Anthropic upstream HTTP errors pass through. Proxy exceptions return -JSON 502 with `error: proxy_error`. Bedrock `ClientError` bodies are reconstructed -with the upstream status and are not byte-for-byte passthrough. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md b/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md deleted file mode 100644 index 8520c4a..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/packaging.md +++ /dev/null @@ -1,81 +0,0 @@ -# Packaging and import roots - -Read this reference when an adapter imports nested source code, the source uses a -`src/` layout, or V031 reports an import-root problem. - -## What `/app` can import - -CanyonOS Core preserves project-relative paths in the image and starts Python at -`/app`. Without an editable install, Python resolves names rooted there: - -- `/app/tools.py` as `import tools` -- `/app/pkg/__init__.py` as `import pkg` -- `/app/src/agents/...` as `import src.agents`, including PEP 420 namespace - directories without `__init__.py` - -It does not resolve `/app/source/pkg` as `import pkg`; `/app/source` must become -an import root first. - -## Detect support, do not infer it from release history - -Run: - -```bash -python /validate.py . -``` - -Read the `editable_install` capability. If it is unavailable and the original -import cannot resolve from `/app`, report a runtime capability blocker and stop. -Do not add a `sys.path` hack or relocate source files. - -## Root metadata is the trigger - -When editable install is supported, only packaging metadata at the **port root** -triggers `pip install -e .`: - -```text -port-root/pyproject.toml detected -port-root/source/pyproject.toml ignored as an install trigger -``` - -A nested source repository may remain untouched. Add minimal root scaffolding -that points package discovery at the existing source package: - -```toml -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "canyonos-port" -version = "0.0.0" -dependencies = [] - -[tool.setuptools.packages.find] -where = ["source/src"] -include = ["pkg*"] -namespaces = true -``` - -Set `where` and `include` from the actual tree and original import spelling. Do -not reference a README or license from this wrapper metadata; file sweeps differ -by runtime capability and a missing referenced file makes the image build fail. - -## Dependencies in nested metadata - -A nested `pyproject.toml` is not installed merely because its Python files are -copied. Keep the source declaration unchanged and repeat its runtime -distributions in each relevant config entry's `requirements` list. This is -compatibility scaffolding, not permission to drop, move, or reclassify declared -dependencies. - -If source metadata is already at the port root, do not create a wrapper. Its -project dependencies participate in the same resolver as config requirements. -Report declared-but-unused toolchain dependencies and their image cost; let the -owner decide whether source metadata should change. - -## Validation boundary - -`ventis build` owns packaging syntax and installation errors. `validate.py` -checks only whether adapter imports appear to require a nested root that the -runtime will not expose. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md deleted file mode 100644 index 94f7401..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md +++ /dev/null @@ -1,187 +0,0 @@ -# CanyonOS Core runtime contract - -The product is CanyonOS Core. Compatibility identifiers remain `ventis` for the -CLI and Python package, `VENTIS_*` for runtime variables, and `ventis-*` for -Docker resources. - -Read this reference when implementing an adapter or explaining a validator -finding. Runtime-dependent behavior is expressed as capabilities; run -`validate.py` against the target environment instead of inferring support from -release history. - -## Project root and discovery - -`ventis build` uses the current working directory as the project root. - -| Input | Discovery | -|---|---| -| `agents/*.yaml` | direct yaml glob under the project root | -| `config/global_controller.yaml` | default config, overridable with `-c` | -| workflow | `workflow_file` on a `type: workflow` entry | -| policy | `policy.yaml` beside the selected config file | -| generated files | `stubs/`, `grpc_stubs/`, `docker_container/` | - -The config name, yaml `agent.name`, and entrypoint class name form one binding: - -```text -config entry name == yaml agent.name == entrypoint class name -``` - -A missing match may skip an image while the command continues, so inspect build -output and generated image tags. - -## Agent yaml and generated stubs - -The consumed yaml shape is: - -```yaml -agent: - name: ExampleAgent - functions: - - name: work - arguments: - - name: query - type: str - returns: - type: dict -``` - -Argument annotations are generated from bare names without adding imports. Use -builtins. Generated methods have no defaults, so every declared argument is -required at the stub call site. `returns` does not control runtime conversion; -it documents whether workflow code should parse the returned string. - -Stub destinations differ by runtime capability and entrypoint layout. Workflow -code in this port convention imports the generated class from -`agents.`. The validator checks that import against declarations -before the workflow image starts. - -## Agent loading and execution - -The local controller effectively performs: - -```python -module = load(entrypoint) -agent_class = getattr(module, configured_name) -agent = agent_class() -result = getattr(agent, method_name)(**args) -``` - -Consequences: - -- The class is module-level and named exactly as configured. -- Construction takes no arguments. -- Declared methods accept yaml argument names as keyword arguments. -- Methods are synchronous; this path does not await a coroutine. -- Dicts and lists are JSON-encoded before entering Redis; other results become - strings. -- A remote Future's `.value()` returns text, not the original Python object. - -Agent import and construction exceptions are caught by the controller. A failed -agent may still advertise healthy because health is written independently of -successful agent loading. That is why image probes import both the runtime and -the entrypoint explicitly. - -## Workflow execution - -The workflow file is executed, not imported. Therefore: - -- module-level code runs at container startup; -- `__name__ == "__main__"`; -- `deploy()` blocks in the web server; -- the workflow function runs once per request; -- its function name determines the REST route exposed by the compatibility - runtime. - -The deployment platform additionally expects `/main` with a `{query: string}` -body. This platform constraint is stricter than the underlying transport. - -Each stub method returns a Future immediately. `.value()` blocks. Dispatching -and resolving inside one comprehension serializes work without raising an -error; dispatch all calls first, then resolve them. - -The workflow container also starts runtime controller code and has its own -package resolution. Probe it independently from agent images. - -## Build context and collisions - -The runtime copies project files while preserving relative paths, then writes -shared runtime modules, generated stubs, and entrypoints into the image. Later -writes can shadow project files. - -Avoid root project modules named like runtime files, including: - -```text -future.py -ventis_context.py -local_controller.py -local_controller_frontend.py -redis_client.py -grpc_options.py -bedrock.py -deploy.py -session_logging.py -workflow_launcher.py -``` - -Also avoid a yaml basename that shadows a different source module imported by an -adapter. The validator checks deterministic flat-name collisions. - -File sweep and editable-install behavior are runtime capabilities. For nested -imports, follow [packaging.md](packaging.md). - -## Dependencies and protobuf - -Agent and workflow images include a small runtime dependency set. Config -`requirements` adds source-specific distributions. A malformed requirements -value can be normalized away while image generation continues; missing imports -then surface only when the agent loads. - -The build compiles gRPC Python stubs on the host and copies them into images. -The image resolver does not necessarily know the generated-code version. A -source dependency that constrains protobuf below the host generator version can -produce a green image build that dies on: - -```text -import local_controller -``` - -Always run that probe before probing the entrypoint. Treat a generated-code / -runtime-version mismatch as a CanyonOS Core runtime issue, not a reason to alter -source dependencies silently. - -## Credentials capability - -When `env_file` capability is available, the top-level config path is resolved -against the project root and passed at container start. Hidden env files are not -copied into images. Invalid paths are deploy-preflight errors. - -When the capability is unavailable, declaring `env_file` has no effect. If the -source needs credentials, report the capability blocker rather than hardcoding -or vendoring a secret. - -A source that constructs its client at import time works only when credentials -are already in the container environment. Image entrypoint probes therefore use -the same env file as deployment. - -For proxy-specific credential separation, read [llm-proxy.md](llm-proxy.md). - -## Policy and provider behavior - -No policy file means unrestricted service access. If a policy exists, it needs a -non-empty rules list. Rules are evaluated by specificity and first match; -services excluded from the selected rule fail after request acceptance. - -Local provider handling is case-sensitive: use lowercase `local`. EC2 behavior -and remote networking are covered in [ec2.md](ec2.md). - -## Cleanup boundary - -Stopping foreground deploy normally invokes controller cleanup for recorded -containers and Redis. Hard kills and failures before resource registration may -leave resources behind. - -`ventis clean` removes generated `stubs/`, `grpc_stubs/`, and -`docker_container/`. It does not remove containers or images. Remove exact -leftovers explicitly and preserve source, port scaffolding, and requested -evidence. diff --git a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md b/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md deleted file mode 100644 index 361acf9..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/references/troubleshooting.md +++ /dev/null @@ -1,60 +0,0 @@ -# Troubleshooting - -Read this after a failed build, image probe, deploy, or request. For mechanisms, -read [runtime-contract.md](runtime-contract.md). For proxy or remote-host failures, -read [llm-proxy.md](llm-proxy.md) or [ec2.md](ec2.md). - -## Build or deploy stops early - -| Symptom | Likely cause | -|---|---| -| Agent image is missing | Config name matched no yaml name, entrypoint is absent, or build skipped it; inspect build warnings | -| Two services produce one image | Config names collide after lowercase normalization | -| `generated grpc_stubs are missing` | Build did not complete on this host, or generated output was cleaned before deploy | -| `int(... NoneType)` during local deploy | Local provider was not written exactly as lowercase `local` | -| Replica conversion `TypeError` | `replicas` is not an integer | -| Policy `AttributeError` | Policy exists but is empty or has null/non-list rules; remove it when unrestricted | -| Port or container name already in use | A previous deployment did not complete cleanup | - -## Container exits or serves nothing - -| Symptom | Likely cause | -|---|---| -| `import local_controller` raises protobuf version error | Host-generated gRPC code is newer than the image's protobuf runtime | -| First request says `No agent loaded` | Entrypoint import, class lookup, or constructor failed and the controller swallowed the exception; inspect container logs | -| Replica is healthy but serves nothing | Health publication does not prove successful agent loading | -| Missing credentials while loading | Env injection is unavailable/misconfigured or the source reads another variable | -| Source module is missing | Original import does not resolve from `/app`; read [packaging.md](packaging.md) | -| Third-party module is missing | Distribution is absent from source metadata and config requirements | -| Stub import raises `NameError` | yaml argument type is not a bare builtin | -| Source module behaves like an empty stub | A generated stub basename shadowed the source module | -| Runtime-named project module disappears | Shared runtime copy overwrote a root project module with the same name | - -## Request is accepted, then fails - -| Symptom | Likely cause | -|---|---| -| Unexpected keyword argument | yaml argument name differs from adapter parameter name | -| Required argument missing | yaml does not declare a required adapter parameter, or workflow platform sent only `query` | -| Unauthorized service | The first matching policy rule excludes that service | -| `.value()` returns dict-like text | Expected; remote values are strings, so use `json.loads` | -| Object is not JSON serializable | Adapter returned framework objects without its JSON-safe serializer | -| Redis contains a coroutine repr | Adapter method is async and the controller did not await it | -| Fan-out is no faster | Dispatch and `.value()` were fused, serializing the calls | -| Debug block runs at startup | Workflow is executed with `__name__ == "__main__"` | - -## Deployment platform endpoint - -| Symptom | Likely cause | -|---|---| -| 404 while workflow container is healthy | Workflow function is not named `main` | -| 400 before host receives request | Body is not the platform's `{query: string}` shape | -| Extra workflow argument is missing | Platform sends only `query`; extra parameters need defaults | - -## Cleanup - -| Symptom | Likely cause | -|---|---| -| `ventis clean` succeeds but containers remain | The command removes generated directories only | -| `ventis clean` succeeds but images remain | Image deletion is separate and requires exact tags | -| Next deployment collides with old resources | Foreground deploy was killed or crashed before controller cleanup | diff --git a/cli/.claude/skills/porting-to-canyonos-core/validate.py b/cli/.claude/skills/porting-to-canyonos-core/validate.py deleted file mode 100755 index 04baf37..0000000 --- a/cli/.claude/skills/porting-to-canyonos-core/validate.py +++ /dev/null @@ -1,1257 +0,0 @@ -#!/usr/bin/env python3 -"""Preflight the runtime traps that `ventis build` cannot see. - -This deliberately does not duplicate build-time validation such as malformed -YAML, missing entrypoints, or config-to-yaml matching. `ventis build` owns those -checks. This script parses Python without importing it and catches failures that -otherwise stay hidden until a container loads an agent, starts a workflow, or -serves its first request. A replica is not evidence: the controller writes -`healthy` to Redis before `_load_agent` runs. - - python validate.py [project_dir] [-c config/global_controller.yaml] - [--json] [--strict] - -Exit 1 if any ERROR was reported, 0 otherwise. --strict also fails on warnings. - -Runtime capabilities vary across CanyonOS Core installations. This script probes -the importable `ventis` package directly. A capability-gated check reports -UNAVAILABLE when its behavior cannot be proven. -""" - -import argparse -import ast -import builtins -import json -import os -import re -import sys -from typing import ClassVar - -try: - import yaml -except ImportError: # pragma: no cover - pyyaml is a CanyonOS Core dependency - sys.stderr.write("validate.py needs pyyaml: pip install pyyaml\n") - raise SystemExit(2) from None - - -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" - -# Copied flat into every image over the swept project tree, so a project module -# landing flat under one of these names is overwritten. -# ventis/stub_generator.py generate_docker / generate_workflow_docker. -RUNTIME_FLAT_NAMES = frozenset( - { - "future.py", - "ventis_context.py", - "local_controller.py", - "local_controller_frontend.py", - "redis_client.py", - "grpc_options.py", - "gpu_metrics.py", - "bedrock.py", - "deploy.py", - "session_logging.py", - "workflow_launcher.py", - } -) - -# ventis/stub_generator.py BASE_AGENT_REQUIREMENTS / BASE_WORKFLOW_REQUIREMENTS. -BASE_AGENT_REQUIREMENTS = [ - "grpcio", - "grpcio-tools", - "redis", - "pyyaml", - "psutil", - "ipdb", - "ipython", - "boto3", -] -# Import name -> distribution name, for the handful where they differ and the -# mismatch would otherwise be reported as a missing requirement. -IMPORT_TO_DISTRIBUTION = { - "attr": "attrs", - "bs4": "beautifulsoup4", - "cv2": "opencv-python", - "dateutil": "python-dateutil", - "dotenv": "python-dotenv", - "grpc": "grpcio", - "grpc_tools": "grpcio-tools", - "jwt": "pyjwt", - "PIL": "pillow", - "psycopg": "psycopg", - "psycopg2": "psycopg2-binary", - "pydantic_settings": "pydantic-settings", - "sklearn": "scikit-learn", - "typing_extensions": "typing-extensions", - "yaml": "pyyaml", -} - -SECRET_PATTERNS = [ - (re.compile(r"sk-[A-Za-z0-9_-]{20,}"), "an OpenAI-style secret key"), - (re.compile(r"AKIA[0-9A-Z]{16}"), "an AWS access key id"), - (re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}"), "a GitHub token"), - (re.compile(r"AIza[0-9A-Za-z_-]{30,}"), "a Google API key"), -] -SECRET_NAME = re.compile(r"(API_KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)", re.IGNORECASE) - -ERROR = "ERROR" -WARN = "WARN" -INFO = "INFO" - - -# ------------------------------------------------------------------ # -# Capabilities # -# ------------------------------------------------------------------ # -# -# Stable labels for behavior detected from the importable runtime. They contain -# no external development metadata. - -CAPABILITY_SOURCE = { - "env_file": "runtime env-file injection", - "editable_install": "editable project installation", - "sweeps_all_files": "full project-file sweep", - "stub_two_destinations": "flat and package stub destinations", -} - - -def probe_capabilities(): - """Ask the importable ventis package what it actually supports.""" - caps = dict.fromkeys(CAPABILITY_SOURCE, False) - caps["ventis"] = False - try: - from ventis import stub_generator - except Exception: # noqa: BLE001 - a broken install must not crash the check - return caps - - caps["ventis"] = True - caps["editable_install"] = hasattr(stub_generator, "_install_step") - caps["sweeps_all_files"] = hasattr(stub_generator, "_sweep_project_files") - caps["stub_two_destinations"] = hasattr(stub_generator, "_stub_destinations") - - import importlib - - for module_name in ("ventis.controller.utils.env_file", "ventis.utils.env_file"): - try: - module = importlib.import_module(module_name) - except Exception: # noqa: BLE001,S112 - the other path is the live one - continue - if hasattr(module, "resolve_env_file"): - caps["env_file"] = True - break - return caps - - -# ------------------------------------------------------------------ # -# YAML with line numbers # -# ------------------------------------------------------------------ # - - -class LineDict(dict): - """A mapping that remembers where it and each of its keys were written.""" - - line = 0 - key_lines: ClassVar[dict] = {} - - -class LineLoader(yaml.SafeLoader): - pass - - -def _construct_mapping(loader, node): - data = LineDict() - yield data - data.update(loader.construct_mapping(node, deep=False)) - data.line = node.start_mark.line + 1 - data.key_lines = { - key.value: key.start_mark.line + 1 - for key, _ in node.value - if isinstance(key, yaml.ScalarNode) - } - - -LineLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping -) - - -def line_of(mapping, key=None): - """The source line of `key` inside `mapping`, or of the mapping itself.""" - if not isinstance(mapping, LineDict): - return 0 - if key is not None: - return mapping.key_lines.get(key, mapping.line) - return mapping.line - - -def load_yaml(path): - """Parse `path`, returning (data, error). Never raises.""" - try: - with open(path, "r", encoding="utf-8") as handle: - return yaml.load(handle, Loader=LineLoader), None - except Exception as exc: # noqa: BLE001 - any parse failure is a finding - return None, str(exc) - - -# ------------------------------------------------------------------ # -# Findings # -# ------------------------------------------------------------------ # - - -class Report: - def __init__(self, project_dir, capabilities): - self.project_dir = project_dir - self.capabilities = capabilities - self.findings = [] - # A peer agent is imported by the name of its generated stub, which the - # build copies flat into every image. Those are not project modules and - # need no requirement. - self.stub_module_names = set() - - def add(self, check, level, path, line, summary, mechanism): - self.findings.append( - { - "check": check, - "level": level, - "path": self.rel(path) if path else "", - "line": line or 0, - "summary": summary, - "mechanism": mechanism, - } - ) - - def error(self, check, path, line, summary, mechanism): - self.add(check, ERROR, path, line, summary, mechanism) - - def warn(self, check, path, line, summary, mechanism): - self.add(check, WARN, path, line, summary, mechanism) - - def unavailable(self, check, summary): - self.add(check, INFO, "", 0, summary, "") - - def rel(self, path): - try: - return os.path.relpath(path, self.project_dir) - except ValueError: - return path - - def counts(self): - errors = sum(1 for f in self.findings if f["level"] == ERROR) - warnings = sum(1 for f in self.findings if f["level"] == WARN) - return errors, warnings - - -# ------------------------------------------------------------------ # -# Python source helpers # -# ------------------------------------------------------------------ # - - -def parse_python(path): - """AST for `path`, or (None, error). The port is never imported.""" - try: - with open(path, "r", encoding="utf-8") as handle: - source = handle.read() - except OSError as exc: - return None, str(exc) - try: - return ast.parse(source, filename=path), None - except SyntaxError as exc: - return None, f"{exc.msg} (line {exc.lineno})" - - -def find_class(tree, name): - for node in tree.body: - if isinstance(node, ast.ClassDef) and node.name == name: - return node - return None - - -def class_methods(class_node): - return { - node.name: node - for node in class_node.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - - -def parameter_names(func_node): - """Every parameter a caller can pass by keyword, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - return positional + [a.arg for a in args.kwonlyargs] - - -def required_parameters(func_node): - """Parameters with no default, minus self.""" - args = func_node.args - positional = [a.arg for a in args.posonlyargs + args.args] - if positional and positional[0] in ("self", "cls"): - positional = positional[1:] - if args.defaults: - positional = positional[: len(positional) - len(args.defaults)] - kwonly = [ - arg.arg - for arg, default in zip(args.kwonlyargs, args.kw_defaults) - if default is None - ] - return positional + kwonly - - -def toplevel_import_names(tree): - """Top-level package name of every import in the module, with line numbers.""" - names = {} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - names.setdefault(alias.name.split(".")[0], node.lineno) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - names.setdefault(node.module.split(".")[0], node.lineno) - return names - - -# ------------------------------------------------------------------ # -# V006-V010 adapter failures hidden by _load_agent # -# ------------------------------------------------------------------ # - -BUILTIN_TYPE_NAMES = frozenset( - name - for name in dir(builtins) - if isinstance(getattr(builtins, name), type) and not name[0].isupper() -) - - -def check_adapter(report, agent_yaml_path, agent_block, entry, project_dir): - """V006 V007 V008 V009 V010.""" - name = agent_block["name"] - functions = agent_block.get("functions") or [] - check_argument_types(report, agent_yaml_path, functions) - - entrypoint = entry.get("entrypoint") - if not entrypoint: - return - entrypoint_path = os.path.join(project_dir, entrypoint) - if not os.path.isfile(entrypoint_path): - return - - tree, error = parse_python(entrypoint_path) - if tree is None: - report.error( - "V006", - entrypoint_path, - 0, - f"the entrypoint does not parse: {error}", - "_load_agent exec_module's it and swallows the exception; the first " - "request answers 'No agent loaded'.", - ) - return - - class_node = find_class(tree, name) - if class_node is None: - classes = [n.name for n in tree.body if isinstance(n, ast.ClassDef)] - found = ", ".join(classes) if classes else "no classes at all" - report.error( - "V006", - entrypoint_path, - 1, - f"no class named `{name}` at module level (found: {found})", - "_load_agent does getattr(module, VENTIS_AGENT_NAME) and swallows " - "the AttributeError. The class name must equal agent.name exactly.", - ) - return - - methods = class_methods(class_node) - check_constructor(report, entrypoint_path, name, methods) - - for func in functions: - if not isinstance(func, dict) or not isinstance(func.get("name"), str): - continue - check_method(report, entrypoint_path, agent_yaml_path, name, func, methods) - - -def check_argument_types(report, agent_yaml_path, functions): - """V010 -- the type string is pasted into an ast.Name, never checked.""" - for func in functions: - if not isinstance(func, dict): - continue - for arg in func.get("arguments") or []: - if not isinstance(arg, dict) or "type" not in arg: - continue - declared = arg.get("type") - if not isinstance(declared, str): - continue # stub generation reports malformed type values - if declared in BUILTIN_TYPE_NAMES: - continue - report.error( - "V010", - agent_yaml_path, - line_of(arg, "type"), - f"`type: {declared}` is not a builtin", - "stub_generator pastes it verbatim into the generated " - "annotation, and the stub module imports only Future and " - "inspect. Anything else raises NameError when the stub is " - "imported -- after a green build. Use str int float bool dict " - "list.", - ) - - -def check_constructor(report, entrypoint_path, name, methods): - """V007 -- _load_agent calls agent_class() with no arguments.""" - init = methods.get("__init__") - if init is None: - return - required = required_parameters(init) - if required: - report.error( - "V007", - entrypoint_path, - init.lineno, - f"`{name}.__init__` requires {', '.join(required)}", - "_load_agent calls agent_class() with no arguments; the TypeError is " - "swallowed and the first request answers 'No agent loaded'. Read " - "configuration from the environment inside __init__ instead.", - ) - - -def check_method(report, entrypoint_path, agent_yaml_path, class_name, func, methods): - """V008 V009.""" - func_name = func["name"] - method = methods.get(func_name) - if method is None: - report.error( - "V008", - entrypoint_path, - 0, - f"`{class_name}` has no method `{func_name}`", - "The yaml declares it, so callers get a stub for it; the controller " - f"then answers \"Agent {class_name} has no method '{func_name}'\".", - ) - return - - # V009 -- nothing on the execution path awaits. - if isinstance(method, ast.AsyncFunctionDef): - report.error( - "V009", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` is `async def`", - "The executor calls method(**args) with no await, so Redis receives " - "''. Keep the signature synchronous and call " - "asyncio.run(...) inside the body.", - ) - - # V008 -- the controller calls method(**args) with the yaml's names. - declared = [ - arg["name"] - for arg in func.get("arguments") or [] - if isinstance(arg, dict) and isinstance(arg.get("name"), str) - ] - actual = parameter_names(method) - required = required_parameters(method) - - missing = [arg for arg in declared if arg not in actual] - if missing: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` has no parameter " - f"{', '.join(repr(m) for m in missing)}, but the yaml declares it", - "LocalController does method(**args) with the yaml's argument names. " - "A mismatch is TypeError: unexpected keyword argument, at request " - f"time. See {os.path.basename(agent_yaml_path)}.", - ) - - unfilled = [arg for arg in required if arg not in declared] - if unfilled: - report.error( - "V008", - entrypoint_path, - method.lineno, - f"`{class_name}.{func_name}` requires {', '.join(unfilled)}, which " - "the yaml does not declare", - "Only declared arguments are ever sent, and the generated stub gives " - "none of them a default. Declare them in the yaml or default them " - "in the signature.", - ) - - - -# ------------------------------------------------------------------ # -# V016-V018 the workflow # -# ------------------------------------------------------------------ # - - -def check_stub_imports(report, workflow_path, tree, stub_classes): - """V023 -- the workflow must import a stub as `from agents. import `. - - The build copies each stub to exactly one path, and for the workflow image - that path is agents/.py. Two ways of writing this line fail, and - the project walks you into both: the flat form is what examples/ uses, and - the class name is the one `ventis build` prints, which is not the one it - writes. - """ - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - base = alias.name.split(".")[0] - if base in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`import {alias.name}` -- the stub is at " - f"agents/{base}.py, not flat", - "The build copies a stub to one path, and for the " - "workflow that path is under agents/. This is a " - "ModuleNotFoundError the moment the workflow runs. " - f"Write `from agents.{base} import {stub_classes[base]}`.", - ) - continue - - if not isinstance(node, ast.ImportFrom) or not node.module: - continue - - module = node.module - if module in stub_classes: - report.error( - "V023", workflow_path, node.lineno, - f"`from {module} import ...` -- the stub is at " - f"agents/{module}.py, not flat", - "The build copies a stub to one path, and for the workflow " - "that path is under agents/. The flat form is what this " - "repository's own examples use and it raises " - "ModuleNotFoundError in the workflow image. Write " - f"`from agents.{module} import {stub_classes[module]}`.", - ) - continue - - if not module.startswith("agents."): - continue - base = module.split(".", 1)[1] - expected = stub_classes.get(base) - if expected is None: - continue - for alias in node.names: - if alias.name == expected: - continue - if alias.name == f"{expected}Stub": - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is the name the build prints, not the " - f"class it writes", - "generate_agent_stub sets class_name = agent_config['name'] " - "and then recomputes it with a 'Stub' suffix for the log " - "line only. The message names a class that does not exist; " - f"the class is `{expected}`.", - ) - else: - report.error( - "V023", workflow_path, node.lineno, - f"`{alias.name}` is not a class the stub for {base} defines", - f"The stub's class carries the agent's own name: `{expected}`.", - ) - - -def check_workflow(report, workflow_path, stub_classes=None): - """V016 V017 V018 V023.""" - tree, error = parse_python(workflow_path) - if tree is None: - report.error("V016", workflow_path, 0, f"does not parse: {error}", "") - return - - main = None - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and ( - node.name == "main" - ): - main = node - break - - if main is None: - defined = [ - n.name - for n in tree.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) - ] - found = ", ".join(defined) if defined else "no top-level functions" - report.error( - "V016", - workflow_path, - 1, - f"no top-level function named `main` (found: {found})", - "CanyonOS Core serves POST /, but the deployment platform's " - "test endpoint posts to a hardcoded /main. A differently named " - "workflow builds, deploys and stays unreachable -- 404, container " - "healthy.", - ) - else: - check_main_signature(report, workflow_path, main) - - if stub_classes: - check_stub_imports(report, workflow_path, tree, stub_classes) - - # V016 -- deploy() is what starts Flask. - if not any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "deploy" - for node in ast.walk(tree) - ): - report.error( - "V016", - workflow_path, - 1, - "the workflow never calls `deploy(...)`", - "workflow_launcher.py exec's this file and nothing else starts the " - "HTTP server; the container comes up serving nothing.", - ) - - check_main_guard(report, workflow_path, tree) - check_fused_fanout(report, workflow_path, tree) - - -def check_main_signature(report, workflow_path, main): - """V016 -- the platform sends exactly {"query": ...}.""" - if isinstance(main, ast.AsyncFunctionDef): - report.error( - "V016", - workflow_path, - main.lineno, - "`main` is `async def`", - "deploy() calls workflow_fn(**kwargs) on a Flask worker thread with " - "no await; the response body would be a coroutine repr.", - ) - params = parameter_names(main) - if not params: - report.error( - "V016", - workflow_path, - main.lineno, - "`main` takes no arguments", - 'The platform posts {"query": "..."} and deploy() splats the ' - "body in as kwargs -- TypeError on every request.", - ) - return - if params[0] != "query": - report.error( - "V016", - workflow_path, - main.lineno, - f"`main`'s first parameter is `{params[0]}`, not `query`", - "The platform's body schema is strictly validated as " - "{query: string}; any other key is rejected with 400 in the control " - "plane, before the request reaches the host.", - ) - extra = [p for p in required_parameters(main) if p != "query"] - if extra: - report.error( - "V016", - workflow_path, - main.lineno, - f"`main` requires {', '.join(extra)} beyond `query`", - "Only `query` is ever sent, so every other parameter needs a " - "default or the call raises on every request. Pack richer input " - "into `query`.", - ) - - -def check_main_guard(report, workflow_path, tree): - """V017 -- the workflow is exec'd, so __name__ == "__main__".""" - for node in tree.body: - if not isinstance(node, ast.If): - continue - test = node.test - if ( - isinstance(test, ast.Compare) - and isinstance(test.left, ast.Name) - and test.left.id == "__name__" - and any( - isinstance(c, ast.Constant) and c.value == "__main__" - for c in test.comparators - ) - ): - report.error( - "V017", - workflow_path, - node.lineno, - '`if __name__ == "__main__":` block in the workflow', - "workflow_launcher.py runs exec(open().read()), so " - "__name__ IS '__main__' here and this block executes in " - "production, at container start.", - ) - - -def check_fused_fanout(report, workflow_path, tree): - """V018 -- .value() blocks, so dispatching and resolving in one - comprehension runs the fan-out one call at a time.""" - comprehensions = (ast.ListComp, ast.SetComp, ast.GeneratorExp) - for node in ast.walk(tree): - if not isinstance(node, comprehensions + (ast.DictComp,)): - continue - elements = ( - [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt] - ) - for element in elements: - for inner in ast.walk(element): - if ( - isinstance(inner, ast.Call) - and isinstance(inner.func, ast.Attribute) - and inner.func.attr == "value" - and isinstance(inner.func.value, ast.Call) - ): - report.error( - "V018", - workflow_path, - node.lineno, - "one comprehension both dispatches a call and resolves " - "it with .value()", - ".value() blocks, so each call completes before the " - "next is dispatched. It does not error -- the fan-out " - "is just silently serial, and with it the reason to be " - "on CanyonOS Core. Dispatch every call first, then resolve: " - "futures = [a.work(i) for i in items] then " - "[f.value() for f in futures].", - ) - return - - -# ------------------------------------------------------------------ # -# V019-V020 what the copy order overwrites # -# ------------------------------------------------------------------ # - - -def check_flat_collisions(report, project_dir, yaml_paths, entrypoints): - """V019 V020 -- later copies land on earlier ones at the context root.""" - for entry in sorted(os.listdir(project_dir)): - path = os.path.join(project_dir, entry) - if not os.path.isfile(path) or not entry.endswith(".py"): - continue - - # V019 -- the shared runtime is copied flat, after the project sweep. - if entry in RUNTIME_FLAT_NAMES: - report.error( - "V019", - path, - 1, - f"a project module named `{entry}` sits at the project root", - "The shared CanyonOS Core runtime is copied flat into the image after " - "the project sweep, so this file is overwritten by CanyonOS Core's own " - f"{entry}. Rename it or move it into a package directory.", - ) - - # V020 -- a stub is copied flat under its yaml's basename. - entrypoint_basenames = {os.path.basename(e) for e in entrypoints if e} - for yaml_path in yaml_paths: - stem = os.path.splitext(os.path.basename(yaml_path))[0] - module = f"{stem}.py" - if module in entrypoint_basenames: - continue # the entrypoint is copied last and wins its flat name back - candidate = os.path.join(project_dir, module) - if os.path.isfile(candidate): - report.error( - "V020", - candidate, - 1, - f"`{os.path.basename(yaml_path)}` generates a stub that lands on " - f"`{module}`", - "The yaml's basename names the stub, and the stub is copied flat " - "over the swept tree. Anything importing this module inside the " - "container gets the generated stub instead of the real code. " - "Rename the yaml to match its own entrypoint.", - ) - - -# ------------------------------------------------------------------ # -# V030-V031 capability-gated rules # -# ------------------------------------------------------------------ # - - -def check_env_file(report, config, config_path, project_dir): - """V030 -- gated on detected env-file injection support.""" - declared = config.get("env_file") - supported = report.capabilities.get("env_file") - - if not supported: - if declared: - report.error( - "V030", - config_path, - line_of(config, "env_file"), - f"`env_file: {declared}` is set, but this CanyonOS Core never reads it", - "No resolve_env_file in the importable ventis package, so the " - "key is silently dropped and the container answers a provider " - "credential error on the first request. This port requires the " - "`env_file` runtime capability.", - ) - else: - report.unavailable( - "V030", - "env_file is not supported by the importable `ventis` runtime. " - "Credentials have no declared path into a container on this tree.", - ) - return - - if not declared: - report.warn( - "V030", - config_path, - line_of(config), - "no `env_file:` in the config", - "Only runtime-managed VENTIS_* variables are guaranteed without it. " - "If the source reads credentials from the environment, the first " - "request fails on a provider error.", - ) - return - - # Path existence and readability are deploy-preflight checks. Do not - # duplicate them here. - - -def check_import_root(report, project_dir, entrypoint_paths): - """V031 -- gated on detected editable-install support.""" - supported = report.capabilities.get("editable_install") - has_metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - - non_flat = [] - for path in entrypoint_paths: - tree, _ = parse_python(path) - if tree is None: - continue - for name, lineno in toplevel_import_names(tree).items(): - if name in report.stub_module_names or f"{name}.py" in RUNTIME_FLAT_NAMES: - continue - if _resolves_flat(project_dir, name): - continue - location = _resolves_nested(project_dir, name) - if location: - non_flat.append((path, lineno, name, location)) - - if not supported: - report.unavailable( - "V031", - "the editable install (`-e .`) is not supported by the importable " - "`ventis` runtime. Only names rooted at /app import inside a container.", - ) - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, which is not at the " - "project root", - "sys.path[0] is /app and this CanyonOS Core runs no editable install, " - "so only modules swept to the root import. The adapter raises " - "ModuleNotFoundError inside _load_agent and the first request " - "answers 'No agent loaded'.", - ) - return - - if non_flat and not has_metadata: - for path, lineno, name, location in non_flat: - report.error( - "V031", - path, - lineno, - f"`import {name}` resolves to {location}, and the project root " - "has no packaging metadata", - "A pyproject.toml, setup.py or setup.cfg at the port root is " - "what adds `-e .`; metadata nested inside the untouched source " - "tree is ignored. Add minimal root metadata pointing at the " - "existing package directory. Without it the install is skipped " - "silently.", - ) - - -def _resolves_flat(project_dir, name): - """Whether Python can resolve `name` with /app as its import root. - - A directory does not need __init__.py: PEP 420 namespace packages resolve - from sys.path just like regular packages. - """ - return os.path.isfile(os.path.join(project_dir, f"{name}.py")) or os.path.isdir( - os.path.join(project_dir, name) - ) - - -def _resolves_nested(project_dir, name): - """Where below /app `name` lives but cannot resolve as a top-level name.""" - for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if not d.startswith(".") and d != "__pycache__"] - if root == project_dir: - continue - if f"{name}.py" in files: - return os.path.relpath(os.path.join(root, f"{name}.py"), project_dir) - if name in dirs: - return os.path.relpath(os.path.join(root, name), project_dir) - return None - - -# ------------------------------------------------------------------ # -# W003, W006 secrets and imports a green build does not reject # -# ------------------------------------------------------------------ # - - -def check_secrets(report, port_paths): - """W003 -- env_file is the way in; nothing else is.""" - for path in port_paths: - try: - with open(path, "r", encoding="utf-8") as handle: - lines = handle.read().splitlines() - except OSError: - continue - for number, line in enumerate(lines, start=1): - for pattern, description in SECRET_PATTERNS: - if pattern.search(line): - report.warn( - "W003", - path, - number, - f"this line looks like {description}", - "Never put a secret in the source tree or the build " - "context. The build sweeps the project into every " - "image.", - ) - break - - tree, _ = parse_python(path) - if tree is None: - continue - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - if not ( - isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and node.value.value.strip() - ): - continue - for target in node.targets: - if isinstance(target, ast.Name) and SECRET_NAME.search(target.id): - report.warn( - "W003", - path, - node.lineno, - f"`{target.id}` is assigned a literal string", - "Read it from the environment instead; the build sweeps " - "this file into every image.", - ) - - -def _pyproject_dependencies(project_dir): - """What `-e .` installs alongside `requirements:`, or None if unreadable. - - None and the empty set mean different things here: empty means the project - declares no dependencies, None means we could not find out -- a setup.py, or - a tomllib this interpreter does not have. The caller must not treat the - second as the first, or it warns about imports the install would satisfy. - """ - path = os.path.join(project_dir, "pyproject.toml") - if not os.path.isfile(path): - return None - try: - import tomllib - except ImportError: # < 3.11 - return None - try: - with open(path, "rb") as handle: - data = tomllib.load(handle) - except Exception: # noqa: BLE001 - malformed metadata is uv's error to give - return None - deps = (data.get("project") or {}).get("dependencies") - if not isinstance(deps, list): - return None - return {_normalize_distribution(d) for d in deps if isinstance(d, str)} - - -def check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path -): - """W006 -- an import the container cannot satisfy.""" - tree, _ = parse_python(entrypoint_path) - if tree is None: - return - - declared = { - _normalize_distribution(item) - for item in (entry.get("requirements") or []) - if isinstance(item, str) - } - - # Where the editable install exists, `-e .` resolves the project's own - # [project.dependencies] in the same pass as `requirements:`. Warning about - # those is a false positive, and a false warning about a dependency is worse - # than none: it teaches the reader to dismiss this check. - editable = report.capabilities.get("editable_install") - metadata = any( - os.path.isfile(os.path.join(project_dir, name)) - for name in ("pyproject.toml", "setup.py", "setup.cfg") - ) - unreadable_metadata = False - if editable and metadata: - project_deps = _pyproject_dependencies(project_dir) - if project_deps is None: - unreadable_metadata = True - else: - declared |= project_deps - base = {_normalize_distribution(item) for item in BASE_AGENT_REQUIREMENTS} - stdlib = getattr(sys, "stdlib_module_names", frozenset()) - - for name, lineno in sorted(toplevel_import_names(tree).items()): - if name in stdlib or name == "ventis": - continue - # Provided by the image itself: the shared runtime is copied flat, and - # every agents/*.yaml generates a stub that is copied flat too. - if f"{name}.py" in RUNTIME_FLAT_NAMES or name in report.stub_module_names: - continue - if _resolves_flat(project_dir, name) or _resolves_nested(project_dir, name): - continue - distribution = _normalize_distribution(IMPORT_TO_DISTRIBUTION.get(name, name)) - if distribution in base or distribution in declared: - continue - if unreadable_metadata: - mechanism = ( - "The container installs the base list, `requirements:`, and -- " - "since this project declares packaging metadata -- whatever " - "`-e .` resolves from it. That metadata could not be read here, " - f"so if it already requires `{name}` this line is noise; " - "otherwise it is a ModuleNotFoundError inside _load_agent and " - "'No agent loaded' on the first request." - ) - else: - mechanism = ( - "The container installs the base list plus `requirements:` and " - "nothing else, so this is a ModuleNotFoundError inside " - "_load_agent and 'No agent loaded' on the first request. If the " - f"distribution is named something other than `{name}`, declare " - f"that name in {report.rel(config_path)}." - ) - report.warn( - "W006", - entrypoint_path, - lineno, - f"`import {name}` is in neither the runtime's base list nor this " - "entry's `requirements:`", - mechanism, - ) - - -def _normalize_distribution(name): - return re.split(r"[<>=!\[;\s]", name.strip().lower(), maxsplit=1)[0].replace( - "_", "-" - ) - - -# ------------------------------------------------------------------ # -# Driver # -# ------------------------------------------------------------------ # - - -def validate(project_dir, config_path, capabilities): - """Inspect only failures hidden behind a successful image build.""" - report = Report(project_dir, capabilities) - - # The build owns config/YAML syntax and shape validation. We read only enough - # valid structure to locate code for the deeper checks below. - config, error = load_yaml(config_path) - if error is not None or not isinstance(config, dict): - report.unavailable( - "BUILD", - "runtime preflight skipped because the config cannot be read; " - "ventis build owns and reports this error.", - ) - return report - - import glob - - yaml_paths = sorted(glob.glob(os.path.join(project_dir, "agents", "*.yaml"))) - report.stub_module_names = { - os.path.splitext(os.path.basename(path))[0] for path in yaml_paths - } - - agents_by_name = {} - stub_classes = {} - for path in yaml_paths: - data, yaml_error = load_yaml(path) - agent = data.get("agent") if isinstance(data, dict) else None - name = agent.get("name") if isinstance(agent, dict) else None - if yaml_error is not None or not isinstance(name, str): - continue # ventis build reports malformed agent declarations - agents_by_name[name] = (path, agent) - stub_classes[os.path.splitext(os.path.basename(path))[0]] = name - - entries = config.get("agents") - if not isinstance(entries, list): - report.unavailable( - "BUILD", - "runtime preflight skipped because `agents:` is not a list; " - "ventis build owns and reports this error.", - ) - return report - - entrypoints = [] - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") == "workflow": - continue - name = entry.get("name") - entrypoint = entry.get("entrypoint") - if isinstance(entrypoint, str): - entrypoints.append(entrypoint) - if name in agents_by_name: - yaml_path, agent_block = agents_by_name[name] - check_adapter(report, yaml_path, agent_block, entry, project_dir) - entrypoint_path = os.path.join(project_dir, entrypoint or "") - if isinstance(entrypoint, str) and os.path.isfile(entrypoint_path): - check_requirements_coverage( - report, project_dir, entry, entrypoint_path, config_path - ) - - for entry in entries: - if not isinstance(entry, dict) or entry.get("type", "agent") != "workflow": - continue - workflow_file = entry.get("workflow_file") - if not isinstance(workflow_file, str): - continue - workflow_path = os.path.join(project_dir, workflow_file) - if os.path.isfile(workflow_path): - check_workflow(report, workflow_path, stub_classes) - - # These survive a green build and otherwise surface only in a container or - # on its first request. - check_flat_collisions(report, project_dir, yaml_paths, entrypoints) - check_env_file(report, config, config_path, project_dir) - - entrypoint_paths = [ - os.path.join(project_dir, e) - for e in entrypoints - if os.path.isfile(os.path.join(project_dir, e)) - ] - check_import_root(report, project_dir, entrypoint_paths) - - port_paths = list(entrypoint_paths) - for entry in entries: - if isinstance(entry, dict) and isinstance(entry.get("workflow_file"), str): - candidate = os.path.join(project_dir, entry["workflow_file"]) - if os.path.isfile(candidate): - port_paths.append(candidate) - - # Secret detection remains because a green image build would permanently - # bake the credential into every image. - check_secrets(report, port_paths) - return report - - - -# ------------------------------------------------------------------ # -# Output # -# ------------------------------------------------------------------ # - -LEVEL_ORDER = {ERROR: 0, WARN: 1, INFO: 2} - - -def _wrap(text, width, indent): - words = text.split() - lines = [] - current = "" - for word in words: - candidate = f"{current} {word}".strip() - if len(candidate) + len(indent) > width and current: - lines.append(indent + current) - current = word - else: - current = candidate - if current: - lines.append(indent + current) - return lines - - -def print_report(report, project_dir): - caps = report.capabilities - if not caps.get("ventis"): - print("ventis is not importable here -- capability-gated rules are") - print("reported UNAVAILABLE rather than checked.\n") - else: - print("CanyonOS Core capabilities detected:") - for key, source in CAPABILITY_SOURCE.items(): - mark = "yes" if caps.get(key) else "no " - print(f" {mark} {key:<22} {source}") - print() - - findings = sorted( - report.findings, - key=lambda f: (LEVEL_ORDER[f["level"]], f["check"], f["path"], f["line"]), - ) - for finding in findings: - where = finding["path"] - if where and finding["line"]: - where = f"{where}:{finding['line']}" - header = f"{finding['check']} {finding['level']:<5}" - print(f"{header} {where}" if where else header) - for line in _wrap(finding["summary"], 78, " "): - print(line) - if finding["mechanism"]: - for line in _wrap(finding["mechanism"], 78, " "): - print(line) - print() - - errors, warnings = report.counts() - if not findings: - print(f"{project_dir}: clean.") - return - print(f"{errors} error(s), {warnings} warning(s).") - - -def main(argv=None): - parser = argparse.ArgumentParser( - description="Check a CanyonOS Core port against the rules in SKILL.md." - ) - parser.add_argument( - "project_dir", nargs="?", default=".", help="the port's project root" - ) - parser.add_argument( - "-c", - "--config", - default=DEFAULT_CONFIG_PATH, - help=f"config path relative to project_dir (default: {DEFAULT_CONFIG_PATH})", - ) - parser.add_argument("--json", action="store_true", help="emit findings as JSON") - parser.add_argument( - "--strict", action="store_true", help="fail on warnings as well as errors" - ) - args = parser.parse_args(argv) - - project_dir = os.path.abspath(args.project_dir) - config_path = ( - args.config - if os.path.isabs(args.config) - else os.path.join(project_dir, args.config) - ) - - capabilities = probe_capabilities() - report = validate(project_dir, config_path, capabilities) - errors, warnings = report.counts() - - if args.json: - print( - json.dumps( - { - "project_dir": project_dir, - "capabilities": capabilities, - "errors": errors, - "warnings": warnings, - "findings": report.findings, - }, - indent=2, - ) - ) - else: - print_report(report, report.rel(project_dir) or project_dir) - - if errors or (args.strict and warnings): - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From ced549d713f04931915886662978cfd3c7eb75ff Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 15:49:52 -0700 Subject: [PATCH 4/6] cli --- cli/README.md | 13 +- cli/canyonos/build.py | 205 +++++++++++++++++++ cli/canyonos/clean.py | 24 +-- cli/canyonos/config.py | 29 ++- cli/canyonos/constants.py | 53 ++++- cli/canyonos/dashboard.compose.yml | 11 +- cli/canyonos/dashboard_stack.py | 130 ++---------- cli/canyonos/deploy.py | 110 +++++++--- cli/canyonos/doctor.py | 96 +++++++++ cli/canyonos/gc.py | 83 ++++++++ cli/canyonos/init.py | 108 +++++++++- cli/canyonos/integrate.py | 82 -------- cli/canyonos/logs.py | 24 +-- cli/canyonos/quit.py | 18 +- cli/canyonos/serve.py | 4 +- cli/canyonos/stop.py | 37 +--- cli/canyonos/sync.py | 18 +- cli/canyonos/test.py | 173 ++++++++++++++++ cli/cli.py | 190 ++++++----------- cli/utils/help_screen.py | 60 ++++++ cli/utils/tui.py | 2 - {cli/tests => tests}/test_dashboard_stack.py | 0 22 files changed, 987 insertions(+), 483 deletions(-) create mode 100644 cli/canyonos/build.py create mode 100644 cli/canyonos/doctor.py create mode 100644 cli/canyonos/gc.py delete mode 100644 cli/canyonos/integrate.py create mode 100644 cli/canyonos/test.py create mode 100644 cli/utils/help_screen.py rename {cli/tests => tests}/test_dashboard_stack.py (100%) diff --git a/cli/README.md b/cli/README.md index de66cd0..76cd4e1 100644 --- a/cli/README.md +++ b/cli/README.md @@ -4,11 +4,18 @@ Serves as a thin API layer, connecting to the global controller container. ## Serve -`canyonos serve -c config/global_controller.yaml` starts the local CanyonOS dashboard. It reads -`database.url` from the config and writes only `CANYONOS_`-prefixed settings to the current `.env`, -leaving other lines unchanged. +`canyonos serve` starts the local CanyonOS dashboard against the Postgres bundled in its compose +stack — it reads no project config, so it takes no arguments. It writes only `CANYONOS_`-prefixed +settings into the current directory's `.env`, leaving every other line unchanged. +## Requirements +Need a coding agent(Claude Code, Codex, Cursor) +Need uv or pip +Need docker and docker compose +If you have a workflow running, and want to make a config change, canyonos config automatically would reload the project with your config. If you change the workflow files itself though and want the changes to take effect, you need to redeploy from scratch, running canyonos build for good measure + +# Use: canyonos -h ### To Republish to PyPi ```Terminal diff --git a/cli/canyonos/build.py b/cli/canyonos/build.py new file mode 100644 index 0000000..f5292aa --- /dev/null +++ b/cli/canyonos/build.py @@ -0,0 +1,205 @@ +""" +Logic for `canyonos build`: install the CanyonOS skill on a coding agent, +then launch that agent with a prompt to apply it to the current project. +""" + +import os +import shutil +import subprocess +import tarfile +import tempfile +import urllib.request + +from rich.console import Console + +from utils.tui import select_menu + +SKILL_OWNER = "CanyonCodeCoreAI" +SKILL_REPO = "canyoncodecore" +# The .car-aware skill lives only on this branch; the copies on main and every +# other branch are the older flat-layout `porting-to-canyonos-core`. Repoint at +# main once this merges -- and rename SKILL_NAME with it, since the two +# variants declare different `name:` frontmatter. +SKILL_REF = "nickhuo/porting-skill-car-layout" +SKILL_NAME = "porting-to-canyonos" +SKILL_PATH = f".claude/skills/{SKILL_NAME}" + +REPO_URL = f"https://github.com/{SKILL_OWNER}/{SKILL_REPO}" +TREE_URL = f"{REPO_URL}/tree/{SKILL_REF}/{SKILL_PATH}" +TARBALL_URL = f"https://codeload.github.com/{SKILL_OWNER}/{SKILL_REPO}/tar.gz/refs/heads/{SKILL_REF}" + +# The porting skill emits no otel config; without this the dashboard stays empty. +OTEL_BLOCK = """otel: + destinations: + - name: local + protocol: http + endpoint: http://host.docker.internal:3000/v1/traces + headers: {}""" + +BUILD_PROMPT = ( + f"Use the CanyonOS {SKILL_NAME} skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." + "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," + " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" + " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK +) + +AGENTS = { + "claude": { + "label": "Claude Code", + "cli": "claude", + # Claude Code auto-loads project-local skills from here. The leaf name + # must match the skill's own `name:` frontmatter or it won't resolve. + "skill_dir": SKILL_PATH, + }, + "codex": { + "label": "Codex", + "cli": "codex", + # Codex only auto-loads skills from the user's home directory, not per-project. + "skill_dir": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + }, +} + + +def prompt_agent(): + options = [(key, spec["label"]) for key, spec in AGENTS.items()] + return select_menu(options, title="Which coding agent do you want to build on?") + + +def _replace_dir(source, dest): + """Move `source` onto `dest`, replacing whatever was there.""" + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + if os.path.isdir(dest): + shutil.rmtree(dest) + shutil.move(source, dest) + + +def _fetch_with_git(dest): + """Sparse-checkout just the skill path -- no full-repo download, no Node.""" + if not shutil.which("git"): + return False + + with tempfile.TemporaryDirectory() as tmp: + clone = os.path.join(tmp, "repo") + cloned = subprocess.run( + ["git", "clone", "--depth", "1", "--filter=blob:none", "--sparse", + "--branch", SKILL_REF, REPO_URL, clone], + capture_output=True, + ) + if cloned.returncode != 0: + return False + + sparse = subprocess.run( + ["git", "-C", clone, "sparse-checkout", "set", SKILL_PATH], + capture_output=True, + ) + skill = os.path.join(clone, SKILL_PATH) + if sparse.returncode != 0 or not os.path.isdir(skill): + return False + + _replace_dir(skill, dest) + return True + + +def _fetch_with_tarball(dest): + """Stdlib-only fallback: pull the ref's tarball and keep the skill members. + + Needs no external tool at all, at the cost of downloading the whole repo. + """ + prefix = f"{SKILL_PATH}/" + with tempfile.TemporaryDirectory() as tmp: + archive = os.path.join(tmp, "repo.tar.gz") + try: + with urllib.request.urlopen(TARBALL_URL, timeout=60) as response: + with open(archive, "wb") as out: + shutil.copyfileobj(response, out) + except OSError: + return False + + staged = os.path.join(tmp, "skill") + found = False + with tarfile.open(archive, "r:gz") as tar: + for member in tar.getmembers(): + # Drop the archive's own top-level directory, whose name + # depends on how GitHub mangles the ref. + _, _, path = member.name.partition("/") + if not path.startswith(prefix) or not member.isfile(): + continue + relative = os.path.relpath(path, SKILL_PATH) + target = os.path.join(staged, relative) + # Never let an archive entry write outside the staging dir. + if not os.path.abspath(target).startswith(os.path.abspath(staged) + os.sep): + continue + extracted = tar.extractfile(member) + if extracted is None: + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "wb") as out: + shutil.copyfileobj(extracted, out) + found = True + + if not found: + return False + _replace_dir(staged, dest) + return True + + +def _fetch_with_npx(dest): + """Last resort, and the only strategy that needs Node.""" + if not shutil.which("npx"): + return False + # -f overwrites an existing skill dir; without it gitpick exits 1 when the + # target already exists and is non-empty (e.g. re-running `build`). + return subprocess.run( + ["npx", "-y", "gitpick", "-f", TREE_URL, dest], capture_output=True + ).returncode == 0 + + +FETCH_STRATEGIES = ( + ("git", _fetch_with_git), + ("tarball", _fetch_with_tarball), + ("npx", _fetch_with_npx), +) + + +def install_skill(agent, console): + """Fetch the skill into the agent's skill dir. Returns True on success.""" + dest = AGENTS[agent]["skill_dir"] + for name, fetch in FETCH_STRATEGIES: + try: + if fetch(dest): + console.print(f"Fetched the CanyonOS skill via {name}.") + return True + except OSError: + pass + console.print(f"[dim]{name} fetch unavailable, trying the next option...[/dim]") + + console.print( + f"Could not fetch the CanyonOS skill from {TREE_URL}.\n" + "Install git or Node, or check network access, then run `canyonos doctor`." + ) + return False + + +def launch_agent(agent, prompt): + spec = AGENTS[agent] + if not shutil.which(spec["cli"]): + print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + return + # No check=True: the agent exiting non-zero (including the user quitting it) + # is an ordinary outcome, not something to raise a traceback over. + subprocess.run([spec["cli"], prompt]) + + +def run_build(): + console = Console() + agent = prompt_agent() + if agent is None: + console.print("Cancelled.") + return + + console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") + if not install_skill(agent, console): + return + + console.print(f"Launching {AGENTS[agent]['label']}...") + launch_agent(agent, BUILD_PROMPT) diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py index aabf105..d7cc6f5 100644 --- a/cli/canyonos/clean.py +++ b/cli/canyonos/clean.py @@ -1,7 +1,5 @@ """ -Remove generated stubs, gRPC files, and Docker build contexts. - -Ported directly over from canyonos, moving the logic into here. +Logic for `canyonos clean`: remove the generated .car artifact directory. """ import os @@ -9,20 +7,12 @@ def run_clean(): - project_dir = os.getcwd() - - paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), - ] + car_dir = os.path.join(os.getcwd(), ".car") - for path in paths_to_clean: - if os.path.exists(path): - print(f"Cleaning {path}...") - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.remove(path) + if not os.path.isdir(car_dir): + print("Nothing to clean, no .car folder in root") + return + print(f"Cleaning {car_dir}...") + shutil.rmtree(car_dir) print("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py index 226312e..25d2d3d 100644 --- a/cli/canyonos/config.py +++ b/cli/canyonos/config.py @@ -7,9 +7,8 @@ import yaml from rich.console import Console from rich.table import Table -from ruamel.yaml import YAML -from canyonos.constants import default_config_path +from canyonos.constants import default_config_path, round_trip_yaml from canyonos.theme import GREEN, WHITE from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu @@ -99,12 +98,19 @@ def _kv_table(title, data): return table -def run_view_config(config_path=None): +def _require_config(config_path, console): + """Resolved config path, or None after reporting that it's missing.""" config_path = config_path or default_config_path() - console = Console() - if not os.path.isfile(config_path): console.print(f"[red]Config file not found: {config_path}[/red]") + return None + return config_path + + +def run_view_config(config_path=None): + console = Console() + config_path = _require_config(config_path, console) + if config_path is None: return with open(config_path) as f: @@ -288,19 +294,12 @@ def _navigate(screen, node, breadcrumb): def run_change_config(config_path=None): - config_path = config_path or default_config_path() console = Console() - - if not os.path.isfile(config_path): - console.print(f"[red]Config file not found: {config_path}[/red]") + config_path = _require_config(config_path, console) + if config_path is None: return - # Round-trip loader preserves comments, key order, quoting and ${ENV} refs. - yaml_rt = YAML() - yaml_rt.preserve_quotes = True - # Match the project's YAML style so edits don't reflow list indentation: - # block sequences indented under their key (` - item`). - yaml_rt.indent(mapping=2, sequence=4, offset=2) + yaml_rt = round_trip_yaml() with open(config_path) as f: data = yaml_rt.load(f) diff --git a/cli/canyonos/constants.py b/cli/canyonos/constants.py index d34316e..455e860 100644 --- a/cli/canyonos/constants.py +++ b/cli/canyonos/constants.py @@ -1,9 +1,60 @@ -"""Shared constants for the canyonos CLI.""" +"""Shared helpers for the canyonos CLI.""" import os +import yaml +from ruamel.yaml import YAML + +DEFAULT_API_PORT = 8080 + +# The workflow entrypoint is always exposed as POST /main with a {"query": ...} +# body, regardless of what the workflow function is called in the project. +WORKFLOW_ROUTE = "main" + def default_config_path(): """Global controller config for the current directory, preferring the .car artifact layout.""" car = os.path.join(".car", "config", "global_controller.yaml") return car if os.path.isfile(car) else os.path.join("config", "global_controller.yaml") + + +def workflow_api_port(config_path): + """Host port the workflow answers on, or None if there isn't one to read.""" + try: + with open(config_path) as f: + config = yaml.safe_load(f) or {} + except (OSError, yaml.YAMLError): + return None + + for agent in config.get("agents") or []: + if agent.get("type") == "workflow": + return agent.get("api_port", DEFAULT_API_PORT) + return None + + +def workspace_relative(config_path): + """`config_path` relative to the cwd, or None if it falls outside it. + + The container only ever receives a copy of the current directory, and it + resolves what it's given against /workspace -- so an absolute path silently + discards that prefix and a `../` one escapes it. Both then 404 naming a + path that exists on the host, which reads as a bug in the wrong place. + """ + # realpath on both sides: a symlinked project dir (or macOS's /tmp -> + # /private/tmp) otherwise makes an in-project absolute path look external. + relative = os.path.relpath(os.path.realpath(config_path), os.path.realpath(os.getcwd())) + if relative == ".." or relative.startswith(f"..{os.sep}"): + return None + return relative + + +def round_trip_yaml(): + """Loader that preserves comments, key order, quoting and ${ENV} refs. + + The indent settings match the project's YAML style, so edits don't reflow + list indentation: block sequences stay indented under their key. + """ + yaml_rt = YAML() + yaml_rt.preserve_quotes = True + yaml_rt.indent(mapping=2, sequence=4, offset=2) + return yaml_rt diff --git a/cli/canyonos/dashboard.compose.yml b/cli/canyonos/dashboard.compose.yml index db775aa..0689749 100644 --- a/cli/canyonos/dashboard.compose.yml +++ b/cli/canyonos/dashboard.compose.yml @@ -19,11 +19,14 @@ services: depends_on: db: condition: service_healthy - # Published so a GC container can POST OTLP spans to /v1/traces via - # host.docker.internal; that route also renames ventis' `project_id` - # attribute to the `canyon.project.id` every dashboard query filters on. + # Published on all interfaces (not just 127.0.0.1) so a GC container can + # actually reach this via host.docker.internal -- Docker's host-gateway + # route doesn't reach ports bound to loopback only, so a 127.0.0.1 bind + # here silently black-holed every OTLP span export. That route also + # renames ventis' `project_id` attribute to the `canyon.project.id` every + # dashboard query filters on. ports: - - "127.0.0.1:3000:3000" + - "3000:3000" environment: DATABASE_URL: postgresql://canyonos:canyonos@db:5432/canyonos JWT_SECRET: ${CANYONOS_JWT_SECRET} diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py index 6abcfb1..c606357 100644 --- a/cli/canyonos/dashboard_stack.py +++ b/cli/canyonos/dashboard_stack.py @@ -19,11 +19,6 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable -from urllib.parse import urlsplit, urlunsplit - -import yaml - -from canyonos.constants import default_config_path COMPOSE_PROJECT = "canyonos-dashboard" STACK_VERSION = "v0.1.0-rc.2" @@ -32,7 +27,6 @@ HOST_GATEWAY = "host.docker.internal" REDIS_HOST = HOST_GATEWAY REDIS_PORT = "6379" -ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") @dataclass(frozen=True) @@ -54,7 +48,6 @@ def __init__(self, phase: str, message: str, *, had_containers: bool | None = No @dataclass(frozen=True) class DashboardStack: - database_url: str | None state_dir: Path project_dir: Path web_port: int = 8080 @@ -73,9 +66,6 @@ def _state_dir() -> Path: def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: - # Absolute, not "./.env": `canyonos serve` may cd into .car/ before - # running, so a cwd-relative path would miss the project root .env that - # `prepare()` actually writes to (stack.env_path). return [ "docker", "compose", @@ -88,25 +78,6 @@ def _compose_argv(stack: DashboardStack, manifest: Path) -> list[str]: ] -def _managed_database_url(database_url: str) -> tuple[str, str | None]: - parsed = urlsplit(database_url) - if parsed.hostname not in {"localhost", "127.0.0.1"}: - return database_url, None - - hostname = parsed.hostname - credentials = "" - if parsed.username is not None: - credentials = parsed.username - if parsed.password is not None: - credentials = f"{credentials}:{parsed.password}" - credentials = f"{credentials}@" - port = f":{parsed.port}" if parsed.port is not None else "" - rewritten = urlunsplit( - (parsed.scheme, f"{credentials}{HOST_GATEWAY}{port}", parsed.path, parsed.query, parsed.fragment) - ) - return rewritten, hostname - - def _existing_dashboard_port() -> int | None: """The host port an already-running dashboard `web` container owns, if any -- so re-running `canyonos serve` reconnects to the same stack instead of @@ -143,9 +114,9 @@ def _port_is_free(port: int) -> bool: def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: - """First free port at or after `start` -- same retry-on-conflict shape as - init.py's GC port selection, so an unrelated process/container squatting - on 8080 (e.g. a deployed Workflow's own api_port) doesn't hard-block serve. + """First free port at or after `start`, so an unrelated process or container + squatting on 8080 (e.g. a deployed Workflow's own api_port) doesn't + hard-block serve. """ for port in range(start, start + max_attempts): if _port_is_free(port): @@ -155,42 +126,7 @@ def _find_web_port(start: int = 8080, max_attempts: int = 50) -> int: ) -def _load_project_config(config_path: str) -> tuple[object, Path]: - project_root = Path(os.path.abspath(os.path.join(os.path.dirname(config_path), ".."))) - # `canyonos serve` cds into .car/ before calling here, so the naive - # parent-of-parent lands on .car itself -- go up one more level to reach - # the actual project root, where .env lives. - if project_root.name == ".car": - project_root = project_root.parent - dotenv_path = project_root / ".env" - if dotenv_path.is_file(): - with dotenv_path.open(encoding="utf-8") as dotenv: - for line in dotenv: - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - key = key.strip() - value = _env_value(value) - if key and key not in os.environ: - os.environ[key] = value - - with open(config_path, encoding="utf-8") as config_file: - config = yaml.safe_load(config_file) - return _expand_env_value(config), project_root - - -def _expand_env_value(value: object) -> object: - if isinstance(value, str): - return ENV_REFERENCE.sub(lambda match: os.environ.get(match.group(1), match.group(0)), value) - if isinstance(value, dict): - return {key: _expand_env_value(item) for key, item in value.items()} - if isinstance(value, list): - return [_expand_env_value(item) for item in value] - return value - - -def validate(config_path: str) -> DashboardStack: +def validate() -> DashboardStack: if shutil.which("docker") is None: raise PhaseFailure("validate", "docker is not on PATH") @@ -202,26 +138,10 @@ def validate(config_path: str) -> DashboardStack: except OSError: raise PhaseFailure("validate", "docker daemon or socket is unavailable") - try: - # Keep interpolation consistent with GlobalController._load_config in ventis/controller/global_controller.py. - config, project_root = _load_project_config(config_path) - except (OSError, yaml.YAMLError): - raise PhaseFailure("validate", f"config file is not readable: {config_path}") - - # database.url is optional -- the dashboard works without a database configured - # (e.g. OTLP-only setups); if present, it still needs to actually be usable. - database = config.get("database") if isinstance(config, dict) else None - database_url = database.get("url") if isinstance(database, dict) else None - if database_url is not None: - if not isinstance(database_url, str) or not database_url.strip(): - raise PhaseFailure("validate", "database.url must be a non-empty string") - unresolved = ENV_REFERENCE.search(database_url) - if unresolved: - name = unresolved.group(1) - raise PhaseFailure( - "validate", f"database.url needs ${{{name}}}, which is not set in the project .env" - ) - database_url = database_url.strip() + # The dashboard reads no project config -- it always runs against the + # bundled Postgres on this machine -- so the project root is just the cwd, + # the same assumption sync/clean/build already make. + project_root = Path.cwd() state_dir = _state_dir() try: @@ -235,7 +155,7 @@ def validate(config_path: str) -> DashboardStack: web_port = _existing_dashboard_port() or _find_web_port() - return DashboardStack(database_url, state_dir, project_root, web_port) + return DashboardStack(state_dir, project_root, web_port) def _env_value(value: str) -> str: @@ -319,10 +239,6 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: "CANYONOS_WEB_IMAGE": WEB_IMAGE, "CANYONOS_WEB_PORT": str(stack.web_port), } - rewritten_host = None - if stack.database_url is not None: - managed_database_url, rewritten_host = _managed_database_url(stack.database_url) - managed_env["CANYONOS_DATABASE_URL"] = managed_database_url _write_project_env(stack.env_path, managed_env) (stack.state_dir / "stack.json").write_text( json.dumps( @@ -338,20 +254,7 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: except (OSError, ValueError): raise PhaseFailure("prepare", "could not prepare the dashboard state directory") - message = "dashboard state prepared" - if rewritten_host: - message = ( - f"database host {rewritten_host} is reachable from the stack as host.docker.internal" - ) - return managed_env, message - - -def _redact_stack_text(text: str, stack: DashboardStack, managed_env: dict[str, str]) -> str: - secret = managed_env["CANYONOS_JWT_SECRET"] - redacted = redact_logs(text, secret) - if stack.database_url is not None: - redacted = redact_logs(redacted.replace(stack.database_url, "[redacted]"), secret) - return redacted + return managed_env, "dashboard state prepared" def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: @@ -361,13 +264,12 @@ def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: def _command_failure_message( message: str, result: subprocess.CompletedProcess[str], - stack: DashboardStack, managed_env: dict[str, str], ) -> str: detail = _last_stderr_line(result) if detail is None: return message - return f"{message}: {_redact_stack_text(detail, stack, managed_env)}" + return f"{message}: {redact_logs(detail, managed_env['CANYONOS_JWT_SECRET'])}" def pull( @@ -383,7 +285,7 @@ def pull( if result.returncode != 0: raise PhaseFailure( "pull", - _command_failure_message("docker compose pull failed", result, stack, managed_env), + _command_failure_message("docker compose pull failed", result, managed_env), had_containers=had_containers, ) return "dashboard images pulled" @@ -412,7 +314,7 @@ def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> if result.returncode != 0: raise PhaseFailure( "start", - _command_failure_message("docker compose up failed", result, stack, managed_env), + _command_failure_message("docker compose up failed", result, managed_env), had_containers=had_containers, ) return had_containers @@ -462,7 +364,7 @@ def _capture_failure_logs(stack: DashboardStack, manifest: Path, managed_env: di log_path = log_dir / f"serve-{timestamp}.log" _write_private_file( log_path, - _redact_stack_text(logs, stack, managed_env), + redact_logs(logs, managed_env["CANYONOS_JWT_SECRET"]), ) return log_path @@ -475,10 +377,8 @@ def _cleanup(stack: DashboardStack, manifest: Path) -> None: def run_dashboard( - config_path: str | None = None, phase_reporter: Callable[[str, str], None] | None = None, ) -> ServeResult: - config_path = config_path or default_config_path() def report(result: ServeResult) -> None: if phase_reporter is not None: phase_reporter(result.phase, result.message) @@ -489,7 +389,7 @@ def report(result: ServeResult) -> None: had_containers = False with ExitStack() as resources: try: - stack = validate(config_path) + stack = validate() report(ServeResult(True, "validate", "dashboard prerequisites validated")) managed_env, prepare_message = prepare(stack) diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index a2239a2..57c32ad 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -10,12 +10,20 @@ manual step. """ -import json import subprocess -import urllib.error -import urllib.request -from canyonos.constants import default_config_path +from rich.console import Console +from rich.panel import Panel +from rich.text import Text + +from canyonos.constants import ( + WORKFLOW_ROUTE, + default_config_path, + workflow_api_port, + workspace_relative, +) +from canyonos.gc import GCError, post_deploy +from canyonos.theme import GREEN, WHITE from canyonos.init import load_state, run_init from canyonos.serve import run_serve from canyonos.sync import run_sync @@ -27,7 +35,13 @@ def run_deploy(config_path=None, serve=True): - config_path = config_path or default_config_path() + # Left as None when unset: ventis resolves the artifact layout itself. + if config_path is not None: + config_path = workspace_relative(config_path) + if config_path is None: + print("Config must be inside the project directory being synced.") + return + run_init() # Copy the current project into the container before building/deploying. @@ -36,29 +50,53 @@ def run_deploy(config_path=None, serve=True): state = load_state() - url = f"http://127.0.0.1:{state['port']}/deploy" - body = json.dumps({"config_path": config_path}).encode() - req = urllib.request.Request( - url, data=body, headers={"Content-Type": "application/json"}, method="POST" - ) + # Read for display only -- ventis resolves the path it actually deploys. + api_port = workflow_api_port(config_path or default_config_path()) try: - with urllib.request.urlopen(req) as resp: - json.loads(resp.read()) - _stream_logs_and_autoserve(state["container_id"], serve=serve) - except urllib.error.HTTPError as e: - data = json.loads(e.read()) - print(f"Deploy failed: {data.get('error')}") - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") - - -def _stream_logs_and_autoserve(container_id, serve=True): - """Tail the GC container's logs (same as before), and -- unless disabled - via `serve=False` -- launch `canyonos serve` the moment they show the - workflow is up, so the dashboard is ready alongside it. Log tailing - continues afterwards exactly as before. + post_deploy(state["port"], config_path) + _stream_logs_and_autoserve(state["container_id"], api_port, serve=serve) + except GCError as e: + print(e) + + +def print_workflow_endpoint(console, api_port): + """The one thing you need after a deploy: where to send requests. + + Printed at the workflow-up marker and again on exit, because `deploy` keeps + tailing logs afterwards and would otherwise scroll it out of sight. + """ + if api_port is None: + return + + url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" + body = Text.assemble( + ("POST ", "dim"), + (url, f"bold {GREEN}"), + ("\nbody ", "dim"), + ('{"query": "your question here"}', WHITE), + ("\npoll ", "dim"), + (f"http://127.0.0.1:{api_port}/status/", WHITE), + ) + console.print() + console.print( + Panel( + body, + title=f"[bold {GREEN}]Workflow is live[/]", + title_align="left", + border_style=GREEN, + padding=(1, 4), + ) + ) + console.print() + + +def _stream_logs_and_autoserve(container_id, api_port, serve=True): + """Tail the GC container's logs, and once they show the workflow is up, + print where to reach it -- plus, unless disabled via `serve=False`, launch + `canyonos serve`. Log tailing continues afterwards. """ + console = Console() process = subprocess.Popen( ["docker", "logs", "-f", container_id], stdout=subprocess.PIPE, @@ -67,20 +105,26 @@ def _stream_logs_and_autoserve(container_id, serve=True): bufsize=1, ) served = not serve + workflow_up = False try: for line in process.stdout: print(line, end="") - if not served and _WORKFLOW_UP_MARKER in line: - served = True - print("\nWorkflow is up -- starting the local dashboard (canyonos serve)...") - try: - run_serve() - except Exception as e: - print(f"Could not start the dashboard automatically: {e}") - print("Run `canyonos serve` manually to view it.") + if not workflow_up and _WORKFLOW_UP_MARKER in line: + workflow_up = True + print_workflow_endpoint(console, api_port) + if not served: + served = True + print("Starting the local dashboard (canyonos serve)...") + try: + run_serve() + except Exception as e: + print(f"Could not start the dashboard automatically: {e}") + print("Run `canyonos serve` manually to view it.") except KeyboardInterrupt: print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") print("To resubscribe to log stream run `canyonos logs`.") + if workflow_up: + print_workflow_endpoint(console, api_port) finally: if process.poll() is None: process.terminate() diff --git a/cli/canyonos/doctor.py b/cli/canyonos/doctor.py new file mode 100644 index 0000000..7dd9883 --- /dev/null +++ b/cli/canyonos/doctor.py @@ -0,0 +1,96 @@ +""" +Logic for `canyonos doctor`: a simple checklist of environment checks +(Docker installed/running, Compose available). Each check just reports +pass/fail plus a suggested fix -- nothing here attempts to auto-fix anything. +""" + +import shutil +import subprocess + +from canyonos.build import AGENTS +from canyonos.init import docker_start_command + + +def _docker_installed(): + return shutil.which("docker") is not None + + +def _docker_daemon_running(): + result = subprocess.run(["docker", "info"], capture_output=True) + return result.returncode == 0 + + +def _compose_available(): + result = subprocess.run(["docker", "compose", "version"], capture_output=True) + return result.returncode == 0 + + +def _git_available(): + return shutil.which("git") is not None + + +def _docker_daemon_fix(): + """Names the command for the active docker context, since `canyonos deploy` + would run exactly that itself.""" + command = docker_start_command() + if command: + return f"run `{' '.join(command)}` -- or just run `canyonos deploy`, which starts it for you" + return "start your Docker runtime (on Linux: `sudo systemctl start docker`)" + + +def _coding_agent_available(): + return any(shutil.which(spec["cli"]) for spec in AGENTS.values()) + + +def _checks(): + """Built fresh on each call (not a module-level constant) so tests can + patch the individual `_check_*` functions by name and have it take effect. + """ + return [ + ( + "Docker installed", + _docker_installed, + "install Docker: https://docs.docker.com/get-docker/", + ), + ( + "Docker daemon running", + _docker_daemon_running, + _docker_daemon_fix(), + ), + ( + "Docker Compose available", + _compose_available, + "update Docker to a version that includes Compose v2 (needed for `canyonos serve`)", + ), + ( + "git available", + _git_available, + "install git (`canyonos build` fetches the porting skill with it; " + "without git it falls back to a full-repo tarball download)", + ), + ( + "Coding agent available", + _coding_agent_available, + "install one of " + + " or ".join(spec["label"] for spec in AGENTS.values()) + + " (`canyonos build` runs the port through it)", + ), + ] + + +def run_doctor(): + """Run every check, print a pass/fail checklist, and return True iff all passed.""" + all_ok = True + for label, check, fix in _checks(): + try: + passed = bool(check()) + except OSError as e: + passed = False + fix = f"{fix} (error: {e})" + + print(f"{'✓' if passed else '✗'} {label}") + if not passed: + print(f" -> {fix}") + all_ok = False + + return all_ok diff --git a/cli/canyonos/gc.py b/cli/canyonos/gc.py new file mode 100644 index 0000000..b5af7eb --- /dev/null +++ b/cli/canyonos/gc.py @@ -0,0 +1,83 @@ +""" +Shared request helpers for the Global Controller container, so the commands +that talk to it don't each restate the same routes, payloads and failure modes. +""" + +import json +import urllib.error +import urllib.request + +from canyonos.init import load_state + +_DEPLOY_CONFLICT = "Run `canyonos stop` to stop the running deploy first." + + +class GCError(Exception): + """A failed Global Controller request, carrying a message fit to print.""" + + def __init__(self, message, code=None): + super().__init__(message) + self.code = code + + +def _error_detail(e): + """The server's `error` field, falling back to the raw body when it isn't JSON.""" + body = e.read().decode(errors="replace").strip() + try: + return json.loads(body).get("error", body) + except ValueError: + return body or f"HTTP {e.code}" + + +def _request(url, action, data=None, method="GET"): + headers = {"Content-Type": "application/json"} if data is not None else {} + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + raise GCError(f"{action} failed: {_error_detail(e)}", code=e.code) from None + except urllib.error.URLError as e: + raise GCError(f"Could not reach Global Controller container: {e.reason}") from None + + +def require_state(): + """Recorded container state, or None after reporting that there is none.""" + try: + return load_state() + except FileNotFoundError: + print("No Global Controller container is running. Run `canyonos deploy` first.") + return None + + +def post_deploy(port, config_path=None): + """Start a deploy inside the container. Raises GCError on failure. + + Omitting config_path lets ventis resolve it against the synced workspace. + """ + body = json.dumps({"config_path": config_path} if config_path else {}).encode() + try: + return _request(f"http://127.0.0.1:{port}/deploy", "Deploy", data=body, method="POST") + except GCError as e: + if e.code == 409: + raise GCError(f"{e}\n{_DEPLOY_CONFLICT}", code=409) from None + raise + + +def post_clean(port): + """Tear down the running deploy: SIGTERMs the in-container `ventis deploy` + process, whose handler calls GlobalController.stop() and blocks until it + returns. This is what actually removes the local controller and Redis + containers a deploy spawned via docker-outside-of-docker. + """ + return _request(f"http://127.0.0.1:{port}/clean", "Stop", method="POST") + + +def deploy_status(port): + """Parsed /status payload, or None if the container is unreachable.""" + url = f"http://127.0.0.1:{port}/status" + try: + with urllib.request.urlopen(url, timeout=5) as resp: + return json.loads(resp.read()) + except OSError: + return None diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py index 3d9cee9..69e8fbb 100644 --- a/cli/canyonos/init.py +++ b/cli/canyonos/init.py @@ -7,7 +7,10 @@ import json import os +import shutil import subprocess +import sys +import time import urllib.error import urllib.request @@ -23,22 +26,95 @@ GC_IMAGE = "saakeths/canyonos:latest" GC_CONTAINER_PORT = 8000 -# Named docker volume mounted at /workspace inside the container. Unlike a bind -# mount, this lives in the container's docker volume (not the host filesystem): -# it persists across `canyonos quit` (docker rm leaves named volumes intact) and -# is unaffected by host-side changes. Files are copied in via `canyonos sync` -# (docker cp), not mounted live. +# Named docker volume mounted at /workspace inside the container. Files are +# copied in via `canyonos sync` (docker cp), not mounted live, so host-side +# edits don't reach a running build. `canyonos quit` removes the volume, and +# since every deploy quits any previous controller first, each deploy starts +# from an empty workspace. GC_WORKSPACE_VOLUME = "canyonos-workspace" GC_WORKSPACE_PATH = "/workspace" STATE_DIR = os.path.expanduser("~/.canyonos") STATE_PATH = os.path.join(STATE_DIR, "state.json") +# How to start the daemon behind each docker context, as (CLI command, macOS +# app). Keyed off the *active context* rather than which app is installed: with +# both Docker Desktop and OrbStack present, guessing by app bundle starts the +# wrong daemon and then waits out the timeout against a socket nothing is +# listening on. +DOCKER_RUNTIMES = { + "orbstack": (["orb", "start"], "OrbStack"), + "colima": (["colima", "start"], None), + "desktop-linux": (None, "Docker"), + "default": (None, "Docker"), +} +DOCKER_START_TIMEOUT = 60 + + +def docker_running(): + try: + return subprocess.run(["docker", "info"], capture_output=True).returncode == 0 + except OSError: + return False + + +def docker_start_command(): + """The command that starts the daemon for the active context, or None.""" + try: + result = subprocess.run( + ["docker", "context", "show"], capture_output=True, text=True + ) + except OSError: + return None + + context = result.stdout.strip() if result.returncode == 0 else "default" + command, app = DOCKER_RUNTIMES.get(context, (None, "Docker")) + if command and shutil.which(command[0]): + return command + if app and sys.platform == "darwin" and os.path.isdir(f"/Applications/{app}.app"): + return ["open", "-a", app] + return None + + +def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): + if docker_running(): + return + + command = docker_start_command() + if command is None: + # Linux/systemd wants root here; escalating on the user's behalf is not + # this CLI's call to make. + raise RuntimeError( + "Docker isn't running, and there's no way to start it for the current " + "docker context. Start it (on Linux: `sudo systemctl start docker`) and re-run." + ) + + console.print(f"Docker isn't running -- starting it with `{' '.join(command)}`...") + subprocess.run(command, capture_output=True) + + deadline = time.time() + timeout + with console.status("Waiting for the Docker daemon..."): + while time.time() < deadline: + if docker_running(): + console.print("Docker is running.") + return + time.sleep(1) + + raise RuntimeError( + f"Docker did not become ready within {timeout}s. Start it manually and re-run." + ) + def pull_image(image=GC_IMAGE): # Capture output so the rich status spinner isn't clobbered by docker's own - # layer-progress printing. - subprocess.run(["docker", "pull", image], check=True, capture_output=True) + # layer-progress printing -- but surface it on failure (auth, network, + # rate-limit, missing arch, etc. all otherwise look like the same opaque + # "exit status 1"). + result = subprocess.run(["docker", "pull", image], capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + f"docker pull {image} failed: {result.stderr.strip() or result.stdout.strip()}" + ) def _port_reachable(port, attempts=10, delay=0.5): @@ -49,8 +125,6 @@ def _port_reachable(port, attempts=10, delay=0.5): trigger it), which looks fine at the Docker level but resets every real connection. Confirm the container is actually reachable before trusting it. """ - import time - url = f"http://127.0.0.1:{port}/status" for _ in range(attempts): try: @@ -113,6 +187,19 @@ def load_state(): return json.load(f) +def quit_existing(): + """Tear down a previously started Global Controller, if state records one. + + Without this each run starts another container on the next free port and + orphans the last one, which then can't be reached through state.json. + """ + # Deferred: quit.py imports from this module, so a top-level import cycles. + from canyonos.quit import run_quit + + if os.path.isfile(STATE_PATH): + run_quit() + + def run_init(): console = Console() banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) @@ -120,6 +207,9 @@ def run_init(): for line, color in zip(banner.splitlines(), GRADIENT): console.print(line, style=color) + # Before quit_existing(), which shells out to docker itself. + ensure_docker_running(console) + quit_existing() with console.status("Pulling Global Controller image..."): pull_image() diff --git a/cli/canyonos/integrate.py b/cli/canyonos/integrate.py deleted file mode 100644 index 23e61b1..0000000 --- a/cli/canyonos/integrate.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Logic for `canyonos integrate`: install the CanyonOS skill on a coding agent, -then launch that agent with a prompt to apply it to the current project. -""" - -import os -import shutil -import subprocess - -from rich.console import Console - -from utils.tui import select_menu - -# Points at the skill's folder, so SKILL.md and references/ both come along. -SKILL_SOURCE_URL = "https://github.com/CanyonCodeCoreAI/canyoncodecore/tree/nickhuo/porting-skill-car-layout/.claude/skills/porting-to-canyonos" - -# The porting skill emits no otel config; without this the dashboard stays empty. -OTEL_BLOCK = """otel: - destinations: - - name: local - protocol: http - endpoint: http://host.docker.internal:3000/v1/traces - headers: {}""" - -INTEGRATE_PROMPT = ( - "Use the CanyonOS porting-to-canyonos-core skill to convert the codebase in this directory to a canyonos-compatable format. No changes should be made to the current files, but all modifications should be put into a new .car folder." - "\n\nFinally, add the following block verbatim to the generated config/global_controller.yaml," - " at the top level as a sibling of `agents:`. Copy it exactly -- `protocol` must be http, and" - " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK -) - -AGENTS = { - "claude": { - "label": "Claude Code", - "cli": "claude", - # Claude Code auto-loads project-local skills from here. - "skill_dir": ".claude/skills/porting-to-canyonos-core", - }, - "codex": { - "label": "Codex", - "cli": "codex", - # Codex only auto-loads skills from the user's home directory, not per-project. - "skill_dir": os.path.expanduser("~/.codex/skills/porting-to-canyonos-core"), - }, -} - - -def prompt_agent(): - options = [(key, spec["label"]) for key, spec in AGENTS.items()] - return select_menu(options, title="Which coding agent do you want to integrate with?") - - -def install_skill(agent): - spec = AGENTS[agent] - # -f overwrites an existing skill dir; without it gitpick exits 1 when the - # target already exists and is non-empty (e.g. re-running `integrate`). - subprocess.run( - ["npx", "-y", "gitpick", "-f", SKILL_SOURCE_URL, spec["skill_dir"]], - check=True, - ) - - -def launch_agent(agent, prompt): - spec = AGENTS[agent] - if not shutil.which(spec["cli"]): - print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") - return - subprocess.run([spec["cli"], prompt], check=True) - - -def run_integrate(): - console = Console() - agent = prompt_agent() - if agent is None: - console.print("Cancelled.") - return - - console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") - install_skill(agent) - - console.print(f"Launching {AGENTS[agent]['label']}...") - launch_agent(agent, INTEGRATE_PROMPT) diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py index 9b2b1d8..9839134 100644 --- a/cli/canyonos/logs.py +++ b/cli/canyonos/logs.py @@ -2,32 +2,22 @@ Logic for `canyonos logs`: re-subscribe to the running deploy's log stream. """ -import json import subprocess -import urllib.error -import urllib.request -from canyonos.init import load_state +from canyonos.gc import deploy_status, require_state def run_logs(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return - url = f"http://127.0.0.1:{state['port']}/status" - req = urllib.request.Request(url, method="GET") - - try: - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") + status = deploy_status(state["port"]) + if status is None: + print("Could not reach Global Controller container.") return - if not data.get("running"): + if not status.get("running"): print("No deploy running, run `canyonos deploy` to deploy project.") return diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py index 15aff2c..bb2e11d 100644 --- a/cli/canyonos/quit.py +++ b/cli/canyonos/quit.py @@ -10,8 +10,8 @@ from rich.console import Console -from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH, load_state -from canyonos.stop import _post_clean +from canyonos.gc import GCError, post_clean, require_state +from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH def _container_exists(container_id): @@ -22,10 +22,8 @@ def _container_exists(container_id): def run_quit(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running.") + state = require_state() + if state is None: return container_id = state["container_id"] @@ -36,11 +34,9 @@ def run_quit(): # too. Removing the GC container itself doesn't touch them -- they're # sibling containers on the host, not nested inside it. try: - _post_clean(state["port"]) - except OSError: - # Covers urllib.error.HTTPError/URLError (both subclass OSError) - # plus raw connection errors -- nothing was running, or the GC is - # already unreachable/gone. + post_clean(state["port"]) + except GCError: + # Nothing was running, or the GC is already unreachable/gone. pass # state.json can go stale (daemon restarted, container removed by diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py index ddfd7d6..9515699 100644 --- a/cli/canyonos/serve.py +++ b/cli/canyonos/serve.py @@ -3,11 +3,11 @@ from .dashboard_stack import run_dashboard -def run_serve(config_path: str | None = None) -> int: +def run_serve() -> int: def report(phase: str, message: str) -> None: print(f"[serve] {phase}: {message}") - result = run_dashboard(config_path, report) + result = run_dashboard(report) if result.ok: print(f"Dashboard: {result.url}") return 0 diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py index 2f6ad40..a3f96f3 100644 --- a/cli/canyonos/stop.py +++ b/cli/canyonos/stop.py @@ -3,45 +3,20 @@ Controller container (SIGTERM, same teardown as Ctrl+C would trigger). """ -import json -import urllib.error -import urllib.request - from rich.console import Console -from canyonos.init import load_state - - -def _post_clean(port): - """POST /clean to the Global Controller container. - - This is what actually tears down the local controller and Redis - containers a deploy spawned via docker-outside-of-docker: it sends - SIGTERM to the in-container `ventis deploy` process, whose handler calls - `GlobalController.stop()` and blocks until it returns. Shared with - `canyonos quit`, which needs the same teardown before removing the GC - container itself. - """ - url = f"http://127.0.0.1:{port}/clean" - req = urllib.request.Request(url, method="POST") - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) +from canyonos.gc import GCError, post_clean, require_state def run_stop(): - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return console = Console() try: with console.status("Stopping deploy..."): - _post_clean(state["port"]) + post_clean(state["port"]) print("Deploy stopped.") - except urllib.error.HTTPError as e: - data = json.loads(e.read()) - print(f"Stop failed: {data.get('error')}") - except urllib.error.URLError as e: - print(f"Could not reach Global Controller container: {e}") + except GCError as e: + print(e) diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py index f350a1c..20ac3c1 100644 --- a/cli/canyonos/sync.py +++ b/cli/canyonos/sync.py @@ -3,24 +3,24 @@ Controller container's /workspace volume via `docker cp`. Files live inside the container's named volume (see `init.py`), not on a live -bind mount -- so they persist across `canyonos quit` and survive host-side -changes. `docker cp` is additive: it overwrites/adds files but never deletes, -so build outputs generated inside the container (stubs/, grpc_stubs/, -docker_container/) survive a re-sync of the host source. +bind mount, so host-side edits don't reach a running build. `docker cp` is +additive -- it overwrites and adds but never deletes -- so a standalone +re-sync leaves behind anything removed from the host since the last one. +That can't accumulate across deploys: `canyonos deploy` quits any previous +controller first, which removes the volume. """ import os import subprocess -from canyonos.init import GC_WORKSPACE_PATH, load_state +from canyonos.gc import require_state +from canyonos.init import GC_WORKSPACE_PATH def run_sync(): """Copy the current directory into the container. Returns True on success.""" - try: - state = load_state() - except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos init` first.") + state = require_state() + if state is None: return False container_id = state["container_id"] diff --git a/cli/canyonos/test.py b/cli/canyonos/test.py new file mode 100644 index 0000000..468527a --- /dev/null +++ b/cli/canyonos/test.py @@ -0,0 +1,173 @@ +""" +Logic for `canyonos test`: smoke-test a project end to end on this machine. + +Every agent's `provider` is rewritten to `local` for the duration of the run +(the original file is restored verbatim afterwards), the project is deployed +into the Global Controller container, one query is sent to the workflow's +`/main` endpoint, and its result -- or the error that came back -- is printed. +""" + +import json +import os +import time +import urllib.error +import urllib.request + +from rich.console import Console + +from canyonos.constants import ( + WORKFLOW_ROUTE, + default_config_path, + round_trip_yaml, + workflow_api_port, + workspace_relative, +) +from canyonos.gc import GCError, deploy_status, post_deploy +from canyonos.init import load_state, quit_existing, run_init +from canyonos.sync import run_sync + +DEFAULT_QUERY = "hello" +# Generous: the first deploy of a project builds every agent image from scratch. +READY_TIMEOUT = 900 +REQUEST_TIMEOUT = 600 +POLL_INTERVAL = 2 + + + +def _force_local_providers(config_path): + """Set every agent's provider to `local`. Returns the original file text.""" + with open(config_path) as f: + original = f.read() + + yaml_rt = round_trip_yaml() + data = yaml_rt.load(original) + + for agent in data.get("agents") or []: + agent["provider"] = "local" + + with open(config_path, "w") as f: + yaml_rt.dump(data, f) + + return original + + +def _workflow_ready(api_port): + """True once the workflow's REST API answers at all. + + Any HTTP response counts -- /status/ 404s, which still proves + the server is up and listening. + """ + url = f"http://127.0.0.1:{api_port}/status/canyonos-test-probe" + try: + urllib.request.urlopen(url, timeout=2) + return True + except urllib.error.HTTPError: + return True + except OSError: + return False + + +def _wait_for_workflow(gc_port, api_port, console): + deadline = time.time() + READY_TIMEOUT + with console.status("Building and starting containers..."): + while time.time() < deadline: + if _workflow_ready(api_port): + return True + if not (deploy_status(gc_port) or {}).get("running", False): + return False + time.sleep(POLL_INTERVAL) + print(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") + return False + + +def _send_query(api_port, query): + url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" + body = json.dumps({"query": query}).encode() + req = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"}, method="POST" + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read())["request_id"] + + +def _await_result(api_port, request_id, console): + url = f"http://127.0.0.1:{api_port}/status/{request_id}" + deadline = time.time() + REQUEST_TIMEOUT + with console.status("Running query..."): + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=10) as resp: + data = json.loads(resp.read()) + if data.get("status") in ("done", "error"): + return data + except OSError: + # A blip while the workflow is busy; keep polling until the deadline. + pass + time.sleep(POLL_INTERVAL) + return {"status": "timeout"} + + +def run_test(config_path=None, query=None): + console = Console() + config_path = config_path or default_config_path() + query = query or DEFAULT_QUERY + + config_path = workspace_relative(config_path) + if config_path is None: + print("Config must be inside the project directory being synced.") + return 1 + + if not os.path.isfile(config_path): + print(f"Config file not found: {config_path}. Run `canyonos build` first.") + return 1 + + api_port = workflow_api_port(config_path) + if api_port is None: + print(f"No agent with `type: workflow` in {config_path}; nothing to test.") + return 1 + + print(f"Testing {config_path} locally (query: {query!r})") + original_config = _force_local_providers(config_path) + + try: + run_init() + if not run_sync(): + return 1 + + state = load_state() + try: + post_deploy(state["port"], config_path) + except GCError as e: + print(e) + return 1 + + if not _wait_for_workflow(state["port"], api_port, console): + print("The deploy did not come up. Run `canyonos logs` to see why.") + return 1 + + try: + request_id = _send_query(api_port, query) + except OSError as e: + print(f"Could not reach the workflow on port {api_port}: {e}") + return 1 + result = _await_result(api_port, request_id, console) + except KeyboardInterrupt: + print("\nTest cancelled.") + return 1 + finally: + with open(config_path, "w") as f: + f.write(original_config) + # A smoke test leaves nothing behind: run_init() started this container. + quit_existing() + + status = result.get("status") + if status == "done": + print("Test passed.") + print(json.dumps(result.get("result"), indent=2)) + return 0 + + if status == "error": + print(f"Test failed: {result.get('error')}") + else: + print(f"Test failed: workflow did not finish within {REQUEST_TIMEOUT}s.") + return 1 diff --git a/cli/cli.py b/cli/cli.py index 4c78043..770741f 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1,42 +1,33 @@ """ -Most of the commands will be executed by code in the canyonos container. -Anything executing in this CLI pertains to file/folder modification +Almost all commands will be executing on the canyonos container spawned by deploy +Commands like doctor, version, and new_app will not though """ import argparse +import importlib.metadata import sys from canyonos.clean import run_clean from canyonos.constants import default_config_path from canyonos.config import run_config from canyonos.deploy import run_deploy -from canyonos.integrate import run_integrate +from canyonos.build import run_build +from canyonos.doctor import run_doctor from canyonos.logs import run_logs from canyonos.new_app import run_new_app from canyonos.quit import run_quit from canyonos.serve import run_serve from canyonos.stop import run_stop -from canyonos.sync import run_sync - -try: - from rich.console import Console - from rich.panel import Panel - from rich.text import Text - from rich.table import Table - RICH_AVAILABLE = True -except ImportError: - RICH_AVAILABLE = False - -def cmd_connect(args): - pass +from canyonos.test import DEFAULT_QUERY, run_test +from utils.help_screen import DESCRIPTIONS, print_custom_help def cmd_quit(args): run_quit() def cmd_new_app(args): + # Note, not tested much, keeping this in the back burner for now while we flesh out the main path run_new_app() -# Executed in canyonos: syncs files, then builds + deploys def cmd_deploy(args): run_deploy(args.config, serve=args.serve) @@ -49,111 +40,24 @@ def cmd_stop(args): def cmd_logs(args): run_logs() -def cmd_sync(args): - run_sync() - def cmd_config(args): run_config() -def cmd_integrate(args): - run_integrate() +def cmd_build(args): + run_build() def cmd_doctor(args): - pass + sys.exit(0 if run_doctor() else 1) def cmd_serve(args): - sys.exit(run_serve(args.config)) + sys.exit(run_serve()) -# Executed in canyonos def cmd_test(args): - pass - -# Executed in canyonos -def cmd_mega_build(args): - pass + sys.exit(run_test(args.config, query=args.query)) def cmd_version(args): - pass - - -def print_custom_help(): - """Print a custom, visually appealing help screen.""" - if RICH_AVAILABLE: - console = Console() - - # Header - title = Text("CanyonOS CLI", style="bold cyan") - subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") - - console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) - - # Core commands - console.print("\n[bold yellow]Core Commands[/bold yellow]") - core_table = Table(show_header=False, border_style="dim", padding=(0, 2)) - core_table.add_column(style="cyan", width=20) - core_table.add_column(style="white") - core_table.add_row("integrate", "Sync source files to .car/app/") - core_table.add_row("deploy", "Build and deploy agents to configured hosts") - core_table.add_row("config", "Configure project settings") - console.print(core_table) - - # Utils commands - console.print("\n[bold yellow]Utils[/bold yellow]") - utils_table = Table(show_header=False, border_style="dim", padding=(0, 2)) - utils_table.add_column(style="cyan", width=20) - utils_table.add_column(style="white") - utils_table.add_row("new-app", "Create a new CanyonOS project") - utils_table.add_row("serve", "Start local CanyonOS dashboard") - utils_table.add_row("sync", "Sync files with container") - utils_table.add_row("stop", "Stop running containers") - utils_table.add_row("clean", "Remove generated files") - utils_table.add_row("logs", "View container logs") - utils_table.add_row("doctor", "Check system health") - utils_table.add_row("connect", "Connect to remote host") - utils_table.add_row("quit", "Shut down CanyonOS services") - console.print(utils_table) - - # Quick start - console.print("\n[bold green]Quick Start:[/bold green]") - console.print(" [dim]1.[/dim] canyonos new-app [cyan]my-app[/cyan]") - console.print(" [dim]2.[/dim] cd [cyan]my-app[/cyan]") - console.print(" [dim]3.[/dim] canyonos integrate") - console.print(" [dim]4.[/dim] canyonos deploy") - console.print(" [dim]5.[/dim] canyonos serve\n") - - console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") - else: - # Fallback to simple text if rich is not available - print("\n" + "="*60) - print(" " * 20 + "CanyonOS CLI") - print(" " * 10 + "Build, deploy, and manage agentic workflows") - print("="*60 + "\n") - - print("CORE COMMANDS:") - print(" integrate Sync source files to .car/app/") - print(" deploy Build and deploy agents to configured hosts") - print(" config Configure project settings\n") - - print("UTILS:") - print(" new-app Create a new CanyonOS project") - print(" serve Start local CanyonOS dashboard") - print(" sync Sync files with container") - print(" stop Stop running containers") - print(" clean Remove generated files") - print(" logs View container logs") - print(" doctor Check system health") - print(" connect Connect to remote host") - print(" quit Shut down CanyonOS services\n") - - print("QUICK START:") - print(" 1. canyonos new-app my-app") - print(" 2. cd my-app") - print(" 3. canyonos integrate") - print(" 4. canyonos deploy") - print(" 5. canyonos serve\n") - - print("For command-specific help: canyonos --help\n") + print(f"canyonos {importlib.metadata.version('canyonos')}") def _parse_bool(value): @@ -164,18 +68,30 @@ def _parse_bool(value): raise argparse.ArgumentTypeError(f"expected true/false, got: {value!r}") +class _RootParser(argparse.ArgumentParser): + """Routes the top-level -h/--help through the custom help screen.""" + + def print_help(self, file=None): + print_custom_help() + + def main(): - parser = argparse.ArgumentParser(prog="canyonos") - subparsers = parser.add_subparsers(dest="command") + parser = _RootParser(prog="canyonos") + # Subparsers keep the stock argparse help, so `canyonos -h` still + # describes that command instead of reprinting the top-level screen. + subparsers = parser.add_subparsers(dest="command", parser_class=argparse.ArgumentParser) config_default = default_config_path() - subparsers.add_parser("new-app").set_defaults(func=cmd_new_app) - deploy = subparsers.add_parser("deploy") + def add(name): + # A KeyError here means the command has no entry on the help screen. + return subparsers.add_parser(name, help=DESCRIPTIONS[name]) + + add("new-app").set_defaults(func=cmd_new_app) + deploy = add("deploy") deploy.add_argument( "-c", "--config", - default=config_default, - help=f"Path to global controller config (default: {config_default})", + help="Path to global controller config (default: resolved by ventis inside the container)", ) deploy.add_argument( "--serve", @@ -185,32 +101,42 @@ def main(): help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", ) deploy.set_defaults(func=cmd_deploy) - subparsers.add_parser("clean").set_defaults(func=cmd_clean) - subparsers.add_parser("stop").set_defaults(func=cmd_stop) - subparsers.add_parser("logs").set_defaults(func=cmd_logs) - subparsers.add_parser("quit").set_defaults(func=cmd_quit) - subparsers.add_parser("connect").set_defaults(func=cmd_connect) - subparsers.add_parser("sync").set_defaults(func=cmd_sync) - subparsers.add_parser("config").set_defaults(func=cmd_config) - subparsers.add_parser("integrate").set_defaults(func=cmd_integrate) - subparsers.add_parser("doctor").set_defaults(func=cmd_doctor) - serve = subparsers.add_parser("serve") - serve.add_argument( + add("clean").set_defaults(func=cmd_clean) + add("stop").set_defaults(func=cmd_stop) + add("logs").set_defaults(func=cmd_logs) + add("quit").set_defaults(func=cmd_quit) + add("config").set_defaults(func=cmd_config) + add("build").set_defaults(func=cmd_build) + add("doctor").set_defaults(func=cmd_doctor) + add("version").set_defaults(func=cmd_version) + add("serve").set_defaults(func=cmd_serve) + test = add("test") + test.add_argument( "-c", "--config", default=config_default, help=f"Path to global controller config (default: {config_default})", ) - serve.set_defaults(func=cmd_serve) - subparsers.add_parser("test").set_defaults(func=cmd_test) - subparsers.add_parser("mega-build").set_defaults(func=cmd_mega_build) + test.add_argument( + "-q", + "--query", + default=DEFAULT_QUERY, + help=f"Query sent to the workflow (default: {DEFAULT_QUERY!r})", + ) + test.set_defaults(func=cmd_test) args = parser.parse_args() if not getattr(args, "command", None): - print_custom_help() + parser.print_help() return - args.func(args) + try: + args.func(args) + except RuntimeError as e: + # Docker unreachable, image pull failed, no free port -- all already + # carry a readable message, so print it rather than a traceback. + print(e) + sys.exit(1) if __name__ == "__main__": diff --git a/cli/utils/help_screen.py b/cli/utils/help_screen.py new file mode 100644 index 0000000..94b94a0 --- /dev/null +++ b/cli/utils/help_screen.py @@ -0,0 +1,60 @@ +"""Custom help screen for the canyonos CLI.""" + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +# The single source of truth for command descriptions: cli.py registers every +# subparser through this table, so a command can't be added to one and missed +# in the other. +CORE_COMMANDS = ( + ("build", "Build a compatable workflow with an agent"), + ("deploy", "Deploy agents"), + ("config", "Configure project settings"), +) + +UTIL_COMMANDS = ( + ("clean", "Remove the generated .car folder from build"), + ("doctor", "See if all required tools are up"), + ("logs", "View canyonos logs"), + ("new-app", "Create a barebones CanyonOS project"), + ("quit", "Shut down CanyonOS services"), + ("serve", "Start local CanyonOS dashboard"), + ("stop", "Stop running containers"), + ("test", "Run the deployed workflow locally with a test query"), + ("version", "Print canyonos version"), +) + +DESCRIPTIONS = dict(CORE_COMMANDS + UTIL_COMMANDS) + + +def _command_table(commands): + table = Table(show_header=False, border_style="dim", padding=(0, 2)) + table.add_column(style="cyan", width=20) + table.add_column(style="white") + for name, description in commands: + table.add_row(name, description) + return table + + +def print_custom_help(): + """Print a custom, visually appealing help screen.""" + console = Console() + + title = Text("CanyonOS CLI", style="bold cyan") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") + console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + + console.print("\n[bold yellow]Core Commands[/bold yellow]") + console.print(_command_table(CORE_COMMANDS)) + + console.print("\n[bold yellow]Utils[/bold yellow]") + console.print(_command_table(UTIL_COMMANDS)) + + console.print("\n[bold green]Quick Start:[/bold green]") + console.print(" [dim]1.[/dim] cd [cyan]into-your-workflow-root-dir[/cyan]") + console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") + console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") + + console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") diff --git a/cli/utils/tui.py b/cli/utils/tui.py index 3fcdad5..2f0f15d 100644 --- a/cli/utils/tui.py +++ b/cli/utils/tui.py @@ -53,8 +53,6 @@ def select_menu(options, title, deletable=False, quittable=False): `QUIT_ACTION` -- distinct from None -- so the caller can unwind an entire nested session rather than just this one menu. """ - if len(options) == 1: - return options[0][0] if not options or not sys.stdin.isatty(): return None diff --git a/cli/tests/test_dashboard_stack.py b/tests/test_dashboard_stack.py similarity index 100% rename from cli/tests/test_dashboard_stack.py rename to tests/test_dashboard_stack.py From 9447e348a511466397194fda3265f875dbe895f6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 4 Sep 2026 23:13:04 -0700 Subject: [PATCH 5/6] cleanup --- README.md | 4 +- cli/canyonos/build.py | 71 +-- cli/canyonos/clean.py | 10 +- cli/canyonos/config.py | 41 +- cli/canyonos/dashboard_stack.py | 78 ++-- cli/canyonos/deploy.py | 401 ++++++++++++++--- cli/canyonos/doctor.py | 39 +- cli/canyonos/gc.py | 16 +- cli/canyonos/init.py | 28 +- cli/canyonos/logs.py | 8 +- cli/canyonos/new_app.py | 6 +- cli/canyonos/quit.py | 10 +- cli/canyonos/serve.py | 39 +- cli/canyonos/status.py | 55 +++ cli/canyonos/stop.py | 10 +- cli/canyonos/sync.py | 19 +- cli/canyonos/test.py | 365 ++++++++++++--- cli/canyonos/ui.py | 67 +++ cli/canyonos/verify.py | 291 ++++++++++++ cli/cli.py | 100 ++-- cli/pyproject.toml | 2 +- cli/utils/help_screen.py | 60 ++- cli/utils/tui.py | 8 +- examples/finance/agents/finance_agent.py | 9 +- examples/finance/workflow/example_workflow.py | 8 +- examples/helloworld/README.md | 4 +- .../helloworld/workflow/example_workflow.py | 6 +- examples/portfolio/agents/metrics_agent.py | 14 +- .../text2sql/agents/sql_generator_agent.py | 9 +- .../text2sql/workflow/text2sql_workflow.py | 12 +- pyproject.toml | 5 + tests/test_canyonos_test.py | 426 ++++++++++++++++++ tests/test_dashboard_stack.py | 146 +----- tests/test_deploy_progress.py | 235 ++++++++++ tests/test_integration.py | 2 +- uv.lock | 77 +++- .../cloud_provider_logic/EC2/_runtime.py | 11 +- .../cloud_provider_logic/Local/_runtime.py | 2 + ventis/controller/instance_manager.py | 9 +- ventis/server.py | 80 +++- 40 files changed, 2213 insertions(+), 570 deletions(-) create mode 100644 cli/canyonos/status.py create mode 100644 cli/canyonos/ui.py create mode 100644 cli/canyonos/verify.py create mode 100644 tests/test_canyonos_test.py create mode 100644 tests/test_deploy_progress.py diff --git a/README.md b/README.md index 1d61db8..0f1ea27 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The Readme in the newly created project directory provides a quick overview of t #### Step 2: Define Your Agents Agent declarations live under `.car/config/`. The source used for builds is -copied to `.car/app/` by `canyonos integrate`. +copied to `.car/app/` by `canyonos build`. - **`.car/config/my_agent.yaml`**: Defines methods and schemas. - **`.car/app/path/to/my_agent.py`**: Contains the agent implementation. @@ -102,7 +102,7 @@ Users can send requests to this endpoint to trigger the workflow. For this examp curl -X POST http://localhost:8080/main \ -H "Content-Type: application/json" \ -d '{ - "ticker": "AAPL" + "query": "AAPL" }' ``` The request is asynchronous. To get the result, you use the following URL- diff --git a/cli/canyonos/build.py b/cli/canyonos/build.py index f5292aa..c9b56b4 100644 --- a/cli/canyonos/build.py +++ b/cli/canyonos/build.py @@ -10,8 +10,7 @@ import tempfile import urllib.request -from rich.console import Console - +from canyonos import ui from utils.tui import select_menu SKILL_OWNER = "CanyonCodeCoreAI" @@ -43,19 +42,24 @@ " the endpoint must keep the /v1/traces path:\n\n" + OTEL_BLOCK ) +# The leaf name of every install path must match the skill's own `name:` +# frontmatter or the agent won't resolve it. AGENTS = { "claude": { "label": "Claude Code", "cli": "claude", - # Claude Code auto-loads project-local skills from here. The leaf name - # must match the skill's own `name:` frontmatter or it won't resolve. - "skill_dir": SKILL_PATH, + "skill_dirs": { + "local": SKILL_PATH, + "global": os.path.expanduser(f"~/.claude/skills/{SKILL_NAME}"), + }, }, "codex": { "label": "Codex", "cli": "codex", - # Codex only auto-loads skills from the user's home directory, not per-project. - "skill_dir": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + "skill_dirs": { + "local": f".codex/skills/{SKILL_NAME}", + "global": os.path.expanduser(f"~/.codex/skills/{SKILL_NAME}"), + }, }, } @@ -65,6 +69,15 @@ def prompt_agent(): return select_menu(options, title="Which coding agent do you want to build on?") +def prompt_scope(agent): + dirs = AGENTS[agent]["skill_dirs"] + options = [ + ("local", f"This project only ({dirs['local']})"), + ("global", f"Globally ({dirs['global']})"), + ] + return select_menu(options, title="Where should the CanyonOS skill be installed?") + + def _replace_dir(source, dest): """Move `source` onto `dest`, replacing whatever was there.""" os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) @@ -143,47 +156,32 @@ def _fetch_with_tarball(dest): return True -def _fetch_with_npx(dest): - """Last resort, and the only strategy that needs Node.""" - if not shutil.which("npx"): - return False - # -f overwrites an existing skill dir; without it gitpick exits 1 when the - # target already exists and is non-empty (e.g. re-running `build`). - return subprocess.run( - ["npx", "-y", "gitpick", "-f", TREE_URL, dest], capture_output=True - ).returncode == 0 - - FETCH_STRATEGIES = ( ("git", _fetch_with_git), ("tarball", _fetch_with_tarball), - ("npx", _fetch_with_npx), ) -def install_skill(agent, console): - """Fetch the skill into the agent's skill dir. Returns True on success.""" - dest = AGENTS[agent]["skill_dir"] +def install_skill(dest): + """Fetch the skill into `dest`. Returns True on success.""" for name, fetch in FETCH_STRATEGIES: try: if fetch(dest): - console.print(f"Fetched the CanyonOS skill via {name}.") + ui.ok(f"Fetched the CanyonOS skill via {name}.") return True except OSError: pass - console.print(f"[dim]{name} fetch unavailable, trying the next option...[/dim]") + ui.hint(f"{name} fetch unavailable, trying the next option...") - console.print( - f"Could not fetch the CanyonOS skill from {TREE_URL}.\n" - "Install git or Node, or check network access, then run `canyonos doctor`." - ) + ui.fail(f"Could not fetch the CanyonOS skill from {TREE_URL}.") + ui.hint("Install git, or check network access, then run `canyonos doctor`.") return False def launch_agent(agent, prompt): spec = AGENTS[agent] if not shutil.which(spec["cli"]): - print(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") + ui.fail(f"`{spec['cli']}` not found on PATH; install {spec['label']} first.") return # No check=True: the agent exiting non-zero (including the user quitting it) # is an ordinary outcome, not something to raise a traceback over. @@ -191,15 +189,20 @@ def launch_agent(agent, prompt): def run_build(): - console = Console() agent = prompt_agent() if agent is None: - console.print("Cancelled.") + ui.say("Cancelled.") + return + + scope = prompt_scope(agent) + if scope is None: + ui.say("Cancelled.") return - console.print(f"Installing CanyonOS skill for {AGENTS[agent]['label']}...") - if not install_skill(agent, console): + dest = AGENTS[agent]["skill_dirs"][scope] + ui.say(f"Installing CanyonOS skill for {AGENTS[agent]['label']} into {dest}...") + if not install_skill(dest): return - console.print(f"Launching {AGENTS[agent]['label']}...") + ui.say(f"Launching {AGENTS[agent]['label']}...") launch_agent(agent, BUILD_PROMPT) diff --git a/cli/canyonos/clean.py b/cli/canyonos/clean.py index d7cc6f5..9c974b4 100644 --- a/cli/canyonos/clean.py +++ b/cli/canyonos/clean.py @@ -5,14 +5,16 @@ import os import shutil +from canyonos import ui + def run_clean(): car_dir = os.path.join(os.getcwd(), ".car") if not os.path.isdir(car_dir): - print("Nothing to clean, no .car folder in root") + ui.warn("Nothing to clean, no .car folder in root") return - print(f"Cleaning {car_dir}...") - shutil.rmtree(car_dir) - print("Clean complete.") + with ui.status(f"Cleaning {car_dir}..."): + shutil.rmtree(car_dir) + ui.ok("Clean complete.") diff --git a/cli/canyonos/config.py b/cli/canyonos/config.py index 25d2d3d..aec5b12 100644 --- a/cli/canyonos/config.py +++ b/cli/canyonos/config.py @@ -5,11 +5,11 @@ import os import yaml -from rich.console import Console from rich.table import Table from canyonos.constants import default_config_path, round_trip_yaml from canyonos.theme import GREEN, WHITE +from canyonos import ui from utils.tui import DELETE_ACTION, QUIT_ACTION, select_menu BACK = "__back__" @@ -98,30 +98,29 @@ def _kv_table(title, data): return table -def _require_config(config_path, console): +def _require_config(config_path): """Resolved config path, or None after reporting that it's missing.""" config_path = config_path or default_config_path() if not os.path.isfile(config_path): - console.print(f"[red]Config file not found: {config_path}[/red]") + ui.fail(f"Config file not found: {config_path}") return None return config_path def run_view_config(config_path=None): - console = Console() - config_path = _require_config(config_path, console) + config_path = _require_config(config_path) if config_path is None: return with open(config_path) as f: config = yaml.safe_load(f) or {} - console.print(_agents_table(config.get("agents") or [])) - console.print() + ui.console.print(_agents_table(config.get("agents") or [])) + ui.blank() if config.get("otel"): - console.print(_otel_table(config["otel"])) - console.print() + ui.console.print(_otel_table(config["otel"])) + ui.blank() # Every other top-level key: dicts get their own table, bare scalars are # gathered into a single "General" table. @@ -130,13 +129,13 @@ def run_view_config(config_path=None): if key in STRUCTURED_KEYS: continue if isinstance(value, dict): - console.print(_kv_table(key, value)) - console.print() + ui.console.print(_kv_table(key, value)) + ui.blank() else: general[key] = value if general: - console.print(_kv_table("General", general)) + ui.console.print(_kv_table("General", general)) def _is_leaf(value): @@ -294,8 +293,7 @@ def _navigate(screen, node, breadcrumb): def run_change_config(config_path=None): - console = Console() - config_path = _require_config(config_path, console) + config_path = _require_config(config_path) if config_path is None: return @@ -304,14 +302,14 @@ def run_change_config(config_path=None): data = yaml_rt.load(f) if not data: - console.print("[yellow]Config is empty; nothing to change.[/yellow]") + ui.warn("Config is empty; nothing to change.") return - screen = _Screen(console) + screen = _Screen(ui.console) saves = 0 # Alternate screen: the whole session replaces the view, and the terminal # scrollback is restored untouched on exit. - console.set_alt_screen(True) + ui.console.set_alt_screen(True) try: while True: changed = _navigate(screen, data, ["config"]) @@ -323,19 +321,18 @@ def run_change_config(config_path=None): saves += 1 screen.status = f"Saved to {config_path}" finally: - console.set_alt_screen(False) + ui.console.set_alt_screen(False) if saves: - console.print(f"[{GREEN}]Saved {saves} change(s) to {config_path}[/]") + ui.ok(f"Saved {saves} change(s) to {config_path}") else: - console.print("No changes made.") + ui.say("No changes made.") def run_config(): - console = Console() choice = select_menu(OPTIONS, title="What do you want to do?") if choice is None: - console.print("Cancelled.") + ui.say("Cancelled.") return if choice == "view": diff --git a/cli/canyonos/dashboard_stack.py b/cli/canyonos/dashboard_stack.py index c606357..e26fbcf 100644 --- a/cli/canyonos/dashboard_stack.py +++ b/cli/canyonos/dashboard_stack.py @@ -10,7 +10,6 @@ import shutil import socket import subprocess -import tempfile import time import urllib.error import urllib.request @@ -39,11 +38,10 @@ class ServeResult: class PhaseFailure(Exception): - def __init__(self, phase: str, message: str, *, had_containers: bool | None = None): + def __init__(self, phase: str, message: str): super().__init__(message) self.phase = phase self.message = message - self.had_containers = had_containers @dataclass(frozen=True) @@ -179,8 +177,14 @@ def _read_existing_secret(env_path: Path) -> str | None: def _write_private_file(path: Path, contents: str) -> None: + """Write 0600 from the start, so the contents are never briefly world-readable. + + The open mode only applies when creating, so an already-loose file (a `.env` + the user wrote by hand) is tightened explicitly rather than left as it was. + """ descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as output: + os.fchmod(output.fileno(), 0o600) output.write(contents) @@ -191,40 +195,27 @@ def _env_line(key: str, value: str) -> str: def _write_project_env(env_path: Path, managed_env: dict[str, str]) -> None: + """Rewrite only the CANYONOS_* keys, leaving every other line of the user's .env alone.""" try: lines = env_path.read_text(encoding="utf-8").splitlines(keepends=True) except FileNotFoundError: lines = [] - managed_keys = set(managed_env) replaced: set[str] = set() updated_lines: list[str] = [] for line in lines: key, separator, _ = line.partition("=") - if separator and key in managed_keys: + if separator and key in managed_env: if key not in replaced: updated_lines.append(_env_line(key, managed_env[key])) replaced.add(key) continue updated_lines.append(line) - for key, value in managed_env.items(): - if key not in replaced: - updated_lines.append(_env_line(key, value)) - - descriptor, temporary_name = tempfile.mkstemp(prefix=".env.", dir=env_path.parent) - temporary_path = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as output: - os.fchmod(output.fileno(), 0o600) - output.writelines(updated_lines) - os.replace(temporary_path, env_path) - except Exception: - try: - temporary_path.unlink() - except FileNotFoundError: - pass - raise + updated_lines.extend( + _env_line(key, value) for key, value in managed_env.items() if key not in replaced + ) + _write_private_file(env_path, "".join(updated_lines)) def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: @@ -257,36 +248,27 @@ def prepare(stack: DashboardStack) -> tuple[dict[str, str], str]: return managed_env, "dashboard state prepared" -def _last_stderr_line(result: subprocess.CompletedProcess[str]) -> str | None: - return next((line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None) - - def _command_failure_message( message: str, result: subprocess.CompletedProcess[str], managed_env: dict[str, str], ) -> str: - detail = _last_stderr_line(result) + detail = next( + (line.strip() for line in reversed(result.stderr.splitlines()) if line.strip()), None + ) if detail is None: return message return f"{message}: {redact_logs(detail, managed_env['CANYONOS_JWT_SECRET'])}" -def pull( - stack: DashboardStack, - manifest: Path, - managed_env: dict[str, str], - had_containers: bool, -) -> str: +def pull(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> str: try: result = _run([*_compose_argv(stack, manifest), "pull"]) except OSError: - raise PhaseFailure("pull", "could not run docker compose pull", had_containers=had_containers) + raise PhaseFailure("pull", "could not run docker compose pull") if result.returncode != 0: raise PhaseFailure( - "pull", - _command_failure_message("docker compose pull failed", result, managed_env), - had_containers=had_containers, + "pull", _command_failure_message("docker compose pull failed", result, managed_env) ) return "dashboard images pulled" @@ -299,8 +281,7 @@ def _project_has_running_containers(stack: DashboardStack, manifest: Path) -> bo return result.returncode == 0 and bool(result.stdout.strip()) -def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> bool: - had_containers = _project_has_running_containers(stack, manifest) +def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> None: # The api reads the controller's Redis identity once at startup to create # its project row, so a surviving container keeps serving whichever project # was deployed before it. Replace it every serve rather than reuse it. @@ -310,14 +291,11 @@ def start(stack: DashboardStack, manifest: Path, managed_env: dict[str, str]) -> [*_compose_argv(stack, manifest), "up", "-d", "--wait", "--wait-timeout", "180"] ) except OSError: - raise PhaseFailure("start", "could not run docker compose up", had_containers=had_containers) + raise PhaseFailure("start", "could not run docker compose up") if result.returncode != 0: raise PhaseFailure( - "start", - _command_failure_message("docker compose up failed", result, managed_env), - had_containers=had_containers, + "start", _command_failure_message("docker compose up failed", result, managed_env) ) - return had_containers def verify(port: int) -> str: @@ -386,6 +364,8 @@ def report(result: ServeResult) -> None: stack: DashboardStack | None = None managed_env: dict[str, str] | None = None manifest: Path | None = None + # Whether the stack predates this serve, so a failure only tears down what + # this run brought up. Read once, before anything here can change it. had_containers = False with ExitStack() as resources: try: @@ -397,11 +377,11 @@ def report(result: ServeResult) -> None: manifest_resource = importlib.resources.files("canyonos").joinpath("dashboard.compose.yml") manifest = resources.enter_context(importlib.resources.as_file(manifest_resource)) - had_containers_before_pull = _project_has_running_containers(stack, manifest) - pull_message = pull(stack, manifest, managed_env, had_containers_before_pull) - report(ServeResult(True, "pull", pull_message)) + had_containers = _project_has_running_containers(stack, manifest) + + report(ServeResult(True, "pull", pull(stack, manifest, managed_env))) - had_containers = start(stack, manifest, managed_env) + start(stack, manifest, managed_env) report(ServeResult(True, "start", "dashboard stack started")) url = verify(stack.web_port) @@ -411,7 +391,7 @@ def report(result: ServeResult) -> None: log_path = None if failure.phase in {"pull", "start", "verify"} and stack and managed_env and manifest: log_path = _capture_failure_logs(stack, manifest, managed_env) - if not (failure.had_containers if failure.had_containers is not None else had_containers): + if not had_containers: _cleanup(stack, manifest) return ServeResult( False, failure.phase, failure.message, None, str(log_path) if log_path else None diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index 57c32ad..048cd3d 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -3,43 +3,162 @@ volume (via `canyonos sync`), then tell the Global Controller container to build and deploy it. The container's `ventis deploy` handles both the build (stubs, protos, Docker images) and the launch -- the CLI just ships files, -triggers it, and streams the logs. +triggers it, and watches the logs. + +That log stream is mostly noise the user didn't ask for (a whole `docker buildx +bake` transcript, among other things), so by default only the phase transitions +worth seeing are rendered and everything else is dropped. `-v` streams it all, +and a failure reveals the output it had been hiding. Once the deploy's logs report the workflow is actually up, `canyonos serve` is kicked off automatically so the local dashboard is ready without an extra manual step. """ +import queue +import re import subprocess +import threading +import time +from collections import deque -from rich.console import Console from rich.panel import Panel from rich.text import Text +from canyonos import ui from canyonos.constants import ( WORKFLOW_ROUTE, default_config_path, workflow_api_port, workspace_relative, ) -from canyonos.gc import GCError, post_deploy +from canyonos.gc import GCError, deploy_status, post_deploy, workflow_endpoints from canyonos.theme import GREEN, WHITE from canyonos.init import load_state, run_init -from canyonos.serve import run_serve +from canyonos.serve import serve_dashboard from canyonos.sync import run_sync +LOCAL_HOSTS = ("127.0.0.1", "localhost") + # Logged exactly once by GlobalController.run(), right after `_wait_for_healthy()` # returns -- the signal that the workflow finished coming up and entered its # steady-state polling loop. _WORKFLOW_UP_MARKER = "Global controller started, polling every" +# Substrings that mean the in-container deploy hit something fatal. `WARNING:` is +# deliberately absent: the OTel-not-configured notice and stub_generator's +# "Warning:" lines are benign and fire on nearly every run. +_ERROR_MARKERS = ( + "ERROR:", + "Traceback (most recent call last):", + "ERROR: failed to solve", + "process did not complete successfully", +) + +# (substring, spinner message, completed message). A None spinner message keeps +# whatever the spinner already shows; a None completed message prints nothing. +# Matched by substring against the raw line, so a phase that never runs is simply +# never matched -- nothing here assumes a phase happens, or happens in order. +_PHASES = ( + ("Generating stub:", "Generating stubs and Docker contexts...", None), + ("Compiling gRPC proto:", "Generating stubs and Docker contexts...", None), + ("Generating Docker context", "Generating stubs and Docker contexts...", None), + ("Building Docker image:", "Building images...", None), + ("No Docker images to build.", None, "No images to build"), + ("Build complete.", None, "Build complete"), + ("Deploying from config:", "Starting deploy...", None), + ("Checking for stale containers", "Cleaning up stale containers...", None), + ("Redis launched on", None, "Redis ready"), + ("Docker container(s) across", "Starting agents...", None), +) + +_IMAGE_COUNT = re.compile(r"Building (\d+) Docker image\(s\) via") +_REPLICA_COUNT = re.compile(r"Waiting for (\d+) replica\(s\) to become healthy") +# The name repeats across replicas of one agent, so the endpoint is what makes a +# ready line unique. +_READY = re.compile(r"Controller (\S+ \([^)]+\)) is ready\.") + +# Enough to hold a buildx failure block plus a Python traceback; 40 (what +# `canyonos test` tails) truncates both. +_RECENT_LINES = 200 + +# The container logs every request the CLI makes to it, so its own polling shows +# up in the stream it is reading. +_OWN_REQUEST_MARKER = "GET /status HTTP/1.1" + +_STATUS_POLL_SECONDS = 2.0 + +# Upper bound on how long to keep collecting output after a failure is spotted. +_REVEAL_GRACE_SECONDS = 30.0 + + +class PhaseTracker: + """Turns the container's log lines into the handful of events worth showing. + + `feed()` returns (spinner_message, completed_message, is_error) -- any of + which may be None -- so the caller owns all printing. + """ + + def __init__(self): + self.spinner = None + self.replicas_total = 0 + self.replicas_ready = set() + + def _agent_progress(self): + if self.replicas_total: + return f"Starting agents ({len(self.replicas_ready)}/{self.replicas_total} ready)..." + return "Starting agents..." + + def feed(self, line): + if any(marker in line for marker in _ERROR_MARKERS): + return None, None, True + + count = _IMAGE_COUNT.search(line) + if count: + self.spinner = f"Building {count.group(1)} images..." + return self.spinner, None, False + + replicas = _REPLICA_COUNT.search(line) + if replicas: + self.replicas_total = int(replicas.group(1)) + self.spinner = self._agent_progress() + return self.spinner, None, False + + ready = _READY.search(line) + if ready: + self.replicas_ready.add(ready.group(1)) + self.spinner = self._agent_progress() + return self.spinner, None, False -def run_deploy(config_path=None, serve=True): + for marker, spinner, done in _PHASES: + if marker in line: + # Repeats (one `Generating stub:` per agent) collapse: the + # spinner is only re-emitted when the message actually changes. + if spinner and spinner != self.spinner: + self.spinner = spinner + return spinner, done, False + return None, done, False + + return None, None, False + + def agents_ready_message(self): + """(message, all_ready). `_wait_for_healthy` gives up after its timeout and + lets the controller start anyway, so the workflow can come up short. + """ + ready = len(self.replicas_ready) + if not self.replicas_total: + return "Workflow ready", True + if ready < self.replicas_total: + return f"Workflow up, but only {ready}/{self.replicas_total} agents reported healthy", False + return f"{ready} agent(s) ready", True + + +def run_deploy(config_path=None, serve=True, verbose=False): # Left as None when unset: ventis resolves the artifact layout itself. if config_path is not None: config_path = workspace_relative(config_path) if config_path is None: - print("Config must be inside the project directory being synced.") + ui.fail("Config must be inside the project directory being synced.") return run_init() @@ -55,76 +174,254 @@ def run_deploy(config_path=None, serve=True): try: post_deploy(state["port"], config_path) - _stream_logs_and_autoserve(state["container_id"], api_port, serve=serve) + _stream_logs_and_autoserve(state, api_port, serve=serve, verbose=verbose) except GCError as e: - print(e) + ui.fail(e) -def print_workflow_endpoint(console, api_port): - """The one thing you need after a deploy: where to send requests. +def workflow_targets(gc_port, api_port): + """(name, host, port) for each deployed workflow. - Printed at the workflow-up marker and again on exit, because `deploy` keeps - tailing logs afterwards and would otherwise scroll it out of sight. + The container reports the address it actually placed each workflow at, so a + workflow running on another machine shows that machine's public IP. The + local port mapping is the fallback when it reports nothing. """ - if api_port is None: - return + targets = [ + ( + endpoint.get("name"), + "127.0.0.1" if endpoint["host"] in LOCAL_HOSTS else endpoint["host"], + endpoint["port"], + ) + for endpoint in workflow_endpoints(gc_port) + if endpoint.get("host") and endpoint.get("port") + ] + if targets: + return targets + return [(None, "127.0.0.1", api_port)] if api_port else [] - url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" - body = Text.assemble( - ("POST ", "dim"), - (url, f"bold {GREEN}"), - ("\nbody ", "dim"), - ('{"query": "your question here"}', WHITE), - ("\npoll ", "dim"), - (f"http://127.0.0.1:{api_port}/status/", WHITE), - ) - console.print() - console.print( + +def _summary_body(dashboard_url, targets): + body = Text() + body.append("Dashboard ", "dim") + if dashboard_url: + body.append(dashboard_url, f"bold {GREEN}") + else: + body.append("not running -- start it with `canyonos serve`", WHITE) + + for name, host, port in targets: + base = f"http://{host}:{port}" + body.append("\n") + if name: + body.append(f"\n{name}", f"bold {WHITE}") + body.append("\nPOST ", "dim") + body.append(f"{base}/{WORKFLOW_ROUTE}", f"bold {GREEN}") + body.append("\nbody ", "dim") + body.append('{"query": "your question here"}', WHITE) + body.append("\npoll ", "dim") + body.append(f"{base}/status/", WHITE) + if host not in LOCAL_HOSTS: + body.append(f"\n needs inbound TCP {port} open on {host}", "dim") + return body + + +def print_deploy_summary(dashboard_url, targets): + """The one screen printed once everything is up: dashboard and workflow endpoints. + + Under `-v` it is printed again on exit, because the log tail continues + afterwards and would otherwise scroll it out of sight. Quiet mode prints + nothing after it, so once is enough. + """ + ui.blank() + ui.panel( Panel( - body, - title=f"[bold {GREEN}]Workflow is live[/]", + _summary_body(dashboard_url, targets), + title=f"[bold {GREEN}]Deploy is live[/]", title_align="left", border_style=GREEN, padding=(1, 4), ) ) - console.print() + ui.blank() + + +def _start_dashboard(): + """The dashboard's URL, or None -- a dashboard that won't start doesn't fail the deploy.""" + try: + return serve_dashboard().url + except Exception as e: + ui.fail(f"Could not start the dashboard automatically: {e}") + ui.hint("Run `canyonos serve` manually to view it.") + return None + + +def _deploy_summary(state, api_port, serve): + summary = ( + _start_dashboard() if serve else None, + workflow_targets(state["port"], api_port), + ) + print_deploy_summary(*summary) + return summary -def _stream_logs_and_autoserve(container_id, api_port, serve=True): +def _interrupted(summary=None): + ui.blank() + ui.say("Stopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + ui.hint("To resubscribe to log stream run `canyonos logs`.") + if summary is not None: + print_deploy_summary(*summary) + + +def _tail_verbose(stream, state, api_port, serve): + """Every log line, verbatim -- what `-v` restores. + + Ctrl+C reprints the summary here but not in quiet mode: only this tail keeps + printing past it, so only here has it scrolled out of sight. + """ + summary = None + try: + for line in stream: + print(line, end="") + if summary is None and _WORKFLOW_UP_MARKER in line: + summary = _deploy_summary(state, api_port, serve) + except KeyboardInterrupt: + _interrupted(summary) + + +def _tail_quiet(lines, state, api_port, serve): + """Only the phase transitions, until the workflow is up or something fails. + + Nothing is echoed raw: the buildx transcript, ventis' bare prints and grpc's + stderr have no common prefix to filter on, so anything unrecognized is + dropped rather than allow-listed. `-v` and `canyonos logs` still have it all. + """ + tracker = PhaseTracker() + recent = deque(maxlen=_RECENT_LINES) + reached_up_marker = False + + # The spinner is exited before the summary panel or the dashboard's own + # spinner is drawn, and on the way out of a Ctrl+C, so the cursor is restored. + # A nested spinner wouldn't raise, it would silently render nothing. + with ui.status("Starting build...") as spinner: + for line in _drain(lines, state): + recent.append(line) + message, done, is_error = tracker.feed(line) + if is_error: + break + if done: + ui.ok(done) + if message: + spinner.update(message) + if _WORKFLOW_UP_MARKER in line: + summary_line, all_ready = tracker.agents_ready_message() + (ui.ok if all_ready else ui.warn)(summary_line) + reached_up_marker = True + break + + if reached_up_marker: + return _deploy_summary(state, api_port, serve) + + _reveal_failure(lines, recent, state) + return None + + +def _queued_lines(stream): + """Feed `stream` into a queue, terminated by None, so reads can time out. + + A failed build leaves the log stream open and silent -- the deploy is only a + subprocess of the container being tailed -- so blocking on the next line + would wait forever with nothing left to report. + """ + lines = queue.Queue() + + def read(): + for line in stream: + lines.put(line) + lines.put(None) + + threading.Thread(target=read, daemon=True).start() + return lines + + +def _drain(lines, state, deadline=None): + """Yield log lines until the stream ends, the deploy dies, or `deadline` passes. + + The container's /status is polled on the read timeout rather than per line, + because the container logs each of those requests into the very stream being + read -- which would otherwise feed itself. + """ + misses = 0 + while deadline is None or time.monotonic() < deadline: + try: + line = lines.get(timeout=_STATUS_POLL_SECONDS) + except queue.Empty: + # Nothing for a while: check the deploy is still alive, since a + # build that died takes the output with it but not the stream. + dead, misses = _deploy_is_dead(state, misses) + if dead: + return + continue + if line is None: + return + misses = 0 + if _OWN_REQUEST_MARKER not in line: + yield line + + +def _deploy_is_dead(state, misses): + """Whether the in-container deploy has stopped, over two consecutive checks. + + An unreachable container counts as a miss rather than a verdict, so one + dropped request doesn't end a deploy that is merely busy. + """ + status = deploy_status(state["port"]) + if status is not None and status.get("running"): + return False, 0 + misses += 1 + return misses >= 2, misses + + +def _reveal_failure(lines, recent, state): + """Stop hiding: replay what was suppressed, then keep echoing. + + The cause is usually still in flight when the verdict lands, so this keeps + draining until the container confirms the deploy is gone. + """ + ui.fail("Deploy failed.") + ui.blank() + for buffered in recent: + print(buffered, end="") + + for line in _drain(lines, state, deadline=time.monotonic() + _REVEAL_GRACE_SECONDS): + print(line, end="") + + ui.blank() + ui.hint("Run `canyonos deploy -v` or `canyonos logs` for the full container log.") + + +def _stream_logs_and_autoserve(state, api_port, serve=True, verbose=False): """Tail the GC container's logs, and once they show the workflow is up, - print where to reach it -- plus, unless disabled via `serve=False`, launch - `canyonos serve`. Log tailing continues afterwards. + start the dashboard (unless disabled via `serve=False`) and print where + everything lives. Log tailing continues afterwards. """ - console = Console() process = subprocess.Popen( - ["docker", "logs", "-f", container_id], + ["docker", "logs", "-f", state["container_id"]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, ) - served = not serve - workflow_up = False try: - for line in process.stdout: - print(line, end="") - if not workflow_up and _WORKFLOW_UP_MARKER in line: - workflow_up = True - print_workflow_endpoint(console, api_port) - if not served: - served = True - print("Starting the local dashboard (canyonos serve)...") - try: - run_serve() - except Exception as e: - print(f"Could not start the dashboard automatically: {e}") - print("Run `canyonos serve` manually to view it.") + if verbose: + _tail_verbose(process.stdout, state, api_port, serve) + return + lines = _queued_lines(process.stdout) + if _tail_quiet(lines, state, api_port, serve) is not None: + # Quiet mode stays attached after the summary so Ctrl+C means the + # same thing in both modes -- it just swallows what arrives. + while lines.get() is not None: + pass except KeyboardInterrupt: - print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") - print("To resubscribe to log stream run `canyonos logs`.") - if workflow_up: - print_workflow_endpoint(console, api_port) + _interrupted() finally: if process.poll() is None: process.terminate() diff --git a/cli/canyonos/doctor.py b/cli/canyonos/doctor.py index 7dd9883..15c36af 100644 --- a/cli/canyonos/doctor.py +++ b/cli/canyonos/doctor.py @@ -7,17 +7,9 @@ import shutil import subprocess +from canyonos import ui from canyonos.build import AGENTS -from canyonos.init import docker_start_command - - -def _docker_installed(): - return shutil.which("docker") is not None - - -def _docker_daemon_running(): - result = subprocess.run(["docker", "info"], capture_output=True) - return result.returncode == 0 +from canyonos.init import docker_running, docker_start_command def _compose_available(): @@ -25,10 +17,6 @@ def _compose_available(): return result.returncode == 0 -def _git_available(): - return shutil.which("git") is not None - - def _docker_daemon_fix(): """Names the command for the active docker context, since `canyonos deploy` would run exactly that itself.""" @@ -38,23 +26,16 @@ def _docker_daemon_fix(): return "start your Docker runtime (on Linux: `sudo systemctl start docker`)" -def _coding_agent_available(): - return any(shutil.which(spec["cli"]) for spec in AGENTS.values()) - - def _checks(): - """Built fresh on each call (not a module-level constant) so tests can - patch the individual `_check_*` functions by name and have it take effect. - """ return [ ( "Docker installed", - _docker_installed, + lambda: shutil.which("docker") is not None, "install Docker: https://docs.docker.com/get-docker/", ), ( "Docker daemon running", - _docker_daemon_running, + docker_running, _docker_daemon_fix(), ), ( @@ -64,13 +45,13 @@ def _checks(): ), ( "git available", - _git_available, + lambda: shutil.which("git") is not None, "install git (`canyonos build` fetches the porting skill with it; " "without git it falls back to a full-repo tarball download)", ), ( "Coding agent available", - _coding_agent_available, + lambda: any(shutil.which(spec["cli"]) for spec in AGENTS.values()), "install one of " + " or ".join(spec["label"] for spec in AGENTS.values()) + " (`canyonos build` runs the port through it)", @@ -88,9 +69,11 @@ def run_doctor(): passed = False fix = f"{fix} (error: {e})" - print(f"{'✓' if passed else '✗'} {label}") - if not passed: - print(f" -> {fix}") + if passed: + ui.ok(label) + else: + ui.fail(label) + ui.hint(f" -> {fix}") all_ok = False return all_ok diff --git a/cli/canyonos/gc.py b/cli/canyonos/gc.py index b5af7eb..a8778b3 100644 --- a/cli/canyonos/gc.py +++ b/cli/canyonos/gc.py @@ -7,6 +7,7 @@ import urllib.error import urllib.request +from canyonos import ui from canyonos.init import load_state _DEPLOY_CONFLICT = "Run `canyonos stop` to stop the running deploy first." @@ -46,7 +47,7 @@ def require_state(): try: return load_state() except FileNotFoundError: - print("No Global Controller container is running. Run `canyonos deploy` first.") + ui.warn("No Global Controller container is running. Run `canyonos deploy` first.") return None @@ -73,6 +74,19 @@ def post_clean(port): return _request(f"http://127.0.0.1:{port}/clean", "Stop", method="POST") +def workflow_endpoints(port): + """Where the deployed workflows answer, per the container's own instance + records -- for a workflow placed on another machine that is its public IP, + not this host. Empty when the container can't say (an older image has no + /endpoints route), which leaves the caller on its local-port fallback. + """ + try: + data = _request(f"http://127.0.0.1:{port}/endpoints", "Endpoints") + except GCError: + return [] + return data.get("workflows") or [] + + def deploy_status(port): """Parsed /status payload, or None if the container is unreachable.""" url = f"http://127.0.0.1:{port}/status" diff --git a/cli/canyonos/init.py b/cli/canyonos/init.py index 69e8fbb..6b07f84 100644 --- a/cli/canyonos/init.py +++ b/cli/canyonos/init.py @@ -16,9 +16,8 @@ # Formatting from pyfiglet import figlet_format -from rich.console import Console -from canyonos.theme import GRADIENT +from canyonos import ui @@ -76,7 +75,7 @@ def docker_start_command(): return None -def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): +def ensure_docker_running(timeout=DOCKER_START_TIMEOUT): if docker_running(): return @@ -89,14 +88,14 @@ def ensure_docker_running(console, timeout=DOCKER_START_TIMEOUT): "docker context. Start it (on Linux: `sudo systemctl start docker`) and re-run." ) - console.print(f"Docker isn't running -- starting it with `{' '.join(command)}`...") + ui.say(f"Docker isn't running -- starting it with `{' '.join(command)}`...") subprocess.run(command, capture_output=True) deadline = time.time() + timeout - with console.status("Waiting for the Docker daemon..."): + with ui.status("Waiting for the Docker daemon..."): while time.time() < deadline: if docker_running(): - console.print("Docker is running.") + ui.ok("Docker is running.") return time.sleep(1) @@ -200,20 +199,17 @@ def quit_existing(): run_quit() -def run_init(): - console = Console() - banner = figlet_format("CANYON OS", font="ansi_shadow", width=200) - - for line, color in zip(banner.splitlines(), GRADIENT): - console.print(line, style=color) +def run_init(banner=True): + if banner: + ui.gradient(figlet_format("CANYON OS", font="ansi_shadow", width=200)) # Before quit_existing(), which shells out to docker itself. - ensure_docker_running(console) + ensure_docker_running() quit_existing() - with console.status("Pulling Global Controller image..."): + with ui.status("Pulling Global Controller image..."): pull_image() - with console.status("Starting Global Controller container..."): + with ui.status("Starting Global Controller container..."): container_id, port = run_container() save_state(container_id, port) - print(f"Global Controller running in container {container_id[:12]} on port {port}") + ui.ok(f"Global Controller running in container {container_id[:12]} on port {port}") diff --git a/cli/canyonos/logs.py b/cli/canyonos/logs.py index 9839134..3969af6 100644 --- a/cli/canyonos/logs.py +++ b/cli/canyonos/logs.py @@ -4,6 +4,7 @@ import subprocess +from canyonos import ui from canyonos.gc import deploy_status, require_state @@ -14,14 +15,15 @@ def run_logs(): status = deploy_status(state["port"]) if status is None: - print("Could not reach Global Controller container.") + ui.fail("Could not reach Global Controller container.") return if not status.get("running"): - print("No deploy running, run `canyonos deploy` to deploy project.") + ui.warn("No deploy running, run `canyonos deploy` to deploy project.") return try: subprocess.run(["docker", "logs", "-f", state["container_id"]]) except KeyboardInterrupt: - print("\nStopped monitoring log stream. Run `canyonos stop` to stop the deploy.") + ui.blank() + ui.say("Stopped monitoring log stream. Run `canyonos stop` to stop the deploy.") diff --git a/cli/canyonos/new_app.py b/cli/canyonos/new_app.py index 30e93b0..31aeb44 100644 --- a/cli/canyonos/new_app.py +++ b/cli/canyonos/new_app.py @@ -5,10 +5,12 @@ import os +from canyonos import ui + def run_new_app(): if os.listdir("."): - print("Directory is not empty. Run `canyonos new-app` in an empty directory.") + ui.fail("Directory is not empty. Run `canyonos new-app` in an empty directory.") return for folder in ("agents", "config", "workflow"): @@ -18,4 +20,4 @@ def run_new_app(): for filename in ("global_controller.yaml", "policy.yaml"): open(os.path.join("config", filename), "w").close() - print("Created new CanyonOS project.") + ui.ok("Created new CanyonOS project.") diff --git a/cli/canyonos/quit.py b/cli/canyonos/quit.py index bb2e11d..9af5dcc 100644 --- a/cli/canyonos/quit.py +++ b/cli/canyonos/quit.py @@ -8,8 +8,7 @@ import os import subprocess -from rich.console import Console - +from canyonos import ui from canyonos.gc import GCError, post_clean, require_state from canyonos.init import GC_WORKSPACE_VOLUME, STATE_PATH @@ -27,8 +26,7 @@ def run_quit(): return container_id = state["container_id"] - console = Console() - with console.status("Tearing down..."): + with ui.status("Tearing down..."): # Stop any running deploy first, so the local controller and Redis # containers it spawned via docker-outside-of-docker get torn down # too. Removing the GC container itself doesn't touch them -- they're @@ -54,6 +52,6 @@ def run_quit(): os.remove(STATE_PATH) if already_gone: - print(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") + ui.warn(f"Global Controller container {container_id[:12]} was already gone; cleaned up local state.") else: - print(f"Global Controller container {container_id[:12]} torn down (volume removed)") + ui.ok(f"Global Controller container {container_id[:12]} torn down (volume removed)") diff --git a/cli/canyonos/serve.py b/cli/canyonos/serve.py index 9515699..c96e336 100644 --- a/cli/canyonos/serve.py +++ b/cli/canyonos/serve.py @@ -1,18 +1,37 @@ """CLI output for the local dashboard stack.""" -from .dashboard_stack import run_dashboard +from canyonos import ui +from .dashboard_stack import ServeResult, run_dashboard -def run_serve() -> int: - def report(phase: str, message: str) -> None: - print(f"[serve] {phase}: {message}") +def serve_dashboard() -> ServeResult: + """Bring the dashboard up, reporting progress. Returns the stack's result.""" + # Phases drive the spinner while the stack comes up; the trace itself is + # only printed when something fails and the user needs to see how far it got. + trace = [] + + with ui.status("Starting the dashboard...") as spinner: + def report(phase: str, message: str) -> None: + trace.append((phase, message)) + spinner.update(message) + + result = run_dashboard(report) - result = run_dashboard(report) if result.ok: - print(f"Dashboard: {result.url}") - return 0 + return result - print(f"serve failed in {result.phase}: {result.message}") + for phase, message in trace: + ui.hint(f"{phase}: {message}") + ui.fail(f"serve failed in {result.phase}: {result.message}") if result.log_path: - print(f"log: {result.log_path}") - return 1 + ui.hint(f"log: {result.log_path}") + return result + + +def run_serve() -> int: + result = serve_dashboard() + if not result.ok: + return 1 + + ui.ok(f"Dashboard: {result.url}") + return 0 diff --git a/cli/canyonos/status.py b/cli/canyonos/status.py new file mode 100644 index 0000000..071297e --- /dev/null +++ b/cli/canyonos/status.py @@ -0,0 +1,55 @@ +""" +Logic for `canyonos status`: reports whether a deploy is currently running, +and if so, where the workflow (and, if up, the dashboard) answer. +""" + +from canyonos import ui +from canyonos.constants import WORKFLOW_ROUTE, default_config_path, workflow_api_port +from canyonos.dashboard_stack import _existing_dashboard_port +from canyonos.deploy import workflow_targets +from canyonos.gc import deploy_status, require_state + + +def run_status(): + state = require_state() + if state is None: + return + + status = deploy_status(state["port"]) + if not status or not status.get("running"): + ui.warn("No deploy is currently running.") + return + + ui.ok("Deploy is running.") + + # Same resolution `deploy` uses, so both report the address the container + # actually placed the workflow at and fall back to the configured api_port + # rather than a guess. + targets = workflow_targets(state["port"], workflow_api_port(default_config_path())) + for name, target_host, target_port in targets: + label = f"Workflow {name}" if name else "Workflow" + ui.say(f"{label}: {target_host}:{target_port}") + if not targets: + ui.hint("No workflow endpoints reported yet.") + + dashboard_port = _existing_dashboard_port() + if dashboard_port: + ui.say(f"Dashboard: 127.0.0.1:{dashboard_port}") + else: + ui.hint("Dashboard is not running. Run `canyonos serve` to start it.") + + if not targets: + return + + # The body is splatted into the workflow entrypoint as kwargs, so its keys + # are that function's parameter names -- `query` for every bundled example, + # but swap in whatever yours actually takes. + _, host, port = targets[0] + ui.blank() + ui.hint("Query the workflow:") + ui.say(f" curl -X POST http://{host}:{port}/{WORKFLOW_ROUTE} \\") + ui.say(' -H "Content-Type: application/json" \\') + ui.say(" -d '{\"query\": \"your question here\"}'") + ui.blank() + ui.hint("Check a request's result:") + ui.say(f" curl http://{host}:{port}/status/") diff --git a/cli/canyonos/stop.py b/cli/canyonos/stop.py index a3f96f3..519cf49 100644 --- a/cli/canyonos/stop.py +++ b/cli/canyonos/stop.py @@ -3,8 +3,7 @@ Controller container (SIGTERM, same teardown as Ctrl+C would trigger). """ -from rich.console import Console - +from canyonos import ui from canyonos.gc import GCError, post_clean, require_state @@ -13,10 +12,9 @@ def run_stop(): if state is None: return - console = Console() try: - with console.status("Stopping deploy..."): + with ui.status("Stopping deploy..."): post_clean(state["port"]) - print("Deploy stopped.") + ui.ok("Deploy stopped.") except GCError as e: - print(e) + ui.fail(e) diff --git a/cli/canyonos/sync.py b/cli/canyonos/sync.py index 20ac3c1..84f875c 100644 --- a/cli/canyonos/sync.py +++ b/cli/canyonos/sync.py @@ -13,6 +13,7 @@ import os import subprocess +from canyonos import ui from canyonos.gc import require_state from canyonos.init import GC_WORKSPACE_PATH @@ -27,14 +28,18 @@ def run_sync(): # Trailing "/." copies the *contents* of the current directory into # /workspace, rather than nesting it under /workspace/. src = os.path.join(os.getcwd(), ".") - print(f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH} ...") - - result = subprocess.run( - ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"] - ) + label = f"Syncing {os.getcwd()} -> {container_id[:12]}:{GC_WORKSPACE_PATH}" + + with ui.status(f"{label}..."): + # Captured so docker's own progress output doesn't clobber the spinner. + result = subprocess.run( + ["docker", "cp", src, f"{container_id}:{GC_WORKSPACE_PATH}"], + capture_output=True, + text=True, + ) if result.returncode != 0: - print("Sync failed.") + ui.fail(f"Sync failed: {result.stderr.strip() or result.stdout.strip()}") return False - print("Sync complete.") + ui.ok("Sync complete.") return True diff --git a/cli/canyonos/test.py b/cli/canyonos/test.py index 468527a..2409965 100644 --- a/cli/canyonos/test.py +++ b/cli/canyonos/test.py @@ -1,20 +1,28 @@ """ -Logic for `canyonos test`: smoke-test a project end to end on this machine. +Logic for `canyonos test`: check a project end to end on this machine. -Every agent's `provider` is rewritten to `local` for the duration of the run -(the original file is restored verbatim afterwards), the project is deployed -into the Global Controller container, one query is sent to the workflow's -`/main` endpoint, and its result -- or the error that came back -- is printed. +Four phases, each ending the run if it fails: the `.car/` artifact `canyonos +build` produced is verified statically, the project is deployed locally (every +agent's `provider` rewritten to `local` for the duration, the original file +restored verbatim afterwards), the running containers are checked against what +the config declared, and one prompt is sent to the workflow's `/main` endpoint. + +A passing run leaves nothing behind. A failing one leaves the Global Controller +container up, with the tail of its log, so there is something left to debug. """ import json import os +import socket +import subprocess import time import urllib.error import urllib.request -from rich.console import Console +from rich.panel import Panel +from rich.text import Text +from canyonos import ui from canyonos.constants import ( WORKFLOW_ROUTE, default_config_path, @@ -22,16 +30,25 @@ workflow_api_port, workspace_relative, ) +from canyonos.deploy import workflow_targets from canyonos.gc import GCError, deploy_status, post_deploy from canyonos.init import load_state, quit_existing, run_init from canyonos.sync import run_sync +from canyonos.theme import GREEN, WHITE +from canyonos.verify import ( + ARTIFACT_DIR, + VerificationError, + verify_build_artifact, + verify_runtime, +) DEFAULT_QUERY = "hello" # Generous: the first deploy of a project builds every agent image from scratch. READY_TIMEOUT = 900 REQUEST_TIMEOUT = 600 +SUBMIT_TIMEOUT = 30 POLL_INTERVAL = 2 - +LOG_TAIL_LINES = 40 def _force_local_providers(config_path): @@ -51,13 +68,21 @@ def _force_local_providers(config_path): return original -def _workflow_ready(api_port): +def _port_in_use(port): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return True + except OSError: + return False + + +def _workflow_ready(host, port): """True once the workflow's REST API answers at all. Any HTTP response counts -- /status/ 404s, which still proves the server is up and listening. """ - url = f"http://127.0.0.1:{api_port}/status/canyonos-test-probe" + url = f"http://{host}:{port}/status/canyonos-test-probe" try: urllib.request.urlopen(url, timeout=2) return True @@ -67,33 +92,32 @@ def _workflow_ready(api_port): return False -def _wait_for_workflow(gc_port, api_port, console): +def _wait_for_workflow(gc_port, api_port): deadline = time.time() + READY_TIMEOUT - with console.status("Building and starting containers..."): + with ui.status("Building images and starting containers..."): while time.time() < deadline: - if _workflow_ready(api_port): - return True + if _workflow_ready("127.0.0.1", api_port): + return if not (deploy_status(gc_port) or {}).get("running", False): - return False + raise _TestFailed("The deploy stopped before the workflow came up.") time.sleep(POLL_INTERVAL) - print(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") - return False + raise _TestFailed(f"Timed out after {READY_TIMEOUT}s waiting for the workflow to come up.") -def _send_query(api_port, query): - url = f"http://127.0.0.1:{api_port}/{WORKFLOW_ROUTE}" +def _send_query(host, port, query): + url = f"http://{host}:{port}/{WORKFLOW_ROUTE}" body = json.dumps({"query": query}).encode() req = urllib.request.Request( url, data=body, headers={"Content-Type": "application/json"}, method="POST" ) - with urllib.request.urlopen(req, timeout=30) as resp: + with urllib.request.urlopen(req, timeout=SUBMIT_TIMEOUT) as resp: return json.loads(resp.read())["request_id"] -def _await_result(api_port, request_id, console): - url = f"http://127.0.0.1:{api_port}/status/{request_id}" +def _await_result(host, port, request_id): + url = f"http://{host}:{port}/status/{request_id}" deadline = time.time() + REQUEST_TIMEOUT - with console.status("Running query..."): + with ui.status("Running query..."): while time.time() < deadline: try: with urllib.request.urlopen(url, timeout=10) as resp: @@ -107,67 +131,266 @@ def _await_result(api_port, request_id, console): return {"status": "timeout"} -def run_test(config_path=None, query=None): - console = Console() - config_path = config_path or default_config_path() - query = query or DEFAULT_QUERY +def _log_tail(container_id): + result = subprocess.run( + ["docker", "logs", "--tail", str(LOG_TAIL_LINES), container_id], + capture_output=True, + text=True, + ) + return (result.stdout + result.stderr).strip() or None - config_path = workspace_relative(config_path) - if config_path is None: - print("Config must be inside the project directory being synced.") - return 1 - if not os.path.isfile(config_path): - print(f"Config file not found: {config_path}. Run `canyonos build` first.") - return 1 +class _TestFailed(Exception): + """Ends the run early, carrying a message fit for either output mode.""" - api_port = workflow_api_port(config_path) - if api_port is None: - print(f"No agent with `type: workflow` in {config_path}; nothing to test.") - return 1 - print(f"Testing {config_path} locally (query: {query!r})") - original_config = _force_local_providers(config_path) +class _Run: + """One `canyonos test` invocation: the phases it got through, and what they found.""" + + def __init__(self, query): + self.query = query + self.started = time.monotonic() + # Only once a deploy is under way is the container worth keeping and its + # log worth reading; before that it holds nothing about the failure. + self.deploy_started = False + self.phases = [] + self.validation = None + self.runtime = None + self.endpoint = None + self.result = None + self.error = None + self.log_tail = None + + def begin(self, name, number, title): + """Open a phase, recorded as failed until `done` says otherwise.""" + self.phases.append({"name": name, "ok": False, "detail": None}) + ui.blank() + ui.say(f"[{number}/4] {title}") + + def done(self, detail=None): + self.phases[-1].update(ok=True, detail=detail) + + def failed(self, detail): + if self.phases: + self.phases[-1]["detail"] = detail + + def elapsed(self): + return round(time.monotonic() - self.started, 3) + + +def _verify_build(run, config_path): + run.begin("verify_build", 1, "Verify build artifact") + + # A project ported before the .car layout keeps its config at the top level; + # there is no build artifact to check, so the deploy phases still run. + if not config_path.startswith(f"{ARTIFACT_DIR}{os.sep}"): + ui.warn(f"No `{ARTIFACT_DIR}/` artifact -- deploying {config_path} as it is.") + ui.hint(" -> `canyonos build` produces one, and gives this phase something to check.") + run.done("skipped: no .car/ artifact") + return try: - run_init() - if not run_sync(): - return 1 + run.validation = verify_build_artifact() + except VerificationError as e: + raise _TestFailed(str(e)) from None + stale = len(run.validation["stale"]) + run.done(f"{run.validation['warnings']} warning(s), {stale} stale source(s)") + + +def _deploy_locally(run, config_path, api_port): + run.begin("deploy", 2, "Deploy locally") + run_init(banner=False) + + if not run_sync(): + raise _TestFailed("Could not sync the project into the container.") + + # Only the gRPC host port is bumped when a port is taken (the local runtime's + # launch retry), so an occupied api_port dies 50 attempts later as "no free + # port found". `canyonos serve` also starts looking for its web port at 8080. + if _port_in_use(api_port): + raise _TestFailed( + f"Port {api_port} is already in use, and the workflow needs it. Free it " + f"(`canyonos quit` stops a previous deploy) or change `api_port` in {config_path}." + ) + + state = load_state() + try: + post_deploy(state["port"], config_path) + except GCError as e: + raise _TestFailed(str(e)) from None + run.deploy_started = True - state = load_state() - try: - post_deploy(state["port"], config_path) - except GCError as e: - print(e) - return 1 + _wait_for_workflow(state["port"], api_port) + run.done(f"Global Controller on port {state['port']}") + return state - if not _wait_for_workflow(state["port"], api_port, console): - print("The deploy did not come up. Run `canyonos logs` to see why.") - return 1 - try: - request_id = _send_query(api_port, query) - except OSError as e: - print(f"Could not reach the workflow on port {api_port}: {e}") - return 1 - result = _await_result(api_port, request_id, console) - except KeyboardInterrupt: - print("\nTest cancelled.") - return 1 +def _verify_runtime(run, config_path, gc_port): + run.begin("verify_runtime", 3, "Verify runtime") + try: + run.runtime = verify_runtime(config_path, gc_port) + except VerificationError as e: + raise _TestFailed(str(e)) from None + run.done(f"{len(run.runtime['agents'])} agent(s) up") + + +def _query(run, gc_port, api_port): + run.begin("query", 4, "Query the workflow") + targets = workflow_targets(gc_port, api_port) + if not targets: + raise _TestFailed("The deploy reported no workflow endpoint to query.") + + _, host, port = targets[0] + run.endpoint = f"http://{host}:{port}/{WORKFLOW_ROUTE}" + ui.say(f"POST {run.endpoint} {json.dumps({'query': run.query})}") + + try: + request_id = _send_query(host, port, run.query) + except OSError as e: + raise _TestFailed(f"Could not reach the workflow at {run.endpoint}: {e}") from None + + data = _await_result(host, port, request_id) + status = data.get("status") + if status == "error": + raise _TestFailed(data.get("error") or "the workflow returned an error.") + if status != "done": + raise _TestFailed(f"The workflow did not finish within {REQUEST_TIMEOUT}s.") + + run.result = data.get("result") + run.done(f"answered in {run.elapsed()}s") + + +def _run_test(run): + """Walk the four phases, restoring the config whatever happens.""" + config_path = workspace_relative(default_config_path()) + if config_path is None: + raise _TestFailed("Config must be inside the project directory being synced.") + if not os.path.isfile(config_path): + raise _TestFailed(f"No config at {config_path}. Run `canyonos build` first.") + + _verify_build(run, config_path) + + api_port = workflow_api_port(config_path) + if api_port is None: + raise _TestFailed(f"No agent with `type: workflow` in {config_path}; nothing to test.") + + original_config = _force_local_providers(config_path) + try: + state = _deploy_locally(run, config_path, api_port) + _verify_runtime(run, config_path, state["port"]) + _query(run, state["port"], api_port) finally: with open(config_path, "w") as f: f.write(original_config) - # A smoke test leaves nothing behind: run_init() started this container. - quit_existing() - status = result.get("status") - if status == "done": - print("Test passed.") - print(json.dumps(result.get("result"), indent=2)) - return 0 - if status == "error": - print(f"Test failed: {result.get('error')}") +# ------------------------------------------------------------------ # +# Output # +# ------------------------------------------------------------------ # + + +def _summary_body(run): + body = Text() + body.append("Query ", "dim") + body.append(run.query, WHITE) + if run.endpoint: + body.append("\nEndpoint ", "dim") + body.append(run.endpoint, WHITE) + body.append("\nElapsed ", "dim") + body.append(f"{run.elapsed()}s", WHITE) + + body.append("\n") + for phase in run.phases: + body.append("\n") + body.append("✓ " if phase["ok"] else "✗ ", GREEN if phase["ok"] else "bold red") + body.append(f"{phase['name']:<16}", WHITE) + # The failing phase's detail is the error, spelled out below in full. + body.append(phase["detail"] if phase["ok"] else "", "dim") + + body.append("\n\n") + if run.error is None: + body.append("Result ", "dim") + body.append(json.dumps(run.result, indent=2), WHITE) else: - print(f"Test failed: workflow did not finish within {REQUEST_TIMEOUT}s.") - return 1 + body.append(run.error, "bold red") + return body + + +def _print_summary(run): + passed = run.error is None + ui.blank() + ui.panel( + Panel( + _summary_body(run), + title=f"[bold {GREEN}]Test passed[/]" if passed else "[bold red]Test failed[/]", + title_align="left", + border_style=GREEN if passed else "red", + padding=(1, 4), + ) + ) + ui.blank() + + +def _print_failure_logs(run): + if run.log_tail: + ui.hint(f"last {LOG_TAIL_LINES} lines of the Global Controller log:") + ui.say(run.log_tail) + ui.blank() + ui.hint("Containers left running for inspection: `canyonos logs` | `canyonos quit`") + + +def _payload(run): + return { + "ok": run.error is None, + "query": run.query, + "elapsed_s": run.elapsed(), + "phases": run.phases, + "validation": run.validation, + "runtime": run.runtime, + "result": run.result, + "error": run.error, + "log_tail": run.log_tail, + } + + +def run_test(prompt=None, as_json=False): + run = _Run(prompt or DEFAULT_QUERY) + ui.set_quiet(as_json) + + try: + container_live = False + try: + _run_test(run) + except _TestFailed as e: + run.error = str(e) + except KeyboardInterrupt: + run.error = "cancelled by user" + except RuntimeError as e: + # Docker unreachable, image pull failed, no free port: all carry a + # readable message, and `--json` needs it inside the payload. + run.error = str(e) + + if run.error is not None: + run.failed(run.error) + + if run.error is not None and run.deploy_started: + # Read the log before anything else touches the container, and leave + # it running -- a torn-down deploy can't be diagnosed. + try: + run.log_tail = _log_tail(load_state()["container_id"]) + container_live = True + except (FileNotFoundError, OSError): + pass + else: + quit_existing() + + if as_json: + print(json.dumps(_payload(run), indent=2)) + else: + _print_summary(run) + if container_live: + _print_failure_logs(run) + + return 0 if run.error is None else 1 + finally: + ui.set_quiet(False) diff --git a/cli/canyonos/ui.py b/cli/canyonos/ui.py new file mode 100644 index 0000000..056999a --- /dev/null +++ b/cli/canyonos/ui.py @@ -0,0 +1,67 @@ +""" +The CLI's one output surface: every user-facing line goes through here so the +whole tool speaks with the same palette, symbols and spinner. + +Messages are emitted as literal text, never as rich markup, so a path or an +error containing square brackets can't be swallowed as a style tag. +""" + +from contextlib import contextmanager + +from rich.console import Console +from rich.text import Text + +from canyonos.theme import GRADIENT, GREEN, WHITE + +console = Console() + + +def set_quiet(quiet): + """Silence every helper here, so `canyonos test --json` emits only its payload.""" + console.quiet = quiet + + +def _emit(message, style, symbol=None): + parts = [(f"{symbol} ", style)] if symbol else [] + parts.append((str(message), WHITE if symbol else style)) + console.print(Text.assemble(*parts)) + + +def say(message): + _emit(message, WHITE) + + +def ok(message): + _emit(message, GREEN, "✓") + + +def fail(message): + _emit(message, "bold red", "✗") + + +def warn(message): + _emit(message, "yellow", "!") + + +def hint(message): + _emit(message, "dim") + + +def blank(): + console.print() + + +def gradient(text): + """Print `text` line by line down the brand ramp (the `init` banner).""" + for line, color in zip(text.splitlines(), GRADIENT): + console.print(line, style=color) + + +def panel(renderable): + console.print(renderable) + + +@contextmanager +def status(message): + with console.status(message) as spinner: + yield spinner diff --git a/cli/canyonos/verify.py b/cli/canyonos/verify.py new file mode 100644 index 0000000..d047885 --- /dev/null +++ b/cli/canyonos/verify.py @@ -0,0 +1,291 @@ +""" +The two verification passes behind `canyonos test`. + +`verify_build_artifact` checks the `.car/` tree a `canyonos build` produced, +before any container is started: the layout, the porting skill's own validator, +and whether the sources have moved on since the port was taken. + +`verify_runtime` checks a running local deploy against what the config declared +-- every image built, every replica up -- because the controller logs a warning +and carries on when an agent never becomes healthy, so a workflow that answers +is not on its own proof that the deploy is complete. +""" + +import hashlib +import json +import os +import subprocess +import sys + +import yaml +from rich.table import Table + +from canyonos import gc, ui +from canyonos.build import AGENTS, install_skill +from canyonos.constants import DEFAULT_API_PORT +from canyonos.init import STATE_DIR +from canyonos.theme import GREEN + +ARTIFACT_DIR = ".car" +SOURCE_DIR = "app" +CONFIG_REL = "config/global_controller.yaml" +PORTING_STATE_REL = "config/.porting-state.json" + +VALIDATOR_NAME = "validate.py" +SKILL_CACHE_DIR = os.path.join(STATE_DIR, "skill") + +# These two rules decide their verdict by importing `ventis` and probing it for +# env-file injection and editable-install support. The runtime lives in the +# Global Controller image, not on the host running this CLI, so the probe always +# comes back empty here and the rules report a failure that isn't one. +CAPABILITY_GATED_CHECKS = frozenset({"V030", "V031"}) + +RUNTIME_PREFIX = "ventis-local-" + + +class VerificationError(Exception): + """A check that should end the run, carrying a message fit to print.""" + + +# ------------------------------------------------------------------ # +# Build artifact # +# ------------------------------------------------------------------ # + + +def _find_validator(project_root): + """Path to the porting skill's validate.py, fetching the skill if needed.""" + for spec in AGENTS.values(): + for skill_dir in spec["skill_dirs"].values(): + if not os.path.isabs(skill_dir): + skill_dir = os.path.join(project_root, skill_dir) + candidate = os.path.join(skill_dir, VALIDATOR_NAME) + if os.path.isfile(candidate): + return candidate + + cached = os.path.join(SKILL_CACHE_DIR, VALIDATOR_NAME) + if os.path.isfile(cached): + return cached + if install_skill(SKILL_CACHE_DIR) and os.path.isfile(cached): + return cached + return None + + +def _run_validator(validator, artifact_dir): + """The validator's parsed --json report, or None if it produced no report.""" + result = subprocess.run( + [sys.executable, validator, artifact_dir, "-c", CONFIG_REL, "--json"], + capture_output=True, + text=True, + ) + try: + return json.loads(result.stdout) + except ValueError: + detail = (result.stderr or result.stdout).strip().splitlines() + ui.warn(f" The porting validator did not run: {detail[-1] if detail else 'no output'}") + return None + + +def _drop_unprobeable(report): + """Remove the rules that can only be judged with `ventis` importable. + + Their verdict without it is not merely uncertain, it is wrong: V030 reports + that the runtime never reads `env_file` when the container's runtime does. + """ + if report.get("capabilities", {}).get("ventis"): + return 0 + + kept = [] + dropped = 0 + for finding in report.get("findings") or []: + if finding["check"] in CAPABILITY_GATED_CHECKS: + if finding["level"] == "ERROR": + report["errors"] = max(report.get("errors", 0) - 1, 0) + elif finding["level"] == "WARN": + report["warnings"] = max(report.get("warnings", 0) - 1, 0) + dropped += 1 + continue + kept.append(finding) + report["findings"] = kept + return dropped + + +_LEVEL_EMITTER = {"ERROR": ui.fail, "WARN": ui.warn} + + +def _report_findings(findings): + for finding in sorted(findings, key=lambda f: (f["level"] != "ERROR", f["check"])): + where = finding.get("path") or "" + if where and finding.get("line"): + where = f"{where}:{finding['line']}" + parts = [finding["check"], where, finding["summary"]] + line = " ".join(part for part in parts if part) + _LEVEL_EMITTER.get(finding["level"], ui.hint)(f" {line}") + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(65536), b""): + digest.update(block) + return digest.hexdigest() + + +def _stale_sources(project_root, artifact_dir): + """Recorded sources that changed or vanished since the port was taken.""" + try: + with open(os.path.join(artifact_dir, PORTING_STATE_REL)) as f: + state = json.load(f) + except (OSError, ValueError): + return [] + + stale = [] + for relative, expected in (state.get("source_files") or {}).items(): + # The skill's own files are recorded alongside the project's; a newer + # skill would otherwise read as the application having changed. + if relative.startswith(".claude/"): + continue + path = os.path.join(project_root, relative) + if not os.path.isfile(path) or _sha256(path) != expected: + stale.append(relative) + return sorted(stale) + + +def verify_build_artifact(project_root="."): + """Check the `.car/` tree. Raises VerificationError if it can't be deployed.""" + artifact_dir = os.path.join(project_root, ARTIFACT_DIR) + config_path = os.path.join(artifact_dir, CONFIG_REL) + + if not os.path.isfile(config_path) or not os.path.isdir( + os.path.join(artifact_dir, SOURCE_DIR) + ): + raise VerificationError( + f"No `{ARTIFACT_DIR}/` artifact here (expected {CONFIG_REL} beside " + f"{SOURCE_DIR}/). Run `canyonos build` first." + ) + ui.ok(f"{ARTIFACT_DIR}/ layout (config/ + {SOURCE_DIR}/)") + + summary = {"errors": 0, "warnings": 0, "findings": [], "stale": []} + + validator = _find_validator(project_root) + if validator is None: + ui.warn("Could not fetch the porting validator; skipping artifact checks.") + ui.hint(" The deploy below still runs -- `canyonos doctor` checks the fetch path.") + else: + report = _run_validator(validator, os.path.abspath(artifact_dir)) + if report is not None: + skipped = _drop_unprobeable(report) + summary.update( + errors=report.get("errors", 0), + warnings=report.get("warnings", 0), + findings=report.get("findings", []), + ) + counts = f"{summary['errors']} error(s), {summary['warnings']} warning(s)" + (ui.fail if summary["errors"] else ui.ok)(f"porting validator: {counts}") + _report_findings(summary["findings"]) + if skipped: + ui.hint(f" {skipped} rule(s) need the ventis runtime to judge and were skipped") + + summary["stale"] = _stale_sources(project_root, artifact_dir) + for relative in summary["stale"]: + ui.warn(f" source changed since the port: {relative}") + if summary["stale"]: + ui.hint(" -> re-run `canyonos build` to bring the artifact back in step") + + if summary["errors"]: + raise VerificationError( + f"The build artifact has {summary['errors']} validation error(s); fix them " + "or re-run `canyonos build`." + ) + return summary + + +# ------------------------------------------------------------------ # +# Runtime # +# ------------------------------------------------------------------ # + + +def _built_images(): + result = subprocess.run( + ["docker", "images", "--format", "{{.Repository}}"], capture_output=True, text=True + ) + return set(result.stdout.split()) + + +def _running_containers(): + result = subprocess.run( + ["docker", "ps", "--filter", f"name={RUNTIME_PREFIX}", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + return result.stdout.split() + + +def _runtime_table(rows): + table = Table(border_style=GREEN, header_style=f"bold {GREEN}", title_style=f"bold {GREEN}") + for column in ("Agent", "Image", "Replicas", "Endpoint"): + table.add_column(column) + for row in rows: + replicas = f"{row['running']}/{row['expected']}" + style = "" if row["ok"] else "bold red" + table.add_row( + row["name"], + row["image"] if row["image_built"] else f"{row['image']} (missing)", + replicas, + row["endpoint"] or "-", + style=style, + ) + return table + + +def verify_runtime(config_path, gc_port): + """Check the running deploy against the config. Raises VerificationError on a gap.""" + with open(config_path) as f: + config = yaml.safe_load(f) or {} + + images = _built_images() + containers = _running_containers() + endpoints = { + endpoint.get("name"): f"{endpoint['host']}:{endpoint['port']}" + for endpoint in gc.workflow_endpoints(gc_port) + if endpoint.get("host") and endpoint.get("port") + } + + rows = [] + problems = [] + for agent in config.get("agents") or []: + name = agent.get("name") + if not name: + continue + # Image and container names the local provider derives from the agent name. + image = f"ventis-{name.lower()}" + expected = int(agent.get("replicas", 1) or 1) + running = sum(1 for c in containers if c.startswith(f"{RUNTIME_PREFIX}{name.lower()}-")) + image_built = image in images + + if not image_built: + problems.append(f"{name}: image {image} was never built") + elif running < expected: + problems.append(f"{name}: {running} of {expected} replicas running") + + endpoint = endpoints.get(name) + if endpoint is None and agent.get("type") == "workflow": + # The container only reports endpoints it has instance records for; + # locally the published port is the one the config asked for. + endpoint = f"127.0.0.1:{agent.get('api_port', DEFAULT_API_PORT)}" + + rows.append( + { + "name": name, + "image": image, + "image_built": image_built, + "expected": expected, + "running": running, + "endpoint": endpoint, + "ok": image_built and running >= expected, + } + ) + + ui.panel(_runtime_table(rows)) + if problems: + raise VerificationError("The deploy is incomplete -- " + "; ".join(problems)) + return {"agents": rows} diff --git a/cli/cli.py b/cli/cli.py index 770741f..a6bd390 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -7,8 +7,8 @@ import importlib.metadata import sys +from canyonos import ui from canyonos.clean import run_clean -from canyonos.constants import default_config_path from canyonos.config import run_config from canyonos.deploy import run_deploy from canyonos.build import run_build @@ -17,49 +17,11 @@ from canyonos.new_app import run_new_app from canyonos.quit import run_quit from canyonos.serve import run_serve +from canyonos.status import run_status from canyonos.stop import run_stop from canyonos.test import DEFAULT_QUERY, run_test from utils.help_screen import DESCRIPTIONS, print_custom_help -def cmd_quit(args): - run_quit() - -def cmd_new_app(args): - # Note, not tested much, keeping this in the back burner for now while we flesh out the main path - run_new_app() - -def cmd_deploy(args): - run_deploy(args.config, serve=args.serve) - -def cmd_clean(args): - run_clean() - -def cmd_stop(args): - run_stop() - -def cmd_logs(args): - run_logs() - -def cmd_config(args): - run_config() - -def cmd_build(args): - run_build() - -def cmd_doctor(args): - sys.exit(0 if run_doctor() else 1) - -def cmd_serve(args): - sys.exit(run_serve()) - -def cmd_test(args): - sys.exit(run_test(args.config, query=args.query)) - - -def cmd_version(args): - print(f"canyonos {importlib.metadata.version('canyonos')}") - - def _parse_bool(value): if value.lower() in ("true", "1", "yes"): return True @@ -80,14 +42,16 @@ def main(): # Subparsers keep the stock argparse help, so `canyonos -h` still # describes that command instead of reprinting the top-level screen. subparsers = parser.add_subparsers(dest="command", parser_class=argparse.ArgumentParser) - config_default = default_config_path() - def add(name): + def add(name, run): # A KeyError here means the command has no entry on the help screen. - return subparsers.add_parser(name, help=DESCRIPTIONS[name]) + command = subparsers.add_parser(name, help=DESCRIPTIONS[name]) + command.set_defaults(func=run) + return command - add("new-app").set_defaults(func=cmd_new_app) - deploy = add("deploy") + # Note, not tested much, keeping this in the back burner for now while we flesh out the main path + add("new-app", lambda args: run_new_app()) + deploy = add("deploy", lambda args: run_deploy(args.config, serve=args.serve, verbose=args.verbose)) deploy.add_argument( "-c", "--config", @@ -100,30 +64,34 @@ def add(name): metavar="true|false", help="Automatically launch the local dashboard (canyonos serve) once the workflow is up (default: true)", ) - deploy.set_defaults(func=cmd_deploy) - add("clean").set_defaults(func=cmd_clean) - add("stop").set_defaults(func=cmd_stop) - add("logs").set_defaults(func=cmd_logs) - add("quit").set_defaults(func=cmd_quit) - add("config").set_defaults(func=cmd_config) - add("build").set_defaults(func=cmd_build) - add("doctor").set_defaults(func=cmd_doctor) - add("version").set_defaults(func=cmd_version) - add("serve").set_defaults(func=cmd_serve) - test = add("test") - test.add_argument( - "-c", - "--config", - default=config_default, - help=f"Path to global controller config (default: {config_default})", + deploy.add_argument( + "-v", + "--verbose", + action="store_true", + help="Stream the container's full build and deploy logs instead of a progress summary", ) + add("clean", lambda args: run_clean()) + add("stop", lambda args: run_stop()) + add("logs", lambda args: run_logs()) + add("quit", lambda args: run_quit()) + add("config", lambda args: run_config()) + add("build", lambda args: run_build()) + add("doctor", lambda args: sys.exit(0 if run_doctor() else 1)) + add("version", lambda args: ui.say(f"canyonos {importlib.metadata.version('canyonos')}")) + add("serve", lambda args: sys.exit(run_serve())) + add("status", lambda args: run_status()) + test = add("test", lambda args: sys.exit(run_test(args.prompt, as_json=args.json))) test.add_argument( - "-q", - "--query", + "prompt", + nargs="?", default=DEFAULT_QUERY, - help=f"Query sent to the workflow (default: {DEFAULT_QUERY!r})", + help=f"Prompt sent to the workflow (default: {DEFAULT_QUERY!r})", + ) + test.add_argument( + "--json", + action="store_true", + help="Print a single JSON result object and nothing else (for CI)", ) - test.set_defaults(func=cmd_test) args = parser.parse_args() if not getattr(args, "command", None): @@ -135,7 +103,7 @@ def add(name): except RuntimeError as e: # Docker unreachable, image pull failed, no free port -- all already # carry a readable message, so print it rather than a traceback. - print(e) + ui.fail(e) sys.exit(1) diff --git a/cli/pyproject.toml b/cli/pyproject.toml index e85c825..f11a241 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "canyonos" -version = "0.1.4" +version = "0.1.5" description = "CanyonOS CLI" requires-python = ">=3.10" dependencies = [ diff --git a/cli/utils/help_screen.py b/cli/utils/help_screen.py index 94b94a0..81ea3ca 100644 --- a/cli/utils/help_screen.py +++ b/cli/utils/help_screen.py @@ -1,38 +1,50 @@ """Custom help screen for the canyonos CLI.""" -from rich.console import Console from rich.panel import Panel from rich.table import Table from rich.text import Text +from canyonos import ui +from canyonos.theme import GREEN, WHITE + # The single source of truth for command descriptions: cli.py registers every # subparser through this table, so a command can't be added to one and missed # in the other. CORE_COMMANDS = ( - ("build", "Build a compatable workflow with an agent"), - ("deploy", "Deploy agents"), + ("build", "Port an existing project into a CanyonOS workflow"), + ("deploy", "Build and launch the workflow, then open the dashboard"), ("config", "Configure project settings"), ) +# The three teardown commands differ only in what they leave behind, so each +# description says so explicitly rather than all three reading as "stop stuff". UTIL_COMMANDS = ( - ("clean", "Remove the generated .car folder from build"), - ("doctor", "See if all required tools are up"), - ("logs", "View canyonos logs"), + ("clean", "Delete the generated .car folder from this project"), + ("doctor", "Check Docker, git and a coding agent are all available"), + ("logs", "Follow the running deploy's logs"), ("new-app", "Create a barebones CanyonOS project"), - ("quit", "Shut down CanyonOS services"), + ("quit", "Stop the deploy and remove the container and its files"), ("serve", "Start local CanyonOS dashboard"), - ("stop", "Stop running containers"), - ("test", "Run the deployed workflow locally with a test query"), + ("status", "Check whether a deploy is running and where it answers"), + ("stop", "Stop the running deploy, keeping the container and files"), + ("test", "Deploy locally and run one prompt end to end"), ("version", "Print canyonos version"), ) DESCRIPTIONS = dict(CORE_COMMANDS + UTIL_COMMANDS) +# Both columns are sized from the widest entry across BOTH tables, so Core and +# Utils line up with each other instead of each shrinking to fit its own rows. +_ALL_COMMANDS = CORE_COMMANDS + UTIL_COMMANDS +_NAME_WIDTH = max(len(name) for name, _ in _ALL_COMMANDS) +_DESCRIPTION_WIDTH = max(len(description) for _, description in _ALL_COMMANDS) + + def _command_table(commands): table = Table(show_header=False, border_style="dim", padding=(0, 2)) - table.add_column(style="cyan", width=20) - table.add_column(style="white") + table.add_column(style=f"bold {GREEN}", width=_NAME_WIDTH) + table.add_column(style=WHITE, width=_DESCRIPTION_WIDTH) for name, description in commands: table.add_row(name, description) return table @@ -40,21 +52,19 @@ def _command_table(commands): def print_custom_help(): """Print a custom, visually appealing help screen.""" - console = Console() - - title = Text("CanyonOS CLI", style="bold cyan") - subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease\n", style="dim") - console.print(Panel(title + subtitle, border_style="cyan", padding=(1, 2))) + title = Text("CanyonOS CLI", style=f"bold {GREEN}") + subtitle = Text("\nBuild, deploy, and manage agentic workflows with ease", style="dim") + ui.console.print(Panel(title + subtitle, border_style=GREEN, padding=(1, 2))) - console.print("\n[bold yellow]Core Commands[/bold yellow]") - console.print(_command_table(CORE_COMMANDS)) + ui.console.print(f"\n[bold {GREEN}]Core Commands[/]") + ui.console.print(_command_table(CORE_COMMANDS)) - console.print("\n[bold yellow]Utils[/bold yellow]") - console.print(_command_table(UTIL_COMMANDS)) + ui.console.print(f"\n[bold {GREEN}]Utils[/]") + ui.console.print(_command_table(UTIL_COMMANDS)) - console.print("\n[bold green]Quick Start:[/bold green]") - console.print(" [dim]1.[/dim] cd [cyan]into-your-workflow-root-dir[/cyan]") - console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") - console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") + ui.console.print(f"\n[bold {GREEN}]Quick Start:[/]") + ui.console.print(f" [dim]1.[/dim] cd [{GREEN}]into-your-workflow-root-dir[/]") + ui.console.print(" [dim]2.[/dim] canyonos build (opens coding agent)") + ui.console.print(" [dim]3.[/dim] canyonos deploy (UI automatically starts)") - console.print("[dim]For command-specific help: [cyan]canyonos --help[/cyan][/dim]\n") + ui.console.print(f"[dim]For command-specific help: [{GREEN}]canyonos --help[/][/dim]\n") diff --git a/cli/utils/tui.py b/cli/utils/tui.py index 2f0f15d..3154062 100644 --- a/cli/utils/tui.py +++ b/cli/utils/tui.py @@ -8,12 +8,18 @@ import termios import tty +from canyonos.theme import GREEN + UP_KEYS = ("\x1b[A", "\x1bOA", "k") DOWN_KEYS = ("\x1b[B", "\x1bOB", "j") CANCEL_KEYS = ("\x03", "\x1b") DELETE_KEYS = ("d", "D") QUIT_KEYS = ("q", "Q") +# The brand green as a raw truecolor escape: this menu writes ANSI directly +# rather than going through rich, but shares the CLI's one palette. +_GREEN = "\x1b[38;2;{};{};{}m".format(*(int(GREEN[i:i + 2], 16) for i in (1, 3, 5))) + # Sentinel returned (paired with the hovered value) when the delete key is # pressed and `deletable=True`. Callers check `result[0] is DELETE_ACTION`. DELETE_ACTION = object() @@ -65,7 +71,7 @@ def select_menu(options, title, deletable=False, quittable=False): def frame(): lines = [f"\x1b[1m{title}\x1b[0m", ""] for i, (_, label) in enumerate(options): - lines.append(f"\x1b[36m❯ {label}\x1b[0m" if i == idx else f" {label}") + lines.append(f"{_GREEN}❯ {label}\x1b[0m" if i == idx else f" {label}") hint = "↑/↓ move · 1-9 jump · enter select" if deletable: hint += " · d delete" diff --git a/examples/finance/agents/finance_agent.py b/examples/finance/agents/finance_agent.py index 7c36ef5..db70b01 100644 --- a/examples/finance/agents/finance_agent.py +++ b/examples/finance/agents/finance_agent.py @@ -1,4 +1,11 @@ -from vllm_agent import VllmAgent +# `agents.vllm_agent` is where the generated VllmAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `vllm_agent` fallback covers running outside that layout. +try: + from agents.vllm_agent import VllmAgent +except ImportError: + from vllm_agent import VllmAgent # Example of a simple finance agent diff --git a/examples/finance/workflow/example_workflow.py b/examples/finance/workflow/example_workflow.py index e4b6ce5..644e483 100644 --- a/examples/finance/workflow/example_workflow.py +++ b/examples/finance/workflow/example_workflow.py @@ -4,7 +4,7 @@ # Start agents first: python src/controller/global_controller.py # Then run this file: python examples/workflow.py # Test: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"ticker": "AAPL"}' +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"query": "AAPL"}' # curl http://localhost:8080/status/ import sys @@ -22,13 +22,13 @@ from agents.market_agent import MarketResearchAgent -def main(ticker: str = "AAPL"): +def main(query: str = "AAPL"): finance = FinanceAgent() market = MarketResearchAgent() # Call finance agent functions - price = finance.get_stock_price(ticker=ticker) - company = finance.get_company_name(ticker=ticker) + price = finance.get_stock_price(ticker=query) + company = finance.get_company_name(ticker=query) # Call market agent functions trend = market.get_market_trend(sector="tech") diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md index a182896..38483be 100644 --- a/examples/helloworld/README.md +++ b/examples/helloworld/README.md @@ -14,7 +14,7 @@ ventis deploy # Test with curl curl -X POST http://:8080/main \ -H 'Content-Type: application/json' \ - -d '{"name": "World"}' + -d '{"query": "World"}' # Check result curl http://:8080/status/ @@ -52,5 +52,5 @@ Pass `_context` in your curl request to set the caller identity: ```bash curl -X POST http://localhost:8080/main \ -H 'Content-Type: application/json' \ - -d '{"name": "World", "_context": {"origin": "admin"}}' + -d '{"query": "World", "_context": {"origin": "admin"}}' ``` diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 6bafff3..693590e 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -2,7 +2,7 @@ # This file demonstrates how to call agent stubs and deploy as a REST API. # # After running `ventis build` and `ventis deploy`: -# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"name": "World"}' +# curl -X POST http://localhost:8080/main -H 'Content-Type: application/json' -d '{"query": "World"}' # curl http://localhost:8080/status/ import sys @@ -18,9 +18,9 @@ from agents.example_agent import ExampleAgent -def main(name: str = "World"): +def main(query: str = "World"): agent = ExampleAgent() - greeting = agent.hello(name=name) + greeting = agent.hello(name=query) return {"greeting": greeting.value()} diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 28069ac..253a2d2 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,14 +8,18 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. -import os -import sys - import json import math -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from price_agent import PriceAgent +# `agents.price_agent` is where the generated PriceAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `price_agent` fallback covers running outside that layout (e.g. local +# dev, where `ventis build` only emits a flat stubs/ directory). +try: + from agents.price_agent import PriceAgent +except ImportError: + from price_agent import PriceAgent TRADING_DAYS = 252 diff --git a/examples/text2sql/agents/sql_generator_agent.py b/examples/text2sql/agents/sql_generator_agent.py index 6320089..4964ce6 100644 --- a/examples/text2sql/agents/sql_generator_agent.py +++ b/examples/text2sql/agents/sql_generator_agent.py @@ -9,7 +9,14 @@ # These calls sit on the request's critical path, so the scheduler should # prioritize them over background work. -from vllm_agent import VllmAgent +# `agents.vllm_agent` is where the generated VllmAgent stub actually lands +# inside this agent's own Docker container (stubs are copied to their source +# agent's own entrypoint-mirrored path -- see ventis/stub_generator.py). The +# bare `vllm_agent` fallback covers running outside that layout. +try: + from agents.vllm_agent import VllmAgent +except ImportError: + from vllm_agent import VllmAgent class SQLGeneratorAgent(object): diff --git a/examples/text2sql/workflow/text2sql_workflow.py b/examples/text2sql/workflow/text2sql_workflow.py index ac9801d..b8ce45b 100644 --- a/examples/text2sql/workflow/text2sql_workflow.py +++ b/examples/text2sql/workflow/text2sql_workflow.py @@ -11,7 +11,7 @@ # Test: # curl -X POST http://localhost:8080/main \ # -H 'Content-Type: application/json' \ -# -d '{"question": "total order amount per customer region"}' +# -d '{"query": "total order amount per customer region"}' # curl http://localhost:8080/status/ import json @@ -32,7 +32,7 @@ from agents.production_agent import ProductionExecutorAgent -def main(question: str = "total order amount per customer region", n_candidates: int = 3): +def main(query: str = "total order amount per customer region", n_candidates: int = 3): schema_agent = SchemaRetrievalAgent() generator = SQLGeneratorAgent() validator = SQLValidatorAgent() @@ -44,12 +44,12 @@ def main(question: str = "total order amount per customer region", n_candidates: # whichever node created it -- here, this workflow's own -- so resolving # it where it was created is always safe, regardless of which node ends # up running the next stage. - schema = json.loads(schema_agent.get_relevant_schema(question=question).value()) + schema = json.loads(schema_agent.get_relevant_schema(question=query).value()) # Stage 2: fan out candidate SQL queries (LLM calls happen inside). candidates = json.loads( generator.generate_candidates( - question=question, schema=schema, n=n_candidates + question=query, schema=schema, n=n_candidates ).value() ) @@ -70,7 +70,7 @@ def main(question: str = "total order amount per customer region", n_candidates: survivors.append(sql) if not survivors: - return {"question": question, "error": "no candidate passed static validation"} + return {"question": query, "error": "no candidate passed static validation"} # Stage 4: execute survivors on the sampled replica, then vote. sample_results = [ @@ -87,7 +87,7 @@ def main(question: str = "total order amount per customer region", n_candidates: ) return { - "question": question, + "question": query, "candidates": candidates, "costs": costs, "survivors": survivors, diff --git a/pyproject.toml b/pyproject.toml index 40cb43a..41169a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ include = ["ventis*"] ventis = [ "templates/**/*", "controller/proto/*.proto", + "controller/utils/aws_pricing_chart.db", ] @@ -62,4 +63,8 @@ allowed-unresolved-imports = [ [dependency-groups] dev = [ "pytest>=9.1.1", + "canyonos", ] + +[tool.uv.sources] +canyonos = { path = "cli", editable = true } diff --git a/tests/test_canyonos_test.py b/tests/test_canyonos_test.py new file mode 100644 index 0000000..d2d64e5 --- /dev/null +++ b/tests/test_canyonos_test.py @@ -0,0 +1,426 @@ +import hashlib +import json +import subprocess + +import pytest + +from canyonos import test as test_cmd +from canyonos import ui, verify + +CONFIG = """\ +agents: + - name: EchoAgent + entrypoint: echo_agent.py + provider: EC2 + replicas: 2 + + - name: Workflow + type: workflow + workflow_file: echo_workflow.py + api_port: 8080 + provider: EC2 + replicas: 1 +""" + + +@pytest.fixture(autouse=True) +def loud(): + """Every test starts with output enabled; `--json` runs flip it and restore it.""" + ui.set_quiet(False) + yield + ui.set_quiet(False) + + +@pytest.fixture +def project(monkeypatch, tmp_path): + car = tmp_path / ".car" + (car / "config").mkdir(parents=True) + (car / "app").mkdir() + (car / "config" / "global_controller.yaml").write_text(CONFIG) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def report(errors=0, warnings=0, findings=(), ventis=False): + return { + "capabilities": {"ventis": ventis}, + "errors": errors, + "warnings": warnings, + "findings": list(findings), + } + + +def finding(check, level="ERROR"): + return {"check": check, "level": level, "path": "config/x.yaml", "line": 1, "summary": "s"} + + +# ------------------------------------------------------------------ # +# Locating the porting validator # +# ------------------------------------------------------------------ # + + +def test_validator_prefers_the_project_skill(monkeypatch, project, tmp_path): + codex = tmp_path / "codex-skill" + codex.mkdir() + (codex / "validate.py").write_text("") + local = project / ".claude" / "skills" / "porting-to-canyonos" + local.mkdir(parents=True) + (local / "validate.py").write_text("") + + monkeypatch.setattr( + verify, + "AGENTS", + { + "claude": {"skill_dirs": {"local": ".claude/skills/porting-to-canyonos"}}, + "codex": {"skill_dirs": {"global": str(codex)}}, + }, + ) + assert verify._find_validator(str(project)) == str(local / "validate.py") + + +def test_validator_falls_back_to_the_codex_skill(monkeypatch, project, tmp_path): + codex = tmp_path / "codex-skill" + codex.mkdir() + (codex / "validate.py").write_text("") + + monkeypatch.setattr( + verify, + "AGENTS", + { + "claude": {"skill_dirs": {"local": ".claude/skills/porting-to-canyonos"}}, + "codex": {"skill_dirs": {"global": str(codex)}}, + }, + ) + assert verify._find_validator(str(project)) == str(codex / "validate.py") + + +def test_validator_is_fetched_when_nothing_is_installed(monkeypatch, project, tmp_path): + cache = tmp_path / "cache" + monkeypatch.setattr(verify, "AGENTS", {}) + monkeypatch.setattr(verify, "SKILL_CACHE_DIR", str(cache)) + + def fake_install(dest): + assert dest == str(cache) + cache.mkdir() + (cache / "validate.py").write_text("") + return True + + monkeypatch.setattr(verify, "install_skill", fake_install) + assert verify._find_validator(str(project)) == str(cache / "validate.py") + + +def test_a_validator_that_cannot_be_fetched_does_not_stop_the_run(monkeypatch, project, tmp_path): + monkeypatch.setattr(verify, "AGENTS", {}) + monkeypatch.setattr(verify, "SKILL_CACHE_DIR", str(tmp_path / "empty-cache")) + monkeypatch.setattr(verify, "install_skill", lambda _dest: False) + + summary = verify.verify_build_artifact(str(project)) + + assert summary == {"errors": 0, "warnings": 0, "findings": [], "stale": []} + + +# ------------------------------------------------------------------ # +# Reading the validator's report # +# ------------------------------------------------------------------ # + + +def test_validator_errors_fail_the_phase(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, "_run_validator", lambda *_: report(errors=1, findings=[finding("V002")]) + ) + + with pytest.raises(verify.VerificationError): + verify.verify_build_artifact(str(project)) + + +def test_validator_warnings_pass(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(warnings=1, findings=[finding("V018", "WARN")]), + ) + + summary = verify.verify_build_artifact(str(project)) + + assert (summary["errors"], summary["warnings"]) == (0, 1) + + +def test_rules_needing_ventis_are_dropped_when_it_is_not_importable(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(errors=1, findings=[finding("V030"), finding("V031", "INFO")]), + ) + + summary = verify.verify_build_artifact(str(project)) + + assert summary["errors"] == 0 + assert summary["findings"] == [] + + +def test_rules_needing_ventis_are_kept_when_it_is_importable(monkeypatch, project): + monkeypatch.setattr(verify, "_find_validator", lambda _root: "/validate.py") + monkeypatch.setattr( + verify, + "_run_validator", + lambda *_: report(errors=1, findings=[finding("V030")], ventis=True), + ) + + with pytest.raises(verify.VerificationError): + verify.verify_build_artifact(str(project)) + + +def test_a_missing_car_directory_is_an_error(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + with pytest.raises(verify.VerificationError, match="Run `canyonos build` first"): + verify.verify_build_artifact(str(tmp_path)) + + +# ------------------------------------------------------------------ # +# Source drift # +# ------------------------------------------------------------------ # + + +def write_porting_state(project, entries): + (project / ".car" / "config" / ".porting-state.json").write_text( + json.dumps({"version": 1, "source_files": entries}) + ) + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_unchanged_sources_are_not_reported_as_stale(project): + source = project / "echo_agent.py" + source.write_text("x = 1\n") + write_porting_state(project, {"echo_agent.py": sha256(source)}) + + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +def test_changed_and_deleted_sources_are_reported(project): + source = project / "echo_agent.py" + source.write_text("x = 2\n") + write_porting_state( + project, {"echo_agent.py": "0" * 64, "gone.py": "0" * 64} + ) + + assert verify._stale_sources(str(project), str(project / ".car")) == [ + "echo_agent.py", + "gone.py", + ] + + +def test_the_skills_own_files_are_not_reported_as_drift(project): + write_porting_state(project, {".claude/skills/porting-to-canyonos/SKILL.md": "0" * 64}) + + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +def test_a_hand_written_artifact_has_no_state_to_compare(project): + assert verify._stale_sources(str(project), str(project / ".car")) == [] + + +# ------------------------------------------------------------------ # +# Runtime verification # +# ------------------------------------------------------------------ # + + +@pytest.fixture +def runtime(monkeypatch): + monkeypatch.setattr(verify.gc, "workflow_endpoints", lambda _port: []) + + def install(images, containers): + monkeypatch.setattr(verify, "_built_images", lambda: set(images)) + monkeypatch.setattr(verify, "_running_containers", lambda: list(containers)) + + return install + + +ALL_UP = [ + "ventis-local-echoagent-0", + "ventis-local-echoagent-1", + "ventis-local-workflow-0", +] + + +def test_a_complete_deploy_passes(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP) + + result = verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + assert [(a["name"], a["running"], a["expected"]) for a in result["agents"]] == [ + ("EchoAgent", 2, 2), + ("Workflow", 1, 1), + ] + + +def test_a_short_replica_count_fails(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP[1:]) + + with pytest.raises(verify.VerificationError, match="1 of 2 replicas"): + verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + +def test_an_image_that_was_never_built_fails(project, runtime): + runtime({"ventis-workflow"}, ["ventis-local-workflow-0"]) + + with pytest.raises(verify.VerificationError, match="ventis-echoagent was never built"): + verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + +def test_the_workflow_endpoint_falls_back_to_the_configured_port(project, runtime): + runtime({"ventis-echoagent", "ventis-workflow"}, ALL_UP) + + result = verify.verify_runtime(str(project / ".car" / "config" / "global_controller.yaml"), 8000) + + assert result["agents"][0]["endpoint"] is None + assert result["agents"][1]["endpoint"] == "127.0.0.1:8080" + + +# ------------------------------------------------------------------ # +# The command itself # +# ------------------------------------------------------------------ # + + +@pytest.fixture +def deployable(monkeypatch, project): + """A project where every step past the build check succeeds unless overridden.""" + calls = {"post_deploy": 0, "quit": 0} + + monkeypatch.setattr(test_cmd, "verify_build_artifact", lambda *a: {"warnings": 0, "stale": []}) + monkeypatch.setattr(test_cmd, "run_init", lambda banner=True: None) + monkeypatch.setattr(test_cmd, "run_sync", lambda: True) + monkeypatch.setattr(test_cmd, "load_state", lambda: {"container_id": "abc", "port": 8000}) + monkeypatch.setattr(test_cmd, "_port_in_use", lambda _port: False) + monkeypatch.setattr(test_cmd, "_wait_for_workflow", lambda *a: None) + monkeypatch.setattr(test_cmd, "verify_runtime", lambda *a: {"agents": []}) + monkeypatch.setattr(test_cmd, "workflow_targets", lambda *a: [("Workflow", "127.0.0.1", 8080)]) + monkeypatch.setattr(test_cmd, "_send_query", lambda *a: "req-1") + monkeypatch.setattr(test_cmd, "_await_result", lambda *a: {"status": "done", "result": {"r": 1}}) + monkeypatch.setattr(test_cmd, "_log_tail", lambda _cid: "boom") + + def post_deploy(*_a, **_k): + calls["post_deploy"] += 1 + + def quit_existing(): + calls["quit"] += 1 + + monkeypatch.setattr(test_cmd, "post_deploy", post_deploy) + monkeypatch.setattr(test_cmd, "quit_existing", quit_existing) + return calls + + +def test_a_passing_run_tears_everything_down(deployable): + assert test_cmd.run_test("hi") == 0 + assert deployable["quit"] == 1 + + +def test_the_provider_is_restored_after_the_run(project, deployable): + config = project / ".car" / "config" / "global_controller.yaml" + + test_cmd.run_test("hi") + + assert config.read_text() == CONFIG + + +def test_an_occupied_api_port_fails_before_the_deploy(monkeypatch, deployable, capsys): + monkeypatch.setattr(test_cmd, "_port_in_use", lambda _port: True) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["post_deploy"] == 0 + assert "8080 is already in use" in payload["error"] + + +def test_a_failed_deploy_keeps_the_container_and_reads_its_log(monkeypatch, deployable, capsys): + def boom(*_a): + raise test_cmd._TestFailed("the deploy did not come up") + + monkeypatch.setattr(test_cmd, "_wait_for_workflow", boom) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["quit"] == 0 + assert payload["log_tail"] == "boom" + + +def test_a_failure_before_the_deploy_leaves_nothing_running(monkeypatch, deployable, capsys): + monkeypatch.setattr(test_cmd, "run_sync", lambda: False) + + assert test_cmd.run_test("hi", as_json=True) == 1 + payload = json.loads(capsys.readouterr().out) + + assert deployable["quit"] == 1 + assert payload["log_tail"] is None + + +def test_json_mode_prints_one_object_and_nothing_else(deployable, capsys): + assert test_cmd.run_test("a prompt", as_json=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["ok"] is True + assert payload["query"] == "a prompt" + assert payload["result"] == {"r": 1} + assert payload["error"] is None + assert [(p["name"], p["ok"]) for p in payload["phases"]] == [ + ("verify_build", True), + ("deploy", True), + ("verify_runtime", True), + ("query", True), + ] + + +def test_a_workflow_error_is_reported_as_a_failure(monkeypatch, deployable, capsys): + monkeypatch.setattr( + test_cmd, "_await_result", lambda *a: {"status": "error", "error": "agent blew up"} + ) + + assert test_cmd.run_test("hi", as_json=True) == 1 + + assert json.loads(capsys.readouterr().out)["error"] == "agent blew up" + + +def test_a_flat_layout_project_skips_the_build_check(monkeypatch, tmp_path, deployable, capsys): + legacy = tmp_path / "legacy" / "config" + legacy.mkdir(parents=True) + (legacy / "global_controller.yaml").write_text(CONFIG) + monkeypatch.chdir(tmp_path / "legacy") + + def unexpected(*_a): + raise AssertionError("the artifact validator should not run without a .car/") + + monkeypatch.setattr(test_cmd, "verify_build_artifact", unexpected) + + assert test_cmd.run_test("hi", as_json=True) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["phases"][0] == { + "name": "verify_build", + "ok": True, + "detail": "skipped: no .car/ artifact", + } + + +# ------------------------------------------------------------------ # +# Docker plumbing # +# ------------------------------------------------------------------ # + + +def test_running_containers_are_filtered_to_the_local_provider(monkeypatch): + seen = [] + + def fake_run(argv, **_): + seen.append(argv) + return subprocess.CompletedProcess(argv, 0, "ventis-local-echoagent-0\n", "") + + monkeypatch.setattr(verify.subprocess, "run", fake_run) + + assert verify._running_containers() == ["ventis-local-echoagent-0"] + assert "name=ventis-local-" in seen[0] diff --git a/tests/test_dashboard_stack.py b/tests/test_dashboard_stack.py index 2e62480..7be86ac 100644 --- a/tests/test_dashboard_stack.py +++ b/tests/test_dashboard_stack.py @@ -16,9 +16,7 @@ def project(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path / "home")) monkeypatch.chdir(tmp_path) for key in ( - "DATABASE_URL", "JWT_SECRET", - "CANYONOS_DATABASE_URL", "CANYONOS_JWT_SECRET", "CANYONOS_REDIS_HOST", "CANYONOS_REDIS_PORT", @@ -26,13 +24,9 @@ def project(monkeypatch, tmp_path): "CANYONOS_WEB_IMAGE", ): monkeypatch.delenv(key, raising=False) - config_dir = tmp_path / "config" - config_dir.mkdir() - config = config_dir / "global_controller.yaml" - config.write_text("database:\n url: postgres://user:password@db.example/canyonos\n") monkeypatch.setattr(dashboard_stack.shutil, "which", lambda _: "/usr/bin/docker") monkeypatch.setattr(dashboard_stack, "_port_is_free", lambda _port: True) - return config + return tmp_path def install_docker(monkeypatch, calls, responses=None): @@ -76,86 +70,21 @@ def test_docker_validation_failures_do_not_pull(monkeypatch, project, prepare, m prepare(monkeypatch, responses) install_docker(monkeypatch, calls, responses) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result == dashboard_stack.ServeResult(False, "validate", message) assert all(command[-1] != "pull" for command in calls) -def test_empty_database_url_fails_validation_but_absent_one_does_not(monkeypatch, project): - project.write_text("database:\n url: ''\n") - calls = [] - install_docker(monkeypatch, calls) - - result = dashboard_stack.run_dashboard(str(project)) - - assert result == dashboard_stack.ServeResult( - False, "validate", "database.url must be a non-empty string" - ) - assert all(command[-1] != "pull" for command in calls) - - -def test_missing_database_section_is_not_a_validation_failure(monkeypatch, project): - project.write_text("") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - - assert stack.database_url is None - - -def test_prepare_omits_database_env_when_not_configured(project): - project.write_text("") - stack = dashboard_stack.DashboardStack( - None, dashboard_stack._state_dir(), Path.cwd() - ) - - managed_env, message = dashboard_stack.prepare(stack) - - assert message == "dashboard state prepared" - assert "CANYONOS_DATABASE_URL" not in managed_env - assert "CANYONOS_DATABASE_URL" not in stack.env_path.read_text() - - -def test_config_substitutes_quoted_dotenv_value_without_replacing_source(monkeypatch, project): - source_line = 'DATABASE_URL="postgres://user:password@db.example/canyonos"\n' - Path.cwd().joinpath(".env").write_text(source_line) - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - managed_env, _ = dashboard_stack.prepare(stack) - - assert stack.database_url == "postgres://user:password@db.example/canyonos" - assert managed_env["CANYONOS_DATABASE_URL"] == stack.database_url - assert stack.env_path.read_text().startswith(source_line) -def test_missing_database_url_variable_fails_without_pulling(monkeypatch, project): - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - result = dashboard_stack.run_dashboard(str(project)) - - assert result == dashboard_stack.ServeResult( - False, - "validate", - "database.url needs ${DATABASE_URL}, which is not set in the project .env", - ) - assert all(command[-1] != "pull" for command in calls) def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): source_line = "JWT_SECRET=user-value\n" Path.cwd().joinpath(".env").write_text(source_line) - stack = dashboard_stack.DashboardStack( - "postgres://user:password@db.example/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) + stack = dashboard_stack.DashboardStack(dashboard_stack._state_dir(), Path.cwd()) first_env, _ = dashboard_stack.prepare(stack) second_env, _ = dashboard_stack.prepare(stack) @@ -166,26 +95,6 @@ def test_user_jwt_secret_is_untouched_while_canyonos_secret_is_stable(project): assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] -def test_process_environment_database_url_wins_over_project_dotenv(monkeypatch, project): - Path.cwd().joinpath(".env").write_text("DATABASE_URL=postgres://from-file/canyonos\n") - monkeypatch.setenv("DATABASE_URL", "postgres://from-process/canyonos") - project.write_text("database:\n url: ${DATABASE_URL}\n") - calls = [] - install_docker(monkeypatch, calls) - - stack = dashboard_stack.validate(str(project)) - - assert stack.database_url == "postgres://from-process/canyonos" - - -def test_unreadable_config_does_not_pull(monkeypatch, project): - calls = [] - install_docker(monkeypatch, calls) - - result = dashboard_stack.run_dashboard(str(project.with_name("missing.yaml"))) - - assert result.message == f"config file is not readable: {project.with_name('missing.yaml')}" - assert all(command[-1] != "pull" for command in calls) def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, project, tmp_path): @@ -195,7 +104,7 @@ def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, p blocked_state_dir.write_text("not a directory") monkeypatch.setattr(dashboard_stack, "_state_dir", lambda: blocked_state_dir) - state_result = dashboard_stack.run_dashboard(str(project)) + state_result = dashboard_stack.run_dashboard() assert state_result.message == "dashboard state directory is not writable" assert all(command[-1] != "pull" for command in calls) @@ -205,7 +114,7 @@ def test_state_directory_and_port_validation_failures_do_not_pull(monkeypatch, p monkeypatch.setattr(dashboard_stack, "_find_web_port", lambda start=8080, max_attempts=50: (_ for _ in ()).throw( dashboard_stack.PhaseFailure("validate", "no free port found for the dashboard after 50 attempts starting at 8080") )) - port_result = dashboard_stack.run_dashboard(str(project)) + port_result = dashboard_stack.run_dashboard() assert port_result.message == "no free port found for the dashboard after 50 attempts starting at 8080" assert all(command[-1] != "pull" for command in calls) @@ -215,15 +124,11 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): Path.cwd().joinpath(".env").write_text( "OTHER=one\n# preserved\nJWT_SECRET=kept-secret\nLAST=two\n" ) - stack = dashboard_stack.DashboardStack( - "postgres://user:password@localhost:5432/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) + stack = dashboard_stack.DashboardStack(dashboard_stack._state_dir(), Path.cwd()) managed_env, message = dashboard_stack.prepare(stack) - assert message == "database host localhost is reachable from the stack as host.docker.internal" + assert message == "dashboard state prepared" env_lines = stack.env_path.read_text().splitlines() assert env_lines[:2] == ["OTHER=one", "# preserved"] assert env_lines[2] == "JWT_SECRET=kept-secret" @@ -232,7 +137,6 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): "OTHER", "JWT_SECRET", "LAST", - "CANYONOS_DATABASE_URL", "CANYONOS_JWT_SECRET", "CANYONOS_REDIS_HOST", "CANYONOS_REDIS_PORT", @@ -240,35 +144,11 @@ def test_prepare_preserves_unrelated_env_lines_and_mode(project): "CANYONOS_WEB_IMAGE", "CANYONOS_WEB_PORT", } - assert "CANYONOS_DATABASE_URL=postgres://user:password@host.docker.internal:5432/canyonos" in env_lines assert stack.env_path.stat().st_mode & 0o777 == 0o600 assert stack.state_dir.stat().st_mode & 0o777 == 0o700 assert sorted(path.name for path in stack.state_dir.iterdir()) == ["stack.json"] -def test_prepare_reuses_secret_and_rewrites_only_local_hosts(project): - stack = dashboard_stack.DashboardStack( - "postgres://user:password@localhost/canyonos", - dashboard_stack._state_dir(), - Path.cwd(), - ) - first_env, first_message = dashboard_stack.prepare(stack) - second_env, second_message = dashboard_stack.prepare(stack) - - assert first_message.startswith("database host localhost") - assert second_message.startswith("database host localhost") - assert first_env["CANYONOS_JWT_SECRET"] == second_env["CANYONOS_JWT_SECRET"] - assert ( - first_env["CANYONOS_DATABASE_URL"] - == "postgres://user:password@host.docker.internal/canyonos" - ) - - remote_stack = dashboard_stack.DashboardStack( - "postgres://db.example/canyonos", dashboard_stack._state_dir(), Path.cwd() - ) - remote_env, _ = dashboard_stack.prepare(remote_stack) - assert remote_env["CANYONOS_DATABASE_URL"] == "postgres://db.example/canyonos" - def test_redaction_removes_urls_secrets_and_credentials(): database_url = "postgres://user:password@db.example/canyonos" @@ -297,7 +177,7 @@ def response(argv): return completed(argv) install_docker(monkeypatch, calls, response) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok is False assert result.phase == "start" @@ -328,7 +208,7 @@ def response(argv): return completed(argv) install_docker(monkeypatch, calls, response) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.phase == "pull" assert "pull unauthorized" in result.message @@ -355,7 +235,7 @@ def urlopen(*_args, **_kwargs): monkeypatch.setattr(dashboard_stack.time, "monotonic", lambda: next(clock)) monkeypatch.setattr(dashboard_stack.time, "sleep", lambda _: None) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok is False assert result.phase == "verify" @@ -383,7 +263,7 @@ def urlopen(endpoint, timeout): return Response() monkeypatch.setattr(dashboard_stack.urllib.request, "urlopen", urlopen) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result == dashboard_stack.ServeResult( True, "verify", "dashboard health checks passed", "http://127.0.0.1:8080" @@ -391,7 +271,7 @@ def urlopen(endpoint, timeout): pull_index = next(index for index, command in enumerate(calls) if command[-1] == "pull") up_index = next(index for index, command in enumerate(calls) if "up" in command) assert pull_index < up_index - assert calls[pull_index][4:6] == ["--env-file", str(project.parent.parent / ".env")] + assert calls[pull_index][4:6] == ["--env-file", str(project / ".env")] assert calls[up_index][-5:] == ["up", "-d", "--wait", "--wait-timeout", "180"] assert endpoints == [ ("http://127.0.0.1:8080/healthz", 5), @@ -426,7 +306,7 @@ def response(argv): lambda *_args, **_kwargs: type("Response", (), {"status": 200, "close": lambda self: None})(), ) - result = dashboard_stack.run_dashboard(str(project)) + result = dashboard_stack.run_dashboard() assert result.ok assert result.url == "http://127.0.0.1:8080" diff --git a/tests/test_deploy_progress.py b/tests/test_deploy_progress.py new file mode 100644 index 0000000..d1c8cf6 --- /dev/null +++ b/tests/test_deploy_progress.py @@ -0,0 +1,235 @@ +import pytest + +from canyonos import deploy as deploy_cmd +from canyonos.deploy import PhaseTracker + + +def drive(lines): + """Feed lines to a tracker, returning (spinners, completions, errored).""" + tracker = PhaseTracker() + spinners, done = [], [] + errored = False + for line in lines: + message, completed, is_error = tracker.feed(line) + if is_error: + errored = True + if message: + spinners.append(message) + if completed: + done.append(completed) + return tracker, spinners, done, errored + + +def test_a_full_run_reports_each_phase_once(): + _, spinners, done, errored = drive( + [ + "INFO:ventis:Generating stub: a.yaml -> a_stub.py\n", + "INFO:ventis:Compiling gRPC proto: a.proto\n", + "INFO:ventis:Building 3 Docker image(s) via `docker buildx bake`.\n", + "#5 [4/7] RUN pip install -r requirements.txt\n", + "INFO:ventis:Build complete.\n", + "INFO:ventis:Deploying from config: config.yaml\n", + "INFO:ventis.controller.global_controller:Redis launched on 1 node(s).\n", + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Intent (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Metrics (127.0.0.1:50052) is ready.\n", + ] + ) + assert not errored + assert done == ["Build complete", "Redis ready"] + assert "Building 3 images..." in spinners + assert spinners[-1] == "Starting agents (2/2 ready)..." + + +def test_phases_are_matched_in_the_order_the_container_emits_them(): + """Redis and stale-container cleanup are logged by GlobalController.__init__, + which runs before `Deploying from config:` -- so the matcher must not assume + the config line comes first. + """ + _, spinners, done, _ = drive( + [ + "INFO:ventis.controller.global_controller:Checking for stale containers from previous runs...\n", + "INFO:ventis.controller.global_controller:Redis launched on 1 node(s).\n", + "INFO:ventis:Deploying from config: config.yaml\n", + ] + ) + assert done == ["Redis ready"] + assert spinners == ["Cleaning up stale containers...", "Starting deploy..."] + + +def test_repeated_build_lines_collapse_to_one_spinner_update(): + _, spinners, _, _ = drive( + [ + "INFO:ventis:Generating stub: a.yaml -> a_stub.py\n", + "INFO:ventis:Generating stub: b.yaml -> b_stub.py\n", + "INFO:ventis:Generating Docker context for 'b'\n", + ] + ) + assert spinners == ["Generating stubs and Docker contexts..."] + + +def test_a_run_with_nothing_to_build_still_reports_the_phase(): + _, _, done, _ = drive( + [ + "INFO:ventis:No Docker images to build.\n", + "INFO:ventis:Build complete.\n", + ] + ) + assert done == ["No images to build", "Build complete"] + + +def test_agent_progress_counts_up_against_the_announced_total(): + tracker, spinners, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 3 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n", + "INFO:ventis.controller.global_controller:Controller B (127.0.0.1:2) is ready.\n", + ] + ) + assert spinners[-1] == "Starting agents (2/3 ready)..." + assert tracker.replicas_total == 3 + + +def test_replicas_of_one_agent_are_counted_separately(): + """`Controller %s is ready.` logs the agent name, which repeats across that + agent's replicas -- the endpoint is what distinguishes them. + """ + tracker, spinners, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50052) is ready.\n", + ] + ) + assert spinners[-1] == "Starting agents (2/2 ready)..." + assert tracker.agents_ready_message() == ("2 agent(s) ready", True) + + +def test_a_re_read_ready_line_does_not_double_count(): + tracker, _, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 2 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + "INFO:ventis.controller.global_controller:Controller Echo (127.0.0.1:50051) is ready.\n", + ] + ) + assert tracker.agents_ready_message() == ( + "Workflow up, but only 1/2 agents reported healthy", + False, + ) + + +def test_coming_up_short_of_the_announced_replicas_is_not_reported_as_success(): + """`_wait_for_healthy` gives up after its timeout and the controller starts + anyway, so the up-marker can arrive with agents still unhealthy. + """ + tracker, _, _, _ = drive( + [ + "INFO:ventis.controller.global_controller:Waiting for 3 replica(s) to become healthy (timeout=300s)...\n", + "INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n", + ] + ) + message, all_ready = tracker.agents_ready_message() + assert not all_ready + assert message == "Workflow up, but only 1/3 agents reported healthy" + + +def test_a_run_that_never_announced_replicas_still_reports_ready(): + tracker, _, _, _ = drive(["INFO:ventis:Build complete.\n"]) + assert tracker.agents_ready_message() == ("Workflow ready", True) + + +def test_replicas_ready_without_an_announced_total_still_reports_progress(): + _, spinners, _, _ = drive( + ["INFO:ventis.controller.global_controller:Controller A (127.0.0.1:1) is ready.\n"] + ) + assert spinners == ["Starting agents..."] + + +@pytest.mark.parametrize( + "line", + [ + "ERROR:ventis:Config file not found: missing.yaml\n", + "Traceback (most recent call last):\n", + "ERROR: failed to solve: process \"/bin/sh -c pip install\" did not complete successfully\n", + ], +) +def test_fatal_lines_are_flagged(line): + _, _, _, errored = drive([line]) + assert errored + + +@pytest.mark.parametrize( + "line", + [ + "WARNING:ventis.controller.global_controller:otel.destinations not configured -- no OTel metrics collection will happen.\n", + " Warning: no entrypoint mapping for 'agent'\n", + ], +) +def test_benign_warnings_do_not_trip_the_error_path(line): + _, _, _, errored = drive([line]) + assert not errored + + +def test_the_deploy_is_only_declared_dead_after_two_consecutive_checks(monkeypatch): + """One dropped request shouldn't end a deploy that is merely busy.""" + replies = iter([None, {"running": True}, None, None]) + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _port: next(replies)) + state = {"port": 1} + + misses = 0 + verdicts = [] + for _ in range(4): + dead, misses = deploy_cmd._deploy_is_dead(state, misses) + verdicts.append(dead) + + # a miss, then a recovery that resets the count, then two misses in a row + assert verdicts == [False, False, False, True] + + +def test_a_running_deploy_is_never_declared_dead(monkeypatch): + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _port: {"running": True}) + dead, misses = deploy_cmd._deploy_is_dead({"port": 1}, 1) + assert not dead and misses == 0 + + +def test_the_clis_own_status_requests_are_not_shown_or_buffered(monkeypatch): + """The container logs every request the CLI makes to it, so its own polling + lands in the stream it is reading. + """ + shown = [] + monkeypatch.setattr(deploy_cmd.ui, "ok", lambda m: shown.append(m)) + monkeypatch.setattr(deploy_cmd.ui, "warn", lambda m: shown.append(m)) + monkeypatch.setattr(deploy_cmd, "_deploy_summary", lambda *a: ("url", [])) + + lines = deploy_cmd._queued_lines( + iter( + [ + '172.17.0.1 - - [04/Sep/2026 21:00:00] "GET /status HTTP/1.1" 200 -\n', + "INFO:ventis:Build complete.\n", + "INFO:ventis.controller.global_controller:Global controller started, polling every 5s...\n", + ] + ) + ) + summary = deploy_cmd._tail_quiet(lines, {"port": 1}, 8080, serve=False) + + assert summary == ("url", []) + assert shown == ["Build complete", "Workflow ready"] + + +def test_a_build_that_dies_silently_does_not_hang(monkeypatch, capsys): + """A failed build leaves `docker logs -f` open with nothing more to say, so + the wait has to end on /status rather than on the stream closing. + """ + monkeypatch.setattr(deploy_cmd, "_STATUS_POLL_SECONDS", 0.01) + monkeypatch.setattr(deploy_cmd, "_REVEAL_GRACE_SECONDS", 0.5) + monkeypatch.setattr(deploy_cmd, "deploy_status", lambda _p: {"running": False}) + + lines = deploy_cmd._queued_lines(iter(["INFO:ventis:Building 2 Docker image(s) via `x`.\n"])) + # The queue never yields None: the stream stays open, as it does in reality. + lines.put = lambda *a, **k: None + + summary = deploy_cmd._tail_quiet(lines, {"port": 1}, 8080, serve=False) + + assert summary is None + assert "Building 2 Docker image(s)" in capsys.readouterr().out diff --git a/tests/test_integration.py b/tests/test_integration.py index a2e3747..85f0c20 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,7 +10,7 @@ def run_integration_test(): base_url = "http://localhost:8080" print(f"Submitting query to {base_url}/main...") - response = requests.post(f"{base_url}/main", json={"ticker": "MSFT"}) + response = requests.post(f"{base_url}/main", json={"query": "MSFT"}) if response.status_code != 202: print(f"Error submitting request: HTTP {response.status_code}") diff --git a/uv.lock b/uv.lock index 9b86805..8278401 100644 --- a/uv.lock +++ b/uv.lock @@ -53,6 +53,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, ] +[[package]] +name = "canyonos" +version = "0.1.5" +source = { editable = "cli" } +dependencies = [ + { name = "pyfiglet" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "ruamel-yaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyfiglet" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "ruamel-yaml" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -529,6 +548,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -614,6 +645,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -854,6 +894,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615, upload-time = "2025-08-15T18:32:47.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118, upload-time = "2025-08-15T18:32:45.556Z" }, +] + [[package]] name = "pygments" version = "2.21.0" @@ -984,6 +1033,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +] + [[package]] name = "s3transfer" version = "0.19.2" @@ -1172,6 +1243,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "canyonos" }, { name = "pytest" }, ] @@ -1193,7 +1265,10 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=9.1.1" }] +dev = [ + { name = "canyonos", editable = "cli" }, + { name = "pytest", specifier = ">=9.1.1" }, +] [[package]] name = "werkzeug" diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index a1cbb4d..be3bb18 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -134,12 +134,16 @@ def provision_instance(spec, replica_index, next_host_port=None): raise RuntimeError( f"EC2 instance {instance_id} does not have a reachable IP address." ) + # Kept alongside `host` (a private IP inside the VPC): callers outside the + # VPC, such as the CLI printing where to send requests, need this one. + public_host = instance.get("PublicIpAddress") if instance else None redis_port = spec.get( "redis_port", _controller.config.get("redis", {}).get("port", 6379) ) record = { "host": host, + "public_host": public_host, "runtime_id": runtime_id, "redis_host": host, "redis_port": redis_port, @@ -169,7 +173,7 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): f"{host}:{CONTAINER_PORT}", timeout=cfg.get("controller_health_timeout", 180), ) - return { + instance = { "agent_name": spec["name"], "provider": "EC2", "instance_type": spec["instance_type"], @@ -182,6 +186,11 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "redis_port": str(redis_port), "runtime_id": runtime_id, } + if provisioned.get("public_host"): + instance["public_host"] = provisioned["public_host"] + if spec.get("type") == "workflow": + instance["api_port"] = str(spec.get("api_port", 8080)) + return instance except Exception: terminate_instance(provisioned) raise diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index dda4807..88e4ec3 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -153,6 +153,8 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): } if user: instance["user"] = user + if ctrl_type == "workflow": + instance["api_port"] = str(spec.get("api_port", 8080)) logger.info("Runtime ready: %s -> %s", runtime_id, instance["endpoint"]) return instance diff --git a/ventis/controller/instance_manager.py b/ventis/controller/instance_manager.py index e904c5c..4117fd1 100644 --- a/ventis/controller/instance_manager.py +++ b/ventis/controller/instance_manager.py @@ -128,10 +128,11 @@ def _write_instance(self, instance): "redis_port": str(instance["redis_port"]), "runtime_id": instance["runtime_id"], } - if instance.get("user"): - mapping["user"] = instance["user"] - if instance.get("instance_type"): - mapping["instance_type"] = instance["instance_type"] + # public_host: set by providers whose `host` isn't reachable from outside + # the deployment's network. api_port: workflow replicas only. + for field in ("user", "instance_type", "public_host", "api_port"): + if instance.get(field): + mapping[field] = str(instance[field]) self.redis.hset_multiple(key, mapping) node_redis = self.controller.node_redis.get(instance["host"]) or self.redis diff --git a/ventis/server.py b/ventis/server.py index 8df8b0f..31acba7 100644 --- a/ventis/server.py +++ b/ventis/server.py @@ -3,15 +3,22 @@ import subprocess import sys +import yaml from flask import Flask, jsonify, request +from ventis.cli import _artifact_prefix +from ventis.controller.utils.redis_client import RedisClient + app = Flask("ventis-server") # The project files are copied here (into a named volume) by `canyonos sync` / # `canyonos deploy`. Deploy builds and launches against this path. WORKSPACE_DIR = "/workspace" +DEFAULT_API_PORT = 8080 + _gc_process = None +_config_path = None def _gc_running(): @@ -25,13 +32,17 @@ def new_project(): @app.route("/deploy", methods=["POST"]) def deploy(): - global _gc_process + global _gc_process, _config_path if _gc_running(): return jsonify({"error": "already running"}), 409 data = request.get_json(force=True, silent=True) or {} - config_path = data.get("config_path", "config/global_controller.yaml") + # Resolved with ventis' own artifact-layout rule rather than a second copy + # of it, so a `.car` project works when the client sends no config_path. + config_path = data.get("config_path") or os.path.join( + _artifact_prefix(WORKSPACE_DIR), "config", "global_controller.yaml" + ) full_path = os.path.join(WORKSPACE_DIR, config_path) if not os.path.isfile(full_path): @@ -45,6 +56,7 @@ def deploy(): [sys.executable, "-m", "ventis.cli", "deploy", "-c", config_path], cwd=WORKSPACE_DIR, ) + _config_path = full_path return jsonify({"status": "started", "pid": _gc_process.pid}), 200 @@ -66,5 +78,69 @@ def status(): return jsonify({"running": _gc_running()}), 200 +def _primary_redis(config): + """The Redis the controller writes instance records to: the local node's. + + Mirrors GlobalController._launch_redis_containers(), where a localhost node + is reached through VENTIS_REDIS_HOST when the controller is containerized. + """ + redis_cfg = config.get("redis", {}) + host = redis_cfg.get("host", "localhost") + port = redis_cfg.get("port", 6379) + for agent in config.get("agents") or []: + if str(agent.get("provider", "local")).lower() == "local": + port = agent.get("redis_port", port) + break + if host in ("localhost", "127.0.0.1"): + host = os.environ.get("VENTIS_REDIS_HOST", host) + return RedisClient(host=host, port=int(port)) + + +def _workflow_endpoints(config): + """Address of every running workflow replica, as the caller should reach it.""" + ports = { + agent["name"]: agent.get("api_port", DEFAULT_API_PORT) + for agent in config.get("agents") or [] + if agent.get("type") == "workflow" and agent.get("name") + } + if not ports: + return [] + + redis_client = _primary_redis(config) + endpoints = [] + for key in sorted(redis_client.scan_keys("agent_instance:*")): + record = redis_client.hgetall(key) + name = record.get("agent_name") + if name not in ports: + continue + # public_host wins: `host` is the address the controller routes over, + # which for a workflow on another machine is private to that network. + host = record.get("public_host") or record.get("host") + if not host: + continue + endpoints.append( + { + "name": name, + "host": host, + "port": int(record.get("api_port") or ports[name]), + } + ) + return endpoints + + +@app.route("/endpoints", methods=["GET"]) +def endpoints(): + """Where the deployed workflows answer, so the CLI can print real addresses.""" + if _config_path is None or not os.path.isfile(_config_path): + return jsonify({"workflows": []}), 200 + + try: + with open(_config_path) as f: + config = yaml.safe_load(f) or {} + return jsonify({"workflows": _workflow_endpoints(config)}), 200 + except Exception as e: + return jsonify({"workflows": [], "error": str(e)}), 200 + + if __name__ == "__main__": app.run(host="0.0.0.0", port=8000) From 06700a1e09a24ad31241bab57a1f2e22d39a3cb6 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Sat, 5 Sep 2026 00:57:24 -0700 Subject: [PATCH 6/6] docs(cli): add high-level ARCHITECTURE.md and link from README --- cli/ARCHITECTURE.md | 200 ++++++++++++++++++++++++++++++++++++++++++++ cli/README.md | 6 ++ 2 files changed, 206 insertions(+) create mode 100644 cli/ARCHITECTURE.md diff --git a/cli/ARCHITECTURE.md b/cli/ARCHITECTURE.md new file mode 100644 index 0000000..62f869a --- /dev/null +++ b/cli/ARCHITECTURE.md @@ -0,0 +1,200 @@ +# CanyonOS CLI — Architecture + +## The one idea to keep in your head + +**The CLI does almost nothing. The container does everything.** + +`canyonos` is a thin client. It never builds, compiles, or runs your workflow +itself — it manages a **Global Controller (GC) container**, ships your project +into it, and drives it over a small HTTP API. Everything you see in your +terminal is the CLI *narrating* what the container is doing. + +If you remember only one picture, remember this: + +``` + YOU CLI (host) GLOBAL CONTROLLER (container) + │ │ │ + │ canyonos deploy │ │ + ├──────────────────────▶│ pull + run container │ + │ ├───────────────────────────────▶│ + │ │ copy project in (docker cp) │ + │ ├───────────────────────────────▶│ /workspace + │ │ POST /deploy │ + │ ├───────────────────────────────▶│ ventis build + launch + │ │◀── log stream (docker logs) ───┤ │ + │◀── readable progress ─┤ │ ▼ + │ │ spawns Redis + agents + │ │ (sibling containers) +``` + +--- + +## How the pieces connect + +``` +┌───────────────────────────── your machine ─────────────────────────────┐ +│ │ +│ ┌───────────┐ HTTP :8000 ┌──────────────────────────┐ │ +│ │ canyonos │ ───── /deploy /clean ────▶│ Global Controller │ │ +│ │ CLI │ /status /endpoints │ container │ │ +│ │ │ ───── docker cp ─────────▶│ ├─ /workspace (a copy │ │ +│ │ │ ───── docker logs -f ────▶│ │ of your project) │ │ +│ └─────┬─────┘ │ └─ runs `ventis` │ │ +│ │ └───────────┬──────────────┘ │ +│ │ docker compose │ docker.sock │ +│ ▼ ▼ (spawns siblings)│ +│ ┌───────────────────────┐ ┌───────────────────────────┐ │ +│ │ Dashboard stack │◀── traces ───│ Redis + your agent / │ │ +│ │ web · api · postgres │ (OTLP) │ workflow containers │ │ +│ └───────────────────────┘ └───────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +Three things worth internalizing about this diagram: + +1. **The container talks to the host Docker daemon.** The GC mounts the host's + `docker.sock`, so the Redis and agent/workflow containers it launches are + **siblings on your machine**, not nested inside it. (This is why teardown has + to be explicit — see `stop` vs `quit` below.) +2. **Your project is a *copy*, not a live mount.** Files are `docker cp`'d into + a named volume at `/workspace`. Editing files on the host after a deploy does + **not** reach the running build. +3. **The dashboard is separate.** It's its own compose stack that just *renders* + the OTLP traces your workflow emits — it isn't in the deploy critical path. + +State connecting the CLI to its container is a single file: +`~/.canyonos/state.json` (container id + port). Every command that needs the +container reads it. + +--- + +## The three commands that matter + +### `build` — get your code into CanyonOS shape + +``` + you ──▶ canyonos build ──▶ pick agent (Claude/Codex) + scope + └─▶ fetch the porting skill from GitHub + └─▶ launch your coding agent with it + │ + ▼ + generates .car/ ◀── canyonos-formatted project + (originals untouched) +``` + +A **host-side, agent-driven** step. The CLI installs the CanyonOS porting skill +onto your coding agent and hands it a prompt; the agent produces a `.car/` +folder — the canyonos-ready version of your project plus its config. **No +container is involved yet.** + +### `deploy` — the main path + +``` + canyonos deploy + │ + ├─ 1. start fresh → ensure Docker up, tear down any old controller, + │ pull + run the GC container, save state (previous canyonos init) + │ + ├─ 2. ship code → docker cp your project into /workspace + │ + ├─ 3. trigger → POST /deploy (container runs `ventis`: + │ build stubs/images + launch the workflow) + │ + └─ 4. narrate → tail container logs, boil them down to phases, + and when the workflow reports "up": + • auto-start the dashboard (canyonos serve) + • print where everything lives +``` + +Everything after step 3 happens *inside* the container. The CLI's real job in +step 4 is turning a very noisy log stream (a full image-build transcript, etc.) +into a short, readable progression — and, on failure, revealing the part it had +been hiding so you can see the actual cause. + +When it finishes you get a summary panel: the **dashboard URL** and each +**workflow endpoint** (`POST /main`), using the real address the container +placed the workflow at. + +``` + ┌─ Deploy is live ─────────────────────────────┐ + │ Dashboard http://127.0.0.1:8080 │ + │ POST http://127.0.0.1:8000/main │ + │ body {"query": "..."} │ + └──────────────────────────────────────────────┘ +``` + +### `config` — view or edit settings + +``` + canyonos config ──▶ View → pretty tables of agents / otel / general + └─▶ Change → interactive editor (comments & order preserved) +``` + +The important mental model isn't the editor — it's **what a change costs you**: +canyonos config only allows you to change the config file, changing the source code requires a redeploy. + +``` + change type takes effect by... + ─────────────────────── ─────────────────────────────────── + config value only reloads in place (no rebuild) + workflow *code* changes full redeploy (container holds a copy) +``` + +--- + +## Lifecycle: what stays and what goes + +Because the deploy spawns real sibling containers, "make it stop" has two levels of "stop": + +``` + deploy sibling GC project files + stops? containers? container? (volume)? + ─────────────── ─────── ─────────── ───────── ───────────── + canyonos stop ✅ ✅ keep keep + canyonos quit ✅ ✅ remove remove +``` + +- **`stop`** — pause the show, keep the stage set. Redeploy without re-pulling. +- **`quit`** — full teardown. Removes the container *and* the `/workspace` + volume (your copied files). Every `deploy` quietly does this to any previous + controller, so each deploy starts clean. + +And to observe without changing anything: + +- **`logs`** — re-attach to the same live log stream `deploy` shows. Useful + after you Ctrl+C out of a deploy: the deploy keeps running; you just stopped + *watching*. (Ctrl+C on `logs` likewise only detaches.) + +``` + deploy ──▶ (Ctrl+C) ──▶ still running in the container + │ ▲ + └── logs ─────────────────┘ re-attach anytime +``` + +--- + +## The whole loop, one screen + +``` + cd your-project + │ + ▼ + build port your code → .car/ (opens your coding agent) + │ + ▼ + deploy build + launch in the container (dashboard opens itself) + │ + ├─ status where does the workflow answer? + ├─ config tweak settings (live reload); redeploy for code changes + ├─ logs re-attach to the stream + │ + ▼ + stop halt the deploy, keep container + files + or + quit full teardown, remove everything +``` + +That's the entire system: a thin CLI, one container that does the heavy +lifting, a pile of sibling containers it spawns, and a dashboard watching the +whole thing. diff --git a/cli/README.md b/cli/README.md index 76cd4e1..b72f186 100644 --- a/cli/README.md +++ b/cli/README.md @@ -2,6 +2,12 @@ Lightweight CLI for CanyonOS Serves as a thin API layer, connecting to the global controller container. +## Architecture + +For a full walkthrough of the `build`, `deploy`, and `config` flows — plus how +`logs`, `stop`, and `quit` fit into the container lifecycle — see +[ARCHITECTURE.md](ARCHITECTURE.md). + ## Serve `canyonos serve` starts the local CanyonOS dashboard against the Postgres bundled in its compose