From dea1c2b3297095a9dd2a20697c45a93e4b5f40eb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 17 Sep 2026 00:08:54 +0000 Subject: [PATCH] docs: align links and claims with current CI and StackRun control plane Refresh user-facing docs so GitHub Actions triggers, CRD-primary runs, snapshot PR tags, and current test/Newman counts match the tree. Add a relative Markdown link check to local regression. Co-authored-by: jmjava --- .github/workflows/cluster-regression.yml | 10 +- .github/workflows/intercept-e2e.yml | 5 + .github/workflows/local-regression.yml | 3 + .github/workflows/results-regression.yml | 3 + DO-THIS-LOCAL.md | 2 +- README.md | 86 ++++++++++----- docs/MAINTENANCE.md | 16 +-- docs/README-FULL.md | 2 + docs/README.md | 3 +- docs/REGRESSION.md | 19 +++- docs/SCRIPTS.md | 20 ++-- docs/TESTING-AND-REGRESSION-OVERVIEW.md | 2 +- docs/argocd-architecture-guide.md | 4 +- docs/c4-diagrams.md | 102 +++++++++--------- docs/research/README.md | 4 +- docs/research/evaluation.md | 52 +++++---- docs/research/gaps.md | 18 ++-- docs/research/seip-tasks.md | 8 +- helm/tekton-dag/README.md | 1 + milestones/milestone-16.md | 4 +- milestones/milestone-17.md | 11 +- pipeline/stack-pr-pipeline.yaml | 2 +- scripts/README.md | 3 +- scripts/check-markdown-links.py | 131 +++++++++++++++++++++++ scripts/run-regression.sh | 4 + 25 files changed, 370 insertions(+), 145 deletions(-) create mode 100644 scripts/check-markdown-links.py diff --git a/.github/workflows/cluster-regression.yml b/.github/workflows/cluster-regression.yml index 5e96692..8ff80fc 100644 --- a/.github/workflows/cluster-regression.yml +++ b/.github/workflows/cluster-regression.yml @@ -1,8 +1,12 @@ -# Cluster-backed regression (S33). Not on pull requests — Kind/Tekton is too -# slow and too easy to confuse with the PR-gated --local-only job. +# Cluster-backed regression (S33). Kind/Tekton is too slow to run on every +# pull request — local-regression.yml is the everyday gate. # # Runs: Playwright (no cluster) + Kind (isolation-eval --cluster, Phase 2 -# stack-dag-verify, Newman vs orchestrator). +# stack-dag-verify, Newman vs orchestrator). Isolation-eval is skipped on +# pull_request; nightly / dispatch / tags still measure it. +# +# pull_request paths below exercise this job when Helm or cluster-ci scripts +# change. Everyday product PRs (tasks/, docs/, orchestrator/, …) do not match. name: cluster regression on: diff --git a/.github/workflows/intercept-e2e.yml b/.github/workflows/intercept-e2e.yml index ea8a2ac..fc8ef07 100644 --- a/.github/workflows/intercept-e2e.yml +++ b/.github/workflows/intercept-e2e.yml @@ -1,3 +1,8 @@ +# Weekly (and path-filtered) product-path evidence: trigger → StackRun → +# operator → PR PipelineRun → intercept (Telepresence or mirrord) → tests. +# Not the everyday PR gate — local-regression.yml is. Path filters still +# start Kind when Helm/tasks/operator/orchestrator/pipeline/stacks or the +# listed scripts change. name: intercept product E2E on: diff --git a/.github/workflows/local-regression.yml b/.github/workflows/local-regression.yml index 470227d..51c97cd 100644 --- a/.github/workflows/local-regression.yml +++ b/.github/workflows/local-regression.yml @@ -1,3 +1,6 @@ +# Everyday PR / main-push gate. Does not start Kind. +# Kind jobs live in cluster-regression.yml, intercept-e2e.yml, +# results-regression.yml, and operator.yml (path-filtered or scheduled). name: local regression on: diff --git a/.github/workflows/results-regression.yml b/.github/workflows/results-regression.yml index f0db7f4..e654f08 100644 --- a/.github/workflows/results-regression.yml +++ b/.github/workflows/results-regression.yml @@ -1,3 +1,6 @@ +# Weekly (and path-filtered) pinned Results/Postgres + strict agent regression. +# Not the everyday PR gate. Path filters cover Results installers and +# run-regression-agent-full.sh. name: Tekton Results regression on: diff --git a/DO-THIS-LOCAL.md b/DO-THIS-LOCAL.md index a27bb75..d021467 100644 --- a/DO-THIS-LOCAL.md +++ b/DO-THIS-LOCAL.md @@ -7,7 +7,7 @@ Cloud agents for this PR ran **`--local-only`** (no `kubectl` / Docker / Kind). ## Prerequisites - Kind (or equivalent) cluster with Tekton Pipelines installed -- This repo checked out on branch `cursor/m13-production-hardening-features-b923` (or `main` after merge) +- This repo checked out (typically `main`, or the branch under test) - Python venv + tools from [docs/AGENT-REGRESSION.md](docs/AGENT-REGRESSION.md) ```bash diff --git a/README.md b/README.md index 4f04a38..3eb0d29 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,23 @@ Standalone Tekton pipeline system for **local development and proof-of-concept** **Academic packaging** ([docs/research/](docs/research/)) records claims and evidence for a possible workshop paper. It does **not** set architecture. Control plane direction is CRD-primary (`Stack` / `StackRun`); see [M14](milestones/milestone-14.md). Cite via [`CITATION.cff`](CITATION.cff). License: [Apache-2.0](LICENSE). -[![local regression](https://github.com/jmjava/tekton-dag/actions/workflows/local-regression.yml/badge.svg)](https://github.com/jmjava/tekton-dag/actions/workflows/local-regression.yml) runs `scripts/run-regression.sh --local-only --require-lang-tests` on every pull request (Phase 1, pytest, vitest, isolation-eval protocol, Maven, PHPUnit, operator `go test`). +[![local regression](https://github.com/jmjava/tekton-dag/actions/workflows/local-regression.yml/badge.svg)](https://github.com/jmjava/tekton-dag/actions/workflows/local-regression.yml) +[![cluster regression](https://github.com/jmjava/tekton-dag/actions/workflows/cluster-regression.yml/badge.svg)](https://github.com/jmjava/tekton-dag/actions/workflows/cluster-regression.yml) -[![cluster regression](https://github.com/jmjava/tekton-dag/actions/workflows/cluster-regression.yml/badge.svg)](https://github.com/jmjava/tekton-dag/actions/workflows/cluster-regression.yml) runs Playwright plus Kind (`scripts/run-cluster-ci.sh`: isolation-eval `--cluster`, `stack-dag-verify`, Newman) on **nightly / `workflow_dispatch` / `v*` tags** — not on pull requests. +**Not every workflow runs on every PR.** Cheap jobs (local regression, docgen smoke, dependency review) run on each pull request. Kind jobs are scheduled or path-filtered — see [docs/REGRESSION.md](docs/REGRESSION.md). + +| Workflow | Everyday PR / `main` push | Also runs | +|----------|---------------------------|-----------| +| [local regression](https://github.com/jmjava/tekton-dag/actions/workflows/local-regression.yml) | Yes — `--local-only --require-lang-tests` | `workflow_dispatch` | +| [docgen demo-function](.github/workflows/docgen-demo-function.yml) | Yes (cheap smoke) | | +| [dependency review](.github/workflows/dependency-review.yml) | PRs only | | +| [operator](.github/workflows/operator.yml) | Only if `operator/**` (or `install-tekton.sh`) changed | `workflow_dispatch` | +| [cluster regression](https://github.com/jmjava/tekton-dag/actions/workflows/cluster-regression.yml) | No | Nightly cron, `v*` tags, dispatch, or PRs that touch Helm / `run-cluster-ci.sh` / `bootstrap-namespace.sh` (isolation-eval skipped on those PRs) | +| [intercept product E2E](.github/workflows/intercept-e2e.yml) | No | Weekly cron, dispatch, or PRs/pushes matching that workflow's path filters (Helm, tasks, operator, orchestrator, pipeline, stacks, and listed scripts) | +| [Tekton Results](.github/workflows/results-regression.yml) | No | Weekly cron, dispatch, or PRs that touch Results installers / `run-regression-agent-full.sh` | +| [GitHub Pages](.github/workflows/pages.yml) | No | `main` pushes that touch `docs/**` | + +The Actions sidebar can still list **deleted** workflow names. Disable those in the GitHub UI (Actions → workflow → ⋯ → Disable). **Dependabot Updates** is GitHub-managed, not a repo workflow file. ## What's new (M13 foundations) @@ -60,6 +74,7 @@ Each row links to the **in-browser player** on Pages (`#seg-…`) and to the **c | 16 | Management GUI | Vue 3 + Flask: team switcher, DAG view, runs, triggers, tests, Git browser | ~3:30 | [▶](https://jmjava.github.io/tekton-dag/#seg-16) | [`16-management-gui.mp4`](docs/demos/recordings/16-management-gui.mp4) | | 17 | Extending the GUI | Five-step pattern: Flask route, pytest, Pinia store, Vue component, Playwright | ~2:36 | [▶](https://jmjava.github.io/tekton-dag/#seg-17) | [`17-extending-gui.mp4`](docs/demos/recordings/17-extending-gui.mp4) | | 18 | What's Coming Next | Post-M16 roadmap: retry, sizing, multi-cluster, reliability, observability | ~3:25 | [▶](https://jmjava.github.io/tekton-dag/#seg-18) | [`18-roadmap-forward.mp4`](docs/demos/recordings/18-roadmap-forward.mp4) | +| 19 | Kubernetes Operator | `Stack` / `StackRun` CRDs, operator reconcile, Kind soak | ~2:00 | [▶](https://jmjava.github.io/tekton-dag/#seg-19) | [`19-kubernetes-operator.mp4`](docs/demos/recordings/19-kubernetes-operator.mp4) | ### Concat Demos @@ -86,15 +101,15 @@ Each row links to the **in-browser player** on Pages (`#seg-…`) and to the **c | [M8](milestones/milestone-8.md) | **Partial** | Demo assets: Manim + TTS + composed segments + [GitHub Pages](https://jmjava.github.io/tekton-dag/); VHS terminal recordings, Slidev PDF, full concat still open | | [M9](milestones/milestone-9.md) | **Completed** | Test-trace regression graph + minimal test selection (Neo4j, mock Datadog). 10 Newman requests, 36 assertions. Test filtering in PR pipeline. | | [M10](milestones/milestone-10.md) | **Completed** | Multi-team scaling: orchestration service, Helm chart, ArgoCD, batched builds | -| [M10.1](milestones/milestone-10-1.md) | **Completed** | Orchestration service testing: Postman/Newman (15 requests, 30 assertions), integration validation | -| [M11](milestones/milestone-11.md) | **Completed** | Vue 3 Management GUI + Python/Flask backend (replaces `reporting-gui/`). Multi-team, multi-cluster, DAG visualization. Current regression inventory: 70 Playwright tests and 68 backend pytest tests. | -| [M12](milestones/milestone-12.md) | **Completed** | Architecture customization: shared Python package, Helm ConfigMap/PVC templates, parameterized pipelines (no hardcoded `localhost:5000`), `scripts/common.sh`, build image variants (Java 11/17/21, Node 18/20/22, Python 3.10–3.12, PHP 8.1–8.3), custom pipeline hook tasks (pre/post build/test), stack JSON schema, 62 orchestrator pytest tests, 14 shared-package tests. Full docs: [CUSTOMIZATION.md](docs/CUSTOMIZATION.md), [TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md](docs/TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md), MAINTENANCE.md, Helm README. | +| [M10.1](milestones/milestone-10-1.md) | **Completed** | Orchestration service testing: Postman/Newman (live collection is now 20 requests / 38 assertions). | +| [M11](milestones/milestone-11.md) | **Completed** | Vue 3 Management GUI + Python/Flask backend (replaces `reporting-gui/`). Multi-team, multi-cluster, DAG visualization. Current inventory: 70 Playwright tests and 68 backend pytest tests. | +| [M12](milestones/milestone-12.md) | **Completed** | Architecture customization: shared Python package, Helm ConfigMap/PVC templates, parameterized pipelines (no hardcoded `localhost:5000`), `scripts/common.sh`, build image variants (Java 11/17/21, Node 18/20/22, Python 3.10–3.12, PHP 8.1–8.3), custom pipeline hook tasks (pre/post build/test), stack JSON schema. Current inventory: 108 orchestrator pytest tests, 89 `tekton-dag-common` tests. Full docs: [CUSTOMIZATION.md](docs/CUSTOMIZATION.md), [TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md](docs/TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md), [MAINTENANCE.md](docs/MAINTENANCE.md), Helm README. | | [M12.2](milestones/milestone-12.2.md) | **Partial** | **Part A done:** doc sync + archive. **Part B open:** regression + Management GUI [docs & demo plan](docs/TESTING-AND-REGRESSION-OVERVIEW.md) / [GUI extension](docs/MANAGEMENT-GUI-EXTENSION.md) / [video segments](docs/demos/segments-m12-2-regression-gui.md) | | [doc-generator](milestones/milestone-doc-generator.md) | **Completed** | Reusable Python library ([`docgen`](https://github.com/jmjava/documentation-generator)) extracting the demo pipeline (TTS, Manim, VHS, ffmpeg, validation, Pages). OCR validation, A/V sync, narration linting, auto-generated GitHub Pages. All 18 demo segments regenerated via `docgen`. | | [M13](milestones/milestone-13.md) | **Partial** | Production hardening foundations shipped: webhook HMAC, stack secrets/config + deploy wiring + injection-status APIs, PipelineRun timeouts / task retries / failure classifier / resource profiles, `stack-promote` + registries + approval gate. Open: intercept secret/config wiring, Helm `appConfig` / ESO, GUI panels, observability, cross-cluster deploy. Roadmap video: [segment 18](https://jmjava.github.io/tekton-dag/#seg-18). Local cluster checklist: [DO-THIS-LOCAL.md](DO-THIS-LOCAL.md). | | [M14](milestones/milestone-14.md) | **Partial** | **Kubernetes operator (CRD-primary, default-on):** `Stack` + `StackRun` + `Team` (`tektondag.io/v1alpha1`). Helm `operator.enabled` defaults **true**; Flask/GUI/Triggers/`generate-run.sh` create StackRuns. Kind soak path: `scripts/install-operator-kind.sh`, `run-cluster-ci.sh` (operator on unless `--skip-operator`). Hygiene: [M15](milestones/milestone-15.md). Follow-ons: [M16](milestones/milestone-16.md). | | [M15](milestones/milestone-15.md) | **Completed** | Control-plane hygiene: idempotent StackRun→PipelineRun, GHA `--skip-operator`, Triggers `prNumber`, Flask/GUI promote approval, soak `ready==total`, dead PipelineRun builders. Squash-merged [#19](https://github.com/jmjava/tekton-dag/pull/19). | -| [M16](milestones/milestone-16.md) | **Code-complete** | Team CR overlay, `spec.continueFrom`, Kind webhook installer, PipelineRun escape hatches retired, spoken demo segments 01/08/18/19 rebuilt ([#20](https://github.com/jmjava/tekton-dag/pull/20)–[#24](https://github.com/jmjava/tekton-dag/pull/24)). Remaining: S34 intercept E2E (parked). | +| [M16](milestones/milestone-16.md) | **Code-complete** | Team CR overlay, `spec.continueFrom`, Kind webhook installer, PipelineRun escape hatches retired, spoken demo segments 01/08/18/19 rebuilt ([#20](https://github.com/jmjava/tekton-dag/pull/20)–[#24](https://github.com/jmjava/tekton-dag/pull/24)). Intercept E2E follow-on is [M17.3](milestones/milestone-17.md) (`intercept-e2e.yml`); live matrix evidence is still required. | | [M17](milestones/milestone-17.md) | **In progress** | End-to-end audit closure: least-privilege RBAC, authenticated mutation APIs, full intercept/Results verification, CI quality gates, maintainability consolidation, and documentation accuracy. | Older milestones (M2, M3) are in [milestones/completed/](milestones/completed/). @@ -156,9 +171,9 @@ flowchart LR | **Merge** (`stack-merge-release`) | Promote RC to release, build, tag release images, push next dev cycle version commit. | | **Promote** (`stack-promote`) | Copy release-tagged images to a target registry/environment (`registries.yaml` or API overrides); optional approval gate. | -**Intercept backends:** Telepresence (default) or mirrord, selected via pipeline param `intercept-backend`. Both are implemented; continuously scheduled full intercept E2E is tracked in [M17](milestones/milestone-17.md). +**Intercept backends:** Telepresence (default) or mirrord, selected via pipeline param `intercept-backend`. Both are implemented. Weekly/manual CI is [`.github/workflows/intercept-e2e.yml`](.github/workflows/intercept-e2e.yml) (`scripts/run-product-intercept-e2e.sh`). Treat only a recent successful matrix job and its retained traffic artifact as verification ([M17.3](milestones/milestone-17.md)). -**Orchestration service** (M10): In-cluster Flask service that receives GitHub webhooks, resolves repo-to-stack dynamically, and creates PipelineRuns. Packaged via Helm chart with ArgoCD ApplicationSet for multi-team provisioning. See [docs/m10-multi-team-architecture.md](docs/m10-multi-team-architecture.md). +**Orchestration service** (M10): In-cluster Flask service that receives GitHub webhooks, resolves repo-to-stack dynamically, and creates **StackRun** CRs. The Go operator reconciles each StackRun into a Tekton PipelineRun. Packaged via Helm (`operator.enabled` defaults **true**) with an ArgoCD ApplicationSet for multi-team provisioning. Mutation APIs require a bearer token. See [docs/m10-multi-team-architecture.md](docs/m10-multi-team-architecture.md) and [orchestrator/README.md](orchestrator/README.md). --- @@ -171,6 +186,9 @@ flowchart LR # 2. Tekton + stack tasks/pipelines ./scripts/install-tekton.sh +# 2b. Operator (CRDs + controller). generate-run.sh applies StackRuns, not raw PipelineRuns. +./scripts/install-operator-kind.sh + # 3. Publish build images to Kind registry (one-time) ./scripts/publish-build-images.sh @@ -205,23 +223,26 @@ kubectl port-forward svc/el-stack-event-listener 8080:8080 -n tekton-pipelines & ## Regression testing -**E2E with intercepts** — runs bootstrap (optional) + PR pipeline + Tekton Results verification: +**E2E with intercepts** — two entrypoints: + +- **CI / authenticated product path:** `scripts/run-product-intercept-e2e.sh` (used by `intercept-e2e.yml`). +- **Local Kind helper:** `scripts/run-e2e-with-intercepts.sh` (bootstrap + PR pipeline; optional `--skip-bootstrap`). ```bash -# Full run (bootstrap + PR pipeline) -./scripts/run-e2e-with-intercepts.sh --intercept-backend telepresence -./scripts/run-e2e-with-intercepts.sh --intercept-backend mirrord +# Product path against a cluster that already has the control plane +./scripts/run-product-intercept-e2e.sh --intercept-backend telepresence +./scripts/run-product-intercept-e2e.sh --intercept-backend mirrord -# Skip bootstrap if stack is already deployed (saves ~8-12 min) -./scripts/run-e2e-with-intercepts.sh --intercept-backend telepresence --skip-bootstrap +# Full local helper (bootstrap + PR pipeline) +./scripts/run-e2e-with-intercepts.sh --intercept-backend telepresence ./scripts/run-e2e-with-intercepts.sh --intercept-backend mirrord --skip-bootstrap ``` -**Orchestrator service tests** — Newman suite against the live service (15 requests, 30 assertions): +**Orchestrator service tests** — Newman suite against the live service (20 requests / 38 assertions in `tests/postman/orchestrator-tests.json`): ```bash ./scripts/run-orchestrator-tests.sh -./scripts/run-orchestrator-tests.sh --skip-integration # skip PipelineRun validation +./scripts/run-orchestrator-tests.sh --skip-integration # skip live StackRun/PipelineRun wait ``` --- @@ -263,7 +284,8 @@ C4Container Person(platform, "Platform Engineer") System_Boundary(tekton_std, "Tekton DAG") { - Container(orchestrator, "Orchestrator Service", "Flask + Gunicorn", "Webhook handler, stack resolver, PipelineRun creator") + Container(orchestrator, "Orchestrator Service", "Flask + Gunicorn", "Webhook handler, stack resolver, StackRun creator") + Container(operator, "Operator", "Go / Kubebuilder", "Reconciles Stack + StackRun CRs to PipelineRuns") Container(event_listener, "EventListener", "Tekton Triggers", "Legacy webhook path via Cloudflare Tunnel") Container(pr_pipeline, "stack-pr-test", "Tekton Pipeline", "PR: build, intercept, validate, test") Container(merge_pipeline, "stack-merge-release", "Tekton Pipeline", "Merge: promote, build, tag, push") @@ -284,6 +306,7 @@ C4Container Rel(orchestrator, pr_pipeline, "Creates StackRun; operator reconciles") Rel(orchestrator, bootstrap_pipeline, "Creates StackRun; operator reconciles") Rel(orchestrator, merge_pipeline, "Creates StackRun; operator reconciles") + Rel(orchestrator, operator, "StackRun CR") Rel(event_listener, pr_pipeline, "PR opened") Rel(event_listener, merge_pipeline, "PR merged") Rel(pr_pipeline, stack_defs, "Read") @@ -354,10 +377,12 @@ In-cluster Flask service that replaces script-driven orchestration for productio | `/api/stacks` | GET | List registered stacks | | `/api/teams` | GET | List team configs | | `/api/runs` | GET | List recent StackRuns | -| `/api/run` | POST | Create a StackRun (pr, bootstrap, merge, or promote) | -| `/api/bootstrap` | POST | Trigger bootstrap pipeline | -| `/webhook/github` | POST | GitHub webhook handler | -| `/api/reload` | POST | Hot-reload stack and team configs | +| `/api/run` | POST | Create a StackRun (pr, bootstrap, merge, or promote). Requires `Authorization: Bearer`. | +| `/api/bootstrap` | POST | Trigger bootstrap pipeline. Requires bearer token. | +| `/webhook/github` | POST | GitHub webhook handler (HMAC when `WEBHOOK_SECRET` is set) | +| `/api/reload` | POST | Hot-reload stack and team configs. Requires bearer token. | + +All non-read-only `/api/*` routes require a bearer token (`API_MUTATION_TOKEN`). See [orchestrator/README.md](orchestrator/README.md). Deploy: @@ -493,23 +518,24 @@ Pre-commit hook runs GitGuardian ggshield: `pip install pre-commit && pre-commit | Directory | Contents | |-----------|----------| -| `stacks/` | Stack YAML (DAG definitions), [registry.yaml](stacks/registry.yaml), [versions.yaml](stacks/versions.yaml) | +| `stacks/` | Stack YAML (DAG definitions), [registry.yaml](stacks/registry.yaml) (repo → stack), [registries.yaml](stacks/registries.yaml) (promote targets), [versions.yaml](stacks/versions.yaml) | | `tasks/` | Tekton tasks: resolve-stack, clone-app-repos, build-compile-*, build-containerize, deploy-full-stack, deploy-intercept, deploy-intercept-mirrord, validate-propagation, validate-original-traffic, run-stack-tests, pr-snapshot-tag, version-bump, tag-release-images, post-pr-comment, cleanup-stack | -| `pipeline/` | stack-pr-test, stack-merge-release, stack-bootstrap, stack-pr-continue, stack-dag-verify, triggers | +| `pipeline/` | stack-pr-test, stack-merge-release, stack-bootstrap, stack-promote, stack-pr-continue, stack-dag-verify, triggers | +| `operator/` | Go Kubebuilder operator: `Stack` / `StackRun` / `Team` CRDs (`tektondag.io/v1alpha1`). See [operator/README.md](operator/README.md). | | `orchestrator/` | Flask orchestration service: creates StackRuns through app.py/routes.py; includes resolver, Kubernetes client, and a contract-only legacy PipelineRun builder used by golden tests | -| `helm/tekton-dag/` | Helm chart: packages tasks, pipelines, orchestrator deployment, RBAC | +| `helm/tekton-dag/` | Helm chart: packages tasks, pipelines, orchestrator deployment, operator, RBAC | | `argocd/` | ArgoCD AppProject and ApplicationSet for multi-team provisioning | | `teams/` | Per-team config (team.yaml, values.yaml) for multi-team data model | | `build-images/` | Dockerfiles and build script for pre-built compile images | -| `libs/` | Standalone baggage middleware libraries (Spring Boot, Node, Flask, PHP) | -| `scripts/` | CLI scripts: generate-run, publish-build-images, publish-orchestrator-image, run-e2e-with-intercepts, run-orchestrator-tests, run-valid-pr-flow, kind-with-registry, install-tekton, install-tekton-results, and more | +| `libs/` | Standalone baggage middleware libraries (Spring Boot, Node, Flask, PHP) plus `tekton-dag-common` | +| `scripts/` | CLI scripts: generate-run, publish-build-images, publish-orchestrator-image, run-product-intercept-e2e, run-e2e-with-intercepts, run-orchestrator-tests, run-valid-pr-flow, kind-with-registry, install-tekton, install-operator-kind, install-tekton-results, and more | | `management-gui/` | Vue 3 + Flask management GUI (frontend + backend). See [Management GUI](#management-gui-m11) | | `tests/postman/` | Postman/Newman collections (orchestrator-tests.json, management-gui-tests.json) | | `docs/` | Architecture docs, diagrams, guides. See [docs/README.md](docs/README.md) | | `docs/research/` | Workshop / tool-demo packaging: claims, related work, IEEE draft | | `milestones/` | Milestone planning and status docs | | `session-notes/` | Session notes and debugging logs | -| `reporting-gui/` | Vue + Node reporting GUI. See [reporting-gui/README.md](reporting-gui/README.md) | +| `reporting-gui/` | **Legacy** Vue + Node reporting GUI (M17.17: archive). Canonical UI is `management-gui/`. See [reporting-gui/README.md](reporting-gui/README.md) | | `sample-repos/` | Scripts for creating sample app repos | | `config/` | Kubernetes manifests (Postgres for Tekton Results) | | `.vscode/` | Launch configs and debug setup for all app frameworks | @@ -520,12 +546,16 @@ Pre-commit hook runs GitGuardian ggshield: `pip install pre-commit && pre-commit - [docs/DAG-AND-PROPAGATION.md](docs/DAG-AND-PROPAGATION.md) — stack DAG and header propagation - [docs/c4-diagrams.md](docs/c4-diagrams.md) — full diagram set +- [docs/REGRESSION.md](docs/REGRESSION.md) — platform regression tiers and GitHub Actions triggers - [docs/PR-TEST-FLOW.md](docs/PR-TEST-FLOW.md) — valid PR test flow - [docs/m10-multi-team-architecture.md](docs/m10-multi-team-architecture.md) — multi-team architecture - [docs/argocd-architecture-guide.md](docs/argocd-architecture-guide.md) — ArgoCD + Tekton together - [docs/bootstrap-pipeline-speed-analysis.md](docs/bootstrap-pipeline-speed-analysis.md) — pipeline speed analysis - [docs/m7-mirrord-intercept-task.md](docs/m7-mirrord-intercept-task.md) — mirrord intercept task - [docs/demo-playbook.md](docs/demo-playbook.md) — demo recording playbook -- [docs/README-FULL.md](docs/README-FULL.md) — full design doc +- [docs/README-FULL.md](docs/README-FULL.md) — historical long-form design (pre-operator; see banner there) +- [orchestrator/README.md](orchestrator/README.md) — orchestrator API and env +- [operator/README.md](operator/README.md) — Stack / StackRun operator +- [helm/tekton-dag/README.md](helm/tekton-dag/README.md) — Helm chart - [docs/research/README.md](docs/research/README.md) — academic / workshop packaging - [SHARING-BACK.md](SHARING-BACK.md) — sharing back to reference-architecture diff --git a/docs/MAINTENANCE.md b/docs/MAINTENANCE.md index 6b87982..1d9e298 100644 --- a/docs/MAINTENANCE.md +++ b/docs/MAINTENANCE.md @@ -4,16 +4,17 @@ This document is a map of the **tekton-dag** repository: what each area does, ho ## Architecture overview -The system wires **stack definitions** (YAML) to **Tekton** pipelines, with an in-cluster **orchestrator** that creates `PipelineRun` objects and optional **Neo4j** queries for test planning. +The system wires **stack definitions** (YAML) to **Tekton** pipelines. The in-cluster **orchestrator** creates `StackRun` objects; the **operator** reconciles them to `PipelineRun`s. Optional **Neo4j** queries support test planning. | Area | Role | |------|------| | **`stacks/`** | Stack YAML: applications, build tool per app, downstream dependencies, test configuration. Often synced into the cluster via Helm ConfigMaps. | | **`tasks/`** | Tekton `Task` manifests: resolve stack, clone repos, compile (per tool), containerize, deploy intercepts / full stack, validate propagation, run tests, versioning, cleanup, graph helpers, etc. | -| **`pipeline/`** | Tekton `Pipeline` definitions and trigger bindings. Core flows: PR test, bootstrap deploy, merge/release. Additional pipelines exist for continuation and DAG verification. | +| **`pipeline/`** | Tekton `Pipeline` definitions and trigger bindings. Core flows: PR test, bootstrap deploy, merge/release, promote. Additional pipelines exist for continuation and DAG verification. | +| **`operator/`** | Go Kubebuilder controller for `Stack`, `StackRun`, and `Team` (`tektondag.io/v1alpha1`). Canonical PipelineRun builder. | | **`orchestrator/`** | Flask service: `app.py` (config), `routes.py` (HTTP API), `stack_resolver.py` (repo → stack), `stackrun_builder.py` (StackRun CRs), `k8s_client.py`, `graph_client.py` (Neo4j). The legacy PipelineRun builder is a golden-contract oracle, not the runtime path. | | **`management-gui/`** | Vue 3 (Vite) frontend plus Flask backend for operating and observing pipelines, teams, and repos. | -| **`helm/tekton-dag/`** | Helm chart: orchestrator, management GUI, RBAC, ConfigMaps for stacks/teams, optional PVCs, values for registry and defaults. | +| **`helm/tekton-dag/`** | Helm chart: orchestrator, operator, management GUI, RBAC, ConfigMaps for stacks/teams, optional PVCs, values for registry and defaults. `operator.enabled` defaults **true**. | | **`scripts/`** | Bash utilities; new scripts should `source` **`scripts/common.sh`** for shared defaults (`NAMESPACE`, `GIT_URL`, port-forward helpers, etc.). | | **`build-images/`** | Parameterized Dockerfiles for compile-side images (Maven, Gradle, Node, Python, PHP, mirrord) and scripts to build/push them. | | **`teams/`** | Per-team metadata (`team.yaml`) and related values; orchestrator discovers teams under mounted paths (e.g. `/teams//team.yaml`). | @@ -22,9 +23,10 @@ The system wires **stack definitions** (YAML) to **Tekton** pipelines, with an i | Pipeline | File | Purpose | |----------|------|---------| -| `stack-pr-test` | `pipeline/stack-pr-pipeline.yaml` | PR path: resolve stack, build changed app, intercept deploy, tests, version bump, cleanup. | +| `stack-pr-test` | `pipeline/stack-pr-pipeline.yaml` | PR path: resolve stack, snapshot-tag and build the changed app, intercept deploy, tests, PR comment, cleanup. **No** `versions.yaml` bump. | | `stack-bootstrap` | `pipeline/stack-bootstrap-pipeline.yaml` | Full stack bring-up / environment bootstrap. | -| `stack-merge-release` | `pipeline/stack-merge-pipeline.yaml` | Post-merge release flow (images, tags, etc.). | +| `stack-merge-release` | `pipeline/stack-merge-pipeline.yaml` | Post-merge release flow (images, tags, next-cycle version bump). | +| `stack-promote` | `pipeline/stack-promote-pipeline.yaml` | Copy release images to a target registry (`stacks/registries.yaml`). | Other pipelines in `pipeline/` include **`stack-pr-continue`**, **`stack-dag-verify`**, and **`triggers.yaml`** (EventListener / bindings / templates) for webhook-style automation. @@ -53,7 +55,7 @@ Tasks live under `tasks/` (see filenames for the canonical Tekton `metadata.name ### `orchestrator/` -- **Contains:** Flask app, K8s and Neo4j clients, PipelineRun construction. +- **Contains:** Flask app, K8s and Neo4j clients, StackRun construction (runtime) plus a contract-only PipelineRun builder (tests). - **Modify when:** New HTTP APIs, different default params on created runs, resolver rules, or graph behavior. ### `management-gui/` @@ -140,7 +142,7 @@ Implement a Tekton `Task`, install it in the cluster, then set the pipeline para | Orchestrator unit tests | `cd orchestrator && python3 -m pytest tests/ -v` | **108** tests at the M17 baseline. | | Management GUI backend | `cd management-gui/backend && python3 -m pytest tests/ -v` | **68** tests at the M17 baseline. | | Management GUI frontend (E2E) | `cd management-gui/frontend && npx playwright test` | **70** tests at the M17 baseline. | -| Newman / Postman (cluster) | `./scripts/run-orchestrator-tests.sh --all` | Requires running orchestrator (and Neo4j for graph collection); see script header for prerequisites. | +| Newman / Postman (cluster) | `./scripts/run-orchestrator-tests.sh --all` | 20 requests / 38 assertions in `tests/postman/orchestrator-tests.json`. Requires a running orchestrator (and Neo4j for graph collection); see script header. | --- diff --git a/docs/README-FULL.md b/docs/README-FULL.md index d442174..cad1ad3 100644 --- a/docs/README-FULL.md +++ b/docs/README-FULL.md @@ -1,5 +1,7 @@ # Tekton Job Standardization +> **Historical long-form design.** Runtime control plane is now CRD-primary (`Stack` / `StackRun` via the operator). PR runs use **snapshot image tags** and do **not** bump `versions.yaml`. For current behavior start at the root [README](../README.md), [DAG-AND-PROPAGATION.md](DAG-AND-PROPAGATION.md), and [milestones/milestone-14.md](../milestones/milestone-14.md). + One universal Tekton pipeline system that adapts to any combination of applications — single services, multi-tier stacks, or fan-out graphs — while managing versioning, build toolchains, header propagation, and container image lifecycle automatically. ## Problem diff --git a/docs/README.md b/docs/README.md index e6562a0..d127b71 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,8 @@ This folder contains design, operations, and verification docs for the tekton-da | **Archive (not maintained)** | [archive/](archive/) — obsolete session plans and similar | | **Customization & ops** | [CUSTOMIZATION.md](CUSTOMIZATION.md), [MAINTENANCE.md](MAINTENANCE.md) | | **Teams: stacks + baggage libs** | [TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md](TEAM-ONBOARDING-STACKS-AND-BAGGAGE.md) — header libraries per runtime, new stack checklist | -| **Demo toolchain (M8)** | [demos/](demos/) — [demos/README.md](demos/README.md), `generate-all.sh`; [milestones/milestone-8.md](../milestones/milestone-8.md) | +| **Demo toolchain (M8)** | [demos/](demos/) — [demos/README.md](demos/README.md) (`docgen` CLI; `generate-all.sh` is a thin wrapper); [milestones/milestone-8.md](../milestones/milestone-8.md) | +| **GitHub Actions** | [REGRESSION.md](REGRESSION.md) — which workflows run on every PR vs schedule/path filters | | **GitHub Pages** | [GITHUB-PAGES.md](GITHUB-PAGES.md) — how the demo site is deployed; fix 404 | | **Illustrations & logo** | [assets/README.md](assets/README.md) — composite PNGs + **split panels** in `assets/panels/` for READMEs and demos | | **Agents / regression loop** | [AGENT-REGRESSION.md](AGENT-REGRESSION.md) — iterate until full regression green; [../AGENTS.md](../AGENTS.md) | diff --git a/docs/REGRESSION.md b/docs/REGRESSION.md index 2fcd659..2bf6165 100644 --- a/docs/REGRESSION.md +++ b/docs/REGRESSION.md @@ -11,9 +11,24 @@ Do **not** confuse these: | Scope | What runs | Typical trigger | |-------|-----------|-----------------| | **Application PR** (`stack-pr-test` on an **app** repo) | Stack-defined tests only — e.g. that app’s Newman/Playwright/Artillery as declared in `stacks/*.yaml`, against the intercept build. | Every PR on the **application** repository (when webhooks/Tekton are wired). | -| **Platform regression** (`scripts/run-regression*.sh` on **this** repo) | **System / integration** tiers: Phase 1 + orchestrator + shared libs + GUI pytest, Playwright for **management-gui**, real **`stack-dag-verify`** PipelineRun, Newman against **orchestrator** API, optional Tekton Results, optional Kind E2E. | **PRs / `main`:** [`.github/workflows/local-regression.yml`](../.github/workflows/local-regression.yml) runs **`--local-only --require-lang-tests`**. **Nightly / dispatch / `v*` tags:** [`.github/workflows/cluster-regression.yml`](../.github/workflows/cluster-regression.yml) runs Playwright + Kind. **Weekly / dispatch:** [`intercept-e2e.yml`](../.github/workflows/intercept-e2e.yml) runs both intercept backends and [`results-regression.yml`](../.github/workflows/results-regression.yml) runs strict Results/Postgres verification. | +| **Platform regression** (`scripts/run-regression*.sh` on **this** repo) | **System / integration** tiers: Phase 1 + orchestrator + shared libs + GUI pytest, Playwright for **management-gui**, real **`stack-dag-verify`** PipelineRun, Newman against **orchestrator** API, optional Tekton Results, optional Kind E2E. | See the GitHub Actions table below. | -So: **not all tests run on every PR.** `--local-only` (including Java/PHP/operator) is PR-gated. Playwright, Newman, Phase 2, and Kind isolation measurements run on **cluster-regression** (nightly / `workflow_dispatch` / version tags), not on pull requests. The slower Telepresence and mirrord product paths run weekly and on dispatch. App PRs run a narrower, stack-scoped test stage. +**GitHub Actions — what runs when (Kind is not every PR):** + +| Workflow | Every PR / every `main` push | Also runs | +|----------|------------------------------|-----------| +| **local regression** | Yes (`--local-only --require-lang-tests`) | dispatch | +| **docgen demo-function** | Yes (cheap smoke) | | +| **dependency review** | PRs only | | +| **operator** | Only if `operator/**` (or `install-tekton.sh`) changed — unit/envtest plus Kind domain E2E | dispatch | +| **cluster regression** | No | Nightly cron, `v*` tags, dispatch, or PRs that touch Helm / `scripts/run-cluster-ci.sh` / `scripts/bootstrap-namespace.sh` / this workflow file. Isolation-eval is **skipped** on pull_request. | +| **intercept product E2E** | No | Weekly cron, dispatch, or PRs/pushes matching [intercept-e2e.yml](../.github/workflows/intercept-e2e.yml) path filters (Helm, tasks, operator, orchestrator, pipeline, stacks, listed install/E2E scripts). | +| **Tekton Results** | No | Weekly cron, dispatch, or PRs that touch Results installers / `run-regression-agent-full.sh` / `run-lang-unit-tests.sh` | +| **Pages** | No | `main` pushes that touch `docs/**` | + +The Actions sidebar can still list **deleted** workflow names (compatibility matrix, demo validation, Graph/GUI Newman, static quality, supply-chain scan). Those files are gone; disable them in the repo **Actions → workflow → ⋯ → Disable**. **Dependabot Updates** is GitHub-managed, not a repo workflow. + +So: **not all tests run on every PR.** `--local-only` (including Java/PHP/Go) is the default gate. Playwright (management-gui) plus Kind Phase 2 plus Newman run on **cluster-regression** (nightly / tags / Helm-or-cluster-script PRs). Telepresence/mirrord and Results/Postgres are **weekly**, plus their own path filters. App PRs run a narrower, stack-scoped test stage. The existence of the intercept workflow is not proof that either backend is currently healthy. Treat only a recent successful matrix job and its retained diff --git a/docs/SCRIPTS.md b/docs/SCRIPTS.md index 55d583b..ed1a9cc 100644 --- a/docs/SCRIPTS.md +++ b/docs/SCRIPTS.md @@ -14,7 +14,7 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | [`run-regression.sh`](../scripts/run-regression.sh) | **Main regression driver** — pytest, vitest, isolation-eval protocol, Java/PHP/operator unit tests, Playwright, cluster/Newman, optional DAG verify & Results DB. See [REGRESSION.md](REGRESSION.md). | | [`run-lang-unit-tests.sh`](../scripts/run-lang-unit-tests.sh) | Maven (both baggage modules), PHPUnit, operator `go test ./internal/... ./api/...`. | | [`run-isolation-eval.sh`](../scripts/run-isolation-eval.sh) | Clone-vs-intercept harness: `--offline` plan CSV; `--cluster` Kind measurements. | -| [`run-cluster-ci.sh`](../scripts/run-cluster-ci.sh) | Kind cluster CI: isolation-eval `--cluster`, `stack-dag-verify`, Newman. Used by [cluster-regression.yml](../.github/workflows/cluster-regression.yml) (not on PRs). | +| [`run-cluster-ci.sh`](../scripts/run-cluster-ci.sh) | Kind cluster CI: isolation-eval `--cluster`, `stack-dag-verify`, Newman. Used by [cluster-regression.yml](../.github/workflows/cluster-regression.yml) (nightly / tags / Helm-or-cluster-script PRs; not every PR). | | [`run-regression-agent.sh`](../scripts/run-regression-agent.sh) | Streamed output for **agents** (Cursor); wraps tiers for iterative fix loops. [AGENT-REGRESSION.md](AGENT-REGRESSION.md) | | [`run-regression-agent-full.sh`](../scripts/run-regression-agent-full.sh) | Full agent-oriented run (heavier than default agent script). | | [`run-regression-stream.sh`](../scripts/run-regression-stream.sh) | Regression with streaming log-friendly behavior. | @@ -25,7 +25,7 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | [`verify-dag-phase1.sh`](../scripts/verify-dag-phase1.sh) | Local DAG structure checks (Phase 1). | | [`verify-dag-phase2.sh`](../scripts/verify-dag-phase2.sh) | Cluster: `stack-dag-verify` PipelineRun + CLI match (Phase 2). | | [`verify-m4-stacks-and-labels.sh`](../scripts/verify-m4-stacks-and-labels.sh) | M4-era stack/label checks (still useful for multi-namespace stacks). | -| [`run-artillery-variants.sh`](../scripts/run-artillery-variants.sh) | Load / Artillery variant runs (optional performance testing). | +| [`run-stack-tests-runners.sh`](../scripts/run-stack-tests-runners.sh) | Fixtures for Newman / Playwright / Artillery branches of `run-stack-tests`. | --- @@ -38,6 +38,12 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | [`cloud-agent-start-docker.sh`](../scripts/cloud-agent-start-docker.sh) | Cloud Agent `start`: dockerd with fuse-overlayfs; does not create Kind. | | [`install-kind-default-storage.sh`](../scripts/install-kind-default-storage.sh) | Default StorageClass for Kind. | | [`install-tekton.sh`](../scripts/install-tekton.sh) | Install Tekton Pipelines (and related baseline). | +| [`install-operator-kind.sh`](../scripts/install-operator-kind.sh) | Kind: CRDs + tekton-dag-operator image/Deployment. Required for `generate-run.sh --apply`. | +| [`install-operator-webhook-kind.sh`](../scripts/install-operator-webhook-kind.sh) | Kind: TLS certs + Stack ValidatingWebhookConfiguration (`failurePolicy: Fail`). | +| [`apply-stack-crs.sh`](../scripts/apply-stack-crs.sh) | Apply Stack (and Team) CRs from Git YAML into the cluster. | +| [`check-helm-chart.sh`](../scripts/check-helm-chart.sh) | `helm template` / package sanity for `helm/tekton-dag`. | +| [`check-markdown-links.py`](../scripts/check-markdown-links.py) | Relative Markdown link checker (run from `run-regression.sh`). | +| [`check-ci-policy.py`](../scripts/check-ci-policy.py) | Dependabot ignores + operator Go pin (run from `run-regression.sh`). | | [`install-tekton-dashboard.sh`](../scripts/install-tekton-dashboard.sh) | Install Tekton Dashboard. | | [`uninstall-tekton-dashboard.sh`](../scripts/uninstall-tekton-dashboard.sh) | Remove Tekton Dashboard. | | [`port-forward-tekton-dashboard.sh`](../scripts/port-forward-tekton-dashboard.sh) | `kubectl port-forward` to dashboard. | @@ -45,7 +51,6 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | [`install-postgres-kind.sh`](../scripts/install-postgres-kind.sh) | Postgres in Kind (Results / app DB). | | [`install-neo4j-kind.sh`](../scripts/install-neo4j-kind.sh) | Neo4j in Kind for graph features. | | [`bootstrap-namespace.sh`](../scripts/bootstrap-namespace.sh) | Bootstrap namespace resources with least-privilege pipeline RBAC. `--cluster-admin` is an explicit disposable-cluster escape hatch. | -| [`install-operator-webhook-kind.sh`](../scripts/install-operator-webhook-kind.sh) | Kind: TLS certs + Stack ValidatingWebhookConfiguration (`failurePolicy: Fail`). | --- @@ -55,6 +60,8 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m |--------|---------| | [`publish-build-images.sh`](../scripts/publish-build-images.sh) | Build/push **compile** images (polyglot builders). | | [`publish-orchestrator-image.sh`](../scripts/publish-orchestrator-image.sh) | Build/push **orchestrator** image. | +| [`publish-operator-image.sh`](../scripts/publish-operator-image.sh) | Build/push **operator** image. | +| [`publish-management-gui-image.sh`](../scripts/publish-management-gui-image.sh) | Build/push **management GUI** image. | | [`generate-run.sh`](../scripts/generate-run.sh) | Emit/apply a **StackRun** (operator). `--pipeline-run` was removed in M16. | | [`promote-pipelines.sh`](../scripts/promote-pipelines.sh) | Promote pipeline definitions across environments/namespaces. | | [`create-and-push-sample-repos.sh`](../scripts/create-and-push-sample-repos.sh) | Sample app repos for demos/regression. | @@ -71,7 +78,8 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | [`merge-pr.sh`](../scripts/merge-pr.sh) | Merge helper for test PRs. | | [`rerun-pr-from.sh`](../scripts/rerun-pr-from.sh) | Create a StackRun with `spec.continueFrom` to re-run `stack-pr-continue` from a failed PR. | | [`configure-github-webhooks.sh`](../scripts/configure-github-webhooks.sh) | Wire GitHub webhooks to EventListener. | -| [`run-e2e-with-intercepts.sh`](../scripts/run-e2e-with-intercepts.sh) | End-to-end with **telepresence** or **mirrord** intercept backend. | +| [`run-e2e-with-intercepts.sh`](../scripts/run-e2e-with-intercepts.sh) | Local Kind helper: bootstrap (optional) + PR pipeline with **telepresence** or **mirrord**. | +| [`run-product-intercept-e2e.sh`](../scripts/run-product-intercept-e2e.sh) | Authenticated trigger → StackRun → operator → PR PipelineRun → intercept traffic. Used by [intercept-e2e.yml](../.github/workflows/intercept-e2e.yml). | | [`run-all-setup-and-test.sh`](../scripts/run-all-setup-and-test.sh) | Broad setup + test orchestration (legacy-style “do a lot”). | --- @@ -109,8 +117,8 @@ Shared helpers live in [`scripts/common.sh`](../scripts/common.sh) (sourced by m | Location | Purpose | |----------|---------| -| [`docs/demos/generate-all.sh`](../docs/demos/generate-all.sh) | Regenerate Manim, VHS, TTS, and composed MP4s (M8 + M12.2 segments). | -| [`docs/demos/compose.sh`](../docs/demos/compose.sh) | FFmpeg: merge visuals + narration per segment. | +| [`docs/demos/generate-all.sh`](../docs/demos/generate-all.sh) | Thin wrapper: `docgen generate-all` (canonical: [demos/README.md](demos/README.md)). | +| [`docs/demos/compose.sh`](../docs/demos/compose.sh) | Thin wrapper: `docgen compose`. | See [milestones/milestone-8.md](../milestones/milestone-8.md) and [milestones/milestone-12.2.md](../milestones/milestone-12.2.md). diff --git a/docs/TESTING-AND-REGRESSION-OVERVIEW.md b/docs/TESTING-AND-REGRESSION-OVERVIEW.md index 42a31ae..0fb7fc8 100644 --- a/docs/TESTING-AND-REGRESSION-OVERVIEW.md +++ b/docs/TESTING-AND-REGRESSION-OVERVIEW.md @@ -2,7 +2,7 @@ This document is the **story** behind how tekton-dag is verified end-to-end. Use it for **onboarding**, **release checklists**, and **video voiceover**. Operational detail lives in [REGRESSION.md](REGRESSION.md), [AGENT-REGRESSION.md](AGENT-REGRESSION.md), and [scripts/run-regression.sh](../scripts/run-regression.sh). -**Scope:** An **application PR** only runs that stack’s declared tests inside **stack-pr-test**. The **`run-regression*.sh`** suites are **platform / system** checks (orchestrator, GUI, real PipelineRuns, Newman, Results). They are **not** all expected to run on every GitHub pull request unless you wire CI that way — see [REGRESSION.md](REGRESSION.md) § *Application PR pipeline vs platform (system) regression*. +**Scope:** An **application PR** only runs that stack’s declared tests inside **stack-pr-test**. The **`run-regression*.sh`** suites are **platform / system** checks (orchestrator, GUI, real PipelineRuns, Newman, Results). They are **not** all expected to run on every GitHub pull request — see [REGRESSION.md](REGRESSION.md) for the Actions trigger table (cheap local job every PR; Kind on schedule or path filters). ## Why regression is more than unit tests diff --git a/docs/argocd-architecture-guide.md b/docs/argocd-architecture-guide.md index 2676d42..91cafd1 100644 --- a/docs/argocd-architecture-guide.md +++ b/docs/argocd-architecture-guide.md @@ -126,7 +126,7 @@ tekton-dag - receives PR event - determines pipeline DAG -- creates PipelineRun +- creates a StackRun; the operator reconciles it to a PipelineRun ↓ @@ -150,7 +150,7 @@ tekton-dag --- -## 2 tekton-dag Generates PipelineRun +## 2 tekton-dag Creates a StackRun Example resource created dynamically: diff --git a/docs/c4-diagrams.md b/docs/c4-diagrams.md index 437a4a6..5c52c93 100644 --- a/docs/c4-diagrams.md +++ b/docs/c4-diagrams.md @@ -1,6 +1,6 @@ # C4 Architecture Diagrams — Tekton DAG (current code) -These diagrams reflect the **current** pipeline and task layout: platform repo (tekton-dag) with `stacks/` and `versions.yaml`; app repos are **separate** Git repos cloned per stack. Runs locally on Kind or in cloud; no AWS required. +These diagrams reflect the **current** pipeline and task layout: platform repo (tekton-dag) with `stacks/` and `versions.yaml`; app repos are **separate** Git repos cloned per stack. The control plane is CRD-primary: the orchestrator (or `generate-run.sh`) creates a **StackRun**; the operator reconciles it to a Tekton **PipelineRun**. Runs locally on Kind or in cloud. ## Level 1: System Context @@ -16,16 +16,16 @@ C4Context System(tekton_std, "Tekton DAG", "Universal pipeline: clone platform + app repos, resolve DAG, build per toolchain, deploy intercepts, validate, test, version. Local (Kind) or cloud.") System_Ext(github, "GitHub", "Platform repo (tekton-dag) + app repos (jmjava/tekton-dag-*). Webhooks or manual generate-run.sh") - System_Ext(registry, "Container Registry", "Local (localhost:5000) or ECR. Stores RC and release images") - System_Ext(k8s, "Kubernetes Cluster", "Kind or cloud. Runs app deployments, Telepresence intercepts") + System_Ext(registry, "Container Registry", "Local (localhost:5000) or ECR. Stores snapshot and release images") + System_Ext(k8s, "Kubernetes Cluster", "Kind or cloud. Runs app deployments, Telepresence or mirrord intercepts") System_Ext(argocd, "ArgoCD", "Optional GitOps; syncs from registry") System_Ext(argo_rollouts, "Argo Rollouts", "Optional blue/green production promotion") Rel(dev, github, "Opens PR / merges PR") - Rel(github, tekton_std, "Webhook or manual pipeline run") - Rel(tekton_std, registry, "Pushes RC and release images") - Rel(tekton_std, k8s, "Deploys PR pods, Telepresence intercepts") - Rel(tekton_std, github, "Pushes version bump commits") + Rel(github, tekton_std, "Webhook or StackRun via generate-run.sh / orchestrator") + Rel(tekton_std, registry, "Pushes snapshot and release images") + Rel(tekton_std, k8s, "Deploys PR pods and intercepts") + Rel(tekton_std, github, "Merge pipeline may push version-bump commits") Rel(platform, tekton_std, "Defines stacks, sets version overrides") Rel(registry, argocd, "ArgoCD syncs tagged images") Rel(argocd, argo_rollouts, "Triggers blue/green rollout") @@ -46,10 +46,18 @@ C4Container Container(event_listener, "EventListener", "Tekton Triggers", "Optional. Routes GitHub webhooks to PR or merge pipeline. Manual: generate-run.sh") - Container(pr_pipeline, "stack-pr-test", "Tekton Pipeline", "PR: fetch platform → resolve → clone-app-repos → bump RC → build → deploy intercepts → validate → test → push version → cleanup") + Container(orchestrator, "Orchestrator", "Flask", "Webhooks + API: creates StackRun CRs (bearer-auth mutations)") + + Container(operator, "Operator", "Go", "Reconciles Stack / StackRun / Team to PipelineRuns") + + Container(pr_pipeline, "stack-pr-test", "Tekton Pipeline", "PR: fetch platform → resolve → clone-app-repos → snapshot-tag → build changed app → deploy intercepts → validate → test → PR comment → cleanup") Container(merge_pipeline, "stack-merge-release", "Tekton Pipeline", "Merge: fetch → resolve → clone-app-repos → release version → build → tag-release-images → push version") + Container(bootstrap_pipeline, "stack-bootstrap", "Tekton Pipeline", "Bootstrap: deploy full stack with optional secrets/config injection") + + Container(promote_pipeline, "stack-promote", "Tekton Pipeline", "Promote: copy release images via stacks/registries.yaml") + Container(dag_verify, "stack-dag-verify", "Tekton Pipeline", "Local verification: fetch + resolve only (no build/deploy)") Container(stack_defs, "Stack Definitions", "stacks/*.yaml", "DAG graphs: apps, downstream, propagation, build tool per app") @@ -67,16 +75,22 @@ C4Container Rel(dev, github, "PR / merge") Rel(github, event_listener, "Webhook") + Rel(github, orchestrator, "Webhook / API") Rel(event_listener, pr_pipeline, "PR opened/sync") Rel(event_listener, merge_pipeline, "PR merged") + Rel(orchestrator, operator, "StackRun") + Rel(operator, pr_pipeline, "PipelineRun") + Rel(operator, merge_pipeline, "PipelineRun") + Rel(operator, bootstrap_pipeline, "PipelineRun") + Rel(operator, promote_pipeline, "PipelineRun") Rel(pr_pipeline, stack_defs, "Reads stack graph") - Rel(pr_pipeline, version_reg, "Reads/bumps RC version") + Rel(pr_pipeline, version_reg, "Reads versions (no bump)") Rel(merge_pipeline, stack_defs, "Reads stack graph") Rel(merge_pipeline, version_reg, "Promotes release, bumps next dev") Rel(event_listener, stack_registry, "Resolves repo → stack") - Rel(pr_pipeline, registry, "Pushes v0.1.0-rc.N images") + Rel(pr_pipeline, registry, "Pushes snapshot-tagged images") Rel(pr_pipeline, k8s, "Deploys intercept pods") - Rel(merge_pipeline, registry, "Pushes v0.1.0 release images") + Rel(merge_pipeline, registry, "Pushes release images") Rel(platform, stack_defs, "Defines/updates stacks") Rel(platform, version_reg, "Manual major/minor bumps") Rel(platform, scripts, "Queries graphs, triggers manual runs") @@ -98,17 +112,17 @@ C4Component Component(clone_apps, "clone-app-repos", "clone-app-repos", "Clones each app repo from stack .apps[].repo (e.g. jmjava/tekton-dag-vue-fe) into workspace/ via SSH") - Component(bump_rc, "bump-rc-version", "version-bump", "Increments RC in versions.yaml (0.1.0-rc.3 → rc.4); emits bumped-versions for image tags") + Component(snapshot, "pr-snapshot-tag", "pr-snapshot-tag", "Emits a snapshot image tag for the changed app (not a versions.yaml RC bump)") - Component(build, "build-apps", "build-stack-apps", "Per app: compile (npm/maven/gradle/composer/pip) then Kaniko containerize. Pushes RC-tagged images") + Component(build, "build-apps", "build-stack-apps", "Changed app only: compile (npm/maven/gradle/composer/pip) then Kaniko containerize. Pushes snapshot-tagged images") - Component(deploy, "deploy-intercepts", "deploy-stack-intercepts", "Deploys PR pods for build-apps, Telepresence intercept with header matching") + Component(deploy, "deploy-intercepts", "deploy-stack-intercepts", "Deploys PR pods for build-apps, Telepresence or mirrord intercept with header matching") Component(validate, "validate-propagation", "validate-stack-propagation", "Request through entry; verifies header reaches intercepted app(s)") Component(test, "run-tests", "run-stack-tests", "E2E through entry; per-app Postman/Playwright/Artillery") - Component(push_ver, "push-version-commit", "git-cli", "Pushes RC bump commit to platform repo") + Component(comment, "post-pr-comment", "post-pr-comment", "Posts test summary on the application PR") Component(cleanup, "cleanup", "cleanup-stack-pods", "Finally: deletes PR pods (always)") } @@ -123,18 +137,17 @@ C4Component Rel(resolve, stack_defs, "Reads stack YAML") Rel(resolve, version_reg, "Reads versions, overrides") Rel(resolve, clone_apps, "stack-json, build-apps") - Rel(clone_apps, bump_rc, "workspace with app sources") - Rel(bump_rc, version_reg, "Writes bumped RC") - Rel(bump_rc, build, "bumped-versions (image tags)") - Rel(build, registry, "Pushes v0.1.0-rc.N") + Rel(clone_apps, snapshot, "workspace with app sources") + Rel(snapshot, build, "snapshot image tag") + Rel(build, registry, "Pushes snapshot-tagged images") Rel(build, deploy, "built-images") Rel(deploy, k8s, "Creates PR pods + intercepts") Rel(deploy, validate, " ") Rel(validate, k8s, "Test request through chain") Rel(validate, test, " ") Rel(test, k8s, "Runs test suites") - Rel(test, push_ver, " ") - Rel(push_ver, github, "Pushes version commit") + Rel(test, comment, "test-summary") + Rel(comment, github, "PR comment") Rel(cleanup, k8s, "Deletes PR pods (finally)") ``` @@ -311,46 +324,29 @@ sequenceDiagram ## Dynamic Diagram: Version Lifecycle +PR runs use **snapshot image tags** and do not advance `stacks/versions.yaml`. Merge/release promotes the release tag and bumps the next development cycle. Optional `stack-promote` copies that release into another registry. + ```mermaid stateDiagram-v2 - [*] --> rc0: App onboarded
0.1.0-rc.0 - - rc0 --> rc1: PR #1 passes
bump RC - rc1 --> rc2: PR #2 passes
bump RC - rc2 --> rc3: PR #3 passes
bump RC - - rc3 --> released: PR merged
promote to 0.1.0 - released --> next_rc0: bump patch
0.1.1-rc.0 - - next_rc0 --> next_rc1: PR #4 passes - next_rc1 --> next_released: PR merged
promote to 0.1.1 - - state rc0 { - [*]: v0.1.0-rc.0 - } - state rc1 { - [*]: v0.1.0-rc.1 - } - state rc2 { - [*]: v0.1.0-rc.2 - } - state rc3 { - [*]: v0.1.0-rc.3 + [*] --> snapshot: PR opened
stack-pr-test + snapshot --> snapshot: more PRs
new snapshot tags + snapshot --> released: PR merged
stack-merge-release + released --> next_dev: bump next cycle
e.g. 0.1.1-rc.0 + next_dev --> snapshot: next PR + released --> promoted: stack-promote
copy to target registry + + state snapshot { + [*]: snapshot-tagged image
intercept tests, no versions.yaml bump } state released { [*]: v0.1.0
image tagged, pushed to registry } - state next_rc0 { - [*]: v0.1.1-rc.0 - } - state next_rc1 { - [*]: v0.1.1-rc.1 + state next_dev { + [*]: v0.1.1-rc.0 on versions.yaml } - state next_released { - [*]: v0.1.1
image tagged, pushed to registry + state promoted { + [*]: same release tag
in stacks/registries.yaml target } - - next_released --> [*]: available for
Argo Rollouts promotion ``` ## Dynamic Diagram: Build Toolchain Selection diff --git a/docs/research/README.md b/docs/research/README.md index e6b6e59..c6d74ab 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -14,11 +14,11 @@ It does **not** invent a new platform and it does **not** drive design. Architec | **Contribution claims** | Draft | [`contributions.md`](contributions.md) — honest about engineering vs. research novelty | | **Related work** | Draft | [`related-work.md`](related-work.md) + [`paper/refs.bib`](paper/refs.bib) | | **Multi-repo artifact map** | Draft | [`artifact-map.md`](artifact-map.md) | -| **Evaluation evidence** | Partial | [`evaluation.md`](evaluation.md) — PR CI is `--local-only`. Cluster job exists ([cluster-regression.yml](../../.github/workflows/cluster-regression.yml)) but is **not** on PRs; no site study | +| **Evaluation evidence** | Partial | [`evaluation.md`](evaluation.md) — PR CI is `--local-only`. Cluster, intercept, and Results jobs exist as scheduled/path-filtered workflows; cite only recorded artifacts | | **ACM artifact badges** | Partial | [`artifact-checklist.md`](artifact-checklist.md) — Available is blocked until Zenodo/Software Heritage DOI; Reusable needs a reviewer-timed Kind path | | **SEIP / full research track** | Not this cycle | Standing backlog: [seip-tasks.md](seip-tasks.md). Gate 0 is industrial context. No ICSE 2027 deadline. | -**Short answer: no, the testing work is not “all done.”** Local unit/static tests are PR-gated. Cluster Kind CI is nightly/manual. Intercept E2E and comparative studies are not gated. +**Short answer: no, the testing work is not “all done.”** Local unit/static tests are PR-gated. Kind cluster CI is nightly (plus Helm/cluster-script PRs). Intercept E2E and Results/Postgres are weekly workflows; live evidence is still required before claiming either is continuously verified. See [`evaluation.md`](evaluation.md) for counts from a live `--local-only` run. diff --git a/docs/research/evaluation.md b/docs/research/evaluation.md index e3339c0..2f084b0 100644 --- a/docs/research/evaluation.md +++ b/docs/research/evaluation.md @@ -2,9 +2,9 @@ Workshop and tool-demo reviewers accept **functional** evidence if claims stay inside it. Research-track and SEIP reviewers will not. This inventory lists what already exists in-tree so the paper does not invent numbers. -**Short answer: no, the testing work is not “all done.”** There is a real, passing *local* unit/static suite, now **gated on GitHub PRs**. This Cloud Agent Kind cluster ran isolation-eval, Phase 2, and Newman. Intercept E2E (S34) and GitHub `cluster-regression` artifacts have not. Comparative studies a research PC would ask for remain incomplete. +**Short answer: no, the testing work is not “all done.”** There is a real, passing *local* unit/static suite, now **gated on GitHub PRs**. Kind cluster-regression, intercept E2E, and Results/Postgres jobs exist as scheduled (and path-filtered) GitHub Actions workflows. Comparative studies a research PC would ask for remain incomplete. Do not tell reviewers intercepts are continuously verified without a recent retained matrix artifact. -## What actually ran (this packaging branch) +## What actually ran (packaging snapshot, 2026-09-07) On 2026-09-07, `bash scripts/run-regression-stream.sh --local-only --require-lang-tests` is the CI path (Phase 1 + pytest + vitest + isolation-eval protocol + Maven + PHPUnit + operator `go test`). Playwright later ran locally (**69 passed**, Vite only). Nested Docker + Kind: @@ -12,14 +12,14 @@ On 2026-09-07, `bash scripts/run-regression-stream.sh --local-only --require-lan - `stack-dag-verify` Phase 2: **Succeeded**. - Newman vs in-cluster orchestrator: **18 requests / 36 assertions, 0 failed** (`run-cluster-ci.sh --skip-isolation --skip-phase2` after S39; `kind load` warned overlayfs on this nested VM, image came from `localhost:5000`). - Operator Kind soak (`STACKRUN_VIA_CRD=true`): **18/18 Newman**; **6/6** StackRuns received a `status.pipelineRunName` and a PipelineRun labeled `tektondag.io/stackrun` (bootstrap ×2, pr ×2, merge, promote). Some PipelineRuns then hit Tekton `CouldntGetTask` / `ResolvingTaskRef` (task catalog on this cluster, not operator create). `scripts/install-operator-kind.sh` + `WAIT_STACKRUN_RECONCILE=1`. -- Intercept E2E (S34) was **not** run. GitHub `cluster-regression` was **not** dispatched (no Actions artifact). +- Intercept E2E (S34) was **not** run on that day. GitHub `cluster-regression` had **not** been dispatched yet (no Actions artifact from that session). | Suite | Collected / result | |-------|-------------------| | Phase 1 DAG (`verify-dag-phase1.sh`) | PASSED (stack-one, stack-two-vendor, single-app, single-flask-app) | -| pytest orchestrator | **105 passed** (README still says 62) | -| pytest `tekton-dag-common` | **47 passed** (README still says 14) | -| pytest management-gui backend | **61 passed** (README still says 56) | +| pytest orchestrator | **105 passed** (current tree is 108 `test_` functions) | +| pytest `tekton-dag-common` | **47 passed** (current tree is 89 `test_` functions) | +| pytest management-gui backend | **61 passed** (current tree is 68 `test_` functions) | | pytest baggage-python | **17 passed** | | pytest isolation-eval | **10 passed** | | vitest baggage-node | **15 passed** | @@ -27,22 +27,34 @@ On 2026-09-07, `bash scripts/run-regression-stream.sh --local-only --require-lan | Maven Java baggage | both modules OK (`--require-lang-tests`) | | operator `go test` | `internal/pipeline` + `internal/controller` OK | -`--local-only` **skips** Playwright on purpose. This environment ran `npx playwright test` in `management-gui/frontend`: **69 passed** (Vite only, no cluster). Kind isolation-eval **6/6**, Phase 2 **Succeeded**, Newman **18/18 requests**, operator soak **6/6** StackRun→PipelineRun. S34 intercept E2E was not run. +`--local-only` **skips** Playwright on purpose. That environment ran `npx playwright test` in `management-gui/frontend`: **69 passed** (Vite only, no cluster). Kind isolation-eval **6/6**, Phase 2 **Succeeded**, Newman **18/18 requests**, operator soak **6/6** StackRun→PipelineRun. S34 intercept E2E was not run that day. + +## Current automation (2026-09-16) + +| Workflow | Role | +|----------|------| +| `local-regression.yml` | Every PR / `main` push: `--local-only --require-lang-tests` | +| `cluster-regression.yml` | Nightly + tags + Helm/cluster-script PRs | +| `intercept-e2e.yml` | Weekly Telepresence + mirrord product path | +| `results-regression.yml` | Weekly Results/Postgres + `run-regression-agent-full.sh` | +| `operator.yml` | `operator/**` PRs: unit/envtest + Kind domain E2E | + +Cite a **retained Actions artifact**, not the workflow file, when claiming cluster or intercept verification. ## What exists but is *not* CI-gated on every PR -`--local-only --require-lang-tests` **does** run on pull requests ([`.github/workflows/local-regression.yml`](../../.github/workflows/local-regression.yml)). The suites below are **not** on PRs; they run on [`.github/workflows/cluster-regression.yml`](../../.github/workflows/cluster-regression.yml) (nightly / dispatch / tags) unless noted. +`--local-only --require-lang-tests` **does** run on pull requests ([`.github/workflows/local-regression.yml`](../../.github/workflows/local-regression.yml)). The suites below are **not** on every PR; they run on scheduled or path-filtered Kind workflows (see [REGRESSION.md](../REGRESSION.md)). | Suite | Approx. cases | How you run it today | |-------|---------------|----------------------| -| Go `operator/` e2e | present | Kind soak via `run-cluster-ci.sh` (operator on by default; `--skip-operator` to opt out) | -| Playwright GUI | 69 `test(` | cluster-regression Playwright job; `npx playwright test` locally | -| Newman orchestrator | collection grew past the README “15 requests / 30 assertions” | `run-cluster-ci.sh` (live orchestrator Service) | -| Newman graph (M9) | ~10 requests in `tests/postman/graph-tests.json` | `run-cluster-ci.sh --with-graph` | -| Newman management GUI | optional | `--gui-newman` (not in cluster-regression) | +| Go `operator/` e2e | present | Kind soak via `run-cluster-ci.sh` (operator on by default; `--skip-operator` to opt out); operator.yml Kind domain job on `operator/**` | +| Playwright GUI | 70 `test(` | cluster-regression Playwright job; `npx playwright test` locally | +| Newman orchestrator | 20 requests / 38 assertions | `run-cluster-ci.sh` (live orchestrator Service) | +| Newman graph (M9) | 10 requests / 36 assertions in `tests/postman/graph-tests.json` | `run-cluster-ci.sh --with-graph` | +| Newman management GUI | 24 requests / 54 assertions | `--gui-newman` (not in cluster-regression) | | `stack-dag-verify` PipelineRun | one real Tekton run | `run-cluster-ci.sh` / `--require-dag-verify` | -| Intercept E2E both backends | scripts exist | S34: `run-e2e-with-intercepts.sh` | -| Kind clone-vs-intercept **measurements** | CSV rows | `run-cluster-ci.sh` / `run-isolation-eval.sh --cluster` | +| Intercept E2E both backends | weekly workflow + scripts | [`intercept-e2e.yml`](../../.github/workflows/intercept-e2e.yml) / `run-product-intercept-e2e.sh` (legacy helper: `run-e2e-with-intercepts.sh`) | +| Kind clone-vs-intercept **measurements** | CSV rows | `run-cluster-ci.sh` / `run-isolation-eval.sh --cluster` (skipped on cluster-regression **pull_request**) | | Sample **app-repo** tests (Newman/Playwright/Artillery in the six `tekton-dag-*` repos) | declared in stack YAML | only during `stack-pr-test`, not platform regression | C2 (polyglot baggage **unit** tests) is PR-gated for Python, Node, Java, and PHP. In-cluster hop validation (`validate-stack-propagation`) remains a pipeline task. C3 isolation **probes on Kind** run in cluster-regression (dummy HTTP stacks, not Telepresence). C4 (test-plan) has mocked pytest + a Postman collection; `milestones/milestone-9.md` still says **Planned** even though `query-test-plan` is wired in `stack-pr-pipeline.yaml`. @@ -61,8 +73,10 @@ C2 (polyglot baggage **unit** tests) is PR-gated for Python, Node, Java, and PHP Same header-filter idea is documented as parity with Telepresence `--http-match`. E2E scripts exist for **both** backends: ```bash -./scripts/run-e2e-with-intercepts.sh --intercept-backend telepresence -./scripts/run-e2e-with-intercepts.sh --intercept-backend mirrord +./scripts/run-product-intercept-e2e.sh --intercept-backend telepresence +./scripts/run-product-intercept-e2e.sh --intercept-backend mirrord +# local Kind helper: +./scripts/run-e2e-with-intercepts.sh --intercept-backend telepresence --skip-bootstrap ``` **How to report:** “In a controlled Kind deployment of the three-app exemplar, unmatched requests remained on the baseline replica; matched requests were stolen (5/5 each).” Cite [`docs/mirrord-poc-results.md`](../mirrord-poc-results.md) for that smoke, and `run-isolation-eval.sh --cluster` for dummy-stack probes (this Cloud Agent: 6/6 `isolation_ok`). Neither is a site measurement (S21). @@ -114,7 +128,7 @@ Code exists (`/api/test-plan`, `query-test-plan` task, Neo4j client, Postman gra | Missing study **or** missing engineering gate | Why it matters | Minimum next step | |-----------------------------------------------|----------------|-------------------| | Recorded cluster-regression log on a tag | C1/C3 need a live PipelineRun **artifact**, not only a workflow file | `workflow_dispatch` on [cluster-regression.yml](../../.github/workflows/cluster-regression.yml) or push a `v*` tag after merge | -| Intercept E2E both backends | C3 beyond dummy-stack probes | S34: `run-e2e-with-intercepts.sh` on a chosen tag | +| Intercept E2E both backends | C3 beyond dummy-stack probes | Live `intercept-e2e.yml` matrix artifact (or `run-product-intercept-e2e.sh` on a chosen tag) | | Cost/time vs. namespace-per-PR (**measured**, ≥3 repeats) | Central *research* claim of routing vs. cloning | Dispatch cluster CI with `isolation_repeats=3`; cite the artifact CSV, not the plan | | Concurrent PRs | Multi-tenant intercepts | Two PRs, two headers, no cross-steal | | Developer study | Demo “envisioned users” | Even n=3 think-aloud | @@ -131,7 +145,7 @@ Until cluster E2E and the *studies* exist, keep the paper in **tool / experience **External.** Six first-party sample repos under one GitHub user are not an independent software ecosystem. Polyglot coverage is real (Vue, Spring, Flask, PHP) but all examples were written to fit the platform. -**Reliability.** Cluster E2E is timing-sensitive (image pulls, intercept attach). Report the **script** (`run-e2e-with-intercepts.sh`) rather than a single laptop run as the result. +**Reliability.** Cluster E2E is timing-sensitive (image pulls, intercept attach). Report the **script** (`run-product-intercept-e2e.sh` / `run-e2e-with-intercepts.sh`) and a retained Actions artifact rather than a single laptop run as the result. ## Carbon / sustainability (ICSE encourages a mention) diff --git a/docs/research/gaps.md b/docs/research/gaps.md index ed44169..bfa3708 100644 --- a/docs/research/gaps.md +++ b/docs/research/gaps.md @@ -39,11 +39,11 @@ See **[seip-tasks.md](seip-tasks.md)**. No ICSE 2027 date. Gate 0 is a real site Engineering completeness is separate from the HotCRP PDF. As of this branch: - `--local-only --require-lang-tests` **passes** and is **gated** by [`.github/workflows/local-regression.yml`](../../.github/workflows/local-regression.yml). -- Playwright, Newman, Phase 2, and Kind isolation **measurements** are gated by [`.github/workflows/cluster-regression.yml`](../../.github/workflows/cluster-regression.yml) (**not** on pull requests). A **GitHub Actions** log exists only after that workflow has run (dispatch / nightly on default branch / `v*` tag). This Cloud Agent recorded a local Kind run (isolation 6/6, Phase 2 Succeeded, Newman 18/18); that is not an Actions artifact. -- Intercept E2E (Telepresence + mirrord) is still **out of band** (S34). -- README milestone test counts are **stale**. +- Playwright, Newman, Phase 2, and Kind isolation **measurements** are gated by [`.github/workflows/cluster-regression.yml`](../../.github/workflows/cluster-regression.yml) (nightly / dispatch / `v*` tags / Helm-or-cluster-script PRs — **not** every PR). Isolation-eval is skipped on those PRs. +- Intercept E2E automation is [`.github/workflows/intercept-e2e.yml`](../../.github/workflows/intercept-e2e.yml) (weekly + path filters). Live matrix evidence is still required ([M17.3](../../milestones/milestone-17.md)). +- README milestone test counts were refreshed in the 2026-09 documentation review (orchestrator 108, common 89, GUI backend 68, Playwright 70, Newman 20/38). -Until a **recorded** cluster-regression artifact exists for a tagged commit, do not tell reviewers the *platform* (Tekton/intercepts) is continuously verified on every PR. Local unit/static CI plus an on-demand Kind job is the honest claim. +Until a **recorded** cluster-regression artifact exists for a tagged commit, do not tell reviewers the *platform* (Tekton/intercepts) is continuously verified on every PR. Local unit/static CI plus scheduled Kind jobs is the honest claim. **Kind cluster-ci design debt (2026-09-07 run, not site evidence):** @@ -51,12 +51,12 @@ Until a **recorded** cluster-regression artifact exists for a tagged commit, do |----|---------|--------| | **S38** | `install-tekton.sh` tracked `latest`; v1.6 rejected `taskRef.name: $(params.pre-build-task)`. Hooks use cluster resolver; pin Pipelines/Triggers. | Landed | | **S39** | `common.sh` defaulted host `localhost:5001` while `kind-with-registry.sh` listens on **`:5000`**. Newman image push failed (`connection refused` on 5001). Phase 2 `stack-dag-verify` **Succeeded**. | Landed | -| **S33 local** | `run-cluster-ci.sh` on this Cloud Agent Kind: isolation 6/6, Phase 2 Succeeded, Newman 18 req / 36 asserts. `kind load` overlayfs warning (nested Docker); registry pull worked. | Local log only; GHA not dispatched | -| **M14 soak** | Default-on: Helm `operator.enabled=true`, Kind `STACKRUN_VIA_CRD=true`. Newman 18/18 and 6/6 StackRun → PipelineRun. Team `default` Ready; Stacks `valid` + `injectionNamespace`. Some PRs then `CouldntGetTask` (task catalog). | Landed (product); GHA cluster-regression still not the proof | +| **S33 local** | `run-cluster-ci.sh` on this Cloud Agent Kind: isolation 6/6, Phase 2 Succeeded, Newman 18 req / 36 asserts. `kind load` overlayfs warning (nested Docker); registry pull worked. | Local log (2026-09-07); GHA cluster-regression now exists as a nightly/path-filtered workflow | +| **M14 soak** | Default-on: Helm `operator.enabled=true`, Kind `STACKRUN_VIA_CRD=true`. Newman 18/18 and 6/6 StackRun → PipelineRun. Team `default` Ready; Stacks `valid` + `injectionNamespace`. Some PRs then `CouldntGetTask` (task catalog). | Landed (product); GHA cluster-regression is the scheduled proof path | | **M15** | Idempotent StackRun→PipelineRun, GHA `--skip-operator`, Triggers `prNumber`, Flask/GUI promote wait, soak `ready==total`. | Landed [#19](https://github.com/jmjava/tekton-dag/pull/19) — [milestone-15.md](../../milestones/milestone-15.md) | -| **M16** | Team CR overlay, `continueFrom`, Kind webhook installer, escape hatches retired, spoken demos 01/08/18/19 rebuilt. | Code-complete [#20](https://github.com/jmjava/tekton-dag/pull/20)–[#24](https://github.com/jmjava/tekton-dag/pull/24); S34 parked — [milestone-16.md](../../milestones/milestone-16.md) | -| **S34** | Phase 2 ≠ intercept E2E. Dummy isolation-eval ≠ Telepresence/mirrord. Bootstrap skipped SSH/GitHub secrets. | Open | -| Lessons | Dual-port registry, Kaniko stdout vs results, intercept vs Pod Security: see [seip/lessons-learned.md](seip/lessons-learned.md). | Registry default paid in S39; intercept PSS still S34 | +| **M16** | Team CR overlay, `continueFrom`, Kind webhook installer, escape hatches retired, spoken demos 01/08/18/19 rebuilt. | Code-complete [#20](https://github.com/jmjava/tekton-dag/pull/20)–[#24](https://github.com/jmjava/tekton-dag/pull/24); intercept follow-on is M17.3 — [milestone-16.md](../../milestones/milestone-16.md) | +| **S34 / M17.3** | Phase 2 ≠ intercept E2E. Dummy isolation-eval ≠ Telepresence/mirrord. Weekly `intercept-e2e.yml` exists; live matrix evidence still required. | Automation in-tree; checkbox open until a retained artifact | +| Lessons | Dual-port registry, Kaniko stdout vs results, intercept vs Pod Security: see [seip/lessons-learned.md](seip/lessons-learned.md). | Registry default paid in S39; intercept PSS still tracked with M17.3 | ## Must-not-do diff --git a/docs/research/seip-tasks.md b/docs/research/seip-tasks.md index f290d28..1a04bfe 100644 --- a/docs/research/seip-tasks.md +++ b/docs/research/seip-tasks.md @@ -59,9 +59,9 @@ Each slice is independently useful even if Gate 0 is still open. - [x] **S30** GitHub Actions workflow: `run-regression.sh --local-only` on pull requests (Phase 1 + pytest + vitest). Badge or README line that cites **CI**, not milestone tables. - [x] **S31** Add Java (`mvn test` in both baggage modules) and PHPUnit to that same driver or a second CI job. - [x] **S32** Add `go test ./internal/...` for `operator/`. -- [x] **S33** Nightly or manual cluster job: Playwright + Newman + `stack-dag-verify` (and optionally `run-isolation-eval.sh --cluster`). Record a log on a tagged release; do not pretend every PR ran Kind. Implementation: [`.github/workflows/cluster-regression.yml`](../../.github/workflows/cluster-regression.yml) (`workflow_dispatch`, nightly on default branch, `v*` tags) calling [`scripts/run-cluster-ci.sh`](../../scripts/run-cluster-ci.sh). Artifacts: `cluster-ci.log` + measured CSV. **Not** on pull requests. **Local Kind (2026-09-07 Cloud Agent):** isolation 6/6, Phase 2 Succeeded, Newman 18/18. GitHub Actions cluster-regression has **not** been dispatched. -- [ ] **S34** On a chosen tag: `run-e2e-with-intercepts.sh` for Telepresence **and** mirrord; attach logs to the release. **Not** implied by Phase 2 (`stack-dag-verify` only clones this repo and resolves the DAG). Needs git-ssh (or HTTPS) for app repos, intercept Traffic Manager, Pod Security exceptions. -- [ ] **S35** Refresh README / milestone test counts from CI (orchestrator is already 105 pytest, not 62). +- [x] **S33** Nightly or manual cluster job: Playwright + Newman + `stack-dag-verify` (and optionally `run-isolation-eval.sh --cluster`). Record a log on a tagged release; do not pretend every PR ran Kind. Implementation: [`.github/workflows/cluster-regression.yml`](../../.github/workflows/cluster-regression.yml) (`workflow_dispatch`, nightly on default branch, `v*` tags, plus Helm/cluster-script PRs) calling [`scripts/run-cluster-ci.sh`](../../scripts/run-cluster-ci.sh). Artifacts: `cluster-ci.log` + measured CSV. **Local Kind (2026-09-07 Cloud Agent):** isolation 6/6, Phase 2 Succeeded, Newman 18/18. +- [ ] **S34** Live Telepresence **and** mirrord product-path evidence. Automation: [`.github/workflows/intercept-e2e.yml`](../../.github/workflows/intercept-e2e.yml) + [`scripts/run-product-intercept-e2e.sh`](../../scripts/run-product-intercept-e2e.sh). **Not** implied by Phase 2 (`stack-dag-verify` only clones this repo and resolves the DAG). Needs git-ssh (or HTTPS) for app repos, intercept Traffic Manager, Pod Security exceptions. Keep unchecked until a retained matrix artifact exists. +- [x] **S35** Refresh README / milestone test counts from CI (orchestrator 108, common 89, GUI backend 68, Playwright 70; Newman 20/38). Counts still drift — prefer generating them over hand-edits (M17.20). - [ ] **S36** License + `CITATION.cff` already landed; add a Zenodo DOI when you freeze a “paper artifact” tag (any year). - [x] **S37** Scripted Kind isolation harness: clone-vs-intercept, stack width, probes, CSV (`scripts/run-isolation-eval.sh`). Offline plan is in `--local-only`; `--cluster` is measured Kind data, **not** site evidence (S21). - [x] **S38** Pin Tekton Pipelines / Triggers / git-clone (stop `…/latest/release.yaml`). Keep hook Tasks on a contract that `kubectl apply` accepts on that pin (cluster resolver, not `taskRef.name: $(params.*)`). Pytest must fail if a pipeline reintroduces param substitution in `taskRef.name`. @@ -114,7 +114,7 @@ Do this only when you have picked a specific year/track. Not now. 1. **S00–S03** if you have even a short window with the site — or skip SEIP and stay on the artifact (C). 2. **S38 / S39** (Tekton pin + Kind registry defaults) when cluster-ci is the work in front of you — they unblock Newman and stop `latest` from breaking hook pipelines. -3. **S34** (intercept E2E on a chosen tag) when you next have a Kind environment with Telepresence/mirrord **and** app-repo clone credentials; S30–S33 and S37 already landed. Trigger cluster CI with **workflow_dispatch** on this branch until it is the default. +3. **S34 / M17.3** (live intercept E2E artifact) when you next have a Kind environment with Telepresence/mirrord **and** app-repo clone credentials; S30–S33 and S37 already landed. The weekly workflow is `intercept-e2e.yml`. 4. **S11 / S24 / lessons-learned** whenever you hit a real incident; one paragraph per incident is enough. 5. **S20–S22** once a metrics window exists (needs the site or a real staging fleet). 6. **S40+** last. Prose without B is what gets rejected. diff --git a/helm/tekton-dag/README.md b/helm/tekton-dag/README.md index ba3239c..afc5497 100644 --- a/helm/tekton-dag/README.md +++ b/helm/tekton-dag/README.md @@ -43,6 +43,7 @@ Use `-f` with a per-team values file (for example `teams//values.yaml`) wh | `templates/_helpers.tpl` | Labels, names, service account helper | | `templates/orchestration-deployment.yaml` | Orchestrator Deployment + Service (optional) | | `templates/rbac.yaml` | ServiceAccount and optional cluster-admin binding | +| `templates/operator.yaml` | Go operator Deployment (when `operator.enabled`, default true) | | `templates/pipelines/pipelines.yaml` | Tekton pipelines/triggers from packaged `raw/pipelines/*.yaml` | | `templates/tasks/tasks.yaml` | Tekton tasks from packaged `raw/tasks/*.yaml` | | `templates/configmap-stacks.yaml` | `tekton-dag-stacks`: all `raw/stacks/*.yaml` | diff --git a/milestones/milestone-16.md b/milestones/milestone-16.md index 0f81fb3..ab67c8b 100644 --- a/milestones/milestone-16.md +++ b/milestones/milestone-16.md @@ -1,6 +1,6 @@ # Milestone 16 — Control-plane follow-ons -**Status:** Code-complete except parked S34 (M15 [#19](https://github.com/jmjava/tekton-dag/pull/19); overlay / continueFrom / webhook [#20](https://github.com/jmjava/tekton-dag/pull/20); escape hatches [#21](https://github.com/jmjava/tekton-dag/pull/21); hint polish [#22](https://github.com/jmjava/tekton-dag/pull/22); unused create helpers [#23](https://github.com/jmjava/tekton-dag/pull/23); spoken demo rebuild [#24](https://github.com/jmjava/tekton-dag/pull/24)). +**Status:** Code-complete except parked S34 *as an M16 slice* (M15 [#19](https://github.com/jmjava/tekton-dag/pull/19); overlay / continueFrom / webhook [#20](https://github.com/jmjava/tekton-dag/pull/20); escape hatches [#21](https://github.com/jmjava/tekton-dag/pull/21); hint polish [#22](https://github.com/jmjava/tekton-dag/pull/22); unused create helpers [#23](https://github.com/jmjava/tekton-dag/pull/23); spoken demo rebuild [#24](https://github.com/jmjava/tekton-dag/pull/24)). Intercept product automation moved to [M17.3](milestone-17.md) (`.github/workflows/intercept-e2e.yml`); do not treat that workflow as proof that both backends are currently healthy. **Goal:** Finish the remaining dual sources of truth after M15 hygiene. Still **no new CRD kinds**. @@ -11,7 +11,7 @@ - [x] Retire `--pipeline-run` / Flask `STACKRUN_VIA_CRD=false` (Flask always creates StackRuns; Helm env stays `"true"` for mixed-image rollouts; `--skip-operator` no longer flips the flag) - [x] `spec.continueFrom` on StackRun + operator `stack-pr-continue` builder; `rerun-pr-from.sh` creates a StackRun - [x] Demo **hints** already describe default-on; rebuilt **spoken** `docs/demos/narration/*.md` + MP4s via TTS + Manim + `docgen compose` + `docgen validate --pre-push` (segments 01, 08, 18, 19) — [#24](https://github.com/jmjava/tekton-dag/pull/24) -- [ ] S34 intercept E2E remains a separate stop — do not start unless explicitly requested +- [ ] S34 intercept E2E remains a separate stop in **M16** — automation now lives under M17.3; do not start a new E2E design unless explicitly requested ## Exit criteria diff --git a/milestones/milestone-17.md b/milestones/milestone-17.md index 0fa7582..35140fd 100644 --- a/milestones/milestone-17.md +++ b/milestones/milestone-17.md @@ -43,12 +43,17 @@ regression criteria in `docs/AGENT-REGRESSION.md` are satisfied. PR PipelineRun → build → intercept route → app tests → cleanup. - Cover Telepresence and mirrord over an explicit cadence; neither may be described as continuously verified without current evidence. - - Acceptance: artifacts retain PipelineRun/TaskRun logs and traffic evidence. + - In-tree: [`.github/workflows/intercept-e2e.yml`](../.github/workflows/intercept-e2e.yml) + + [`scripts/run-product-intercept-e2e.sh`](../scripts/run-product-intercept-e2e.sh). + - Acceptance: artifacts retain PipelineRun/TaskRun logs and traffic evidence + from a **live** matrix run (local automation green is not enough). - [ ] **M17.4 Add scheduled Tekton Results verification** - Install Results/Postgres and run the strict Results DB verification. - - Acceptance: `run-regression-agent-full.sh` equivalent exits zero and uploads - diagnostic artifacts on failure. + - In-tree: [`.github/workflows/results-regression.yml`](../.github/workflows/results-regression.yml) + (pinned Results v0.20.0 + ephemeral Postgres + `run-regression-agent-full.sh`). + - Acceptance: a **live** `run-regression-agent-full.sh` equivalent exits zero + and uploads diagnostic artifacts on failure. ## P1 — CI gates and operator assurance diff --git a/pipeline/stack-pr-pipeline.yaml b/pipeline/stack-pr-pipeline.yaml index b81efea..824a29d 100644 --- a/pipeline/stack-pr-pipeline.yaml +++ b/pipeline/stack-pr-pipeline.yaml @@ -15,7 +15,7 @@ spec: 2. Clones and builds ONLY the changed app (snapshot tag) — no build of other apps 3. Deploys intercepts only for that app; rest of stack keeps running as-is 4. Validates header propagation, runs test suites - 5. After tests pass: bumps RC version, pushes version commit + 5. Posts a PR comment; does **not** bump `versions.yaml` (snapshot image tags only) 6. Cleans up PR pods (always) PR builds use snapshot tagging so they are never confused with real releases. diff --git a/scripts/README.md b/scripts/README.md index 3980320..31fd572 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,6 +8,7 @@ This directory contains **bash** automation for Kind/Tekton setup, image builds, - **Shared shell API:** `common.sh` (sourced by install/publish/regression scripts) - **Primary regression:** `./scripts/run-regression.sh` — details in [docs/REGRESSION.md](../docs/REGRESSION.md) - **Cursor / agents:** `./scripts/run-regression-agent.sh` — [docs/AGENT-REGRESSION.md](../docs/AGENT-REGRESSION.md) +- **Product intercept E2E:** `./scripts/run-product-intercept-e2e.sh` ## Obsolete scripts @@ -15,4 +16,4 @@ Retired scripts are moved to **`scripts/archive/`** (see [scripts/archive/README ## Demo video generation -Demo pipelines live under **`docs/demos/`** (e.g. `generate-all.sh`, `compose.sh`), not here. +Demo pipelines live under **`docs/demos/`**. Canonical interface is the **`docgen`** CLI (`docgen generate-all`, `docgen compose`). The `.sh` scripts there are thin wrappers. diff --git a/scripts/check-markdown-links.py b/scripts/check-markdown-links.py new file mode 100644 index 0000000..8212f90 --- /dev/null +++ b/scripts/check-markdown-links.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Fail if relative Markdown links point at missing repo files. + +Skips fenced code blocks (so example paths are not treated as links), +http(s)/mailto targets, and historical trees that are not maintained. +""" + +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path +from urllib.parse import unquote, urlparse + +ROOT = Path(__file__).resolve().parents[1] + +SKIP_DIR_NAMES = { + ".git", + "node_modules", + ".venv", + "venv", + "__pycache__", + ".pytest_cache", +} + +# Not rewritten as product docs; broken historical links should not fail CI. +SKIP_PREFIXES = ( + "session-notes/", + "docs/archive/", + "documentation-generator/", +) + +FENCE_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") +LINK_RE = re.compile(r"(? list[Path]: + files: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(ROOT): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIR_NAMES and not d.startswith(".")] + for name in filenames: + if name.endswith(".md"): + files.append(Path(dirpath) / name) + return sorted(files) + + +def _strip_fences(text: str) -> str: + """Replace fenced code blocks with blank lines so line numbers stay aligned.""" + out: list[str] = [] + fence: str | None = None + for line in text.splitlines(keepends=True): + match = FENCE_RE.match(line.rstrip("\n")) + if match: + marker = match.group(2)[0] + n = len(match.group(2)) + token = marker * n + if fence is None: + fence = token + out.append("\n" if line.endswith("\n") else "") + continue + if line.lstrip().startswith(fence): + fence = None + out.append("\n" if line.endswith("\n") else "") + continue + if fence is not None: + out.append("\n" if line.endswith("\n") else "") + else: + out.append(line) + return "".join(out) + + +def _relative_target(url: str) -> str | None: + raw = url.strip() + if raw.startswith("<") and raw.endswith(">"): + raw = raw[1:-1].strip() + if not raw: + return None + # Optional title: [text](path "title") + if raw.startswith('"') or raw.startswith("'"): + return None + if " " in raw and (raw.endswith('"') or raw.endswith("'")): + raw = raw.rsplit(" ", 1)[0] + parsed = urlparse(raw) + if parsed.scheme in {"http", "https", "mailto", "tel", "data"}: + return None + if raw.startswith("//"): + return None + path_part = raw.split("#", 1)[0] + path_part = unquote(path_part).strip() + if not path_part: + return None # same-file anchor + if path_part.lower() in PLACEHOLDER_TARGETS: + return None + return path_part + + +def main() -> int: + broken: list[str] = [] + checked = 0 + for md in _iter_markdown(): + rel = md.relative_to(ROOT).as_posix() + if rel.startswith(SKIP_PREFIXES): + continue + text = _strip_fences(md.read_text(encoding="utf-8", errors="replace")) + for match in LINK_RE.finditer(text): + target = _relative_target(match.group(2)) + if target is None: + continue + checked += 1 + dest = (md.parent / target).resolve() + try: + dest.relative_to(ROOT.resolve()) + except ValueError: + broken.append(f"{rel}: {target} (outside repo)") + continue + if not dest.exists(): + broken.append(f"{rel}: {target}") + + if broken: + print(f"Broken relative Markdown links ({len(broken)}):", file=sys.stderr) + for item in broken: + print(f" {item}", file=sys.stderr) + return 1 + print(f"Markdown link check passed ({checked} relative links)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-regression.sh b/scripts/run-regression.sh index d63a307..c983edf 100755 --- a/scripts/run-regression.sh +++ b/scripts/run-regression.sh @@ -125,6 +125,10 @@ echo "" echo ">>> CI policy: Dependabot ignores + operator Go pin — scripts/check-ci-policy.py" python3 "$SCRIPT_DIR/check-ci-policy.py" +echo "" +echo ">>> Docs: relative Markdown links — scripts/check-markdown-links.py" +python3 "$SCRIPT_DIR/check-markdown-links.py" + echo "" echo ">>> Phase 1: DAG verification (no cluster) — scripts/verify-dag-phase1.sh" ensure_mikefarah_yq