diff --git a/.github/workflows/vscode-e2e.yml b/.github/workflows/vscode-e2e.yml index 09078d19028..60b385cb5d5 100644 --- a/.github/workflows/vscode-e2e.yml +++ b/.github/workflows/vscode-e2e.yml @@ -138,10 +138,14 @@ jobs: - name: Build extension run: pnpm turbo run build:extension --cache-dir=.turbo - - name: Compile E2E tests + - name: Compile ExTester E2E tests working-directory: apps/vs-code-designer run: npx tsup --config tsup.e2e.test.config.ts + - name: Compile @vscode/test-cli E2E tests + working-directory: apps/vs-code-designer + run: pnpm run test:e2e-cli:compile + - name: Tar build artifacts # Tar preserves symlinks/permissions and dramatically speeds upload # vs uploading thousands of small files in node_modules-adjacent dirs. @@ -678,6 +682,175 @@ jobs: if-no-files-found: ignore retention-days: 30 + # --------------------------------------------------------------------------- + # @vscode/test-cli Create Workspace parity visibility. + # + # This is additive to the ExTester matrix above: ExTester remains the owner for + # full designer/wizard DOM coverage and the VS Code 1.108.0 compatibility pin + # below remains unchanged. These matrix legs exercise the official + # @vscode/test-cli baseline against focused Create Workspace labels so PR CI + # can see latest-stable VS Code extension-host parity without serializing the + # whole suite into one hard-to-diagnose run. + # + # Intentionally excluded from PR CI for now: workspaceLifecycle. That target + # creates workspaces, reopens generated app folders, opens designer webviews, + # builds .NET projects, starts Azurite/runtime, runs triggers, and validates + # Overview results. Keep it as an explicit/manual script until its runtime cost + # and flake profile justify becoming a PR gate. + vscode-e2e-cli-create-workspace: + name: vscode-e2e-cli-create-workspace (${{ matrix.label }}) + needs: setup-extension-build + timeout-minutes: 25 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: createWorkspaceBehavior + - label: createWorkspaceCoreMatrix + - label: createWorkspacePreviewMatrix + - label: createWorkspaceCodeful + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 20.x + package-manager-cache: false + + - name: Cache pnpm store + uses: actions/cache@v5 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Setup pnpm + uses: pnpm/action-setup@v5 + with: + run_install: | + - recursive: true + args: [--frozen-lockfile, --strict-peer-dependencies] + + - name: Download extension build artifact + uses: actions/download-artifact@v7 + with: + name: extension-build-${{ github.sha }} + path: . + + - name: Extract build artifacts + run: tar -xzf extension-build.tar.gz + + - name: Install system dependencies for virtual display + run: | + sudo apt-get update + sudo apt-get install -y xvfb libgbm-dev libgtk-3-0 libnss3 libasound2t64 libxss1 libatk-bridge2.0-0 libatk1.0-0 + + - name: Run @vscode/test-cli Create Workspace label (${{ matrix.label }}) + id: run_cli_create_workspace_label + working-directory: apps/vs-code-designer + run: | + export PATH="$(dirname $(which node)):/usr/local/bin:/usr/bin:/bin:$PATH" + echo "PATH=$PATH" + mkdir -p .vscode-test/results + set +e + xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" \ + pnpm exec node scripts/run-e2e-cli.js --label "${{ matrix.label }}" 2>&1 | tee ".vscode-test/results/${{ matrix.label }}.log" + status=${PIPESTATUS[0]} + set -e + outcome="success" + if [ "$status" -ne 0 ]; then + outcome="failure" + fi + node scripts/summarize-e2e-cli-results.js \ + --label "${{ matrix.label }}" \ + --log ".vscode-test/results/${{ matrix.label }}.log" \ + --out-dir ".vscode-test/results" \ + --outcome "$outcome" + exit "$status" + env: + NODE_OPTIONS: --max-old-space-size=4096 + TEMP: ${{ runner.temp }} + TMPDIR: ${{ runner.temp }} + + - name: Summarize @vscode/test-cli Create Workspace result + if: always() + working-directory: apps/vs-code-designer + run: | + node scripts/summarize-e2e-cli-results.js \ + --append-summary \ + --json ".vscode-test/results/${{ matrix.label }}.json" \ + --github-summary "$GITHUB_STEP_SUMMARY" + + - name: Upload CLI structured results (always) + uses: actions/upload-artifact@v6 + if: always() + with: + name: vscode-e2e-cli-test-results-${{ matrix.label }} + path: | + apps/vs-code-designer/.vscode-test/results/${{ matrix.label }}.json + apps/vs-code-designer/.vscode-test/results/${{ matrix.label }}.junit.xml + apps/vs-code-designer/.vscode-test/results/${{ matrix.label }}.summary.md + if-no-files-found: ignore + retention-days: 30 + + - name: Upload CLI screenshots (always) + uses: actions/upload-artifact@v6 + if: always() + with: + name: vscode-e2e-cli-screenshots-${{ matrix.label }} + path: apps/vs-code-designer/.vscode-test/screenshots/cli/ + if-no-files-found: ignore + retention-days: 30 + + - name: Upload CLI test log (always) + uses: actions/upload-artifact@v6 + if: always() + with: + name: vscode-e2e-cli-log-${{ matrix.label }} + path: apps/vs-code-designer/.vscode-test/results/${{ matrix.label }}.log + if-no-files-found: ignore + retention-days: 30 + + vscode-e2e-cli-create-workspace-report: + name: vscode-e2e-cli-create-workspace-report + needs: vscode-e2e-cli-create-workspace + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download structured CLI results + uses: actions/download-artifact@v7 + continue-on-error: true + with: + pattern: vscode-e2e-cli-test-results-* + path: apps/vs-code-designer/.vscode-test/results/aggregate-input + merge-multiple: true + + - name: Summarize aggregate @vscode/test-cli Create Workspace results + run: | + node apps/vs-code-designer/scripts/summarize-e2e-cli-results.js \ + --aggregate \ + --results-dir apps/vs-code-designer/.vscode-test/results/aggregate-input \ + --out-dir apps/vs-code-designer/.vscode-test/results/aggregate \ + --github-summary "$GITHUB_STEP_SUMMARY" + + - name: Upload aggregate CLI result dashboard + uses: actions/upload-artifact@v6 + if: always() + with: + name: vscode-e2e-cli-test-results-summary + path: apps/vs-code-designer/.vscode-test/results/aggregate/ + if-no-files-found: ignore + retention-days: 30 + # --------------------------------------------------------------------------- # Stage C: codeful debug (F5) as a first-class CI shard on BOTH OSes. # @@ -1855,7 +2028,7 @@ jobs: # ("vscode-e2e-summary") regardless of how many scenarios we add later. vscode-e2e-summary: name: vscode-e2e-summary - needs: [setup-extension-build, setup-fixtures, vscode-e2e, vscode-e2e-windows, vscode-e2e-codeful-ubuntu, vscode-e2e-codeful-windows, vscode-e2e-azurite, vscode-e2e-azurite-windows, vscode-e2e-funcselfheal-windows, vscode-e2e-compat, setup-runtime-deps-windows] + needs: [setup-extension-build, setup-fixtures, vscode-e2e, vscode-e2e-windows, vscode-e2e-cli-create-workspace, vscode-e2e-codeful-ubuntu, vscode-e2e-codeful-windows, vscode-e2e-azurite, vscode-e2e-azurite-windows, vscode-e2e-funcselfheal-windows, vscode-e2e-compat, setup-runtime-deps-windows] if: always() runs-on: ubuntu-latest steps: @@ -1870,6 +2043,7 @@ jobs: echo "setup-fixtures: ${{ needs.setup-fixtures.result }}" echo "vscode-e2e (matrix): ${{ needs.vscode-e2e.result }}" echo "vscode-e2e-windows: ${{ needs.vscode-e2e-windows.result }}" + echo "vscode-e2e-cli-create-workspace: ${{ needs.vscode-e2e-cli-create-workspace.result }}" echo "vscode-e2e-codeful-ubuntu: ${{ needs.vscode-e2e-codeful-ubuntu.result }}" echo "vscode-e2e-codeful-windows: ${{ needs.vscode-e2e-codeful-windows.result }}" echo "vscode-e2e-azurite: ${{ needs.vscode-e2e-azurite.result }}" @@ -1881,6 +2055,7 @@ jobs: [ "${{ needs.setup-fixtures.result }}" != "success" ] || \ [ "${{ needs.vscode-e2e.result }}" != "success" ] || \ [ "${{ needs.vscode-e2e-windows.result }}" != "success" ] || \ + [ "${{ needs.vscode-e2e-cli-create-workspace.result }}" != "success" ] || \ [ "${{ needs.vscode-e2e-codeful-ubuntu.result }}" != "success" ] || \ [ "${{ needs.vscode-e2e-codeful-windows.result }}" != "success" ] || \ [ "${{ needs.vscode-e2e-azurite.result }}" != "success" ] || \ diff --git a/.squad/knowledge/vscode-e2e-testing.md b/.squad/knowledge/vscode-e2e-testing.md index 9c3f08b3ed3..34ca064312b 100644 --- a/.squad/knowledge/vscode-e2e-testing.md +++ b/.squad/knowledge/vscode-e2e-testing.md @@ -23,6 +23,20 @@ Curated durable learnings for VS Code ExTester UI E2E tests. Add entries through ## Current Learnings +### `@vscode/test-cli` Create Workspace CI publishes structured results + +- Learning: `.github/workflows/vscode-e2e.yml` runs the latest-stable `@vscode/test-cli` Create Workspace labels in the `vscode-e2e-cli-create-workspace` matrix and each label must publish structured results, not only logs. +- Why it matters: Without JUnit/JSON summaries, screenshots, and aggregate pass-rate output, failures are discovered by reading long VS Code host logs and users cannot quickly see which label failed. +- Pattern: + 1. Run each label through `apps/vs-code-designer/scripts/run-e2e-cli.js`. + 2. Tee the raw log to `.vscode-test/results/