diff --git a/.github/workflows/vscode-e2e-synthetics.yml b/.github/workflows/vscode-e2e-synthetics.yml new file mode 100644 index 00000000000..1d93d4e3dc5 --- /dev/null +++ b/.github/workflows/vscode-e2e-synthetics.yml @@ -0,0 +1,155 @@ +name: VS Code Extension Synthetics + +# Scheduled synthetic E2E tests for the Logic Apps Standard VS Code extension. +# Validates the published marketplace extension (or latest source build) on a +# recurring schedule using the same ExTester harness as vscode-e2e.yml, but +# against the production-published binary users actually install. + +on: + schedule: + # Weekday mornings at 8 AM UTC + - cron: '0 8 * * 1-5' + workflow_dispatch: + inputs: + vsix_source: + description: 'Extension source to test' + type: choice + options: + - marketplace + - source + default: marketplace + +concurrency: + group: vscode-synthetics + cancel-in-progress: true + +jobs: + synthetics: + name: synthetics (${{ inputs.vsix_source || 'marketplace' }}) + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 20.x + package-manager-cache: false + + - name: Symlink node to system path (for func host child processes) + run: | + NODE_BIN="$(which node)" + NPM_BIN="$(which npm)" + NPX_BIN="$(which npx)" + for dir in /usr/local/bin /usr/bin; do + sudo ln -sf "$NODE_BIN" "$dir/node" + sudo ln -sf "$NPM_BIN" "$dir/npm" + sudo ln -sf "$NPX_BIN" "$dir/npx" + done + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '8.0.x' + + - name: Install system dependencies (xvfb + VS Code prereqs) + run: | + sudo apt-get update + sudo apt-get install -y xvfb libgbm-dev libgtk-3-0 libnss3 libasound2t64 libxss1 libatk-bridge2.0-0 libatk1.0-0 + + - name: Cache pnpm store + uses: actions/cache@v5 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Setup pnpm + uses: pnpm/action-setup@v5 + with: + run_install: | + - recursive: true + args: [--frozen-lockfile, --strict-peer-dependencies] + + - name: Cache turbo build + uses: actions/cache@v5 + with: + path: .turbo + key: ${{ runner.os }}-turbo-synthetics-${{ hashFiles('apps/**/*.ts', 'apps/**/*.tsx', 'libs/**/*.ts', 'libs/**/*.tsx', 'pnpm-lock.yaml', 'turbo.json') }} + restore-keys: | + ${{ runner.os }}-turbo-synthetics- + + # Build the extension from source when vsix_source=source. + # In marketplace mode, we skip the full extension build — the test harness + # only needs tsup compilation (above), and the extension comes from marketplace. + - name: Build extension (source mode only) + if: inputs.vsix_source == 'source' + run: pnpm turbo run build:extension --cache-dir=.turbo + + - name: Compile E2E tests + working-directory: apps/vs-code-designer + run: npx tsup --config tsup.e2e.test.config.ts + + - name: Cache Logic Apps runtime dependencies + uses: actions/cache@v5 + with: + path: ~/.azurelogicapps/dependencies + key: la-runtime-deps-${{ runner.os }}-synthetics-v1 + restore-keys: | + la-runtime-deps-${{ runner.os }}- + + # Phase 1: Run activation smoke test (no workspace fixtures needed) + - name: Run activation smoke (p40) + run: | + export PATH="$(dirname $(which node)):/usr/local/bin:/usr/bin:/bin:$PATH" + xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" \ + node apps/vs-code-designer/out/test/run-e2e.js + env: + LA_E2E_SCENARIO: p40-nonlogicapp + LA_E2E_SCENARIO_RETRIES: '1' + LA_E2E_VSIX_SOURCE: ${{ inputs.vsix_source || 'marketplace' }} + NODE_OPTIONS: --max-old-space-size=4096 + TEMP: ${{ runner.temp }} + TMPDIR: ${{ runner.temp }} + + # Phase 2: Run workspace creation + designer lifecycle + - name: Run workspace creation + designer smoke (p41a + p42) + run: | + export PATH="$(dirname $(which node)):/usr/local/bin:/usr/bin:/bin:$PATH" + xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" \ + node apps/vs-code-designer/out/test/run-e2e.js + env: + LA_E2E_SCENARIO: p41a-fixtures,p42-standard + LA_E2E_SCENARIO_RETRIES: '1' + LA_E2E_VSIX_SOURCE: ${{ inputs.vsix_source || 'marketplace' }} + LA_E2E_STRICT_DEPENDENCY_VALIDATION: '1' + NODE_OPTIONS: --max-old-space-size=4096 + TEMP: ${{ runner.temp }} + TMPDIR: ${{ runner.temp }} + + - name: Upload screenshots on failure + uses: actions/upload-artifact@v6 + if: failure() + with: + name: vscode-synthetics-screenshots + path: | + ${{ runner.temp }}/test-resources/screenshots/ + test-resources/screenshots/ + if-no-files-found: ignore + retention-days: 14 + + - name: Report extension version tested + if: always() + run: | + echo "## Synthetics Run Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **VSIX Source**: ${{ inputs.vsix_source || 'marketplace' }}" >> $GITHUB_STEP_SUMMARY + echo "- **Run Time**: $(date -u +'%Y-%m-%d %H:%M UTC')" >> $GITHUB_STEP_SUMMARY + echo "- **Runner OS**: ${{ runner.os }}" >> $GITHUB_STEP_SUMMARY + echo "- **Node Version**: $(node --version)" >> $GITHUB_STEP_SUMMARY + echo "- **VS Code Version**: $(cat apps/vs-code-designer/out/test/run-e2e.js 2>/dev/null | grep -oP 'DEFAULT_VSCODE_VERSION.*?= .\K[0-9.]+' || echo 'unknown')" >> $GITHUB_STEP_SUMMARY diff --git a/apps/vs-code-designer/src/test/ui/run-e2e.ts b/apps/vs-code-designer/src/test/ui/run-e2e.ts index 098a1c2f7d8..0df67da2fa1 100644 --- a/apps/vs-code-designer/src/test/ui/run-e2e.ts +++ b/apps/vs-code-designer/src/test/ui/run-e2e.ts @@ -662,8 +662,20 @@ async function main(): Promise { } console.log('✓ D-001 lint guard passed (no fs.writeFile/outputJson in *.fixtures.test.ts).'); - // Read extension metadata from dist/package.json - const pkgJson = JSON.parse(fs.readFileSync(path.join(distDir, 'package.json'), 'utf8')); + // Determine extension source: 'source' (default) builds from dist/, 'marketplace' installs the published VSIX. + const vsixSource = (process.env.LA_E2E_VSIX_SOURCE || 'source').toLowerCase(); + const useMarketplaceVsix = vsixSource === 'marketplace'; + if (useMarketplaceVsix) { + console.log('\n=== Marketplace VSIX mode: will install published extension from marketplace ==='); + } + + // Read extension metadata from package.json. + // In marketplace mode, dist/ may not exist — fall back to the project-level package.json. + const distPkgPath = path.join(distDir, 'package.json'); + const projectPkgPath = path.join(projectDir, 'package.json'); + const pkgJsonPath = fs.existsSync(distPkgPath) ? distPkgPath : projectPkgPath; + const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')); + console.log(` Extension metadata read from: ${pkgJsonPath}`); const extDeps: string[] = pkgJson.extensionDependencies || []; const devContainersDependency = 'ms-vscode-remote.remote-containers'; if (extDeps.some((dep) => dep.toLowerCase() === devContainersDependency)) { @@ -825,108 +837,162 @@ async function main(): Promise { console.log('\n=== Step 2: No extension dependencies to install ==='); } - // Step 3: Copy our built extension into test-extensions as an "installed" extension - // Skip the copy if dist/main.js hasn't changed (compare mtime for a fast-path). - console.log('\n=== Step 3: Install our extension from dist/ ==='); + // Step 3: Install our extension into test-extensions const pkgId = `${pkgJson.publisher}.${pkgJson.name}`.toLowerCase(); + const marketplaceExtensionId = `${pkgJson.publisher}.${pkgJson.name}`; - const srcMainJs = path.join(distDir, 'main.js'); - const destMainJs = path.join(ourExtTarget, 'main.js'); - let needsCopy = true; - - if (fs.existsSync(destMainJs) && fs.existsSync(srcMainJs)) { - const srcMtime = fs.statSync(srcMainJs).mtimeMs; - const destMtime = fs.statSync(destMainJs).mtimeMs; - if (srcMtime === destMtime) { - console.log(' ✓ dist/main.js unchanged (mtime match), skipping copy'); - needsCopy = false; - } else { - console.log(` dist/main.js changed (src=${new Date(srcMtime).toISOString()}, dest=${new Date(destMtime).toISOString()})`); - } - } + if (useMarketplaceVsix) { + // Marketplace mode: install the published extension using the VS Code CLI + console.log('\n=== Step 3: Install extension from marketplace ==='); + console.log(` Extension ID: ${marketplaceExtensionId}`); - if (needsCopy) { - // Remove old copies of our extension + // Remove any previously copied source-built extension if (fs.existsSync(extDir)) { for (const entry of fs.readdirSync(extDir)) { if (entry.toLowerCase().startsWith(pkgId)) { const fullPath = path.join(extDir, entry); - console.log(` Removing old copy: ${entry}`); + console.log(` Removing old source-built copy: ${entry}`); fs.rmSync(fullPath, { recursive: true, force: true }); } } } - // Copy dist/ → test-extensions/.-/ - // Skip the test-extensions dir itself to avoid infinite recursion - console.log(` Copying dist/ → ${extDirName}/`); - copyDirSync(distDir, ourExtTarget, extDir); + // Install from marketplace using the VS Code CLI + const cliBase = extest.code.getCliInitCommand(); + const installResult = await installExtensionWithCli(cliBase, marketplaceExtensionId, 'marketplace extension'); + if (!installResult.success) { + // Retry once + console.log(' Retrying marketplace extension install...'); + const retry = await installExtensionWithCli(cliBase, marketplaceExtensionId, 'marketplace extension retry'); + if (!retry.success) { + throw new Error(`Failed to install marketplace extension: ${marketplaceExtensionId}`); + } + } - // Preserve source mtime on main.js so subsequent runs can detect changes - if (fs.existsSync(srcMainJs) && fs.existsSync(destMainJs)) { - const srcStat = fs.statSync(srcMainJs); - fs.utimesSync(destMainJs, srcStat.atime, srcStat.mtime); + // Find the installed extension directory to read its package.json for verification + const installedEntry = fs.readdirSync(extDir).find((entry) => entry.toLowerCase().startsWith(pkgId)); + if (!installedEntry) { + throw new Error(`Marketplace extension not found in ${extDir} after install`); } + const installedPkgPath = path.join(extDir, installedEntry, 'package.json'); + if (!fs.existsSync(installedPkgPath)) { + throw new Error(`package.json not found in installed marketplace extension: ${installedPkgPath}`); + } + const installedPkg = JSON.parse(fs.readFileSync(installedPkgPath, 'utf8')); + console.log(` ✓ Marketplace extension installed: ${installedEntry} (v${installedPkg.version})`); - console.log(' ✓ Extension installed in test-extensions'); - } + // Remove .obsolete file if present + const obsoleteFile = path.join(extDir, '.obsolete'); + if (fs.existsSync(obsoleteFile)) { + console.log(' Removing .obsolete file (was blocking extension loading)'); + fs.rmSync(obsoleteFile); + } - // Verify the copied extension has the necessary files - const copiedPkg = path.join(ourExtTarget, 'package.json'); - const copiedMain = path.join(ourExtTarget, 'main.js'); - if (!fs.existsSync(copiedPkg)) { - console.error(` ✗ ERROR: package.json not found at ${copiedPkg}`); - process.exit(1); - } - if (!fs.existsSync(copiedMain)) { - console.error(` ✗ ERROR: main.js not found at ${copiedMain}`); - process.exit(1); - } - console.log(' ✓ Verified: package.json and main.js present'); - - // CRITICAL: Remove .obsolete file — VS Code uses this to skip loading extensions. - // Previous test runs or ExTester's cleanup step may have marked our extension - // as obsolete, causing VS Code to refuse to load it even though the directory exists. - const obsoleteFile = path.join(extDir, '.obsolete'); - if (fs.existsSync(obsoleteFile)) { - console.log(' Removing .obsolete file (was blocking extension loading)'); - fs.rmSync(obsoleteFile); - } + // Rebuild extensions.json to include the marketplace-installed extension + rebuildExtensionsJson(extDir); + console.log(' ✓ extensions.json rebuilt'); + } else { + // Source mode: copy dist/ into test-extensions (existing behavior) + console.log('\n=== Step 3: Install our extension from dist/ ==='); + + const srcMainJs = path.join(distDir, 'main.js'); + const destMainJs = path.join(ourExtTarget, 'main.js'); + let needsCopy = true; + + if (fs.existsSync(destMainJs) && fs.existsSync(srcMainJs)) { + const srcMtime = fs.statSync(srcMainJs).mtimeMs; + const destMtime = fs.statSync(destMainJs).mtimeMs; + if (srcMtime === destMtime) { + console.log(' ✓ dist/main.js unchanged (mtime match), skipping copy'); + needsCopy = false; + } else { + console.log(` dist/main.js changed (src=${new Date(srcMtime).toISOString()}, dest=${new Date(destMtime).toISOString()})`); + } + } - // CRITICAL: Register our extension in extensions.json so VS Code recognizes it. - // Marketplace-installed extensions get entries automatically, but our manually - // copied extension needs to be added explicitly. - const extensionsJsonPath = path.join(extDir, 'extensions.json'); - let extensionsJson: ExtensionJsonEntry[] = []; - if (fs.existsSync(extensionsJsonPath)) { - try { - extensionsJson = JSON.parse(fs.readFileSync(extensionsJsonPath, 'utf8')); - } catch { - extensionsJson = []; + if (needsCopy) { + // Remove old copies of our extension + if (fs.existsSync(extDir)) { + for (const entry of fs.readdirSync(extDir)) { + if (entry.toLowerCase().startsWith(pkgId)) { + const fullPath = path.join(extDir, entry); + console.log(` Removing old copy: ${entry}`); + fs.rmSync(fullPath, { recursive: true, force: true }); + } + } + } + + // Copy dist/ → test-extensions/.-/ + // Skip the test-extensions dir itself to avoid infinite recursion + console.log(` Copying dist/ → ${extDirName}/`); + copyDirSync(distDir, ourExtTarget, extDir); + + // Preserve source mtime on main.js so subsequent runs can detect changes + if (fs.existsSync(srcMainJs) && fs.existsSync(destMainJs)) { + const srcStat = fs.statSync(srcMainJs); + fs.utimesSync(destMainJs, srcStat.atime, srcStat.mtime); + } + + console.log(' ✓ Extension installed in test-extensions'); } - } - // Remove any existing entry for our extension - extensionsJson = extensionsJson.filter( - (entry) => entry.identifier?.id?.toLowerCase() !== `${pkgJson.publisher}.${pkgJson.name}`.toLowerCase() - ); - // Add our extension entry - const ourExtPath = ourExtTarget.replace(/\\/g, '/'); - extensionsJson.push({ - identifier: { id: `${pkgJson.publisher}.${pkgJson.name}` }, - version: pkgJson.version, - location: { - $mid: 1, - path: `/${ourExtPath}`.replace(/^\/([A-Za-z]):/, (_, d) => `/${d.toLowerCase()}:`), - scheme: 'file', - }, - relativeLocation: extDirName, - metadata: { - installedTimestamp: Date.now(), - source: 'gallery', - }, - }); - fs.writeFileSync(extensionsJsonPath, JSON.stringify(extensionsJson)); - console.log(' ✓ Registered in extensions.json'); + + // Verify the copied extension has the necessary files + const copiedPkg = path.join(ourExtTarget, 'package.json'); + const copiedMain = path.join(ourExtTarget, 'main.js'); + if (!fs.existsSync(copiedPkg)) { + console.error(` ✗ ERROR: package.json not found at ${copiedPkg}`); + process.exit(1); + } + if (!fs.existsSync(copiedMain)) { + console.error(` ✗ ERROR: main.js not found at ${copiedMain}`); + process.exit(1); + } + console.log(' ✓ Verified: package.json and main.js present'); + + // CRITICAL: Remove .obsolete file — VS Code uses this to skip loading extensions. + // Previous test runs or ExTester's cleanup step may have marked our extension + // as obsolete, causing VS Code to refuse to load it even though the directory exists. + const obsoleteFile = path.join(extDir, '.obsolete'); + if (fs.existsSync(obsoleteFile)) { + console.log(' Removing .obsolete file (was blocking extension loading)'); + fs.rmSync(obsoleteFile); + } + + // CRITICAL: Register our extension in extensions.json so VS Code recognizes it. + // Marketplace-installed extensions get entries automatically, but our manually + // copied extension needs to be added explicitly. + const extensionsJsonPath = path.join(extDir, 'extensions.json'); + let extensionsJson: ExtensionJsonEntry[] = []; + if (fs.existsSync(extensionsJsonPath)) { + try { + extensionsJson = JSON.parse(fs.readFileSync(extensionsJsonPath, 'utf8')); + } catch { + extensionsJson = []; + } + } + // Remove any existing entry for our extension + extensionsJson = extensionsJson.filter( + (entry) => entry.identifier?.id?.toLowerCase() !== `${pkgJson.publisher}.${pkgJson.name}`.toLowerCase() + ); + // Add our extension entry + const ourExtPath = ourExtTarget.replace(/\\/g, '/'); + extensionsJson.push({ + identifier: { id: `${pkgJson.publisher}.${pkgJson.name}` }, + version: pkgJson.version, + location: { + $mid: 1, + path: `/${ourExtPath}`.replace(/^\/([A-Za-z]):/, (_, d) => `/${d.toLowerCase()}:`), + scheme: 'file', + }, + relativeLocation: extDirName, + metadata: { + installedTimestamp: Date.now(), + source: 'gallery', + }, + }); + fs.writeFileSync(extensionsJsonPath, JSON.stringify(extensionsJson)); + console.log(' ✓ Registered in extensions.json'); + } // end of source mode (else branch) // List all extensions that will be loaded console.log('\n=== Extensions in test-extensions/ ==='); @@ -943,14 +1009,17 @@ async function main(): Promise { // Remove any OTHER versions of our extension that VS Code may have // auto-downloaded from the marketplace in a previous run. // Only keep our exact version. Also clean up duplicate dependency versions. - for (const entry of fs.readdirSync(extDir)) { - if (entry === 'extensions.json' || entry === '.obsolete') { - continue; - } - if (entry.toLowerCase().startsWith(pkgId) && entry !== extDirName) { - const fullPath = path.join(extDir, entry); - console.log(` Removing stale auto-updated version: ${entry}`); - fs.rmSync(fullPath, { recursive: true, force: true }); + // Skip in marketplace mode — the marketplace-installed version IS the correct one. + if (!useMarketplaceVsix) { + for (const entry of fs.readdirSync(extDir)) { + if (entry === 'extensions.json' || entry === '.obsolete') { + continue; + } + if (entry.toLowerCase().startsWith(pkgId) && entry !== extDirName) { + const fullPath = path.join(extDir, entry); + console.log(` Removing stale auto-updated version: ${entry}`); + fs.rmSync(fullPath, { recursive: true, force: true }); + } } }