From 51f504b989fd1b8cbb905c5c9cc0ff25a2a14796 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:15:51 +0000 Subject: [PATCH 1/4] chore: remove Microsoft.DotNet.Arcade.Sdk and related infrastructure Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com> --- Directory.Build.props | 3 - Directory.Build.targets | 2 - Directory.Packages.props | 2 +- NuGet.config | 24 - eng/AfterSolutionBuild.targets | 27 - eng/CIBuild.cmd | 2 - eng/Publishing.props | 5 - eng/SignCheckExclusionsFile.txt | 6 - eng/Signing.props | 55 - eng/Version.Details.xml | 11 - eng/Versions.props | 13 - eng/cibuild.sh | 19 - .../build-configuration.json | 4 - eng/common/CIBuild.cmd | 2 - eng/common/PSScriptAnalyzerSettings.psd1 | 11 - eng/common/README.md | 28 - eng/common/SetupNugetSources.ps1 | 167 --- eng/common/SetupNugetSources.sh | 171 ---- eng/common/build.ps1 | 166 --- eng/common/build.sh | 247 ----- eng/common/cibuild.sh | 16 - eng/common/cross/arm/sources.list.bionic | 11 - eng/common/cross/arm/sources.list.jessie | 3 - eng/common/cross/arm/sources.list.xenial | 11 - eng/common/cross/arm/sources.list.zesty | 11 - eng/common/cross/arm64/sources.list.bionic | 11 - eng/common/cross/arm64/sources.list.xenial | 11 - eng/common/cross/arm64/sources.list.zesty | 11 - eng/common/cross/armel/armel.jessie.patch | 43 - eng/common/cross/armel/sources.list.jessie | 3 - eng/common/cross/armel/tizen/tizen.patch | 9 - eng/common/cross/armv6/sources.list.buster | 2 - eng/common/cross/build-android-rootfs.sh | 131 --- eng/common/cross/build-rootfs.sh | 648 ------------ eng/common/cross/ppc64le/sources.list.bionic | 11 - eng/common/cross/riscv64/sources.list.sid | 1 - eng/common/cross/s390x/sources.list.bionic | 11 - eng/common/cross/tizen-build-rootfs.sh | 61 -- eng/common/cross/tizen-fetch.sh | 172 ---- eng/common/cross/toolchain.cmake | 380 ------- eng/common/cross/x86/sources.list.bionic | 11 - eng/common/cross/x86/sources.list.xenial | 11 - eng/common/darc-init.ps1 | 47 - eng/common/darc-init.sh | 82 -- eng/common/dotnet-install.cmd | 2 - eng/common/dotnet-install.ps1 | 28 - eng/common/dotnet-install.sh | 91 -- eng/common/enable-cross-org-publishing.ps1 | 13 - eng/common/generate-locproject.ps1 | 189 ---- eng/common/generate-sbom-prep.ps1 | 29 - eng/common/generate-sbom-prep.sh | 39 - eng/common/helixpublish.proj | 26 - eng/common/init-tools-native.cmd | 3 - eng/common/init-tools-native.ps1 | 203 ---- eng/common/init-tools-native.sh | 238 ----- eng/common/internal-feed-operations.ps1 | 132 --- eng/common/internal-feed-operations.sh | 141 --- eng/common/internal/Directory.Build.props | 4 - eng/common/internal/NuGet.config | 7 - eng/common/internal/Tools.csproj | 30 - eng/common/loc/P22DotNetHtmlLocalization.lss | 29 - eng/common/msbuild.ps1 | 28 - eng/common/msbuild.sh | 58 -- eng/common/native/CommonLibrary.psm1 | 400 -------- eng/common/native/common-library.sh | 172 ---- eng/common/native/init-compiler.sh | 137 --- eng/common/native/init-distro-rid.sh | 130 --- eng/common/native/init-os-and-arch.sh | 80 -- eng/common/native/install-cmake-test.sh | 117 --- eng/common/native/install-cmake.sh | 117 --- eng/common/native/install-tool.ps1 | 132 --- eng/common/pipeline-logging-functions.ps1 | 260 ----- eng/common/pipeline-logging-functions.sh | 206 ---- .../post-build/add-build-to-channel.ps1 | 48 - .../post-build/check-channel-consistency.ps1 | 40 - eng/common/post-build/nuget-validation.ps1 | 24 - eng/common/post-build/post-build-utils.ps1 | 91 -- eng/common/post-build/publish-using-darc.ps1 | 53 - .../post-build/sourcelink-validation.ps1 | 319 ------ eng/common/post-build/symbols-validation.ps1 | 339 ------- .../post-build/trigger-subscriptions.ps1 | 64 -- eng/common/retain-build.ps1 | 45 - eng/common/sdk-task.ps1 | 97 -- eng/common/sdl/NuGet.config | 18 - eng/common/sdl/configure-sdl-tool.ps1 | 130 --- eng/common/sdl/execute-all-sdl-tools.ps1 | 165 --- eng/common/sdl/extract-artifact-archives.ps1 | 63 -- eng/common/sdl/extract-artifact-packages.ps1 | 82 -- eng/common/sdl/init-sdl.ps1 | 47 - eng/common/sdl/packages.config | 4 - eng/common/sdl/run-sdl.ps1 | 49 - eng/common/sdl/sdl.ps1 | 40 - eng/common/sdl/trim-assets-version.ps1 | 75 -- eng/common/templates-official/job/job.yml | 265 ----- .../templates-official/job/onelocbuild.yml | 112 -- .../job/publish-build-assets.yml | 160 --- .../templates-official/job/source-build.yml | 75 -- .../job/source-index-stage1.yml | 83 -- .../templates-official/jobs/codeql-build.yml | 31 - eng/common/templates-official/jobs/jobs.yml | 97 -- .../templates-official/jobs/source-build.yml | 54 - .../post-build/common-variables.yml | 22 - .../post-build/post-build.yml | 287 ------ .../post-build/setup-maestro-vars.yml | 70 -- .../post-build/trigger-subscription.yml | 13 - .../steps/add-build-to-channel.yml | 13 - .../templates-official/steps/build-reason.yml | 12 - .../steps/component-governance.yml | 13 - .../steps/enable-internal-runtimes.yml | 28 - .../steps/execute-codeql.yml | 32 - .../templates-official/steps/execute-sdl.yml | 86 -- .../steps/generate-sbom.yml | 48 - .../steps/get-delegation-sas.yml | 52 - .../steps/get-federated-access-token.yml | 40 - .../templates-official/steps/publish-logs.yml | 23 - .../templates-official/steps/retain-build.yml | 28 - .../steps/send-to-helix.yml | 92 -- .../templates-official/steps/source-build.yml | 129 --- .../variables/pool-providers.yml | 45 - .../variables/sdl-variables.yml | 7 - eng/common/templates/job/execute-sdl.yml | 139 --- eng/common/templates/job/job.yml | 259 ----- eng/common/templates/job/onelocbuild.yml | 109 -- .../templates/job/publish-build-assets.yml | 156 --- eng/common/templates/job/source-build.yml | 74 -- .../templates/job/source-index-stage1.yml | 82 -- eng/common/templates/jobs/codeql-build.yml | 31 - eng/common/templates/jobs/jobs.yml | 97 -- eng/common/templates/jobs/source-build.yml | 54 - .../templates/post-build/common-variables.yml | 22 - .../templates/post-build/post-build.yml | 283 ------ .../post-build/setup-maestro-vars.yml | 70 -- .../post-build/trigger-subscription.yml | 13 - .../templates/steps/add-build-to-channel.yml | 13 - eng/common/templates/steps/build-reason.yml | 12 - .../templates/steps/component-governance.yml | 13 - .../steps/enable-internal-runtimes.yml | 28 - eng/common/templates/steps/execute-codeql.yml | 32 - eng/common/templates/steps/execute-sdl.yml | 89 -- eng/common/templates/steps/generate-sbom.yml | 48 - .../templates/steps/get-delegation-sas.yml | 52 - .../steps/get-federated-access-token.yml | 40 - eng/common/templates/steps/publish-logs.yml | 23 - eng/common/templates/steps/retain-build.yml | 28 - eng/common/templates/steps/run-on-unix.yml | 7 - eng/common/templates/steps/run-on-windows.yml | 7 - .../steps/run-script-ifequalelse.yml | 33 - eng/common/templates/steps/send-to-helix.yml | 92 -- eng/common/templates/steps/source-build.yml | 129 --- eng/common/templates/steps/telemetry-end.yml | 102 -- .../templates/steps/telemetry-start.yml | 241 ----- .../templates/variables/pool-providers.yml | 57 -- .../templates/variables/sdl-variables.yml | 7 - eng/common/tools.ps1 | 957 ------------------ eng/common/tools.sh | 592 ----------- eng/targets/Settings.props | 3 +- eng/targets/Versions.targets | 4 - .../build-and-test-job-linux-templates.yml | 41 - .../build-and-test-job-windows-templates.yml | 45 - eng/templates/delete-dump-files-template.yml | 10 - eng/templates/es-metadata.yml | 8 - eng/templates/setup-template.yml | 19 - eng/templates/variables-template.yml | 17 - global.json | 3 - ...icrosoft.TryDotNet.IntegrationTests.csproj | 4 +- .../Microsoft.TryDotNet.Tests.csproj | 2 +- 166 files changed, 5 insertions(+), 13566 deletions(-) delete mode 100644 eng/AfterSolutionBuild.targets delete mode 100644 eng/CIBuild.cmd delete mode 100644 eng/Publishing.props delete mode 100644 eng/SignCheckExclusionsFile.txt delete mode 100644 eng/Signing.props delete mode 100644 eng/Version.Details.xml delete mode 100755 eng/cibuild.sh delete mode 100644 eng/common/BuildConfiguration/build-configuration.json delete mode 100644 eng/common/CIBuild.cmd delete mode 100644 eng/common/PSScriptAnalyzerSettings.psd1 delete mode 100644 eng/common/README.md delete mode 100644 eng/common/SetupNugetSources.ps1 delete mode 100644 eng/common/SetupNugetSources.sh delete mode 100644 eng/common/build.ps1 delete mode 100644 eng/common/build.sh delete mode 100644 eng/common/cibuild.sh delete mode 100644 eng/common/cross/arm/sources.list.bionic delete mode 100644 eng/common/cross/arm/sources.list.jessie delete mode 100644 eng/common/cross/arm/sources.list.xenial delete mode 100644 eng/common/cross/arm/sources.list.zesty delete mode 100644 eng/common/cross/arm64/sources.list.bionic delete mode 100644 eng/common/cross/arm64/sources.list.xenial delete mode 100644 eng/common/cross/arm64/sources.list.zesty delete mode 100644 eng/common/cross/armel/armel.jessie.patch delete mode 100644 eng/common/cross/armel/sources.list.jessie delete mode 100644 eng/common/cross/armel/tizen/tizen.patch delete mode 100644 eng/common/cross/armv6/sources.list.buster delete mode 100644 eng/common/cross/build-android-rootfs.sh delete mode 100644 eng/common/cross/build-rootfs.sh delete mode 100644 eng/common/cross/ppc64le/sources.list.bionic delete mode 100644 eng/common/cross/riscv64/sources.list.sid delete mode 100644 eng/common/cross/s390x/sources.list.bionic delete mode 100644 eng/common/cross/tizen-build-rootfs.sh delete mode 100644 eng/common/cross/tizen-fetch.sh delete mode 100644 eng/common/cross/toolchain.cmake delete mode 100644 eng/common/cross/x86/sources.list.bionic delete mode 100644 eng/common/cross/x86/sources.list.xenial delete mode 100644 eng/common/darc-init.ps1 delete mode 100644 eng/common/darc-init.sh delete mode 100644 eng/common/dotnet-install.cmd delete mode 100644 eng/common/dotnet-install.ps1 delete mode 100644 eng/common/dotnet-install.sh delete mode 100644 eng/common/enable-cross-org-publishing.ps1 delete mode 100644 eng/common/generate-locproject.ps1 delete mode 100644 eng/common/generate-sbom-prep.ps1 delete mode 100644 eng/common/generate-sbom-prep.sh delete mode 100644 eng/common/helixpublish.proj delete mode 100644 eng/common/init-tools-native.cmd delete mode 100644 eng/common/init-tools-native.ps1 delete mode 100644 eng/common/init-tools-native.sh delete mode 100644 eng/common/internal-feed-operations.ps1 delete mode 100644 eng/common/internal-feed-operations.sh delete mode 100644 eng/common/internal/Directory.Build.props delete mode 100644 eng/common/internal/NuGet.config delete mode 100644 eng/common/internal/Tools.csproj delete mode 100644 eng/common/loc/P22DotNetHtmlLocalization.lss delete mode 100644 eng/common/msbuild.ps1 delete mode 100644 eng/common/msbuild.sh delete mode 100644 eng/common/native/CommonLibrary.psm1 delete mode 100644 eng/common/native/common-library.sh delete mode 100644 eng/common/native/init-compiler.sh delete mode 100644 eng/common/native/init-distro-rid.sh delete mode 100644 eng/common/native/init-os-and-arch.sh delete mode 100644 eng/common/native/install-cmake-test.sh delete mode 100644 eng/common/native/install-cmake.sh delete mode 100644 eng/common/native/install-tool.ps1 delete mode 100644 eng/common/pipeline-logging-functions.ps1 delete mode 100644 eng/common/pipeline-logging-functions.sh delete mode 100644 eng/common/post-build/add-build-to-channel.ps1 delete mode 100644 eng/common/post-build/check-channel-consistency.ps1 delete mode 100644 eng/common/post-build/nuget-validation.ps1 delete mode 100644 eng/common/post-build/post-build-utils.ps1 delete mode 100644 eng/common/post-build/publish-using-darc.ps1 delete mode 100644 eng/common/post-build/sourcelink-validation.ps1 delete mode 100644 eng/common/post-build/symbols-validation.ps1 delete mode 100644 eng/common/post-build/trigger-subscriptions.ps1 delete mode 100644 eng/common/retain-build.ps1 delete mode 100644 eng/common/sdk-task.ps1 delete mode 100644 eng/common/sdl/NuGet.config delete mode 100644 eng/common/sdl/configure-sdl-tool.ps1 delete mode 100644 eng/common/sdl/execute-all-sdl-tools.ps1 delete mode 100644 eng/common/sdl/extract-artifact-archives.ps1 delete mode 100644 eng/common/sdl/extract-artifact-packages.ps1 delete mode 100644 eng/common/sdl/init-sdl.ps1 delete mode 100644 eng/common/sdl/packages.config delete mode 100644 eng/common/sdl/run-sdl.ps1 delete mode 100644 eng/common/sdl/sdl.ps1 delete mode 100644 eng/common/sdl/trim-assets-version.ps1 delete mode 100644 eng/common/templates-official/job/job.yml delete mode 100644 eng/common/templates-official/job/onelocbuild.yml delete mode 100644 eng/common/templates-official/job/publish-build-assets.yml delete mode 100644 eng/common/templates-official/job/source-build.yml delete mode 100644 eng/common/templates-official/job/source-index-stage1.yml delete mode 100644 eng/common/templates-official/jobs/codeql-build.yml delete mode 100644 eng/common/templates-official/jobs/jobs.yml delete mode 100644 eng/common/templates-official/jobs/source-build.yml delete mode 100644 eng/common/templates-official/post-build/common-variables.yml delete mode 100644 eng/common/templates-official/post-build/post-build.yml delete mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml delete mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml delete mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml delete mode 100644 eng/common/templates-official/steps/build-reason.yml delete mode 100644 eng/common/templates-official/steps/component-governance.yml delete mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml delete mode 100644 eng/common/templates-official/steps/execute-codeql.yml delete mode 100644 eng/common/templates-official/steps/execute-sdl.yml delete mode 100644 eng/common/templates-official/steps/generate-sbom.yml delete mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml delete mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml delete mode 100644 eng/common/templates-official/steps/publish-logs.yml delete mode 100644 eng/common/templates-official/steps/retain-build.yml delete mode 100644 eng/common/templates-official/steps/send-to-helix.yml delete mode 100644 eng/common/templates-official/steps/source-build.yml delete mode 100644 eng/common/templates-official/variables/pool-providers.yml delete mode 100644 eng/common/templates-official/variables/sdl-variables.yml delete mode 100644 eng/common/templates/job/execute-sdl.yml delete mode 100644 eng/common/templates/job/job.yml delete mode 100644 eng/common/templates/job/onelocbuild.yml delete mode 100644 eng/common/templates/job/publish-build-assets.yml delete mode 100644 eng/common/templates/job/source-build.yml delete mode 100644 eng/common/templates/job/source-index-stage1.yml delete mode 100644 eng/common/templates/jobs/codeql-build.yml delete mode 100644 eng/common/templates/jobs/jobs.yml delete mode 100644 eng/common/templates/jobs/source-build.yml delete mode 100644 eng/common/templates/post-build/common-variables.yml delete mode 100644 eng/common/templates/post-build/post-build.yml delete mode 100644 eng/common/templates/post-build/setup-maestro-vars.yml delete mode 100644 eng/common/templates/post-build/trigger-subscription.yml delete mode 100644 eng/common/templates/steps/add-build-to-channel.yml delete mode 100644 eng/common/templates/steps/build-reason.yml delete mode 100644 eng/common/templates/steps/component-governance.yml delete mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml delete mode 100644 eng/common/templates/steps/execute-codeql.yml delete mode 100644 eng/common/templates/steps/execute-sdl.yml delete mode 100644 eng/common/templates/steps/generate-sbom.yml delete mode 100644 eng/common/templates/steps/get-delegation-sas.yml delete mode 100644 eng/common/templates/steps/get-federated-access-token.yml delete mode 100644 eng/common/templates/steps/publish-logs.yml delete mode 100644 eng/common/templates/steps/retain-build.yml delete mode 100644 eng/common/templates/steps/run-on-unix.yml delete mode 100644 eng/common/templates/steps/run-on-windows.yml delete mode 100644 eng/common/templates/steps/run-script-ifequalelse.yml delete mode 100644 eng/common/templates/steps/send-to-helix.yml delete mode 100644 eng/common/templates/steps/source-build.yml delete mode 100644 eng/common/templates/steps/telemetry-end.yml delete mode 100644 eng/common/templates/steps/telemetry-start.yml delete mode 100644 eng/common/templates/variables/pool-providers.yml delete mode 100644 eng/common/templates/variables/sdl-variables.yml delete mode 100644 eng/common/tools.ps1 delete mode 100644 eng/common/tools.sh delete mode 100644 eng/templates/build-and-test-job-linux-templates.yml delete mode 100644 eng/templates/build-and-test-job-windows-templates.yml delete mode 100644 eng/templates/delete-dump-files-template.yml delete mode 100644 eng/templates/es-metadata.yml delete mode 100644 eng/templates/setup-template.yml delete mode 100644 eng/templates/variables-template.yml diff --git a/Directory.Build.props b/Directory.Build.props index c4c081f23..69c5a4222 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,12 +1,9 @@ - - - true false false diff --git a/Directory.Build.targets b/Directory.Build.targets index f0739ac4e..330fa884b 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,7 +1,5 @@ - - diff --git a/Directory.Packages.props b/Directory.Packages.props index adb454aeb..b6917c00a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,7 +35,7 @@ - + \ No newline at end of file diff --git a/NuGet.config b/NuGet.config index 887291f11..a6909550f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,30 +5,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets deleted file mode 100644 index 59577accb..000000000 --- a/eng/AfterSolutionBuild.targets +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/eng/CIBuild.cmd b/eng/CIBuild.cmd deleted file mode 100644 index b15ea8b1b..000000000 --- a/eng/CIBuild.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -noprofile -executionPolicy RemoteSigned -file "%~dp0build.ps1" -ci -restore -build -pack -publish -binaryLog %* diff --git a/eng/Publishing.props b/eng/Publishing.props deleted file mode 100644 index bb4af0030..000000000 --- a/eng/Publishing.props +++ /dev/null @@ -1,5 +0,0 @@ - - - 3 - - diff --git a/eng/SignCheckExclusionsFile.txt b/eng/SignCheckExclusionsFile.txt deleted file mode 100644 index 90d6d4f9b..000000000 --- a/eng/SignCheckExclusionsFile.txt +++ /dev/null @@ -1,6 +0,0 @@ -*bundle.js -blazor.*.js -editor.worker.js -interop.js -mono.js -trydotnet.min.js diff --git a/eng/Signing.props b/eng/Signing.props deleted file mode 100644 index e250cb960..000000000 --- a/eng/Signing.props +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml deleted file mode 100644 index 7e52ef5e5..000000000 --- a/eng/Version.Details.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 - - - diff --git a/eng/Versions.props b/eng/Versions.props index 582175f7b..c1167ee76 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,12 +1,5 @@ - - false - false - false - - true - true -parallel none @@ -25,14 +18,8 @@ https://github.com/dotnet/try - false - release - 42424242.42 <_BuildNumber>$(OfficialBuildId) <_BuildNumberYY>$(_BuildNumber.Substring(2, 2)) diff --git a/eng/cibuild.sh b/eng/cibuild.sh deleted file mode 100755 index a5e2cd727..000000000 --- a/eng/cibuild.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where - # the symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd)" - -echo "Building this commit:" -git show --no-patch --pretty=raw HEAD - -. "$scriptroot/build.sh" --ci --restore --build --pack --binaryLog "$@" diff --git a/eng/common/BuildConfiguration/build-configuration.json b/eng/common/BuildConfiguration/build-configuration.json deleted file mode 100644 index 3d1cc8989..000000000 --- a/eng/common/BuildConfiguration/build-configuration.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "RetryCountLimit": 1, - "RetryByAnyError": false -} diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd deleted file mode 100644 index 56c2f25ac..000000000 --- a/eng/common/CIBuild.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 deleted file mode 100644 index 4c1ea7c98..000000000 --- a/eng/common/PSScriptAnalyzerSettings.psd1 +++ /dev/null @@ -1,11 +0,0 @@ -@{ - IncludeRules=@('PSAvoidUsingCmdletAliases', - 'PSAvoidUsingWMICmdlet', - 'PSAvoidUsingPositionalParameters', - 'PSAvoidUsingInvokeExpression', - 'PSUseDeclaredVarsMoreThanAssignments', - 'PSUseCmdletCorrectly', - 'PSStandardDSCFunctionsInResource', - 'PSUseIdenticalMandatoryParametersForDSC', - 'PSUseIdenticalParametersForDSC') -} \ No newline at end of file diff --git a/eng/common/README.md b/eng/common/README.md deleted file mode 100644 index ff49c3715..000000000 --- a/eng/common/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Don't touch this folder - - uuuuuuuuuuuuuuuuuuuu - u" uuuuuuuuuuuuuuuuuu "u - u" u$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ - $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ - $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ - $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ - $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$" u" - "u """""""""""""""""" u" - """""""""""""""""""" - -!!! Changes made in this directory are subject to being overwritten by automation !!! - -The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 deleted file mode 100644 index efa2fd72b..000000000 --- a/eng/common/SetupNugetSources.ps1 +++ /dev/null @@ -1,167 +0,0 @@ -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 -# -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing -# -# - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials -# condition: eq(variables['Agent.OS'], 'Windows_NT') -# inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) - -[CmdletBinding()] -param ( - [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][string]$Password -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -. $PSScriptRoot\tools.ps1 - -# Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { - $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") - - if ($packageSource -eq $null) - { - $packageSource = $doc.CreateElement("add") - $packageSource.SetAttribute("key", $SourceName) - $packageSource.SetAttribute("value", $SourceEndPoint) - $sources.AppendChild($packageSource) | Out-Null - } - else { - Write-Host "Package source $SourceName already present." - } - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd -} - -# Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { - # Looks for credential configuration for the given SourceName. Create it if none is found. - $sourceElement = $creds.SelectSingleNode($Source) - if ($sourceElement -eq $null) - { - $sourceElement = $doc.CreateElement($Source) - $creds.AppendChild($sourceElement) | Out-Null - } - - # Add the node to the credential if none is found. - $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") - if ($usernameElement -eq $null) - { - $usernameElement = $doc.CreateElement("add") - $usernameElement.SetAttribute("key", "Username") - $sourceElement.AppendChild($usernameElement) | Out-Null - } - $usernameElement.SetAttribute("value", $Username) - - # Add the to the credential if none is found. - # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. - # -> https://github.com/NuGet/Home/issues/5526 - $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") - if ($passwordElement -eq $null) - { - $passwordElement = $doc.CreateElement("add") - $passwordElement.SetAttribute("key", "ClearTextPassword") - $sourceElement.AppendChild($passwordElement) | Out-Null - } - - $passwordElement.SetAttribute("value", $pwd) -} - -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd - } -} - -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" - # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries - $DisabledPackageSources.RemoveChild($DisabledPackageSource) - } -} - -if (!(Test-Path $ConfigFile -PathType Leaf)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" - ExitWithExitCode 1 -} - -if (!$Password) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' - ExitWithExitCode 1 -} - -# Load NuGet.config -$doc = New-Object System.Xml.XmlDocument -$filename = (Get-Item $ConfigFile).FullName -$doc.Load($filename) - -# Get reference to or create one if none exist already -$sources = $doc.DocumentElement.SelectSingleNode("packageSources") -if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null -} - -# Looks for a node. Create it if none is found. -$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") -if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null -} - -# Check for disabledPackageSources; we'll enable any darc-int ones we find there -$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") -if ($disabledSources -ne $null) { - Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources -} - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password -} - -$dotnetVersions = @('5','6','7','8') - -foreach ($dotnetVersion in $dotnetVersions) { - $feedPrefix = "dotnet" + $dotnetVersion; - $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") - if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password - } -} - -$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh deleted file mode 100644 index d387c7eac..000000000 --- a/eng/common/SetupNugetSources.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash - -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 -# -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro's managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. -# -# - task: Bash@3 -# displayName: Setup Private Feeds Credentials -# inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(Build.SourcesDirectory)/NuGet.config $Token -# condition: ne(variables['Agent.OS'], 'Windows_NT') -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) - -ConfigFile=$1 -CredToken=$2 -NL='\n' -TB=' ' - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ ! -f "$ConfigFile" ]; then - Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" - ExitWithExitCode 1 -fi - -if [ -z "$CredToken" ]; then - Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" - ExitWithExitCode 1 -fi - -if [[ `uname -s` == "Darwin" ]]; then - NL=$'\\\n' - TB='' -fi - -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi - -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile -fi - -PackageSources=() - -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal-transport') -fi - -DotNetVersions=('5' '6' '7' '8') - -for DotNetVersion in ${DotNetVersions[@]} ; do - FeedPrefix="dotnet${DotNetVersion}"; - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") - fi -done - -# I want things split line by line -PrevIFS=$IFS -IFS=$'\n' -PackageSources+="$IFS" -PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') -IFS=$PrevIFS - -for FeedName in ${PackageSources[@]} ; do - # Check if there is no existing credential for this FeedName - grep -i "<$FeedName>" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." - - PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile - fi -done - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 deleted file mode 100644 index 33a6f2d0e..000000000 --- a/eng/common/build.ps1 +++ /dev/null @@ -1,166 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string][Alias('c')]$configuration = "Debug", - [string]$platform = $null, - [string] $projects, - [string][Alias('v')]$verbosity = "minimal", - [string] $msbuildEngine = $null, - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch][Alias('r')]$restore, - [switch] $deployDeps, - [switch][Alias('b')]$build, - [switch] $rebuild, - [switch] $deploy, - [switch][Alias('t')]$test, - [switch] $integrationTest, - [switch] $performanceTest, - [switch] $sign, - [switch] $pack, - [switch] $publish, - [switch] $clean, - [switch][Alias('bl')]$binaryLog, - [switch][Alias('nobl')]$excludeCIBinarylog, - [switch] $ci, - [switch] $prepareMachine, - [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '', - [switch] $excludePrereleaseVS, - [switch] $nativeToolsOnMachine, - [switch] $help, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties -) - -# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file -# some computer has this env var defined (e.g. Some HP) -if($env:Platform) { - $env:Platform="" -} -function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" - Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" - Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" - Write-Host " -binaryLog Output binary log (short: -bl)" - Write-Host " -help Print help and exit" - Write-Host "" - - Write-Host "Actions:" - Write-Host " -restore Restore dependencies (short: -r)" - Write-Host " -build Build solution (short: -b)" - Write-Host " -rebuild Rebuild solution" - Write-Host " -deploy Deploy built VSIXes" - Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" - Write-Host " -test Run all unit tests in the solution (short: -t)" - Write-Host " -integrationTest Run all integration tests in the solution" - Write-Host " -performanceTest Run all performance tests in the solution" - Write-Host " -pack Package build outputs into NuGet packages and Willow components" - Write-Host " -sign Sign build outputs" - Write-Host " -publish Publish artifacts (e.g. symbols)" - Write-Host " -clean Clean the solution" - Write-Host "" - - Write-Host "Advanced settings:" - Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" - Write-Host " -ci Set when running on CI server" - Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" - Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" - Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" - Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" - Write-Host "" - - Write-Host "Command line arguments not listed above are passed thru to msbuild." - Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." -} - -. $PSScriptRoot\tools.ps1 - -function InitializeCustomToolset { - if (-not $restore) { - return - } - - $script = Join-Path $EngRoot 'restore-toolset.ps1' - - if (Test-Path $script) { - . $script - } -} - -function Build { - $toolsetBuildProj = InitializeToolset - InitializeCustomToolset - - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } - $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } - - if ($projects) { - # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. - # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. - [string[]] $msbuildArgs = $properties - - # Resolve relative project paths into full paths - $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';') - - $msbuildArgs += "/p:Projects=$projects" - $properties = $msbuildArgs - } - - MSBuild $toolsetBuildProj ` - $bl ` - $platformArg ` - /p:Configuration=$configuration ` - /p:RepoRoot=$RepoRoot ` - /p:Restore=$restore ` - /p:DeployDeps=$deployDeps ` - /p:Build=$build ` - /p:Rebuild=$rebuild ` - /p:Deploy=$deploy ` - /p:Test=$test ` - /p:Pack=$pack ` - /p:IntegrationTest=$integrationTest ` - /p:PerformanceTest=$performanceTest ` - /p:Sign=$sign ` - /p:Publish=$publish ` - @properties -} - -try { - if ($clean) { - if (Test-Path $ArtifactsDir) { - Remove-Item -Recurse -Force $ArtifactsDir - Write-Host 'Artifacts directory deleted.' - } - exit 0 - } - - if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { - Print-Usage - exit 0 - } - - if ($ci) { - if (-not $excludeCIBinarylog) { - $binaryLog = $true - } - $nodeReuse = $false - } - - if ($nativeToolsOnMachine) { - $env:NativeToolsOnMachine = $true - } - if ($restore) { - InitializeNativeTools - } - - Build -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 diff --git a/eng/common/build.sh b/eng/common/build.sh deleted file mode 100644 index 50af40cdd..000000000 --- a/eng/common/build.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash - -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u - -# Stop script if command returns non-zero exit code. -# Prevents hidden errors caused by missing error code propagation. -set -e - -usage() -{ - echo "Common settings:" - echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" - echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" - echo " --binaryLog Create MSBuild binary log (short: -bl)" - echo " --help Print help and exit (short: -h)" - echo "" - - echo "Actions:" - echo " --restore Restore dependencies (short: -r)" - echo " --build Build solution (short: -b)" - echo " --sourceBuild Source-build the solution (short: -sb)" - echo " Will additionally trigger the following actions: --restore, --build, --pack" - echo " If --configuration is not set explicitly, will also set it to 'Release'" - echo " --rebuild Rebuild solution" - echo " --test Run all unit tests in the solution (short: -t)" - echo " --integrationTest Run all integration tests in the solution" - echo " --performanceTest Run all performance tests in the solution" - echo " --pack Package build outputs into NuGet packages and Willow components" - echo " --sign Sign build outputs" - echo " --publish Publish artifacts (e.g. symbols)" - echo " --clean Clean the solution" - echo "" - - echo "Advanced settings:" - echo " --projects Project or solution file(s) to build" - echo " --ci Set when running on CI server" - echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" - echo " --prepareMachine Prepare machine for CI run, clean up processes after build" - echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" - echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - echo "" - echo "Command line arguments not listed above are passed thru to msbuild." - echo "Arguments can also be passed in with a single hyphen." -} - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -restore=false -build=false -source_build=false -rebuild=false -test=false -integration_test=false -performance_test=false -pack=false -publish=false -sign=false -public=false -ci=false -clean=false - -warn_as_error=true -node_reuse=true -binary_log=false -exclude_ci_binary_log=false -pipelines_log=false - -projects='' -configuration='' -prepare_machine=false -verbosity='minimal' -runtime_source_feed='' -runtime_source_feed_key='' - -properties='' -while [[ $# > 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -help|-h) - usage - exit 0 - ;; - -clean) - clean=true - ;; - -configuration|-c) - configuration=$2 - shift - ;; - -verbosity|-v) - verbosity=$2 - shift - ;; - -binarylog|-bl) - binary_log=true - ;; - -excludeCIBinarylog|-nobl) - exclude_ci_binary_log=true - ;; - -pipelineslog|-pl) - pipelines_log=true - ;; - -restore|-r) - restore=true - ;; - -build|-b) - build=true - ;; - -rebuild) - rebuild=true - ;; - -pack) - pack=true - ;; - -sourcebuild|-sb) - build=true - source_build=true - restore=true - pack=true - ;; - -test|-t) - test=true - ;; - -integrationtest) - integration_test=true - ;; - -performancetest) - performance_test=true - ;; - -sign) - sign=true - ;; - -publish) - publish=true - ;; - -preparemachine) - prepare_machine=true - ;; - -projects) - projects=$2 - shift - ;; - -ci) - ci=true - ;; - -warnaserror) - warn_as_error=$2 - shift - ;; - -nodereuse) - node_reuse=$2 - shift - ;; - -runtimesourcefeed) - runtime_source_feed=$2 - shift - ;; - -runtimesourcefeedkey) - runtime_source_feed_key=$2 - shift - ;; - *) - properties="$properties $1" - ;; - esac - - shift -done - -if [[ -z "$configuration" ]]; then - if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi -fi - -if [[ "$ci" == true ]]; then - pipelines_log=true - node_reuse=false - if [[ "$exclude_ci_binary_log" == false ]]; then - binary_log=true - fi -fi - -. "$scriptroot/tools.sh" - -function InitializeCustomToolset { - local script="$eng_root/restore-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function Build { - InitializeToolset - InitializeCustomToolset - - if [[ ! -z "$projects" ]]; then - properties="$properties /p:Projects=$projects" - fi - - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:\"$log_dir/Build.binlog\"" - fi - - MSBuild $_InitializeToolset \ - $bl \ - /p:Configuration=$configuration \ - /p:RepoRoot="$repo_root" \ - /p:Restore=$restore \ - /p:Build=$build \ - /p:ArcadeBuildFromSource=$source_build \ - /p:Rebuild=$rebuild \ - /p:Test=$test \ - /p:Pack=$pack \ - /p:IntegrationTest=$integration_test \ - /p:PerformanceTest=$performance_test \ - /p:Sign=$sign \ - /p:Publish=$publish \ - $properties - - ExitWithExitCode 0 -} - -if [[ "$clean" == true ]]; then - if [ -d "$artifacts_dir" ]; then - rm -rf $artifacts_dir - echo "Artifacts directory deleted." - fi - exit 0 -fi - -if [[ "$restore" == true ]]; then - InitializeNativeTools -fi - -Build diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh deleted file mode 100644 index 1a02c0dec..000000000 --- a/eng/common/cibuild.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where - # the symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@ \ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.bionic b/eng/common/cross/arm/sources.list.bionic deleted file mode 100644 index 210955740..000000000 --- a/eng/common/cross/arm/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jessie b/eng/common/cross/arm/sources.list.jessie deleted file mode 100644 index 4d142ac9b..000000000 --- a/eng/common/cross/arm/sources.list.jessie +++ /dev/null @@ -1,3 +0,0 @@ -# Debian (sid) # UNSTABLE -deb http://ftp.debian.org/debian/ sid main contrib non-free -deb-src http://ftp.debian.org/debian/ sid main contrib non-free diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial deleted file mode 100644 index 56fbb36a5..000000000 --- a/eng/common/cross/arm/sources.list.xenial +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.zesty b/eng/common/cross/arm/sources.list.zesty deleted file mode 100644 index ea2c14a78..000000000 --- a/eng/common/cross/arm/sources.list.zesty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.bionic b/eng/common/cross/arm64/sources.list.bionic deleted file mode 100644 index 210955740..000000000 --- a/eng/common/cross/arm64/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial deleted file mode 100644 index 56fbb36a5..000000000 --- a/eng/common/cross/arm64/sources.list.xenial +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.zesty b/eng/common/cross/arm64/sources.list.zesty deleted file mode 100644 index ea2c14a78..000000000 --- a/eng/common/cross/arm64/sources.list.zesty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch deleted file mode 100644 index 2d2615619..000000000 --- a/eng/common/cross/armel/armel.jessie.patch +++ /dev/null @@ -1,43 +0,0 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700 -+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700 -@@ -69,10 +69,10 @@ - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: - return __sync_val_compare_and_swap_8(addr, old, _new); -@@ -109,7 +109,7 @@ - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t*) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -@@ -148,7 +148,7 @@ - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t*) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -@@ -187,7 +187,7 @@ - return __sync_add_and_fetch_2(addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t*) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: - return __sync_add_and_fetch_8(addr, val); diff --git a/eng/common/cross/armel/sources.list.jessie b/eng/common/cross/armel/sources.list.jessie deleted file mode 100644 index 3d9c3059d..000000000 --- a/eng/common/cross/armel/sources.list.jessie +++ /dev/null @@ -1,3 +0,0 @@ -# Debian (jessie) # Stable -deb http://ftp.debian.org/debian/ jessie main contrib non-free -deb-src http://ftp.debian.org/debian/ jessie main contrib non-free diff --git a/eng/common/cross/armel/tizen/tizen.patch b/eng/common/cross/armel/tizen/tizen.patch deleted file mode 100644 index ca7c7c1ff..000000000 --- a/eng/common/cross/armel/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster deleted file mode 100644 index f27fc4fb3..000000000 --- a/eng/common/cross/armv6/sources.list.buster +++ /dev/null @@ -1,2 +0,0 @@ -deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi -deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh deleted file mode 100644 index f163fb9da..000000000 --- a/eng/common/cross/build-android-rootfs.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -set -e -__NDK_Version=r21 - -usage() -{ - echo "Creates a toolchain and sysroot used for cross-compiling for Android." - echo. - echo "Usage: $0 [BuildArch] [ApiLevel]" - echo. - echo "BuildArch is the target architecture of Android. Currently only arm64 is supported." - echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html" - echo. - echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior" - echo "by setting the TOOLCHAIN_DIR environment variable" - echo. - echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," - echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." - echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." - exit 1 -} - -__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h -__BuildArch=arm64 -__AndroidArch=aarch64 -__AndroidToolchain=aarch64-linux-android - -for i in "$@" - do - lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" - case $lowerI in - -?|-h|--help) - usage - exit 1 - ;; - arm64) - __BuildArch=arm64 - __AndroidArch=aarch64 - __AndroidToolchain=aarch64-linux-android - ;; - arm) - __BuildArch=arm - __AndroidArch=arm - __AndroidToolchain=arm-linux-androideabi - ;; - *[0-9]) - __ApiLevel=$i - ;; - *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" - ;; - esac -done - -# Obtain the location of the bash script to figure out where the root of the repo is. -__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs" - -if [[ ! -f "$__CrossDir" ]]; then - mkdir -p "$__CrossDir" -fi - -# Resolve absolute path to avoid `../` in build logs -__CrossDir="$( cd "$__CrossDir" && pwd )" - -__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version" -__lldb_Dir="$__CrossDir/lldb" -__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version" - -if [[ -n "$TOOLCHAIN_DIR" ]]; then - __ToolchainDir=$TOOLCHAIN_DIR -fi - -if [[ -n "$NDK_DIR" ]]; then - __NDK_Dir=$NDK_DIR -fi - -echo "Target API level: $__ApiLevel" -echo "Target architecture: $__BuildArch" -echo "NDK location: $__NDK_Dir" -echo "Target Toolchain location: $__ToolchainDir" - -# Download the NDK if required -if [ ! -d $__NDK_Dir ]; then - echo Downloading the NDK into $__NDK_Dir - mkdir -p $__NDK_Dir - wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir -fi - -if [ ! -d $__lldb_Dir ]; then - mkdir -p $__lldb_Dir - echo Downloading LLDB into $__lldb_Dir - wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip - unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir -fi - -echo "Download dependencies..." -__TmpDir=$__CrossDir/tmp/$__BuildArch/ -mkdir -p "$__TmpDir" - -# combined dependencies for coreclr, installer and libraries -__AndroidPackages="libicu" -__AndroidPackages+=" libandroid-glob" -__AndroidPackages+=" liblzma" -__AndroidPackages+=" krb5" -__AndroidPackages+=" openssl" - -for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\ - grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do - - if [[ "$path" != "Filename:" ]]; then - echo "Working on: $path" - wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir" - fi -done - -cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" - -# Generate platform file for build.sh script to assign to __DistroRid -echo "Generating platform file..." -echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform - -echo "Now to build coreclr, libraries and installers; run:" -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory coreclr -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory libraries -echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory installer diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh deleted file mode 100644 index 9caf9b021..000000000 --- a/eng/common/cross/build-rootfs.sh +++ /dev/null @@ -1,648 +0,0 @@ -#!/usr/bin/env bash - -set -e - -usage() -{ - echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" - echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" - echo " for alpine can be specified with version: alpineX.YY or alpineedge" - echo " for FreeBSD can be: freebsd12, freebsd13" - echo " for illumos can be: illumos" - echo " for Haiku can be: haiku." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" - echo "llvmx[.y] - optional, LLVM version for LLVM related packages." - echo "--skipunmount - optional, will skip the unmount of rootfs folder." - echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)." - echo "--use-mirror - optional, use mirror URL to fetch resources, when available." - echo "--jobs N - optional, restrict to N jobs." - exit 1 -} - -__CodeName=xenial -__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__BuildArch=arm -__AlpineArch=armv7 -__FreeBSDArch=arm -__FreeBSDMachineArch=armv7 -__IllumosArch=arm7 -__HaikuArch=arm -__QEMUArch=arm -__UbuntuArch=armhf -__UbuntuRepo="http://ports.ubuntu.com/" -__LLDB_Package="liblldb-3.9-dev" -__SkipUnmount=0 - -# base development support -__UbuntuPackages="build-essential" - -__AlpinePackages="alpine-base" -__AlpinePackages+=" build-base" -__AlpinePackages+=" linux-headers" -__AlpinePackages+=" lldb-dev" -__AlpinePackages+=" python3" -__AlpinePackages+=" libedit" - -# symlinks fixer -__UbuntuPackages+=" symlinks" - -# runtime dependencies -__UbuntuPackages+=" libicu-dev" -__UbuntuPackages+=" liblttng-ust-dev" -__UbuntuPackages+=" libunwind8-dev" -__UbuntuPackages+=" libnuma-dev" - -__AlpinePackages+=" gettext-dev" -__AlpinePackages+=" icu-dev" -__AlpinePackages+=" libunwind-dev" -__AlpinePackages+=" lttng-ust-dev" -__AlpinePackages+=" compiler-rt" -__AlpinePackages+=" numactl-dev" - -# runtime libraries' dependencies -__UbuntuPackages+=" libcurl4-openssl-dev" -__UbuntuPackages+=" libkrb5-dev" -__UbuntuPackages+=" libssl-dev" -__UbuntuPackages+=" zlib1g-dev" - -__AlpinePackages+=" curl-dev" -__AlpinePackages+=" krb5-dev" -__AlpinePackages+=" openssl-dev" -__AlpinePackages+=" zlib-dev" - -__FreeBSDBase="12.4-RELEASE" -__FreeBSDPkg="1.17.0" -__FreeBSDABI="12" -__FreeBSDPackages="libunwind" -__FreeBSDPackages+=" icu" -__FreeBSDPackages+=" libinotify" -__FreeBSDPackages+=" openssl" -__FreeBSDPackages+=" krb5" -__FreeBSDPackages+=" terminfo-db" - -__IllumosPackages="icu" -__IllumosPackages+=" mit-krb5" -__IllumosPackages+=" openssl" -__IllumosPackages+=" zlib" - -__HaikuPackages="gcc_syslibs" -__HaikuPackages+=" gcc_syslibs_devel" -__HaikuPackages+=" gmp" -__HaikuPackages+=" gmp_devel" -__HaikuPackages+=" icu66" -__HaikuPackages+=" icu66_devel" -__HaikuPackages+=" krb5" -__HaikuPackages+=" krb5_devel" -__HaikuPackages+=" libiconv" -__HaikuPackages+=" libiconv_devel" -__HaikuPackages+=" llvm12_libunwind" -__HaikuPackages+=" llvm12_libunwind_devel" -__HaikuPackages+=" mpfr" -__HaikuPackages+=" mpfr_devel" -__HaikuPackages+=" openssl" -__HaikuPackages+=" openssl_devel" -__HaikuPackages+=" zlib" -__HaikuPackages+=" zlib_devel" - -# ML.NET dependencies -__UbuntuPackages+=" libomp5" -__UbuntuPackages+=" libomp-dev" - -# Taken from https://github.com/alpinelinux/alpine-chroot-install/blob/6d08f12a8a70dd9b9dc7d997c88aa7789cc03c42/alpine-chroot-install#L85-L133 -__AlpineKeys=' -4a6a0840:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1yHJxQgsHQREclQu4Ohe\nqxTxd1tHcNnvnQTu/UrTky8wWvgXT+jpveroeWWnzmsYlDI93eLI2ORakxb3gA2O\nQ0Ry4ws8vhaxLQGC74uQR5+/yYrLuTKydFzuPaS1dK19qJPXB8GMdmFOijnXX4SA\njixuHLe1WW7kZVtjL7nufvpXkWBGjsfrvskdNA/5MfxAeBbqPgaq0QMEfxMAn6/R\nL5kNepi/Vr4S39Xvf2DzWkTLEK8pcnjNkt9/aafhWqFVW7m3HCAII6h/qlQNQKSo\nGuH34Q8GsFG30izUENV9avY7hSLq7nggsvknlNBZtFUcmGoQrtx3FmyYsIC8/R+B\nywIDAQAB -5243ef4b:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvNijDxJ8kloskKQpJdx+\nmTMVFFUGDoDCbulnhZMJoKNkSuZOzBoFC94omYPtxnIcBdWBGnrm6ncbKRlR+6oy\nDO0W7c44uHKCFGFqBhDasdI4RCYP+fcIX/lyMh6MLbOxqS22TwSLhCVjTyJeeH7K\naA7vqk+QSsF4TGbYzQDDpg7+6aAcNzg6InNePaywA6hbT0JXbxnDWsB+2/LLSF2G\nmnhJlJrWB1WGjkz23ONIWk85W4S0XB/ewDefd4Ly/zyIciastA7Zqnh7p3Ody6Q0\nsS2MJzo7p3os1smGjUF158s6m/JbVh4DN6YIsxwl2OjDOz9R0OycfJSDaBVIGZzg\ncQIDAQAB -524d27bb:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr8s1q88XpuJWLCZALdKj\nlN8wg2ePB2T9aIcaxryYE/Jkmtu+ZQ5zKq6BT3y/udt5jAsMrhHTwroOjIsF9DeG\ne8Y3vjz+Hh4L8a7hZDaw8jy3CPag47L7nsZFwQOIo2Cl1SnzUc6/owoyjRU7ab0p\niWG5HK8IfiybRbZxnEbNAfT4R53hyI6z5FhyXGS2Ld8zCoU/R4E1P0CUuXKEN4p0\n64dyeUoOLXEWHjgKiU1mElIQj3k/IF02W89gDj285YgwqA49deLUM7QOd53QLnx+\nxrIrPv3A+eyXMFgexNwCKQU9ZdmWa00MjjHlegSGK8Y2NPnRoXhzqSP9T9i2HiXL\nVQIDAQAB -5261cecb:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwlzMkl7b5PBdfMzGdCT0\ncGloRr5xGgVmsdq5EtJvFkFAiN8Ac9MCFy/vAFmS8/7ZaGOXoCDWbYVLTLOO2qtX\nyHRl+7fJVh2N6qrDDFPmdgCi8NaE+3rITWXGrrQ1spJ0B6HIzTDNEjRKnD4xyg4j\ng01FMcJTU6E+V2JBY45CKN9dWr1JDM/nei/Pf0byBJlMp/mSSfjodykmz4Oe13xB\nCa1WTwgFykKYthoLGYrmo+LKIGpMoeEbY1kuUe04UiDe47l6Oggwnl+8XD1MeRWY\nsWgj8sF4dTcSfCMavK4zHRFFQbGp/YFJ/Ww6U9lA3Vq0wyEI6MCMQnoSMFwrbgZw\nwwIDAQAB -58199dcc:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3v8/ye/V/t5xf4JiXLXa\nhWFRozsnmn3hobON20GdmkrzKzO/eUqPOKTpg2GtvBhK30fu5oY5uN2ORiv2Y2ht\neLiZ9HVz3XP8Fm9frha60B7KNu66FO5P2o3i+E+DWTPqqPcCG6t4Znk2BypILcit\nwiPKTsgbBQR2qo/cO01eLLdt6oOzAaF94NH0656kvRewdo6HG4urbO46tCAizvCR\nCA7KGFMyad8WdKkTjxh8YLDLoOCtoZmXmQAiwfRe9pKXRH/XXGop8SYptLqyVVQ+\ntegOD9wRs2tOlgcLx4F/uMzHN7uoho6okBPiifRX+Pf38Vx+ozXh056tjmdZkCaV\naQIDAQAB -58cbb476:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoSPnuAGKtRIS5fEgYPXD\n8pSGvKAmIv3A08LBViDUe+YwhilSHbYXUEAcSH1KZvOo1WT1x2FNEPBEFEFU1Eyc\n+qGzbA03UFgBNvArurHQ5Z/GngGqE7IarSQFSoqewYRtFSfp+TL9CUNBvM0rT7vz\n2eMu3/wWG+CBmb92lkmyWwC1WSWFKO3x8w+Br2IFWvAZqHRt8oiG5QtYvcZL6jym\nY8T6sgdDlj+Y+wWaLHs9Fc+7vBuyK9C4O1ORdMPW15qVSl4Lc2Wu1QVwRiKnmA+c\nDsH/m7kDNRHM7TjWnuj+nrBOKAHzYquiu5iB3Qmx+0gwnrSVf27Arc3ozUmmJbLj\nzQIDAQAB -58e4f17d:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvBxJN9ErBgdRcPr5g4hV\nqyUSGZEKuvQliq2Z9SRHLh2J43+EdB6A+yzVvLnzcHVpBJ+BZ9RV30EM9guck9sh\nr+bryZcRHyjG2wiIEoduxF2a8KeWeQH7QlpwGhuobo1+gA8L0AGImiA6UP3LOirl\nI0G2+iaKZowME8/tydww4jx5vG132JCOScMjTalRsYZYJcjFbebQQolpqRaGB4iG\nWqhytWQGWuKiB1A22wjmIYf3t96l1Mp+FmM2URPxD1gk/BIBnX7ew+2gWppXOK9j\n1BJpo0/HaX5XoZ/uMqISAAtgHZAqq+g3IUPouxTphgYQRTRYpz2COw3NF43VYQrR\nbQIDAQAB -60ac2099:MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwR4uJVtJOnOFGchnMW5Y\nj5/waBdG1u5BTMlH+iQMcV5+VgWhmpZHJCBz3ocD+0IGk2I68S5TDOHec/GSC0lv\n6R9o6F7h429GmgPgVKQsc8mPTPtbjJMuLLs4xKc+viCplXc0Nc0ZoHmCH4da6fCV\ntdpHQjVe6F9zjdquZ4RjV6R6JTiN9v924dGMAkbW/xXmamtz51FzondKC52Gh8Mo\n/oA0/T0KsCMCi7tb4QNQUYrf+Xcha9uus4ww1kWNZyfXJB87a2kORLiWMfs2IBBJ\nTmZ2Fnk0JnHDb8Oknxd9PvJPT0mvyT8DA+KIAPqNvOjUXP4bnjEHJcoCP9S5HkGC\nIQIDAQAB -6165ee59:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAutQkua2CAig4VFSJ7v54\nALyu/J1WB3oni7qwCZD3veURw7HxpNAj9hR+S5N/pNeZgubQvJWyaPuQDm7PTs1+\ntFGiYNfAsiibX6Rv0wci3M+z2XEVAeR9Vzg6v4qoofDyoTbovn2LztaNEjTkB+oK\ntlvpNhg1zhou0jDVYFniEXvzjckxswHVb8cT0OMTKHALyLPrPOJzVtM9C1ew2Nnc\n3848xLiApMu3NBk0JqfcS3Bo5Y2b1FRVBvdt+2gFoKZix1MnZdAEZ8xQzL/a0YS5\nHd0wj5+EEKHfOd3A75uPa/WQmA+o0cBFfrzm69QDcSJSwGpzWrD1ScH3AK8nWvoj\nv7e9gukK/9yl1b4fQQ00vttwJPSgm9EnfPHLAtgXkRloI27H6/PuLoNvSAMQwuCD\nhQRlyGLPBETKkHeodfLoULjhDi1K2gKJTMhtbnUcAA7nEphkMhPWkBpgFdrH+5z4\nLxy+3ek0cqcI7K68EtrffU8jtUj9LFTUC8dERaIBs7NgQ/LfDbDfGh9g6qVj1hZl\nk9aaIPTm/xsi8v3u+0qaq7KzIBc9s59JOoA8TlpOaYdVgSQhHHLBaahOuAigH+VI\nisbC9vmqsThF2QdDtQt37keuqoda2E6sL7PUvIyVXDRfwX7uMDjlzTxHTymvq2Ck\nhtBqojBnThmjJQFgZXocHG8CAwEAAQ== -61666e3f:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlEyxkHggKCXC2Wf5Mzx4\nnZLFZvU2bgcA3exfNPO/g1YunKfQY+Jg4fr6tJUUTZ3XZUrhmLNWvpvSwDS19ZmC\nIXOu0+V94aNgnhMsk9rr59I8qcbsQGIBoHzuAl8NzZCgdbEXkiY90w1skUw8J57z\nqCsMBydAueMXuWqF5nGtYbi5vHwK42PffpiZ7G5Kjwn8nYMW5IZdL6ZnMEVJUWC9\nI4waeKg0yskczYDmZUEAtrn3laX9677ToCpiKrvmZYjlGl0BaGp3cxggP2xaDbUq\nqfFxWNgvUAb3pXD09JM6Mt6HSIJaFc9vQbrKB9KT515y763j5CC2KUsilszKi3mB\nHYe5PoebdjS7D1Oh+tRqfegU2IImzSwW3iwA7PJvefFuc/kNIijfS/gH/cAqAK6z\nbhdOtE/zc7TtqW2Wn5Y03jIZdtm12CxSxwgtCF1NPyEWyIxAQUX9ACb3M0FAZ61n\nfpPrvwTaIIxxZ01L3IzPLpbc44x/DhJIEU+iDt6IMTrHOphD9MCG4631eIdB0H1b\n6zbNX1CXTsafqHRFV9XmYYIeOMggmd90s3xIbEujA6HKNP/gwzO6CDJ+nHFDEqoF\nSkxRdTkEqjTjVKieURW7Swv7zpfu5PrsrrkyGnsRrBJJzXlm2FOOxnbI2iSL1B5F\nrO5kbUxFeZUIDq+7Yv4kLWcCAwEAAQ== -616a9724:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnC+bR4bHf/L6QdU4puhQ\ngl1MHePszRC38bzvVFDUJsmCaMCL2suCs2A2yxAgGb9pu9AJYLAmxQC4mM3jNqhg\n/E7yuaBbek3O02zN/ctvflJ250wZCy+z0ZGIp1ak6pu1j14IwHokl9j36zNfGtfv\nADVOcdpWITFFlPqwq1qt/H3UsKVmtiF3BNWWTeUEQwKvlU8ymxgS99yn0+4OPyNT\nL3EUeS+NQJtDS01unau0t7LnjUXn+XIneWny8bIYOQCuVR6s/gpIGuhBaUqwaJOw\n7jkJZYF2Ij7uPb4b5/R3vX2FfxxqEHqssFSg8FFUNTZz3qNZs0CRVyfA972g9WkJ\nhPfn31pQYil4QGRibCMIeU27YAEjXoqfJKEPh4UWMQsQLrEfdGfb8VgwrPbniGfU\nL3jKJR3VAafL9330iawzVQDlIlwGl6u77gEXMl9K0pfazunYhAp+BMP+9ot5ckK+\nosmrqj11qMESsAj083GeFdfV3pXEIwUytaB0AKEht9DbqUfiE/oeZ/LAXgySMtVC\nsbC4ESmgVeY2xSBIJdDyUap7FR49GGrw0W49NUv9gRgQtGGaNVQQO9oGL2PBC41P\niWF9GLoX30HIz1P8PF/cZvicSSPkQf2Z6TV+t0ebdGNS5DjapdnCrq8m9Z0pyKsQ\nuxAL2a7zX8l5i1CZh1ycUGsCAwEAAQ== -616abc23:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0MfCDrhODRCIxR9Dep1s\neXafh5CE5BrF4WbCgCsevyPIdvTeyIaW4vmO3bbG4VzhogDZju+R3IQYFuhoXP5v\nY+zYJGnwrgz3r5wYAvPnLEs1+dtDKYOgJXQj+wLJBW1mzRDL8FoRXOe5iRmn1EFS\nwZ1DoUvyu7/J5r0itKicZp3QKED6YoilXed+1vnS4Sk0mzN4smuMR9eO1mMCqNp9\n9KTfRDHTbakIHwasECCXCp50uXdoW6ig/xUAFanpm9LtK6jctNDbXDhQmgvAaLXZ\nLvFqoaYJ/CvWkyYCgL6qxvMvVmPoRv7OPcyni4xR/WgWa0MSaEWjgPx3+yj9fiMA\n1S02pFWFDOr5OUF/O4YhFJvUCOtVsUPPfA/Lj6faL0h5QI9mQhy5Zb9TTaS9jB6p\nLw7u0dJlrjFedk8KTJdFCcaGYHP6kNPnOxMylcB/5WcztXZVQD5WpCicGNBxCGMm\nW64SgrV7M07gQfL/32QLsdqPUf0i8hoVD8wfQ3EpbQzv6Fk1Cn90bZqZafg8XWGY\nwddhkXk7egrr23Djv37V2okjzdqoyLBYBxMz63qQzFoAVv5VoY2NDTbXYUYytOvG\nGJ1afYDRVWrExCech1mX5ZVUB1br6WM+psFLJFoBFl6mDmiYt0vMYBddKISsvwLl\nIJQkzDwtXzT2cSjoj3T5QekCAwEAAQ== -616ac3bc:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvaaoSLab+IluixwKV5Od\n0gib2YurjPatGIbn5Ov2DLUFYiebj2oJINXJSwUOO+4WcuHFEqiL/1rya+k5hLZt\nhnPL1tn6QD4rESznvGSasRCQNT2vS/oyZbTYJRyAtFkEYLlq0t3S3xBxxHWuvIf0\nqVxVNYpQWyM3N9RIeYBR/euXKJXileSHk/uq1I5wTC0XBIHWcthczGN0m9wBEiWS\n0m3cnPk4q0Ea8mUJ91Rqob19qETz6VbSPYYpZk3qOycjKosuwcuzoMpwU8KRiMFd\n5LHtX0Hx85ghGsWDVtS0c0+aJa4lOMGvJCAOvDfqvODv7gKlCXUpgumGpLdTmaZ8\n1RwqspAe3IqBcdKTqRD4m2mSg23nVx2FAY3cjFvZQtfooT7q1ItRV5RgH6FhQSl7\n+6YIMJ1Bf8AAlLdRLpg+doOUGcEn+pkDiHFgI8ylH1LKyFKw+eXaAml/7DaWZk1d\ndqggwhXOhc/UUZFQuQQ8A8zpA13PcbC05XxN2hyP93tCEtyynMLVPtrRwDnHxFKa\nqKzs3rMDXPSXRn3ZZTdKH3069ApkEjQdpcwUh+EmJ1Ve/5cdtzT6kKWCjKBFZP/s\n91MlRrX2BTRdHaU5QJkUheUtakwxuHrdah2F94lRmsnQlpPr2YseJu6sIE+Dnx4M\nCfhdVbQL2w54R645nlnohu8CAwEAAQ== -616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ== -616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ== -616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== -' -__Keyring= -__SkipSigCheck=0 -__UseMirror=0 - -__UnprocessedBuildArgs= -while :; do - if [[ "$#" -le 0 ]]; then - break - fi - - lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case $lowerI in - -\?|-h|--help) - usage - exit 1 - ;; - arm) - __BuildArch=arm - __UbuntuArch=armhf - __AlpineArch=armv7 - __QEMUArch=arm - ;; - arm64) - __BuildArch=arm64 - __UbuntuArch=arm64 - __AlpineArch=aarch64 - __QEMUArch=aarch64 - __FreeBSDArch=arm64 - __FreeBSDMachineArch=aarch64 - ;; - armel) - __BuildArch=armel - __UbuntuArch=armel - __UbuntuRepo="http://ftp.debian.org/debian/" - __CodeName=jessie - ;; - armv6) - __BuildArch=armv6 - __UbuntuArch=armhf - __QEMUArch=arm - __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" - __CodeName=buster - __LLDB_Package="liblldb-6.0-dev" - - if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" - fi - ;; - riscv64) - __BuildArch=riscv64 - __AlpineArch=riscv64 - __AlpinePackages="${__AlpinePackages// lldb-dev/}" - __QEMUArch=riscv64 - __UbuntuArch=riscv64 - __UbuntuRepo="http://deb.debian.org/debian-ports" - __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" - unset __LLDB_Package - - if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" - fi - ;; - ppc64le) - __BuildArch=ppc64le - __AlpineArch=ppc64le - __QEMUArch=ppc64le - __UbuntuArch=ppc64el - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp5/}" - unset __LLDB_Package - ;; - s390x) - __BuildArch=s390x - __AlpineArch=s390x - __QEMUArch=s390x - __UbuntuArch=s390x - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp5/}" - unset __LLDB_Package - ;; - x64) - __BuildArch=x64 - __AlpineArch=x86_64 - __UbuntuArch=amd64 - __FreeBSDArch=amd64 - __FreeBSDMachineArch=amd64 - __illumosArch=x86_64 - __HaikuArch=x86_64 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" - ;; - x86) - __BuildArch=x86 - __UbuntuArch=i386 - __AlpineArch=x86 - __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" - ;; - lldb*) - version="${lowerI/lldb/}" - parts=(${version//./ }) - - # for versions > 6.0, lldb has dropped the minor version - if [[ "${parts[0]}" -gt 6 ]]; then - version="${parts[0]}" - fi - - __LLDB_Package="liblldb-${version}-dev" - ;; - no-lldb) - unset __LLDB_Package - ;; - llvm*) - version="${lowerI/llvm/}" - parts=(${version//./ }) - __LLVM_MajorVersion="${parts[0]}" - __LLVM_MinorVersion="${parts[1]}" - - # for versions > 6.0, llvm has dropped the minor version - if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then - __LLVM_MinorVersion=0; - fi - ;; - xenial) # Ubuntu 16.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=xenial - fi - ;; - zesty) # Ubuntu 17.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=zesty - fi - ;; - bionic) # Ubuntu 18.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=bionic - fi - ;; - focal) # Ubuntu 20.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=focal - fi - ;; - jammy) # Ubuntu 22.04 - if [[ "$__CodeName" != "jessie" ]]; then - __CodeName=jammy - fi - ;; - jessie) # Debian 8 - __CodeName=jessie - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi - ;; - stretch) # Debian 9 - __CodeName=stretch - __LLDB_Package="liblldb-6.0-dev" - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi - ;; - buster) # Debian 10 - __CodeName=buster - __LLDB_Package="liblldb-6.0-dev" - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi - ;; - bullseye) # Debian 11 - __CodeName=bullseye - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi - ;; - sid) # Debian sid - __CodeName=sid - - if [[ -z "$__UbuntuRepo" ]]; then - __UbuntuRepo="http://ftp.debian.org/debian/" - fi - ;; - tizen) - __CodeName= - __UbuntuRepo= - __Tizen=tizen - ;; - alpine*) - __CodeName=alpine - __UbuntuRepo= - version="${lowerI/alpine/}" - - if [[ "$version" == "edge" ]]; then - __AlpineVersion=edge - else - parts=(${version//./ }) - __AlpineMajorVersion="${parts[0]}" - __AlpineMinoVersion="${parts[1]}" - __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion" - fi - ;; - freebsd12) - __CodeName=freebsd - __SkipUnmount=1 - ;; - freebsd13) - __CodeName=freebsd - __FreeBSDBase="13.2-RELEASE" - __FreeBSDABI="13" - __SkipUnmount=1 - ;; - illumos) - __CodeName=illumos - __SkipUnmount=1 - ;; - haiku) - __CodeName=haiku - __SkipUnmount=1 - ;; - --skipunmount) - __SkipUnmount=1 - ;; - --skipsigcheck) - __SkipSigCheck=1 - ;; - --rootfsdir|-rootfsdir) - shift - __RootfsDir="$1" - ;; - --use-mirror) - __UseMirror=1 - ;; - --use-jobs) - shift - MAXJOBS=$1 - ;; - *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" - ;; - esac - - shift -done - -case "$__AlpineVersion" in - 3.14) __AlpinePackages+=" llvm11-libs" ;; - 3.15) __AlpinePackages+=" llvm12-libs" ;; - 3.16) __AlpinePackages+=" llvm13-libs" ;; - 3.17) __AlpinePackages+=" llvm15-libs" ;; - edge) __AlpineLlvmLibsLookup=1 ;; - *) - if [[ "$__AlpineArch" =~ s390x|ppc64le ]]; then - __AlpineVersion=3.15 # minimum version that supports lldb-dev - __AlpinePackages+=" llvm12-libs" - elif [[ "$__AlpineArch" == "x86" ]]; then - __AlpineVersion=3.17 # minimum version that supports lldb-dev - __AlpinePackages+=" llvm15-libs" - elif [[ "$__AlpineArch" == "riscv64" ]]; then - __AlpineLlvmLibsLookup=1 - __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive) - else - __AlpineVersion=3.13 # 3.13 to maximize compatibility - __AlpinePackages+=" llvm10-libs" - - if [[ "$__AlpineArch" == "armv7" ]]; then - __AlpinePackages="${__AlpinePackages//numactl-dev/}" - fi - fi -esac - -if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then - # compiler-rt--static was merged in compiler-rt package in alpine 3.16 - # for older versions, we need compiler-rt--static, so replace the name - __AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}" -fi - -if [[ "$__BuildArch" == "armel" ]]; then - __LLDB_Package="lldb-3.5-dev" -fi - -if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then - # libnuma-dev is not available on armhf for xenial - __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}" -fi - -__UbuntuPackages+=" ${__LLDB_Package:-}" - -if [[ -n "$__LLVM_MajorVersion" ]]; then - __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" -fi - -if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then - __RootfsDir="$ROOTFS_DIR" -fi - -if [[ -z "$__RootfsDir" ]]; then - __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" -fi - -if [[ -d "$__RootfsDir" ]]; then - if [[ "$__SkipUnmount" == "0" ]]; then - umount "$__RootfsDir"/* || true - fi - rm -rf "$__RootfsDir" -fi - -mkdir -p "$__RootfsDir" -__RootfsDir="$( cd "$__RootfsDir" && pwd )" - -if [[ "$__CodeName" == "alpine" ]]; then - __ApkToolsVersion=2.12.11 - __ApkToolsSHA512SUM=53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33 - __ApkToolsDir="$(mktemp -d)" - __ApkKeysDir="$(mktemp -d)" - - wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir" - echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c - chmod +x "$__ApkToolsDir/apk.static" - - if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then - mkdir -p "$__RootfsDir"/usr/bin - cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" - fi - - if [[ "$__AlpineVersion" == "edge" ]]; then - version=edge - else - version="v$__AlpineVersion" - fi - - for line in $__AlpineKeys; do - id="${line%%:*}" - content="${line#*:}" - - echo -e "-----BEGIN PUBLIC KEY-----\n$content\n-----END PUBLIC KEY-----" > "$__ApkKeysDir/alpine-devel@lists.alpinelinux.org-$id.rsa.pub" - done - - if [[ "$__SkipSigCheck" == "1" ]]; then - __ApkSignatureArg="--allow-untrusted" - else - __ApkSignatureArg="--keys-dir $__ApkKeysDir" - fi - - # initialize DB - "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add - - if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then - __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ - search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" - fi - - # install all packages in one go - "$__ApkToolsDir/apk.static" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ - -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \ - add $__AlpinePackages - - rm -r "$__ApkToolsDir" -elif [[ "$__CodeName" == "freebsd" ]]; then - mkdir -p "$__RootfsDir"/usr/local/etc - JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf - echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf - mkdir -p "$__RootfsDir"/tmp - # get and build package manager - wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - - cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" - # needed for install to succeed - mkdir -p "$__RootfsDir"/host/etc - ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install - rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" - # install packages we need. - INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update - INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages -elif [[ "$__CodeName" == "illumos" ]]; then - mkdir "$__RootfsDir/tmp" - pushd "$__RootfsDir/tmp" - JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - echo "Downloading sysroot." - wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - - echo "Building binutils. Please wait.." - wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - - mkdir build-binutils && cd build-binutils - ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" - make -j "$JOBS" && make install && cd .. - echo "Building gcc. Please wait.." - wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - - CFLAGS="-fPIC" - CXXFLAGS="-fPIC" - CXXFLAGS_FOR_TARGET="-fPIC" - CFLAGS_FOR_TARGET="-fPIC" - export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET - mkdir build-gcc && cd build-gcc - ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ - --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ - --disable-libquadmath-support --disable-shared --enable-tls - make -j "$JOBS" && make install && cd .. - BaseUrl=https://pkgsrc.smartos.org - if [[ "$__UseMirror" == 1 ]]; then - BaseUrl=https://pkgsrc.smartos.skylime.net - fi - BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" - echo "Downloading manifest" - wget "$BaseUrl" - echo "Downloading dependencies." - read -ra array <<<"$__IllumosPackages" - for package in "${array[@]}"; do - echo "Installing '$package'" - # find last occurrence of package in listing and extract its name - package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" - echo "Resolved name '$package'" - wget "$BaseUrl"/"$package".tgz - ar -x "$package".tgz - tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null - done - echo "Cleaning up temporary files." - popd - rm -rf "$__RootfsDir"/{tmp,+*} - mkdir -p "$__RootfsDir"/usr/include/net - mkdir -p "$__RootfsDir"/usr/include/netpacket - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h - wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h - wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h - wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h -elif [[ "$__CodeName" == "haiku" ]]; then - JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} - - echo "Building Haiku sysroot for $__HaikuArch" - mkdir -p "$__RootfsDir/tmp" - pushd "$__RootfsDir/tmp" - - mkdir "$__RootfsDir/tmp/download" - - echo "Downloading Haiku package tool" - git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 $__RootfsDir/tmp/script - wget -O "$__RootfsDir/tmp/download/hosttools.zip" $($__RootfsDir/tmp/script/fetch.sh --hosttools) - unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin" - - DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" - HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current" - - # Download Haiku packages - echo "Downloading Haiku packages" - read -ra array <<<"$__HaikuPackages" - for package in "${array[@]}"; do - echo "Downloading $package..." - # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 - # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 - hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \ - --header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" - wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl" - done - for package in haiku haiku_devel; do - echo "Downloading $package..." - hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')" - wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg" - done - - # Set up the sysroot - echo "Setting up sysroot and extracting required packages" - mkdir -p "$__RootfsDir/boot/system" - for file in "$__RootfsDir/tmp/download/"*.hpkg; do - echo "Extracting $file..." - LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package" extract -C "$__RootfsDir/boot/system" "$file" - done - - # Download buildtools - echo "Downloading Haiku buildtools" - wget -O "$__RootfsDir/tmp/download/buildtools.zip" $($__RootfsDir/tmp/script/fetch.sh --buildtools --arch=$__HaikuArch) - unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir" - - # Cleaning up temporary files - echo "Cleaning up temporary files" - popd - rm -rf "$__RootfsDir/tmp" -elif [[ -n "$__CodeName" ]]; then - - if [[ "$__SkipSigCheck" == "0" ]]; then - __Keyring="$__Keyring --force-check-gpg" - fi - - debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" - cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" - chroot "$__RootfsDir" apt-get update - chroot "$__RootfsDir" apt-get -f -y install - chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages - chroot "$__RootfsDir" symlinks -cr /usr - chroot "$__RootfsDir" apt-get clean - - if [[ "$__SkipUnmount" == "0" ]]; then - umount "$__RootfsDir"/* || true - fi - - if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then - pushd "$__RootfsDir" - patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch" - popd - fi -elif [[ "$__Tizen" == "tizen" ]]; then - ROOTFS_DIR="$__RootfsDir" "$__CrossDir/tizen-build-rootfs.sh" "$__BuildArch" -else - echo "Unsupported target platform." - usage; - exit 1 -fi diff --git a/eng/common/cross/ppc64le/sources.list.bionic b/eng/common/cross/ppc64le/sources.list.bionic deleted file mode 100644 index 210955740..000000000 --- a/eng/common/cross/ppc64le/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/riscv64/sources.list.sid b/eng/common/cross/riscv64/sources.list.sid deleted file mode 100644 index 65f730d22..000000000 --- a/eng/common/cross/riscv64/sources.list.sid +++ /dev/null @@ -1 +0,0 @@ -deb http://deb.debian.org/debian-ports sid main diff --git a/eng/common/cross/s390x/sources.list.bionic b/eng/common/cross/s390x/sources.list.bionic deleted file mode 100644 index 210955740..000000000 --- a/eng/common/cross/s390x/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/tizen-build-rootfs.sh b/eng/common/cross/tizen-build-rootfs.sh deleted file mode 100644 index ac84173d4..000000000 --- a/eng/common/cross/tizen-build-rootfs.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -set -e - -ARCH=$1 -LINK_ARCH=$ARCH - -case "$ARCH" in - arm) - TIZEN_ARCH="armv7hl" - ;; - armel) - TIZEN_ARCH="armv7l" - LINK_ARCH="arm" - ;; - arm64) - TIZEN_ARCH="aarch64" - ;; - x86) - TIZEN_ARCH="i686" - ;; - x64) - TIZEN_ARCH="x86_64" - LINK_ARCH="x86" - ;; - *) - echo "Unsupported architecture for tizen: $ARCH" - exit 1 -esac - -__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__TIZEN_CROSSDIR="$__CrossDir/${ARCH}/tizen" - -if [[ -z "$ROOTFS_DIR" ]]; then - echo "ROOTFS_DIR is not defined." - exit 1; -fi - -TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp -mkdir -p $TIZEN_TMP_DIR - -# Download files -echo ">>Start downloading files" -VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR $TIZEN_ARCH -echo "<>Start constructing Tizen rootfs" -TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` -cd $ROOTFS_DIR -for f in $TIZEN_RPM_FILES; do - rpm2cpio $f | cpio -idm --quiet -done -echo "<>Start configuring Tizen rootfs" -ln -sfn asm-${LINK_ARCH} ./usr/include/asm -patch -p1 < $__TIZEN_CROSSDIR/tizen.patch -echo "</dev/null; then - VERBOSE=0 -fi - -Log() -{ - if [ $VERBOSE -ge $1 ]; then - echo ${@:2} - fi -} - -Inform() -{ - Log 1 -e "\x1B[0;34m$@\x1B[m" -} - -Debug() -{ - Log 2 -e "\x1B[0;32m$@\x1B[m" -} - -Error() -{ - >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" -} - -Fetch() -{ - URL=$1 - FILE=$2 - PROGRESS=$3 - if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then - CURL_OPT="--progress-bar" - else - CURL_OPT="--silent" - fi - curl $CURL_OPT $URL > $FILE -} - -hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } -hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } -hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } - -TMPDIR=$1 -if [ ! -d $TMPDIR ]; then - TMPDIR=./tizen_tmp - Debug "Create temporary directory : $TMPDIR" - mkdir -p $TMPDIR -fi - -TIZEN_ARCH=$2 - -TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen -BUILD_XML=build.xml -REPOMD_XML=repomd.xml -PRIMARY_XML=primary.xml -TARGET_URL="http://__not_initialized" - -Xpath_get() -{ - XPATH_RESULT='' - XPATH=$1 - XML_FILE=$2 - RESULT=$(xmllint --xpath $XPATH $XML_FILE) - if [[ -z ${RESULT// } ]]; then - Error "Can not find target from $XML_FILE" - Debug "Xpath = $XPATH" - exit 1 - fi - XPATH_RESULT=$RESULT -} - -fetch_tizen_pkgs_init() -{ - TARGET=$1 - PROFILE=$2 - Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" - - TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs - if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi - mkdir -p $TMP_PKG_DIR - - PKG_URL=$TIZEN_URL/$PROFILE/latest - - BUILD_XML_URL=$PKG_URL/$BUILD_XML - TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML - TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML - TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML - TMP_PRIMARYGZ=${TMP_PRIMARY}.gz - - Fetch $BUILD_XML_URL $TMP_BUILD - - Debug "fetch $BUILD_XML_URL to $TMP_BUILD" - - TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" - Xpath_get $TARGET_XPATH $TMP_BUILD - TARGET_PATH=$XPATH_RESULT - TARGET_URL=$PKG_URL/$TARGET_PATH - - REPOMD_URL=$TARGET_URL/repodata/repomd.xml - PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' - - Fetch $REPOMD_URL $TMP_REPOMD - - Debug "fetch $REPOMD_URL to $TMP_REPOMD" - - Xpath_get $PRIMARY_XPATH $TMP_REPOMD - PRIMARY_XML_PATH=$XPATH_RESULT - PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH - - Fetch $PRIMARY_URL $TMP_PRIMARYGZ - - Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" - - gunzip $TMP_PRIMARYGZ - - Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" -} - -fetch_tizen_pkgs() -{ - ARCH=$1 - PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' - - PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' - - for pkg in ${@:2} - do - Inform "Fetching... $pkg" - XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} - XPATH=${XPATH/_ARCH_/$ARCH} - Xpath_get $XPATH $TMP_PRIMARY - PKG_PATH=$XPATH_RESULT - - XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} - XPATH=${XPATH/_ARCH_/$ARCH} - Xpath_get $XPATH $TMP_PRIMARY - CHECKSUM=$XPATH_RESULT - - PKG_URL=$TARGET_URL/$PKG_PATH - PKG_FILE=$(basename $PKG_PATH) - PKG_PATH=$TMPDIR/$PKG_FILE - - Debug "Download $PKG_URL to $PKG_PATH" - Fetch $PKG_URL $PKG_PATH true - - echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null - if [ $? -ne 0 ]; then - Error "Fail to fetch $PKG_URL to $PKG_PATH" - Debug "Checksum = $CHECKSUM" - exit 1 - fi - done -} - -Inform "Initialize ${TIZEN_ARCH} base" -fetch_tizen_pkgs_init standard Tizen-Base -Inform "fetch common packages" -fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils -Inform "fetch coreclr packages" -fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu -Inform "fetch corefx packages" -fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel - -Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard Tizen-Unified -Inform "fetch corefx packages" -fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release - diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake deleted file mode 100644 index f93dc440d..000000000 --- a/eng/common/cross/toolchain.cmake +++ /dev/null @@ -1,380 +0,0 @@ -set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) - -# reset platform variables (e.g. cmake 3.25 sets LINUX=1) -unset(LINUX) -unset(FREEBSD) -unset(ILLUMOS) -unset(ANDROID) -unset(TIZEN) -unset(HAIKU) - -set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) -if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) - set(CMAKE_SYSTEM_NAME FreeBSD) - set(FREEBSD 1) -elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) - set(CMAKE_SYSTEM_NAME SunOS) - set(ILLUMOS 1) -elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) - set(CMAKE_SYSTEM_NAME Haiku) - set(HAIKU 1) -else() - set(CMAKE_SYSTEM_NAME Linux) - set(LINUX 1) -endif() -set(CMAKE_SYSTEM_VERSION 1) - -if(EXISTS ${CROSS_ROOTFS}/etc/tizen-release) - set(TIZEN 1) -elseif(EXISTS ${CROSS_ROOTFS}/android_platform) - set(ANDROID 1) -endif() - -if(TARGET_ARCH_NAME STREQUAL "arm") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) - set(TOOLCHAIN "armv7-alpine-linux-musleabihf") - elseif(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) - set(TOOLCHAIN "armv6-alpine-linux-musleabihf") - else() - set(TOOLCHAIN "arm-linux-gnueabihf") - endif() - if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm64") - set(CMAKE_SYSTEM_PROCESSOR aarch64) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) - set(TOOLCHAIN "aarch64-alpine-linux-musl") - elseif(LINUX) - set(TOOLCHAIN "aarch64-linux-gnu") - if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") - endif() - elseif(FREEBSD) - set(triple "aarch64-unknown-freebsd12") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "armel") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - set(TOOLCHAIN "arm-linux-gnueabi") - if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "armv6") - set(CMAKE_SYSTEM_PROCESSOR armv6l) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) - set(TOOLCHAIN "armv6-alpine-linux-musleabihf") - else() - set(TOOLCHAIN "arm-linux-gnueabihf") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") - set(CMAKE_SYSTEM_PROCESSOR ppc64le) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl) - set(TOOLCHAIN "powerpc64le-alpine-linux-musl") - else() - set(TOOLCHAIN "powerpc64le-linux-gnu") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "riscv64") - set(CMAKE_SYSTEM_PROCESSOR riscv64) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/riscv64-alpine-linux-musl) - set(TOOLCHAIN "riscv64-alpine-linux-musl") - else() - set(TOOLCHAIN "riscv64-linux-gnu") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "s390x") - set(CMAKE_SYSTEM_PROCESSOR s390x) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/s390x-alpine-linux-musl) - set(TOOLCHAIN "s390x-alpine-linux-musl") - else() - set(TOOLCHAIN "s390x-linux-gnu") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "x64") - set(CMAKE_SYSTEM_PROCESSOR x86_64) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/x86_64-alpine-linux-musl) - set(TOOLCHAIN "x86_64-alpine-linux-musl") - elseif(LINUX) - set(TOOLCHAIN "x86_64-linux-gnu") - if(TIZEN) - set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") - endif() - elseif(FREEBSD) - set(triple "x86_64-unknown-freebsd12") - elseif(ILLUMOS) - set(TOOLCHAIN "x86_64-illumos") - elseif(HAIKU) - set(TOOLCHAIN "x86_64-unknown-haiku") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "x86") - set(CMAKE_SYSTEM_PROCESSOR i686) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) - set(TOOLCHAIN "i586-alpine-linux-musl") - else() - set(TOOLCHAIN "i686-linux-gnu") - endif() - if(TIZEN) - set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu") - endif() -else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") -endif() - -if(DEFINED ENV{TOOLCHAIN}) - set(TOOLCHAIN $ENV{TOOLCHAIN}) -endif() - -# Specify include paths -if(TIZEN) - function(find_toolchain_dir prefix) - # Dynamically find the version subdirectory - file(GLOB DIRECTORIES "${prefix}/*") - list(GET DIRECTORIES 0 FIRST_MATCH) - get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME) - - set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE) - endfunction() - - if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") - find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") - else() - find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") - endif() - - message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") - - include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) - include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) -endif() - -if(ANDROID) - if(TARGET_ARCH_NAME STREQUAL "arm") - set(ANDROID_ABI armeabi-v7a) - elseif(TARGET_ARCH_NAME STREQUAL "arm64") - set(ANDROID_ABI arm64-v8a) - endif() - - # extract platform number required by the NDK's toolchain - file(READ "${CROSS_ROOTFS}/android_platform" RID_FILE_CONTENTS) - string(REPLACE "RID=" "" ANDROID_RID "${RID_FILE_CONTENTS}") - string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "${ANDROID_RID}") - - set(ANDROID_TOOLCHAIN clang) - set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository - set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib") - set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include") - - # include official NDK toolchain script - include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(FREEBSD) - # we cross-compile by instructing clang - set(CMAKE_C_COMPILER_TARGET ${triple}) - set(CMAKE_CXX_COMPILER_TARGET ${triple}) - set(CMAKE_ASM_COMPILER_TARGET ${triple}) - set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") -elseif(ILLUMOS) - set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - - include_directories(SYSTEM ${CROSS_ROOTFS}/include) - - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") - - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") -elseif(HAIKU) - set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") - - set(TOOLSET_PREFIX ${TOOLCHAIN}-) - function(locate_toolchain_exec exec var) - string(TOUPPER ${exec} EXEC_UPPERCASE) - if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") - set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) - return() - endif() - - find_program(EXEC_LOCATION_${exec} - NAMES - "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" - "${TOOLSET_PREFIX}${exec}") - - if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") - message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") - endif() - set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) - endfunction() - - set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") - - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - - set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - - # let CMake set up the correct search paths - include(Platform/Haiku) -else() - set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - - set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") - set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") - set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -endif() - -# Specify link flags - -function(add_toolchain_linker_flag Flag) - set(Config "${ARGV1}") - set(CONFIG_SUFFIX "") - if (NOT Config STREQUAL "") - set(CONFIG_SUFFIX "_${Config}") - endif() - set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) - set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) -endfunction() - -if(LINUX) - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") -endif() - -if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") - if(TIZEN) - add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") - endif() -elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") - if(TIZEN) - add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") - - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "s390x") - add_toolchain_linker_flag("--target=${TOOLCHAIN}") -elseif(TARGET_ARCH_NAME STREQUAL "x86") - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) - add_toolchain_linker_flag("--target=${TOOLCHAIN}") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") - endif() - add_toolchain_linker_flag(-m32) - if(TIZEN) - add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") - endif() -elseif(ILLUMOS) - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") -elseif(HAIKU) - add_toolchain_linker_flag("-lnetwork") - add_toolchain_linker_flag("-lroot") -endif() - -# Specify compile options - -if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) - set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) - set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) - set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) -endif() - -if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") - add_compile_options(-mthumb) - if (NOT DEFINED CLR_ARM_FPU_TYPE) - set (CLR_ARM_FPU_TYPE vfpv3) - endif (NOT DEFINED CLR_ARM_FPU_TYPE) - - add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE}) - if (NOT DEFINED CLR_ARM_FPU_CAPABILITY) - set (CLR_ARM_FPU_CAPABILITY 0x7) - endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY) - - add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) - - # persist variables across multiple try_compile passes - list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES CLR_ARM_FPU_TYPE CLR_ARM_FPU_CAPABILITY) - - if(TARGET_ARCH_NAME STREQUAL "armel") - add_compile_options(-mfloat-abi=softfp) - endif() -elseif(TARGET_ARCH_NAME STREQUAL "s390x") - add_compile_options("--target=${TOOLCHAIN}") -elseif(TARGET_ARCH_NAME STREQUAL "x86") - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) - add_compile_options(--target=${TOOLCHAIN}) - endif() - add_compile_options(-m32) - add_compile_options(-Wno-error=unused-command-line-argument) -endif() - -if(TIZEN) - if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|x86)$") - add_compile_options(-Wno-deprecated-declarations) # compile-time option - add_compile_options(-D__extern_always_inline=inline) # compile-time option - endif() -endif() - -# Set LLDB include and library paths for builds that need lldb. -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") - if(TARGET_ARCH_NAME STREQUAL "x86") - set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}") - else() # arm/armel case - set(LLVM_CROSS_DIR "$ENV{LLVM_ARM_HOME}") - endif() - if(LLVM_CROSS_DIR) - set(WITH_LLDB_LIBS "${LLVM_CROSS_DIR}/lib/" CACHE STRING "") - set(WITH_LLDB_INCLUDES "${LLVM_CROSS_DIR}/include" CACHE STRING "") - set(LLDB_H "${WITH_LLDB_INCLUDES}" CACHE STRING "") - set(LLDB "${LLVM_CROSS_DIR}/lib/liblldb.so" CACHE STRING "") - else() - if(TARGET_ARCH_NAME STREQUAL "x86") - set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/i386-linux-gnu" CACHE STRING "") - set(CHECK_LLVM_DIR "${CROSS_ROOTFS}/usr/lib/llvm-3.8/include") - if(EXISTS "${CHECK_LLVM_DIR}" AND IS_DIRECTORY "${CHECK_LLVM_DIR}") - set(WITH_LLDB_INCLUDES "${CHECK_LLVM_DIR}") - else() - set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include") - endif() - else() # arm/armel case - set(WITH_LLDB_LIBS "${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}" CACHE STRING "") - set(WITH_LLDB_INCLUDES "${CROSS_ROOTFS}/usr/lib/llvm-3.6/include" CACHE STRING "") - endif() - endif() -endif() - -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/x86/sources.list.bionic b/eng/common/cross/x86/sources.list.bionic deleted file mode 100644 index a71ccadcf..000000000 --- a/eng/common/cross/x86/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.xenial b/eng/common/cross/x86/sources.list.xenial deleted file mode 100644 index ad9c5a014..000000000 --- a/eng/common/cross/x86/sources.list.xenial +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 deleted file mode 100644 index 8fda30bdc..000000000 --- a/eng/common/darc-init.ps1 +++ /dev/null @@ -1,47 +0,0 @@ -param ( - $darcVersion = $null, - $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16', - $verbosity = 'minimal', - $toolpath = $null -) - -. $PSScriptRoot\tools.ps1 - -function InstallDarcCli ($darcVersion, $toolpath) { - $darcCliPackageName = 'microsoft.dotnet.darc' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list -g - - if ($toolList -like "*$darcCliPackageName*") { - & "$dotnet" tool uninstall $darcCliPackageName -g - } - - # If the user didn't explicitly specify the darc version, - # query the Maestro API for the correct version of darc to install. - if (-not $darcVersion) { - $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content - } - - $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - - Write-Host "Installing Darc CLI version $darcVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g - }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" - } -} - -try { - InstallDarcCli $darcVersion $toolpath -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Darc' -Message $_ - ExitWithExitCode 1 -} \ No newline at end of file diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh deleted file mode 100644 index c305ae6bd..000000000 --- a/eng/common/darc-init.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -darcVersion='' -versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16' -verbosity='minimal' - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - --darcversion) - darcVersion=$2 - shift - ;; - --versionendpoint) - versionEndpoint=$2 - shift - ;; - --verbosity) - verbosity=$2 - shift - ;; - --toolpath) - toolpath=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ -z "$darcVersion" ]; then - darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") -fi - -function InstallDarcCli { - local darc_cli_package_name="microsoft.dotnet.darc" - - InitializeDotNetCli true - local dotnet_root=$_InitializeDotNetCli - - if [ -z "$toolpath" ]; then - local tool_list=$($dotnet_root/dotnet tool list -g) - if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) - fi - else - local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath") - if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath") - fi - fi - - local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" - - echo "Installing Darc CLI version $darcVersion..." - echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) - else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") - fi -} - -InstallDarcCli diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd deleted file mode 100644 index b1c2642e7..000000000 --- a/eng/common/dotnet-install.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 deleted file mode 100644 index 811f0f717..000000000 --- a/eng/common/dotnet-install.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [string] $architecture = '', - [string] $version = 'Latest', - [string] $runtime = 'dotnet', - [string] $RuntimeSourceFeed = '', - [string] $RuntimeSourceFeedKey = '' -) - -. $PSScriptRoot\tools.ps1 - -$dotnetRoot = Join-Path $RepoRoot '.dotnet' - -$installdir = $dotnetRoot -try { - if ($architecture -and $architecture.Trim() -eq 'x86') { - $installdir = Join-Path $installdir 'x86' - } - InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh deleted file mode 100644 index 7e69e3a9e..000000000 --- a/eng/common/dotnet-install.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -version='Latest' -architecture='' -runtime='dotnet' -runtimeSourceFeed='' -runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -version|-v) - shift - version="$1" - ;; - -architecture|-a) - shift - architecture="$1" - ;; - -runtime|-r) - shift - runtime="$1" - ;; - -runtimesourcefeed) - shift - runtimeSourceFeed="$1" - ;; - -runtimesourcefeedkey) - shift - runtimeSourceFeedKey="$1" - ;; - *) - Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1" - exit 1 - ;; - esac - shift -done - -# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples -cpuname=$(uname -m) -case $cpuname in - arm64|aarch64) - buildarch=arm64 - if [ "$(getconf LONG_BIT)" -lt 64 ]; then - # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) - buildarch=arm - fi - ;; - loongarch64) - buildarch=loongarch64 - ;; - amd64|x86_64) - buildarch=x64 - ;; - armv*l) - buildarch=arm - ;; - i[3-6]86) - buildarch=x86 - ;; - *) - echo "Unknown CPU $cpuname detected, treating it as x64" - buildarch=x64 - ;; -esac - -dotnetRoot="${repo_root}.dotnet" -if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then - dotnetRoot="$dotnetRoot/$architecture" -fi - -InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 - ExitWithExitCode $exit_code -} - -ExitWithExitCode 0 diff --git a/eng/common/enable-cross-org-publishing.ps1 b/eng/common/enable-cross-org-publishing.ps1 deleted file mode 100644 index da09da4f1..000000000 --- a/eng/common/enable-cross-org-publishing.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -param( - [string] $token -) - - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Write-PipelineSetVariable will no-op if a variable named $ci is not defined -# Since this script is only ever called in AzDO builds, just universally set it -$ci = $true - -Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 deleted file mode 100644 index 524aaa57f..000000000 --- a/eng/common/generate-locproject.ps1 +++ /dev/null @@ -1,189 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here - [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json - [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one - [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally -) - -# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: -# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task - -Set-StrictMode -Version 2.0 -$ErrorActionPreference = "Stop" -. $PSScriptRoot\pipeline-logging-functions.ps1 - -$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" -$exclusions = @{ Exclusions = @() } -if (Test-Path -Path $exclusionsFilePath) -{ - $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json -} - -Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work - -# Template files -$jsonFiles = @() -$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern -$jsonTemplateFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" - $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern - -$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them -if (-not $wxlFiles) { - $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files - if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } - } -} - -$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files -$macosHtmlFiles = @() -if ($macosHtmlEnFiles) { - $macosHtmlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } -} - -$xlfFiles = @() - -$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" -$langXlfFiles = @() -if ($allXlfFiles) { - $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' - $firstLangCode = $Matches.1 - $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" -} -$langXlfFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" - $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles - -$locJson = @{ - Projects = @( - @{ - LanguageSet = $LanguageSet - LocItems = @( - $locFiles | ForEach-Object { - $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { - Remove-Item -Path $sourceFile - } - if ($continue) - { - if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" - } - } else { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnName" - OutputPath = $outputPath - } - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "WiX_CloneLanguages" - LssFiles = @( "wxl_loc.lss" ) - LocItems = @( - $wxlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if ($continue) - { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "VS_macOS_CloneLanguages" - LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) - LocItems = @( - $macosHtmlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - $lciFile = $sourceFile + ".lci" - if ($continue) { - $result = @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - if (Test-Path $lciFile -PathType Leaf) { - $result["LciFile"] = $lciFile - } - return $result - } - } - ) - } - ) -} - -$json = ConvertTo-Json $locJson -Depth 5 -Write-Host "LocProject.json generated:`n`n$json`n`n" -Pop-Location - -if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json -} -else { - New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - - if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { - Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." - - exit 1 - } - else { - Write-Host "Generated LocProject.json and current LocProject.json are identical." - } -} diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a..000000000 --- a/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh deleted file mode 100644 index bbb492215..000000000 --- a/eng/common/generate-sbom-prep.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. $scriptroot/pipeline-logging-functions.sh - -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" - -manifest_dir=$1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -sbom_generation_dir="$manifest_dir/$safe_artifact_name" - -if [ ! -d "$sbom_generation_dir" ] ; then - mkdir -p "$sbom_generation_dir" - echo "Sbom directory created." $sbom_generation_dir -else - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -fi - -echo "Artifact name before : "$artifact_name -echo "Artifact name after : "$safe_artifact_name -export ARTIFACT_NAME=$safe_artifact_name -echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" - -exit 0 diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj deleted file mode 100644 index d7f185856..000000000 --- a/eng/common/helixpublish.proj +++ /dev/null @@ -1,26 +0,0 @@ - - - - msbuild - - - - - %(Identity) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) - $(WorkItemTimeout) - - - - - - - - - diff --git a/eng/common/init-tools-native.cmd b/eng/common/init-tools-native.cmd deleted file mode 100644 index 438cd548c..000000000 --- a/eng/common/init-tools-native.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" -exit /b %ErrorLevel% \ No newline at end of file diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 deleted file mode 100644 index 27ccdb9ec..000000000 --- a/eng/common/init-tools-native.ps1 +++ /dev/null @@ -1,203 +0,0 @@ -<# -.SYNOPSIS -Entry point script for installing native tools - -.DESCRIPTION -Reads $RepoRoot\global.json file to determine native assets to install -and executes installers for those tools - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER InstallDirectory -Directory to install native toolset. This is a command-line override for the default -Install directory precedence order: -- InstallDirectory command-line override -- NETCOREENG_INSTALL_DIRECTORY environment variable -- (default) %USERPROFILE%/.netcoreeng/native - -.PARAMETER Clean -Switch specifying to not install anything, but cleanup native asset folders - -.PARAMETER Force -Clean and then install tools - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.PARAMETER GlobalJsonFile -File path to global.json file - -.PARAMETER PathPromotion -Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) -or break the build if a native tool is not found on the path (on a local dev machine) - -.NOTES -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', - [string] $InstallDirectory, - [switch] $Clean = $False, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile, - [switch] $PathPromotion -) - -if (!$GlobalJsonFile) { - $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' -} - -Set-StrictMode -version 2.0 -$ErrorActionPreference='Stop' - -. $PSScriptRoot\pipeline-logging-functions.ps1 -Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq 'Continue' - - $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $NativeBaseDir = $InstallDirectory - if (!$NativeBaseDir) { - $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory - } - $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir - $InstallBin = Join-Path $NativeBaseDir 'bin' - $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' - - # Process tools list - Write-Host "Processing $GlobalJsonFile" - If (-Not (Test-Path $GlobalJsonFile)) { - Write-Host "Unable to find '$GlobalJsonFile'" - exit 0 - } - $NativeTools = Get-Content($GlobalJsonFile) -Raw | - ConvertFrom-Json | - Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue - if ($NativeTools) { - if ($PathPromotion -eq $True) { - $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" - if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $InstalledTools = @{} - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - if ($ToolVersion -eq "latest") { - $ToolVersion = "" - } - $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) - if ($ToolDirectories -eq $null) { - Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." - exit 1 - } - $ToolDirectory = $ToolDirectories[0] - $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" - if (-not (Test-Path -Path "$BinPathFile")) { - Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." - exit 1 - } - $BinPath = Get-Content "$BinPathFile" - $ToolPath = Convert-Path -Path $BinPath - Write-Host "Adding $ToolName to the path ($ToolPath)..." - Write-Host "##vso[task.prependpath]$ToolPath" - $env:PATH = "$ToolPath;$env:PATH" - $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } - } - } - return $InstalledTools - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." - } - } - exit 0 - } - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false - } - if ($showNativeToolsWarning) { - Write-Warning $errMsg - } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 - } - } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 - } - } - } - else { - Write-Host 'No native tools defined in global.json' - exit 0 - } - - if ($Clean) { - exit 0 - } - if (Test-Path $InstallBin) { - Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) - Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" - return $InstallBin - } - elseif (-not ($PathPromotion)) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' - exit 1 - } - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh deleted file mode 100644 index 3e6a8d6ac..000000000 --- a/eng/common/init-tools-native.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' -install_directory='' -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 -global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -a native_assets - -. $scriptroot/pipeline-logging-functions.sh -. $scriptroot/native/common-library.sh - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installdirectory) - install_directory=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --donotabortonfailure) - donotabortonfailure=true - shift 1 - ;; - --donotdisplaywarnings) - donotdisplaywarnings=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --installdirectory Directory to install native toolset." - echo " This is a command-line override for the default" - echo " Install directory precedence order:" - echo " - InstallDirectory command-line override" - echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" - echo " - (default) %USERPROFILE%/.netcoreeng/native" - echo "" - echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" - echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" - echo " --force Clean and then install tools" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --baseuri Base URI for where to download native tools from" - echo " --downloadretries Number of times a download should be attempted" - echo " --retrywaittimeseconds Wait time between download attempts" - echo "" - exit 0 - ;; - esac -done - -function ReadGlobalJsonNativeTools { - # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! - if command -v jq &> /dev/null; then - - # jq: read each key/value pair under "native-tools" entry and emit: - # KEY="" VALUE="" - # followed by a null byte. - # - # bash: read line with null byte delimeter and push to array (for later `eval`uation). - - while IFS= read -rd '' line; do - native_assets+=("$line") - done < <(jq -r '. | - select(has("native-tools")) | - ."native-tools" | - keys[] as $k | - @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") - - return - fi - - # Warning: falling back to manually parsing JSON, which is not recommended. - - # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. - # It has been tested with several weird strings with escaped characters in entries (key and value) - # and results were compared with the output of jq(1) in binary representation using xxd(1); - # just before the assignment to 'native_assets' array (above and below). - - # try to capture the section under "native-tools". - if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then - return - fi - - section="${BASH_REMATCH[1]}" - - parseStarted=0 - possibleEnd=0 - escaping=0 - escaped=0 - isKey=1 - - for (( i=0; i<${#section}; i++ )); do - char="${section:$i:1}" - if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi - - if ! ((escaping)) && [[ "$char" == "\\" ]]; then - escaping=1 - elif ((escaping)) && ! ((escaped)); then - escaped=1 - fi - - if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then - parseStarted=1 - possibleEnd=0 - elif [[ "$char" == "'" ]]; then - token="$token'\\\''" - possibleEnd=0 - elif ((escaping)) || [[ "$char" != "\"" ]]; then - token="$token$char" - possibleEnd=1 - fi - - if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then - # Use printf to unescape token to match jq(1)'s @sh formatting rules. - # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. - printf -v token "'$token'" - - if ((isKey)); then - KEY="$token" - isKey=0 - else - line="KEY=$KEY VALUE=$token" - native_assets+=("$line") - isKey=1 - fi - - # reset for next token - parseStarted=0 - token= - elif ((escaping)) && ((escaped)); then - escaping=0 - escaped=0 - fi - done -} - -native_base_dir=$install_directory -if [[ -z $install_directory ]]; then - native_base_dir=$(GetNativeInstallDirectory) -fi - -install_bin="${native_base_dir}/bin" -installed_any=false - -ReadGlobalJsonNativeTools - -if [[ ${#native_assets[@]} -eq 0 ]]; then - echo "No native tools defined in global.json" - exit 0; -else - native_installer_dir="$scriptroot/native" - for index in "${!native_assets[@]}"; do - eval "${native_assets["$index"]}" - - installer_path="$native_installer_dir/install-$KEY.sh" - installer_command="$installer_path" - installer_command+=" --baseuri $base_uri" - installer_command+=" --installpath $install_bin" - installer_command+=" --version $VALUE" - echo $installer_command - - if [[ $force = true ]]; then - installer_command+=" --force" - fi - - if [[ $clean = true ]]; then - installer_command+=" --clean" - fi - - if [[ -a $installer_path ]]; then - $installer_command - if [[ $? != 0 ]]; then - if [[ $donotabortonfailure = true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 - fi - else - $installed_any = true - fi - else - if [[ $donotabortonfailure == true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - exit 1 - fi - fi - done -fi - -if [[ $clean = true ]]; then - exit 0 -fi - -if [[ -d $install_bin ]]; then - echo "Native tools are available from $install_bin" - echo "##vso[task.prependpath]$install_bin" -else - if [[ $installed_any = true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 - fi -fi - -exit 0 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 deleted file mode 100644 index 92b77347d..000000000 --- a/eng/common/internal-feed-operations.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $Operation, - [string] $AuthToken, - [string] $CommitSha, - [string] $RepoName, - [switch] $IsFeedPrivate -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -. $PSScriptRoot\tools.ps1 - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified -# internal builds -function SetupCredProvider { - param( - [string] $AuthToken - ) - - # Install the Cred Provider NuGet plugin - Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' - Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' - - Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 - - Write-Host 'Installing plugin...' - .\installcredprovider.ps1 -Force - - Write-Host "Deleting local copy of 'installcredprovider.ps1'..." - Remove-Item .\installcredprovider.ps1 - - if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { - Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - } - else { - Write-Host 'CredProvider plugin was installed correctly!' - } - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" - - if (-Not (Test-Path -Path $nugetConfigPath)) { - Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 - } - - $endpoints = New-Object System.Collections.ArrayList - $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} - - if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { - foreach ($stableRestoreResource in $nugetConfigPackageSources) { - $trimmedResource = ([string]$stableRestoreResource).Trim() - [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) - } - } - - if (($endpoints | Measure-Object).Count -gt 0) { - $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress - - # Create the environment variables the AzDo way - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ - 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' - 'issecret' = 'false' - } - - # We don't want sessions cached since we will be updating the endpoints quite frequently - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ - 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' - 'issecret' = 'false' - } - } - else - { - Write-Host 'No internal endpoints found in NuGet.config' - } -} - -#Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = Join-Path $RepoRoot "dotnet" - $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - $dotnet = "$dotnetTempDir\dotnet.exe" - $restoreProjPath = "$PSScriptRoot\restore.proj" - - Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - '' | Out-File "$restoreProjPath" - - & $dotnet restore $restoreProjPath - - Write-Host 'Arcade SDK restored!' - - if (Test-Path -Path $restoreProjPath) { - Remove-Item $restoreProjPath - } - - if (Test-Path -Path $dotnetTempDir) { - Remove-Item $dotnetTempDir -Recurse - } -} - -try { - Push-Location $PSScriptRoot - - if ($Operation -like 'setup') { - SetupCredProvider $AuthToken - } - elseif ($Operation -like 'install-restore') { - InstallDotNetSdkAndRestoreArcade - } - else { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" - ExitWithExitCode 1 - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} -finally { - Pop-Location -} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh deleted file mode 100644 index 9378223ba..000000000 --- a/eng/common/internal-feed-operations.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. -# This should ONLY be called from identified internal builds -function SetupCredProvider { - local authToken=$1 - - # Install the Cred Provider NuGet plugin - echo "Setting up Cred Provider NuGet plugin in the agent..."... - echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" - - echo "Writing the contents of 'installcredprovider.ps1' locally..." - local installcredproviderPath="installcredprovider.sh" - if command -v curl > /dev/null; then - curl $url > "$installcredproviderPath" - else - wget -q -O "$installcredproviderPath" "$url" - fi - - echo "Installing plugin..." - . "$installcredproviderPath" - - echo "Deleting local copy of 'installcredprovider.sh'..." - rm installcredprovider.sh - - if [ ! -d "$HOME/.nuget/plugins" ]; then - Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - else - echo "CredProvider plugin was installed correctly!" - fi - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - local nugetConfigPath="{$repo_root}NuGet.config" - - if [ ! "$nugetConfigPath" ]; then - Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" - ExitWithExitCode 1 - fi - - local endpoints='[' - local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` - local pattern="value=\"(.*)\"" - - for value in $nugetConfigPackageValues - do - if [[ $value =~ $pattern ]]; then - local endpoint="${BASH_REMATCH[1]}" - endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," - fi - done - - endpoints=${endpoints%?} - endpoints+=']' - - if [ ${#endpoints} -gt 2 ]; then - local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" - - echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" - echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" - else - echo "No internal endpoints found in NuGet.config" - fi -} - -# Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - local dotnetTempDir="$repo_root/dotnet" - local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - local restoreProjPath="$repo_root/eng/common/restore.proj" - - echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - echo "" > "$restoreProjPath" - - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - local res=`$dotnetTempDir/dotnet restore $restoreProjPath` - echo "Arcade SDK restored!" - - # Cleanup - if [ "$restoreProjPath" ]; then - rm "$restoreProjPath" - fi - - if [ "$dotnetTempDir" ]; then - rm -r $dotnetTempDir - fi -} - -source="${BASH_SOURCE[0]}" -operation='' -authToken='' -repoName='' - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - --operation) - operation=$2 - shift - ;; - --authtoken) - authToken=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ "$operation" = "setup" ]; then - SetupCredProvider $authToken -elif [ "$operation" = "install-restore" ]; then - InstallDotNetSdkAndRestoreArcade -else - echo "Unknown operation '$operation'!" -fi diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props deleted file mode 100644 index dbf99d82a..000000000 --- a/eng/common/internal/Directory.Build.props +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config deleted file mode 100644 index 19d3d311b..000000000 --- a/eng/common/internal/NuGet.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj deleted file mode 100644 index 7f5ce6d60..000000000 --- a/eng/common/internal/Tools.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - - net472 - false - false - - - - - - - - - - - - - - https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json; - - - $(RestoreSources); - https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json; - - - - - - diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss deleted file mode 100644 index 5d892d619..000000000 --- a/eng/common/loc/P22DotNetHtmlLocalization.lss +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 deleted file mode 100644 index f041e5ddd..000000000 --- a/eng/common/msbuild.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch] $ci, - [switch] $prepareMachine, - [switch] $excludePrereleaseVS, - [string] $msbuildEngine = $null, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs -) - -. $PSScriptRoot\tools.ps1 - -try { - if ($ci) { - $nodeReuse = $false - } - - MSBuild @extraArgs -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh deleted file mode 100644 index 20d3dad54..000000000 --- a/eng/common/msbuild.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -verbosity='minimal' -warn_as_error=true -node_reuse=true -prepare_machine=false -extra_args='' - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --verbosity) - verbosity=$2 - shift 2 - ;; - --warnaserror) - warn_as_error=$2 - shift 2 - ;; - --nodereuse) - node_reuse=$2 - shift 2 - ;; - --ci) - ci=true - shift 1 - ;; - --preparemachine) - prepare_machine=true - shift 1 - ;; - *) - extra_args="$extra_args $1" - shift 1 - ;; - esac -done - -. "$scriptroot/tools.sh" - -if [[ "$ci" == true ]]; then - node_reuse=false -fi - -MSBuild $extra_args -ExitWithExitCode 0 diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 deleted file mode 100644 index ca38268c4..000000000 --- a/eng/common/native/CommonLibrary.psm1 +++ /dev/null @@ -1,400 +0,0 @@ -<# -.SYNOPSIS -Helper module to install an archive to a directory - -.DESCRIPTION -Helper module to download and extract an archive to a specified directory - -.PARAMETER Uri -Uri of artifact to download - -.PARAMETER InstallDirectory -Directory to extract artifact contents to - -.PARAMETER Force -Force download / extraction if file or contents already exist. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds. Default = 30 - -.NOTES -Returns False if download or extraction fail, True otherwise -#> -function DownloadAndExtract { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $InstallDirectory, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 - ) - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri - - # Download native tool - $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` - -Path $TempToolPath ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Force:$Force ` - -Verbose:$Verbose - - if ($DownloadStatus -Eq $False) { - Write-Error "Download failed from $Uri" - return $False - } - - # Extract native tool - $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` - -OutputDirectory $InstallDirectory ` - -Force:$Force ` - -Verbose:$Verbose - - if ($UnzipStatus -Eq $False) { - # Retry Download one more time with Force=true - $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri ` - -Path $TempToolPath ` - -DownloadRetries 1 ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Force:$True ` - -Verbose:$Verbose - - if ($DownloadRetryStatus -Eq $False) { - Write-Error "Last attempt of download failed as well" - return $False - } - - # Retry unzip again one more time with Force=true - $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` - -OutputDirectory $InstallDirectory ` - -Force:$True ` - -Verbose:$Verbose - if ($UnzipRetryStatus -Eq $False) - { - Write-Error "Last attempt of unzip failed as well" - # Clean up partial zips and extracts - if (Test-Path $TempToolPath) { - Remove-Item $TempToolPath -Force - } - if (Test-Path $InstallDirectory) { - Remove-Item $InstallDirectory -Force -Recurse - } - return $False - } - } - - return $True -} - -<# -.SYNOPSIS -Download a file, retry on failure - -.DESCRIPTION -Download specified file and retry if attempt fails - -.PARAMETER Uri -Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded - -.PARAMETER Path -Path to download or copy uri file to - -.PARAMETER Force -Overwrite existing file if present. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds Default = 30 - -#> -function Get-File { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $Path, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [switch] $Force = $False - ) - $Attempt = 0 - - if ($Force) { - if (Test-Path $Path) { - Remove-Item $Path -Force - } - } - if (Test-Path $Path) { - Write-Host "File '$Path' already exists, skipping download" - return $True - } - - $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent - if (-Not (Test-Path $DownloadDirectory)) { - New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null - } - - $TempPath = "$Path.tmp" - if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" - Copy-Item -Path $Uri -Destination $TempPath - Write-Verbose "Moving temporary file to '$Path'" - Move-Item -Path $TempPath -Destination $Path - return $? - } - else { - Write-Verbose "Downloading $Uri" - # Don't display the console progress UI - it's a huge perf hit - $ProgressPreference = 'SilentlyContinue' - while($Attempt -Lt $DownloadRetries) - { - try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath - Write-Verbose "Downloaded to temporary location '$TempPath'" - Move-Item -Path $TempPath -Destination $Path - Write-Verbose "Moved temporary file to '$Path'" - return $True - } - catch { - $Attempt++ - if ($Attempt -Lt $DownloadRetries) { - $AttemptsLeft = $DownloadRetries - $Attempt - Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $RetryWaitTimeInSeconds - } - else { - Write-Error $_ - Write-Error $_.Exception - } - } - } - } - - return $False -} - -<# -.SYNOPSIS -Generate a shim for a native tool - -.DESCRIPTION -Creates a wrapper script (shim) that passes arguments forward to native tool assembly - -.PARAMETER ShimName -The name of the shim - -.PARAMETER ShimDirectory -The directory where shims are stored - -.PARAMETER ToolFilePath -Path to file that shim forwards to - -.PARAMETER Force -Replace shim if already present. Default = False - -.NOTES -Returns $True if generating shim succeeds, $False otherwise -#> -function New-ScriptShim { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ShimName, - [Parameter(Mandatory=$True)] - [string] $ShimDirectory, - [Parameter(Mandatory=$True)] - [string] $ToolFilePath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [switch] $Force - ) - try { - Write-Verbose "Generating '$ShimName' shim" - - if (-Not (Test-Path $ToolFilePath)){ - Write-Error "Specified tool file path '$ToolFilePath' does not exist" - return $False - } - - # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs - # Many of the checks for installed programs expect a .exe extension for Windows tools, rather - # than a .bat or .cmd file. - # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer - if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { - $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` - -InstallDirectory $ShimDirectory\WinShimmer ` - -Force:$Force ` - -DownloadRetries 2 ` - -RetryWaitTimeInSeconds 5 ` - -Verbose:$Verbose - } - - if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { - Write-Host "$ShimName.exe already exists; replacing..." - Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") - } - - & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory - return $True - } - catch { - Write-Host $_ - Write-Host $_.Exception - return $False - } -} - -<# -.SYNOPSIS -Returns the machine architecture of the host machine - -.NOTES -Returns 'x64' on 64 bit machines - Returns 'x86' on 32 bit machines -#> -function Get-MachineArchitecture { - $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE - $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 - if($ProcessorArchitecture -Eq "X86") - { - if(($ProcessorArchitectureW6432 -Eq "") -Or - ($ProcessorArchitectureW6432 -Eq "X86")) { - return "x86" - } - $ProcessorArchitecture = $ProcessorArchitectureW6432 - } - if (($ProcessorArchitecture -Eq "AMD64") -Or - ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64") -Or - ($ProcessorArchitecture -Eq "LOONGARCH64")) { - return "x64" - } - return "x86" -} - -<# -.SYNOPSIS -Get the name of a temporary folder under the native install directory -#> -function Get-TempDirectory { - return Join-Path (Get-NativeInstallDirectory) "temp/" -} - -function Get-TempPathFilename { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Path - ) - $TempDir = CommonLibrary\Get-TempDirectory - $TempFilename = Split-Path $Path -leaf - $TempPath = Join-Path $TempDir $TempFilename - return $TempPath -} - -<# -.SYNOPSIS -Returns the base directory to use for native tool installation - -.NOTES -Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable -is set, or otherwise returns an install directory under the %USERPROFILE% -#> -function Get-NativeInstallDirectory { - $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY - if (!$InstallDir) { - $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" - } - return $InstallDir -} - -<# -.SYNOPSIS -Unzip an archive - -.DESCRIPTION -Powershell module to unzip an archive to a specified directory - -.PARAMETER ZipPath (Required) -Path to archive to unzip - -.PARAMETER OutputDirectory (Required) -Output directory for archive contents - -.PARAMETER Force -Overwrite output directory contents if they already exist - -.NOTES -- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. -- Returns True if unzip operation is successful -- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory -- Returns False if unable to extract zip archive -#> -function Expand-Zip { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ZipPath, - [Parameter(Mandatory=$True)] - [string] $OutputDirectory, - [switch] $Force - ) - - Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" - try { - if ((Test-Path $OutputDirectory) -And (-Not $Force)) { - Write-Host "Directory '$OutputDirectory' already exists, skipping extract" - return $True - } - if (Test-Path $OutputDirectory) { - Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" - Remove-Item $OutputDirectory -Force -Recurse - if ($? -Eq $False) { - Write-Error "Unable to remove '$OutputDirectory'" - return $False - } - } - - $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" - if (Test-Path $TempOutputDirectory) { - Remove-Item $TempOutputDirectory -Force -Recurse - } - New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null - - Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") - if ($? -Eq $False) { - Write-Error "Unable to extract '$ZipPath'" - return $False - } - - Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory - } - catch { - Write-Host $_ - Write-Host $_.Exception - - return $False - } - return $True -} - -export-modulemember -function DownloadAndExtract -export-modulemember -function Expand-Zip -export-modulemember -function Get-File -export-modulemember -function Get-MachineArchitecture -export-modulemember -function Get-NativeInstallDirectory -export-modulemember -function Get-TempDirectory -export-modulemember -function Get-TempPathFilename -export-modulemember -function New-ScriptShim diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh deleted file mode 100644 index 080c2c283..000000000 --- a/eng/common/native/common-library.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -function GetNativeInstallDirectory { - local install_dir - - if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then - install_dir=$HOME/.netcoreeng/native/ - else - install_dir=$NETCOREENG_INSTALL_DIRECTORY - fi - - echo $install_dir - return 0 -} - -function GetTempDirectory { - - echo $(GetNativeInstallDirectory)temp/ - return 0 -} - -function ExpandZip { - local zip_path=$1 - local output_directory=$2 - local force=${3:-false} - - echo "Extracting $zip_path to $output_directory" - if [[ -d $output_directory ]] && [[ $force = false ]]; then - echo "Directory '$output_directory' already exists, skipping extract" - return 0 - fi - - if [[ -d $output_directory ]]; then - echo "'Force flag enabled, but '$output_directory' exists. Removing directory" - rm -rf $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" - return 1 - fi - fi - - echo "Creating directory: '$output_directory'" - mkdir -p $output_directory - - echo "Extracting archive" - tar -xf $zip_path -C $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" - return 1 - fi - - return 0 -} - -function GetCurrentOS { - local unameOut="$(uname -s)" - case $unameOut in - Linux*) echo "Linux";; - Darwin*) echo "MacOS";; - esac - return 0 -} - -function GetFile { - local uri=$1 - local path=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - if [[ -f $path ]]; then - if [[ $force = false ]]; then - echo "File '$path' already exists. Skipping download" - return 0 - else - rm -rf $path - fi - fi - - if [[ -f $uri ]]; then - echo "'$uri' is a file path, copying file to '$path'" - cp $uri $path - return $? - fi - - echo "Downloading $uri" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail - else - wget -q -O "$path" "$uri" --tries="$download_retries" - fi - - return $? -} - -function GetTempPathFileName { - local path=$1 - - local temp_dir=$(GetTempDirectory) - local temp_file_name=$(basename $path) - echo $temp_dir$temp_file_name - return 0 -} - -function DownloadAndExtract { - local uri=$1 - local installDir=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - local temp_tool_path=$(GetTempPathFileName $uri) - - echo "downloading to: $temp_tool_path" - - # Download file - GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." - return 1 - fi - - # Extract File - echo "extracting from $temp_tool_path to $installDir" - ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." - return 1 - fi - - return 0 -} - -function NewScriptShim { - local shimpath=$1 - local tool_file_path=$2 - local force=${3:-false} - - echo "Generating '$shimpath' shim" - if [[ -f $shimpath ]]; then - if [[ $force = false ]]; then - echo "File '$shimpath' already exists." >&2 - return 1 - else - rm -rf $shimpath - fi - fi - - if [[ ! -f $tool_file_path ]]; then - # try to see if the path is lower cased - tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" - if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 - fi - fi - - local shim_contents=$'#!/usr/bin/env bash\n' - shim_contents+="SHIMARGS="$'$1\n' - shim_contents+="$tool_file_path"$' $SHIMARGS\n' - - # Write shim file - echo "$shim_contents" > $shimpath - - chmod +x $shimpath - - echo "Finished generating shim '$shimpath'" - - return $? -} - diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh deleted file mode 100644 index 2d5660642..000000000 --- a/eng/common/native/init-compiler.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/sh -# -# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables -# -# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! - -if [ -z "$build_arch" ] || [ -z "$compiler" ]; then - echo "Usage..." - echo "build_arch= compiler= init-compiler.sh" - echo "Specify the target architecture." - echo "Specify the name of compiler (clang or gcc)." - exit 1 -fi - -case "$compiler" in - clang*|-clang*|--clang*) - # clangx.y or clang-x.y - version="$(echo "$compiler" | tr -d '[:alpha:]-=')" - majorVersion="${version%%.*}" - [ -z "${version##*.*}" ] && minorVersion="${version#*.}" - - if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then - minorVersion=0; - fi - compiler=clang - ;; - - gcc*|-gcc*|--gcc*) - # gccx.y or gcc-x.y - version="$(echo "$compiler" | tr -d '[:alpha:]-=')" - majorVersion="${version%%.*}" - [ -z "${version##*.*}" ] && minorVersion="${version#*.}" - compiler=gcc - ;; -esac - -cxxCompiler="$compiler++" - -# clear the existing CC and CXX from environment -CC= -CXX= -LDFLAGS= - -if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi - -check_version_exists() { - desired_version=-1 - - # Set up the environment to be used for building with the desired compiler. - if command -v "$compiler-$1.$2" > /dev/null; then - desired_version="-$1.$2" - elif command -v "$compiler$1$2" > /dev/null; then - desired_version="$1$2" - elif command -v "$compiler-$1$2" > /dev/null; then - desired_version="-$1$2" - fi - - echo "$desired_version" -} - -if [ -z "$CLR_CC" ]; then - - # Set default versions - if [ -z "$majorVersion" ]; then - # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" - elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi - - for version in $versions; do - _major="${version%%.*}" - [ -z "${version##*.*}" ] && _minor="${version#*.}" - desired_version="$(check_version_exists "$_major" "$_minor")" - if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi - done - - if [ -z "$majorVersion" ]; then - if command -v "$compiler" > /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH." - fi - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "No usable version of $compiler found." - exit 1 - fi - else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi - fi - fi - fi - else - desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" - if [ "$desired_version" = "-1" ]; then - echo "Could not find specific version of $compiler: $majorVersion $minorVersion." - exit 1 - fi - fi - - if [ -z "$CC" ]; then - CC="$(command -v "$compiler$desired_version")" - CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi - fi -else - if [ ! -f "$CLR_CC" ]; then - echo "CLR_CC is set but path '$CLR_CC' does not exist" - exit 1 - fi - CC="$CLR_CC" - CXX="$CLR_CXX" -fi - -if [ -z "$CC" ]; then - echo "Unable to find $compiler." - exit 1 -fi - -# Only lld version >= 9 can be considered stable. lld doesn't support s390x. -if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then - if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then - LDFLAGS="-fuse-ld=lld" - fi -fi - -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" - -export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh deleted file mode 100644 index de1687b2c..000000000 --- a/eng/common/native/init-distro-rid.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash - -# getNonPortableDistroRid -# -# Input: -# targetOs: (str) -# targetArch: (str) -# rootfsDir: (str) -# -# Return: -# non-portable rid -getNonPortableDistroRid() -{ - local targetOs="$1" - local targetArch="$2" - local rootfsDir="$3" - local nonPortableRid="" - - if [ "$targetOs" = "linux" ]; then - if [ -e "${rootfsDir}/etc/os-release" ]; then - source "${rootfsDir}/etc/os-release" - - if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then - # remove the last version digit - VERSION_ID="${VERSION_ID%.*}" - fi - - if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then - nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" - else - # Rolling release distros either do not set VERSION_ID, set it as blank or - # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); - # so omit it here to be consistent with everything else. - nonPortableRid="${ID}-${targetArch}" - fi - - elif [ -e "${rootfsDir}/android_platform" ]; then - source "$rootfsDir"/android_platform - nonPortableRid="$RID" - fi - fi - - if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is shell script and it should always work. - __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) - nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then - __android_sdk_version=$(getprop ro.build.version.sdk) - nonPortableRid="android.$__android_sdk_version-${targetArch}" - elif [ "$targetOs" = "illumos" ]; then - __uname_version=$(uname -v) - case "$__uname_version" in - omnios-*) - __omnios_major_version=$(echo "${__uname_version:8:2}") - nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" - ;; - joyent_*) - __smartos_major_version=$(echo "${__uname_version:7:4}") - nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" - ;; - illumos_*) - nonPortableRid=openindiana-"$targetArch" - ;; - esac - elif [ "$targetOs" = "solaris" ]; then - __uname_version=$(uname -v) - __solaris_major_version=$(echo "${__uname_version%.*}") - nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" - elif [ "$targetOs" = "haiku" ]; then - __uname_release=$(uname -r) - nonPortableRid=haiku.r"$__uname_release"-"$targetArch" - fi - - echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" -} - -# initDistroRidGlobal -# -# Input: -# os: (str) -# arch: (str) -# rootfsDir?: (nullable:string) -# -# Return: -# None -# -# Notes: -# -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. -# -initDistroRidGlobal() -{ - local targetOs="$1" - local targetArch="$2" - local rootfsDir="" - if [ "$#" -ge 3 ]; then - rootfsDir="$3" - fi - - if [ -n "${rootfsDir}" ]; then - # We may have a cross build. Check for the existence of the rootfsDir - if [ ! -e "${rootfsDir}" ]; then - echo "Error rootfsDir has been passed, but the location is not valid." - exit 1 - fi - fi - - __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") - - if [ -z "${__PortableTargetOS:-}" ]; then - __PortableTargetOS="$targetOs" - - STRINGS="$(command -v strings || true)" - if [ -z "$STRINGS" ]; then - STRINGS="$(command -v llvm-strings || true)" - fi - - # Check for musl-based distros (e.g Alpine Linux, Void Linux). - if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || - ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then - __PortableTargetOS="linux-musl" - fi - fi - - export __DistroRid __PortableTargetOS -} diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh deleted file mode 100644 index e693617a6..000000000 --- a/eng/common/native/init-os-and-arch.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# Use uname to determine what the OS is. -OSName=$(uname -s | tr '[:upper:]' '[:lower:]') - -if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then - OSName="android" -fi - -case "$OSName" in -freebsd|linux|netbsd|openbsd|sunos|android|haiku) - os="$OSName" ;; -darwin) - os=osx ;; -*) - echo "Unsupported OS $OSName detected!" - exit 1 ;; -esac - -# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html -# and `uname -p` returns processor type (e.g. i386 on amd64). -# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html. -if [ "$os" = "sunos" ]; then - if uname -o 2>&1 | grep -q illumos; then - os="illumos" - else - os="solaris" - fi - CPUName=$(isainfo -n) -else - # For the rest of the operating systems, use uname(1) to determine what the CPU is. - CPUName=$(uname -m) -fi - -case "$CPUName" in - arm64|aarch64) - arch=arm64 - ;; - - loongarch64) - arch=loongarch64 - ;; - - riscv64) - arch=riscv64 - ;; - - amd64|x86_64) - arch=x64 - ;; - - armv7l|armv8l) - if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then - arch=armel - else - arch=arm - fi - ;; - - armv6l) - arch=armv6 - ;; - - i[3-6]86) - echo "Unsupported CPU $CPUName detected, build might not succeed!" - arch=x86 - ;; - - s390x) - arch=s390x - ;; - - ppc64le) - arch=ppc64le - ;; - *) - echo "Unknown CPU $CPUName detected!" - exit 1 - ;; -esac diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh deleted file mode 100644 index 8a5e7cf0d..000000000 --- a/eng/common/native/install-cmake-test.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake-test" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh deleted file mode 100644 index de496beeb..000000000 --- a/eng/common/native/install-cmake.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1 deleted file mode 100644 index 78f2d84a4..000000000 --- a/eng/common/native/install-tool.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -<# -.SYNOPSIS -Install native tool - -.DESCRIPTION -Install cmake native tool from Azure blob storage - -.PARAMETER InstallPath -Base directory to install native tool to - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER CommonLibraryDirectory -Path to folder containing common library modules - -.PARAMETER Force -Force install of tools even if they previously exist - -.PARAMETER Clean -Don't install the tool, just clean up the current install of the tool - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.NOTES -Returns 0 if install succeeds, 1 otherwise -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [Parameter(Mandatory=$True)] - [string] $ToolName, - [Parameter(Mandatory=$True)] - [string] $InstallPath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [Parameter(Mandatory=$True)] - [string] $Version, - [string] $CommonLibraryDirectory = $PSScriptRoot, - [switch] $Force = $False, - [switch] $Clean = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 -) - -. $PSScriptRoot\..\pipeline-logging-functions.ps1 - -# Import common library modules -Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $Arch = CommonLibrary\Get-MachineArchitecture - $ToolOs = "win64" - if($Arch -Eq "x32") { - $ToolOs = "win32" - } - $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" - $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" - $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" - $ShimPath = Join-Path $InstallPath "$ToolName.exe" - - if ($Clean) { - Write-Host "Cleaning $ToolInstallDirectory" - if (Test-Path $ToolInstallDirectory) { - Remove-Item $ToolInstallDirectory -Force -Recurse - } - Write-Host "Cleaning $ShimPath" - if (Test-Path $ShimPath) { - Remove-Item $ShimPath -Force - } - $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri - Write-Host "Cleaning $ToolTempPath" - if (Test-Path $ToolTempPath) { - Remove-Item $ToolTempPath -Force - } - exit 0 - } - - # Install tool - if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { - Write-Verbose "$ToolName ($Version) already exists, skipping install" - } - else { - $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` - -InstallDirectory $ToolInstallDirectory ` - -Force:$Force ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Verbose:$Verbose - - if ($InstallStatus -Eq $False) { - Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" - exit 1 - } - } - - $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } - if (@($ToolFilePath).Length -Gt 1) { - Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" - exit 1 - } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Host "$ToolName was not found in $ToolInstallDirectory." - exit 1 - } - - # Generate shim - # Always rewrite shims so that we are referencing the expected version - $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` - -ShimDirectory $InstallPath ` - -ToolFilePath "$ToolFilePath" ` - -BaseUri $BaseUri ` - -Force:$Force ` - -Verbose:$Verbose - - if ($GenerateShimStatus -Eq $False) { - Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" - return 1 - } - - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ - exit 1 -} diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 deleted file mode 100644 index 8e422c561..000000000 --- a/eng/common/pipeline-logging-functions.ps1 +++ /dev/null @@ -1,260 +0,0 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. - -# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 - -$script:loggingCommandPrefix = '##vso[' -$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? - New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } - New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } - New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } - New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } -) -# TODO: BUG: Escape % ??? -# TODO: Add test to verify don't need to escape "=". - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTelemetryError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Category, - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force) - - $PSBoundParameters.Remove('Category') | Out-Null - - if ($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) - Write-PipelineTaskError @PSBoundParameters -} - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force - ) - - if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { - if ($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return - } - elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return - } - } - - if (($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - $PSBoundParameters.Remove('Force') | Out-Null - if (-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters -} - -function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput, - [bool]$IsMultiJobVariable = $true) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = $IsMultiJobVariable - } -AsOutput:$AsOutput - } -} - -function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Path, - [switch]$AsOutput) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } -} - -function Write-PipelineSetResult { - [CmdletBinding()] - param( - [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] - [Parameter(Mandatory = $true)] - [string]$Result, - [string]$Message) - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ - 'result' = $Result - } - } -} - -<######################################## -# Private functions. -########################################> -function Format-LoggingCommandData { - [CmdletBinding()] - param([string]$Value, [switch]$Reverse) - - if (!$Value) { - return '' - } - - if (!$Reverse) { - foreach ($mapping in $script:loggingCommandEscapeMappings) { - $Value = $Value.Replace($mapping.Token, $mapping.Replacement) - } - } - else { - for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { - $mapping = $script:loggingCommandEscapeMappings[$i] - $Value = $Value.Replace($mapping.Replacement, $mapping.Token) - } - } - - return $Value -} - -function Format-LoggingCommand { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Area, - [Parameter(Mandatory = $true)] - [string]$Event, - [string]$Data, - [hashtable]$Properties) - - # Append the preamble. - [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder - $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) - - # Append the properties. - if ($Properties) { - $first = $true - foreach ($key in $Properties.Keys) { - [string]$value = Format-LoggingCommandData $Properties[$key] - if ($value) { - if ($first) { - $null = $sb.Append(' ') - $first = $false - } - else { - $null = $sb.Append(';') - } - - $null = $sb.Append("$key=$value") - } - } - } - - # Append the tail and output the value. - $Data = Format-LoggingCommandData $Data - $sb.Append(']').Append($Data).ToString() -} - -function Write-LoggingCommand { - [CmdletBinding(DefaultParameterSetName = 'Parameters')] - param( - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Area, - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Event, - [Parameter(ParameterSetName = 'Parameters')] - [string]$Data, - [Parameter(ParameterSetName = 'Parameters')] - [hashtable]$Properties, - [Parameter(Mandatory = $true, ParameterSetName = 'Object')] - $Command, - [switch]$AsOutput) - - if ($PSCmdlet.ParameterSetName -eq 'Object') { - Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput - return - } - - $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties - if ($AsOutput) { - $command - } - else { - Write-Host $command - } -} - -function Write-LogIssue { - [CmdletBinding()] - param( - [ValidateSet('warning', 'error')] - [Parameter(Mandatory = $true)] - [string]$Type, - [string]$Message, - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } - if ($AsOutput) { - return $command - } - - if ($Type -eq 'error') { - $foregroundColor = $host.PrivateData.ErrorForegroundColor - $backgroundColor = $host.PrivateData.ErrorBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Red - $backgroundColor = [System.ConsoleColor]::Black - } - } - else { - $foregroundColor = $host.PrivateData.WarningForegroundColor - $backgroundColor = $host.PrivateData.WarningBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Yellow - $backgroundColor = [System.ConsoleColor]::Black - } - } - - Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh deleted file mode 100644 index 6a0b2255e..000000000 --- a/eng/common/pipeline-logging-functions.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash - -function Write-PipelineTelemetryError { - local telemetry_category='' - local force=false - local function_args=() - local message='' - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -category|-c) - telemetry_category=$2 - shift - ;; - -force|-f) - force=true - ;; - -*) - function_args+=("$1 $2") - shift - ;; - *) - message=$* - ;; - esac - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$message" >&2 - return - fi - - if [[ $force == true ]]; then - function_args+=("-force") - fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") - Write-PipelineTaskError ${function_args[@]} -} - -function Write-PipelineTaskError { - local message_type="error" - local sourcepath='' - local linenumber='' - local columnnumber='' - local error_code='' - local force=false - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-ln) - linenumber=$2 - shift - ;; - -columnnumber|-cn) - columnnumber=$2 - shift - ;; - -errcode|-e) - error_code=$2 - shift - ;; - -force|-f) - force=true - ;; - *) - break - ;; - esac - - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - local message="##vso[task.logissue" - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - echo "$message" -} - -function Write-PipelineSetVariable { - if [[ "$ci" != true ]]; then - return - fi - - local name='' - local value='' - local secret=false - local as_output=false - local is_multi_job_variable=true - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -name|-n) - name=$2 - shift - ;; - -value|-v) - value=$2 - shift - ;; - -secret|-s) - secret=true - ;; - -as_output|-a) - as_output=true - ;; - -is_multi_job_variable|-i) - is_multi_job_variable=$2 - shift - ;; - esac - shift - done - - value=${value/;/%3B} - value=${value/\\r/%0D} - value=${value/\\n/%0A} - value=${value/]/%5D} - - local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" - - if [[ "$as_output" == true ]]; then - $message - else - echo "$message" - fi -} - -function Write-PipelinePrependPath { - local prepend_path='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -path|-p) - prepend_path=$2 - shift - ;; - esac - shift - done - - export PATH="$prepend_path:$PATH" - - if [[ "$ci" == true ]]; then - echo "##vso[task.prependpath]$prepend_path" - fi -} - -function Write-PipelineSetResult { - local result='' - local message='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -result|-r) - result=$2 - shift - ;; - -message|-m) - message=$2 - shift - ;; - esac - shift - done - - if [[ "$ci" == true ]]; then - echo "##vso[task.complete result=$result;]$message" - fi -} diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 deleted file mode 100644 index 49938f0c8..000000000 --- a/eng/common/post-build/add-build-to-channel.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param( - [Parameter(Mandatory=$true)][int] $BuildId, - [Parameter(Mandatory=$true)][int] $ChannelId, - [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - # Check that the channel we are going to promote the build to exist - $channelInfo = Get-MaestroChannel -ChannelId $ChannelId - - if (!$channelInfo) { - Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!" - ExitWithExitCode 1 - } - - # Get info about which channel(s) the build has already been promoted to - $buildInfo = Get-MaestroBuild -BuildId $BuildId - - if (!$buildInfo) { - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!" - ExitWithExitCode 1 - } - - # Find whether the build is already assigned to the channel or not - if ($buildInfo.channels) { - foreach ($channel in $buildInfo.channels) { - if ($channel.Id -eq $ChannelId) { - Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!" - ExitWithExitCode 0 - } - } - } - - Write-Host "Promoting build '$BuildId' to channel '$ChannelId'." - - Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 deleted file mode 100644 index 63f3464c9..000000000 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ /dev/null @@ -1,40 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to - [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - if ($PromoteToChannels -eq "") { - Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." - ExitWithExitCode 0 - } - - # Check that every channel that Maestro told to promote the build to - # is available in YAML - $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } - - $hasErrors = $false - - foreach ($id in $PromoteToChannelsIds) { - if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { - Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." - $hasErrors = $true - } - } - - # The `Write-PipelineTaskError` doesn't error the script and we might report several errors - # in the previous lines. The check below makes sure that we return an error state from the - # script if we reported any validation error - if ($hasErrors) { - ExitWithExitCode 1 - } - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 deleted file mode 100644 index dab3534ab..000000000 --- a/eng/common/post-build/nuget-validation.ps1 +++ /dev/null @@ -1,24 +0,0 @@ -# This script validates NuGet package metadata information using this -# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage - -param( - [Parameter(Mandatory=$true)][string] $PackagesPath, # Path to where the packages to be validated are - [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' - - New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force - - Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 - - & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 deleted file mode 100644 index 534f6988d..000000000 --- a/eng/common/post-build/post-build-utils.ps1 +++ /dev/null @@ -1,91 +0,0 @@ -# Most of the functions in this file require the variables `MaestroApiEndPoint`, -# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') { - Validate-MaestroVars - - $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $headers.Add('Accept', $ContentType) - $headers.Add('Authorization',"Bearer $MaestroApiAccessToken") - return $headers -} - -function Get-MaestroChannel([int]$ChannelId) { - Validate-MaestroVars - - $apiHeaders = Create-MaestroApiRequestHeaders - $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion" - - $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - return $result -} - -function Get-MaestroBuild([int]$BuildId) { - Validate-MaestroVars - - $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion" - - $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - return $result -} - -function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) { - Validate-MaestroVars - - $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository) - $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion" - - $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - return $result -} - -function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { - Validate-MaestroVars - - $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" - Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null -} - -function Trigger-Subscription([string]$SubscriptionId) { - Validate-MaestroVars - - $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" - Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null -} - -function Validate-MaestroVars { - try { - Get-Variable MaestroApiEndPoint | Out-Null - Get-Variable MaestroApiVersion | Out-Null - Get-Variable MaestroApiAccessToken | Out-Null - - if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" - ExitWithExitCode 1 - } - - if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" - ExitWithExitCode 1 - } - } - catch { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.' - Write-Host $_ - ExitWithExitCode 1 - } -} diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 deleted file mode 100644 index 238945cb5..000000000 --- a/eng/common/post-build/publish-using-darc.ps1 +++ /dev/null @@ -1,53 +0,0 @@ -param( - [Parameter(Mandatory=$true)][int] $BuildId, - [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, - [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', - [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, - [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, - [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - $darc = Get-Darc - - $optionalParams = [System.Collections.ArrayList]::new() - - if ("" -ne $ArtifactsPublishingAdditionalParameters) { - $optionalParams.Add("--artifact-publishing-parameters") | Out-Null - $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null - } - - if ("" -ne $SymbolPublishingAdditionalParameters) { - $optionalParams.Add("--symbol-publishing-parameters") | Out-Null - $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null - } - - if ("false" -eq $WaitPublishingFinish) { - $optionalParams.Add("--no-wait") | Out-Null - } - - & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat "$AzdoToken" ` - --bar-uri "$MaestroApiEndPoint" ` - --ci ` - @optionalParams - - if ($LastExitCode -ne 0) { - Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..." - exit 1 - } - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 4011d324e..000000000 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,319 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -. $PSScriptRoot\post-build-utils.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -$MaxRetries = 5 -$RetryWaitTimeInSeconds = 30 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-Host "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') - - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - $attempts = 0 - - while ($attempts -lt $using:MaxRetries) { - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - if ($Link -match "submodules") { - # Skip submodule links until sourcelink properly handles submodules - $Status = 200 - } - elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - # Only GitHub links are valid - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - # If it's not a github link, we want to break out of the loop and not retry. - $Status = 0 - $attempts = $using:MaxRetries - } - } - catch { - Write-Host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - $attempts++ - - if ($attempts -lt $using:MaxRetries) - { - $attemptsLeft = $using:MaxRetries - $attempts - Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $using:RetryWaitTimeInSeconds - } - else { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - else { - break - } - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - Write-Host $_ - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host 'Passed.' - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$ValidationFailures, - [switch]$logErrors) { - if ($result -ne '0') { - if ($logErrors) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." - } - $ValidationFailures.Value++ - } -} - -function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" - ExitWithExitCode 1 - } - - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." - } - } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $ValidationFailures = 0 - - # Process each NuGet package in parallel - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - Write-Host "Starting $($_.FullName)" - Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors - Remove-Job -Id $Job.Id - } - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) - Remove-Job -Id $Job.Id - } - if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." - ExitWithExitCode 1 - } -} - -function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global - } -} - -try { - InstallSourcelinkCli - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - ValidateSourceLinkLinks -} -catch { - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 deleted file mode 100644 index cd2181baf..000000000 --- a/eng/common/post-build/symbols-validation.ps1 +++ /dev/null @@ -1,339 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs - [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols - [Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server -) - -. $PSScriptRoot\..\tools.ps1 -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -# Max number of retries -$MaxRetry = 5 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -# Set error codes -Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 -Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 - -$WindowsPdbVerificationParam = "" -if ($CheckForWindowsPdbs) { - $WindowsPdbVerificationParam = "--windows-pdbs" -} - -$ExclusionSet = New-Object System.Collections.Generic.HashSet[string]; - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No symbols to validate." - ExitWithExitCode 0 -} - -#Check if the path exists -if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){ - [string[]]$Exclusions = Get-Content "$SymbolExclusionFile" - $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} } -} -else{ - Write-Host "Symbol Exclusion file does not exists. No symbols to exclude." -} - -$CountMissingSymbols = { - param( - [string] $PackagePath, # Path to a NuGet package - [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs - ) - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - Write-Host "Validating $PackagePath " - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = $using:ERROR_FILEDOESNOTEXIST - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - - try { - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - } - catch { - Write-Host "Something went wrong extracting $PackagePath" - Write-Host $_ - return [pscustomobject]@{ - result = $using:ERROR_BADEXTRACT - packagePath = $PackagePath - } - } - - Get-ChildItem -Recurse $ExtractPath | - Where-Object { $RelevantExtensions -contains $_.Extension } | - ForEach-Object { - $FileName = $_.FullName - if ($FileName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $FileName - return - } - - $FirstMatchingSymbolDescriptionOrDefault = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs. - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - - $totalRetries = 0 - - while ($totalRetries -lt $using:MaxRetry) { - - # Save the output and get diagnostic output - $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String - - if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) { - return 'Module and PDB for Module' - } - elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) { - return 'Dll, PDB and NGen PDB' - } - elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) { - return 'So and DBG for SO' - } - elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) { - return 'Dylib and Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else - { - $totalRetries++ - } - } - - return $null - } - - $FileRelativePath = $FileName.Replace("$ExtractPath\", "") - if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){ - Write-Host "Skipping $FileName from symbol validation" - } - - else { - $FileGuid = New-Guid - $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid - - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--microsoft-symbol-server' ` - -SymbolsPath "$ExpandedSymbolsPath-msdl" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--internal-server' ` - -SymbolsPath "$ExpandedSymbolsPath-symweb" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - - Write-Host -NoNewLine "`t Checking file " $FileName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } - } - } - } - } - - if ($using:Clean) { - Remove-Item $ExtractPath -Recurse -Force - } - - Pop-Location - - return [pscustomobject]@{ - result = $MissingSymbols - packagePath = $PackagePath - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$DupedSymbols, - [ref]$TotalFailures) { - if ($result -eq $ERROR_BADEXTRACT) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" - $DupedSymbols.Value++ - } - elseif ($result -eq $ERROR_FILEDOESNOTEXIST) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist" - $TotalFailures.Value++ - } - elseif ($result -gt '0') { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" - $TotalFailures.Value++ - } - else { - Write-Host "All symbols verified for package $packagePath" - } -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $TotalPackages = 0 - $TotalFailures = 0 - $DupedSymbols = 0 - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - $FullName = $_.FullName - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - $TotalPackages++ - - Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null - - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) - Remove-Job -Id $Job.Id - } - Write-Host - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) - } - - if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { - if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" - } - - if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" - } - - ExitWithExitCode 1 - } - else { - Write-Host "All symbols validated!" - } -} - -function InstallDotnetSymbol { - $dotnetSymbolPackageName = 'dotnet-symbol' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." - } - else { - Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global - } -} - -try { - . $PSScriptRoot\post-build-utils.ps1 - - InstallDotnetSymbol - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - CheckSymbolsAvailable -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 deleted file mode 100644 index ac9a95778..000000000 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ /dev/null @@ -1,64 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $SourceRepo, - [Parameter(Mandatory=$true)][int] $ChannelId, - [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - # Get all the $SourceRepo subscriptions - $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') - $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId - - if (!$subscriptions) { - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" - ExitWithExitCode 0 - } - - $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] - $failedTriggeredSubscription = $false - - # Get all enabled subscriptions that need dependency flow on 'everyBuild' - foreach ($subscription in $subscriptions) { - if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { - Write-Host "Should trigger this subscription: ${$subscription.id}" - [void]$subscriptionsToTrigger.Add($subscription.id) - } - } - - foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { - try { - Write-Host "Triggering subscription '$subscriptionToTrigger'." - - Trigger-Subscription -SubscriptionId $subscriptionToTrigger - - Write-Host 'done.' - } - catch - { - Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" - Write-Host $_ - Write-Host $_.ScriptStackTrace - $failedTriggeredSubscription = $true - } - } - - if ($subscriptionsToTrigger.Count -eq 0) { - Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." - } - elseif ($failedTriggeredSubscription) { - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...' - ExitWithExitCode 1 - } - else { - Write-Host 'All subscriptions were triggered successfully!' - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/retain-build.ps1 b/eng/common/retain-build.ps1 deleted file mode 100644 index e7ba975ad..000000000 --- a/eng/common/retain-build.ps1 +++ /dev/null @@ -1,45 +0,0 @@ - -Param( -[Parameter(Mandatory=$true)][int] $buildId, -[Parameter(Mandatory=$true)][string] $azdoOrgUri, -[Parameter(Mandatory=$true)][string] $azdoProject, -[Parameter(Mandatory=$true)][string] $token -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -function Get-AzDOHeaders( - [string] $token) -{ - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}")) - $headers = @{"Authorization"="Basic $base64AuthInfo"} - return $headers -} - -function Update-BuildRetention( - [string] $azdoOrgUri, - [string] $azdoProject, - [int] $buildId, - [string] $token) -{ - $headers = Get-AzDOHeaders -token $token - $requestBody = "{ - `"keepForever`": `"true`" - }" - - $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0" - write-Host "Attempting to retain build using the following URI: ${requestUri} ..." - - try { - Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" - Write-Host "Updated retention settings for build ${buildId}." - } - catch { - Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" - exit 1 - } -} - -Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token -exit 0 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 deleted file mode 100644 index 4f0546dce..000000000 --- a/eng/common/sdk-task.ps1 +++ /dev/null @@ -1,97 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $configuration = 'Debug', - [string] $task, - [string] $verbosity = 'minimal', - [string] $msbuildEngine = $null, - [switch] $restore, - [switch] $prepareMachine, - [switch] $help, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties -) - -$ci = $true -$binaryLog = $true -$warnAsError = $true - -. $PSScriptRoot\tools.ps1 - -function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -task Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)" - Write-Host " -restore Restore dependencies" - Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" - Write-Host " -help Print help and exit" - Write-Host "" - - Write-Host "Advanced settings:" - Write-Host " -prepareMachine Prepare machine for CI run" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host "" - Write-Host "Command line arguments not listed above are passed thru to msbuild." -} - -function Build([string]$target) { - $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } - $log = Join-Path $LogDir "$task$logSuffix.binlog" - $outputPath = Join-Path $ToolsetDir "$task\" - - MSBuild $taskProject ` - /bl:$log ` - /t:$target ` - /p:Configuration=$configuration ` - /p:RepoRoot=$RepoRoot ` - /p:BaseIntermediateOutputPath=$outputPath ` - /v:$verbosity ` - @properties -} - -try { - if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { - Print-Usage - exit 0 - } - - if ($task -eq "") { - Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" - Print-Usage - ExitWithExitCode 1 - } - - if( $msbuildEngine -eq "vs") { - # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" - } - - $taskProject = GetSdkTaskProject $task - if (!(Test-Path $taskProject)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" - ExitWithExitCode 1 - } - - if ($restore) { - Build 'Restore' - } - - Build 'Execute' -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config deleted file mode 100644 index 5bfbb02ef..000000000 --- a/eng/common/sdl/NuGet.config +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 deleted file mode 100644 index 27f5a4115..000000000 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ /dev/null @@ -1,130 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - # The list of Guardian tools to configure. For each object in the array: - # - If the item is a [hashtable], it must contain these entries: - # - Name = The tool name as Guardian knows it. - # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique - # among all tool entries with the same Name. - # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' - # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' - [object[]] $ToolsList, - [string] $GuardianLoggerLevel='Standard', - # Optional: Additional params to add to any tool using CredScan. - [string[]] $CrScanAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using PoliCheck. - [string[]] $PoliCheckAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using CodeQL/Semmle. - [string[]] $CodeQLAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using Binskim. - [string[]] $BinskimAdditionalRunConfigParams -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # Normalize tools list: all in [hashtable] form with defined values for each key. - $ToolsList = $ToolsList | - ForEach-Object { - if ($_ -is [string]) { - $_ = @{ Name = $_ } - } - - if (-not ($_['Scenario'])) { $_.Scenario = "" } - if (-not ($_['Args'])) { $_.Args = @() } - $_ - } - - Write-Host "List of tools to configure:" - $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - foreach ($tool in $ToolsList) { - # Put together the name and scenario to make a unique key. - $toolConfigName = $tool.Name - if ($tool.Scenario) { - $toolConfigName += "_" + $tool.Scenario - } - - Write-Host "=== Configuring $toolConfigName..." - - $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" - - # For some tools, add default and automatic args. - switch -Exact ($tool.Name) { - 'credscan' { - if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" - } - $tool.Args += "`"OutputType < pre`"" - $tool.Args += $CrScanAdditionalRunConfigParams - } - 'policheck' { - if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" - } - $tool.Args += $PoliCheckAdditionalRunConfigParams - } - {$_ -in 'semmle', 'codeql'} { - if ($targetDirectory) { - $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" - } - $tool.Args += $CodeQLAdditionalRunConfigParams - } - 'binskim' { - if ($targetDirectory) { - # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. - # We are excluding all `_.pdb` files from the scan. - $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" - } - $tool.Args += $BinskimAdditionalRunConfigParams - } - } - - # Create variable pointing to the args array directly so we can use splat syntax later. - $toolArgs = $tool.Args - - # Configure the tool. If args array is provided or the current tool has some default arguments - # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", - # one per parameter. Doc page for "guardian configure": - # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure - Exec-BlockVerbosely { - & $GuardianCliLocation configure ` - --working-directory $WorkingDirectory ` - --tool $tool.Name ` - --output-path $gdnConfigFile ` - --logger-level $GuardianLoggerLevel ` - --noninteractive ` - --force ` - $(if ($toolArgs) { "--args" }) @toolArgs - Exit-IfNZEC "Sdl" - } - - Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 deleted file mode 100644 index 81ded5b7f..000000000 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ /dev/null @@ -1,165 +0,0 @@ -Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - - # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list - # format. - [object[]] $SourceToolsList, - # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools - # list format. - [object[]] $ArtifactToolsList, - # Optional: list of SDL tools to run without automatically specifying a target directory. See - # 'configure-sdl-tool.ps1' for tools list format. - [object[]] $CustomToolsList, - - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error - [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") - [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1") - [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1") - [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run -) - -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - #Replace repo names to the format of org/repo - if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; - } - else{ - $RepoName = $Repository; - } - - if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd')) - } else { - $guardianCliLocation = $GuardianCliLocation - } - - $workingDirectory = (Split-Path $SourceDirectory -Parent) - $ValidPath = Test-Path $guardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.' - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel - } - $gdnFolder = Join-Path $workingDirectory '.gdn' - - if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Exec-BlockVerbosely { - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.' - ExitWithExitCode 1 - } - } - - # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. - function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { - if ($tools -and $tools.Count -gt 0) { - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $workingDirectory ` - -TargetDirectory $targetDirectory ` - -GdnFolder $gdnFolder ` - -ToolsList $tools ` - -GuardianLoggerLevel $GuardianLoggerLevel ` - -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` - -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` - -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams ` - -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams - if ($BreakOnFailure) { - Exit-IfNZEC "Sdl" - } - } - } - } - - # Configure Artifact and Source tools with default Target directories. - Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory - Configure-ToolsList $SourceToolsList $SourceDirectory - # Configure custom tools with no default Target directory. - Configure-ToolsList $CustomToolsList $null - - # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. - # (If we used "run" multiple times, each run would overwrite data from earlier runs.) - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $SourceDirectory ` - -UpdateBaseline $UpdateBaseline ` - -GdnFolder $gdnFolder - } - - if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Exec-BlockVerbosely { - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.' - ExitWithExitCode 1 - } - } - - if ($BreakOnFailure) { - Write-Host "Failing the build in case of breaking results..." - Exec-BlockVerbosely { - & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - } else { - Write-Host "Letting the build pass even if there were breaking results..." - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - exit 1 -} diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1 deleted file mode 100644 index 68da4fbf2..000000000 --- a/eng/common/sdl/extract-artifact-archives.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -# This script looks for each archive file in a directory and extracts it into the target directory. -# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". -# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. -param( - # Full path to directory where archives are stored. - [Parameter(Mandatory=$true)][string] $InputPath, - # Full path to directory to extract archives into. May be the same as $InputPath. - [Parameter(Mandatory=$true)][string] $ExtractPath -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - Measure-Command { - $jobs = @() - - # Find archive files for non-Windows and Windows builds. - $archiveFiles = @( - Get-ChildItem (Join-Path $InputPath "*.tar.gz") - Get-ChildItem (Join-Path $InputPath "*.zip") - ) - - foreach ($targzFile in $archiveFiles) { - $jobs += Start-Job -ScriptBlock { - $file = $using:targzFile - $fileName = [System.IO.Path]::GetFileName($file) - $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" - - New-Item $extractDir -ItemType Directory -Force | Out-Null - - Write-Host "Extracting '$file' to '$extractDir'..." - - # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. - # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the - # error. Save output so it can be stored in the exception string along with context. - $output = tar -xf $file -C $extractDir 2>&1 - # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we - # don't have access to the outer scope. - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" - } - - Write-Host "Extracted to $extractDir" - } - } - - Receive-Job $jobs -Wait - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b2..000000000 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 deleted file mode 100644 index 588ff8e22..000000000 --- a/eng/common/sdl/init-sdl.ps1 +++ /dev/null @@ -1,47 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $Repository, - [string] $BranchName='master', - [string] $WorkingDirectory, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -# Don't display the console progress UI - it's a huge perf hit -$ProgressPreference = 'SilentlyContinue' - -Add-Type -AssemblyName System.IO.Compression.FileSystem - -try { - # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host 'Initializing Guardian...' - Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" - & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - # We create the mainbaseline so it can be edited later - Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" - & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - ExitWithExitCode 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config deleted file mode 100644 index e5f543ea6..000000000 --- a/eng/common/sdl/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 deleted file mode 100644 index 2eac8c78f..000000000 --- a/eng/common/sdl/run-sdl.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $GdnFolder, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' - Write-Host "Discovered Guardian config files:" - $gdnConfigFiles | Out-String | Write-Host - - Exec-BlockVerbosely { - & $GuardianCliLocation run ` - --working-directory $WorkingDirectory ` - --baseline mainbaseline ` - --update-baseline $UpdateBaseline ` - --logger-level $GuardianLoggerLevel ` - --config @gdnConfigFiles - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 deleted file mode 100644 index 7fe603fe9..000000000 --- a/eng/common/sdl/sdl.ps1 +++ /dev/null @@ -1,40 +0,0 @@ - -function Install-Gdn { - param( - [Parameter(Mandatory=$true)] - [string]$Path, - - [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", - - # If omitted, install the latest version of Guardian, otherwise install that specific version. - [string]$Version - ) - - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") - - if ($Version) { - $argumentList += "-Version $Version" - } - - Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait - - $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path - - if (!$gdnCliPath) - { - Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' - } - - return $gdnCliPath.FullName -} \ No newline at end of file diff --git a/eng/common/sdl/trim-assets-version.ps1 b/eng/common/sdl/trim-assets-version.ps1 deleted file mode 100644 index a2e004877..000000000 --- a/eng/common/sdl/trim-assets-version.ps1 +++ /dev/null @@ -1,75 +0,0 @@ -<# -.SYNOPSIS -Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name. - -.PARAMETER InputPath -Full path to directory where artifact packages are stored - -.PARAMETER Recursive -Search for NuGet packages recursively - -#> - -Param( - [string] $InputPath, - [bool] $Recursive = $true -) - -$CliToolName = "Microsoft.DotNet.VersionTools.Cli" - -function Install-VersionTools-Cli { - param( - [Parameter(Mandatory=$true)][string]$Version - ) - - Write-Host "Installing the package '$CliToolName' with a version of '$version' ..." - $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" - - $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed") - Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait -} - -# ------------------------------------------------------------------- - -if (!(Test-Path $InputPath)) { - Write-Host "Input Path '$InputPath' does not exist" - ExitWithExitCode 1 -} - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -try { - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - - $toolsetVersion = Read-ArcadeSdkVersion - Install-VersionTools-Cli -Version $toolsetVersion - - $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName}) - if ($null -eq $cliToolFound) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed." - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & "$dotnet" $CliToolName trim-assets-version ` - --assets-path $InputPath ` - --recursive $Recursive - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml deleted file mode 100644 index 98ccbd7a9..000000000 --- a/eng/common/templates-official/job/job.yml +++ /dev/null @@ -1,265 +0,0 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - -jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates-official/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: 'Publish logs' - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates-official/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' - artifactName: 'BuildConfiguration' - displayName: 'Publish build retry configuration' - continueOnError: true diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml deleted file mode 100644 index 52b4d05d3..000000000 --- a/eng/common/templates-official/job/onelocbuild.yml +++ /dev/null @@ -1,112 +0,0 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - -jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml deleted file mode 100644 index b2ccd9df6..000000000 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ /dev/null @@ -1,160 +0,0 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - -jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Build Assets - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: > - -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force - $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - Add-Content -Path $filePath -Value $(BARBuildId) - Add-Content -Path $filePath -Value "$(DefaultChannels)" - Add-Content -Path $filePath -Value $(IsStableBuild) - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates-official/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml deleted file mode 100644 index 4217d6d8b..000000000 --- a/eng/common/templates-official/job/source-build.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - - # If set to true and running on a non-public project, - # Internal blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - -jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 - os: linux - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all - - steps: - - ${{ if eq(parameters.enableInternalSources, true) }}: - - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - - template: /eng/common/templates-official/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml deleted file mode 100644 index fb632b71a..000000000 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ /dev/null @@ -1,83 +0,0 @@ -parameters: - runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - -jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexUploadPackageVersion - value: ${{ parameters.sourceIndexUploadPackageVersion }} - - name: SourceIndexProcessBinlogPackageVersion - value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - os: windows - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - displayName: Get stage 1 auth token - inputs: - azureSubscription: 'SourceDotNet Stage1 Publish' - addSpnToEnvironment: true - scriptType: 'ps' - scriptLocation: 'inlineScript' - inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - - - script: | - az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) - displayName: "Login to Azure" - - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml deleted file mode 100644 index b68d3c2f3..000000000 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ /dev/null @@ -1,31 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - -jobs: -- template: /eng/common/templates-official/jobs/jobs.yml - parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true - - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml deleted file mode 100644 index 857a0f8ba..000000000 --- a/eng/common/templates-official/jobs/jobs.yml +++ /dev/null @@ -1,97 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates-official/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates-official/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete - - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml deleted file mode 100644 index b9247be15..000000000 --- a/eng/common/templates-official/jobs/source-build.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates-official/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - - # If set to true and running on a non-public project, - # Internal nuget and blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - -jobs: - -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - enableInternalSources: ${{ parameters.enableInternalSources }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} - enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml deleted file mode 100644 index 173914f23..000000000 --- a/eng/common/templates-official/post-build/common-variables.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - - group: Publish-Build-Assets - - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - - name: runCodesignValidationInjection - value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml deleted file mode 100644 index b81b8770b..000000000 --- a/eng/common/templates-official/post-build/post-build.yml +++ /dev/null @@ -1,287 +0,0 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - -stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml deleted file mode 100644 index 0c87f149a..000000000 --- a/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,70 +0,0 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - -steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml deleted file mode 100644 index da669030d..000000000 --- a/eng/common/templates-official/post-build/trigger-subscription.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Triggering subscriptions - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 - arguments: -SourceRepo $(Build.Repository.Uri) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml deleted file mode 100644 index f67a210d6..000000000 --- a/eng/common/templates-official/steps/add-build-to-channel.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml deleted file mode 100644 index eba58109b..000000000 --- a/eng/common/templates-official/steps/build-reason.yml +++ /dev/null @@ -1,12 +0,0 @@ -# build-reason.yml -# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons -# to include steps (',' separated). -parameters: - conditions: '' - steps: [] - -steps: - - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: - - ${{ parameters.steps }} - - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index cbba05967..000000000 --- a/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml deleted file mode 100644 index 93a8394a6..000000000 --- a/eng/common/templates-official/steps/enable-internal-runtimes.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' -# variable with the base64-encoded SAS token, by default - -parameters: -- name: federatedServiceConnection - type: string - default: 'dotnetbuilds-internal-read' -- name: outputVariableName - type: string - default: 'dotnetbuilds-internal-container-read-token-base64' -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: true - -steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates-official/steps/get-delegation-sas.yml - parameters: - federatedServiceConnection: ${{ parameters.federatedServiceConnection }} - outputVariableName: ${{ parameters.outputVariableName }} - expiryInHours: ${{ parameters.expiryInHours }} - base64Encode: ${{ parameters.base64Encode }} - storageAccount: dotnetbuilds - container: internal - permissions: rl diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml deleted file mode 100644 index 9b4a5ffa3..000000000 --- a/eng/common/templates-official/steps/execute-codeql.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - # Language that should be analyzed. Defaults to csharp - language: csharp - # Build Commands - buildCommands: '' - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - # optional: determines whether to continue the build if the step errors; - sdlContinueOnError: false - -steps: -- template: /eng/common/templates-official/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: '${{ parameters.additionalParameters }} - -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml deleted file mode 100644 index 301d5c591..000000000 --- a/eng/common/templates-official/steps/execute-sdl.yml +++ /dev/null @@ -1,86 +0,0 @@ -parameters: - overrideGuardianVersion: '' - executeAllSdlToolsScript: '' - overrideParameters: '' - additionalParameters: '' - publishGuardianDirectoryToPipeline: false - sdlContinueOnError: false - condition: '' - -steps: -- task: NuGetAuthenticate@1 - -- task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian (Overridden) - -- ${{ if eq(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian - -- ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL (Overridden) - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianCliLocation $(GuardianCliLocation) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() - - # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration - # with the "SARIF SAST Scans Tab" Azure DevOps extension - - task: CopyFiles@2 - displayName: Copy SARIF files - inputs: - flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ - contents: '**/*.sarif' - targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs - condition: succeededOrFailed() - - # Use PublishBuildArtifacts because the SARIF extension only checks this case - # see microsoft/sarif-azuredevops-extension#4 - - task: PublishBuildArtifacts@1 - displayName: Publish SARIF files to CodeAnalysisLogs container - inputs: - pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml deleted file mode 100644 index daf0957b6..000000000 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ /dev/null @@ -1,48 +0,0 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 8.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - -steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) - diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml deleted file mode 100644 index c690cc0a0..000000000 --- a/eng/common/templates-official/steps/get-delegation-sas.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: false -- name: storageAccount - type: string -- name: container - type: string -- name: permissions - type: string - default: 'rl' - -steps: -- task: AzureCLI@2 - displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - # Calculate the expiration of the SAS token and convert to UTC - $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - - # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads - # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 - $sas = "" - do { - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - } while($sas.IndexOf('/') -ne -1) - - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - - if ('${{ parameters.base64Encode }}' -eq 'true') { - $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) - } - - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml deleted file mode 100644 index 55e33bd38..000000000 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ /dev/null @@ -1,40 +0,0 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: stepName - type: string - default: 'getFederatedAccessToken' -- name: condition - type: string - default: '' -# Resource to get a token for. Common values include: -# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps -# - 'https://storage.azure.com/' for storage -# Defaults to Azure DevOps -- name: resource - type: string - default: '499b84ac-1321-427f-aa17-267ca6975798' -- name: isStepOutputVariable - type: boolean - default: false - -steps: -- task: AzureCLI@2 - displayName: 'Getting federated access token for feeds' - name: ${{ parameters.stepName }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" - exit 1 - } - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml deleted file mode 100644 index 04012fed1..000000000 --- a/eng/common/templates-official/steps/publish-logs.yml +++ /dev/null @@ -1,23 +0,0 @@ -parameters: - StageLabel: '' - JobLabel: '' - -steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml deleted file mode 100644 index 83d97a26a..000000000 --- a/eng/common/templates-official/steps/retain-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - -steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml deleted file mode 100644 index 22f250130..000000000 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml deleted file mode 100644 index 829f17c34..000000000 --- a/eng/common/templates-official/steps/source-build.yml +++ /dev/null @@ -1,129 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - -steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/source-build/self/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() - -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml deleted file mode 100644 index 1f308b24e..000000000 --- a/eng/common/templates-official/variables/pool-providers.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates-official/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) -# -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 - -variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml deleted file mode 100644 index dbdd66d4a..000000000 --- a/eng/common/templates-official/variables/sdl-variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. -- name: DefaultGuardianVersion - value: 0.109.0 -- name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml deleted file mode 100644 index 7870f93bc..000000000 --- a/eng/common/templates/job/execute-sdl.yml +++ /dev/null @@ -1,139 +0,0 @@ -parameters: - enable: 'false' # Whether the SDL validation job should execute or not - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - # optional: determines if build artifacts should be downloaded. - downloadArtifacts: true - # optional: determines if this job should search the directory of downloaded artifacts for - # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. - extractArchiveArtifacts: false - dependsOn: '' # Optional: dependencies of the job - artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts - # Usage: - # artifactNames: - # - 'BlobArtifacts' - # - 'Artifacts_Windows_NT_Release' - # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, - # not pipeline artifacts, so doesn't affect the use of this parameter. - pipelineArtifactNames: [] - -jobs: -- job: Run_SDL - dependsOn: ${{ parameters.dependsOn }} - displayName: Run SDL tool - condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) - variables: - - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: ${{ parameters.AzDOProjectName }} - - name: AzDOPipelineId - value: ${{ parameters.AzDOPipelineId }} - - name: AzDOBuildId - value: ${{ parameters.AzDOBuildId }} - - template: /eng/common/templates/variables/sdl-variables.yml - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - template: /eng/common/templates/variables/pool-providers.yml - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - checkout: self - clean: true - - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - - - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - ${{ each artifactName in parameters.pipelineArtifactNames }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - powershell: eng/common/sdl/trim-assets-version.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Trim the version from the NuGet packages - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - displayName: Extract Package Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - - powershell: eng/common/sdl/extract-artifact-archives.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Extract Archive Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: ${{ parameters.additionalParameters }} - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml deleted file mode 100644 index e295031c0..000000000 --- a/eng/common/templates/job/job.yml +++ /dev/null @@ -1,259 +0,0 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - -jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml deleted file mode 100644 index 60ab00c4d..000000000 --- a/eng/common/templates/job/onelocbuild.yml +++ /dev/null @@ -1,109 +0,0 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - -jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} - - - task: PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml deleted file mode 100644 index cc2b346ba..000000000 --- a/eng/common/templates/job/publish-build-assets.yml +++ /dev/null @@ -1,156 +0,0 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - -jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Build Assets - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 - arguments: > - -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml deleted file mode 100644 index c48f95d93..000000000 --- a/eng/common/templates/job/source-build.yml +++ /dev/null @@ -1,74 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - - # If set to true and running on a non-public project, - # Internal blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - -jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all - - steps: - - ${{ if eq(parameters.enableInternalSources, true) }}: - - template: /eng/common/templates/steps/enable-internal-runtimes.yml - - template: /eng/common/templates/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml deleted file mode 100644 index 8538f44ba..000000000 --- a/eng/common/templates/job/source-index-stage1.yml +++ /dev/null @@ -1,82 +0,0 @@ -parameters: - runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - -jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexUploadPackageVersion - value: ${{ parameters.sourceIndexUploadPackageVersion }} - - name: SourceIndexProcessBinlogPackageVersion - value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - displayName: Get stage 1 auth token - inputs: - azureSubscription: 'SourceDotNet Stage1 Publish' - addSpnToEnvironment: true - scriptType: 'ps' - scriptLocation: 'inlineScript' - inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - - - script: | - az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) - displayName: "Login to Azure" - - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml deleted file mode 100644 index f7dc5ea4a..000000000 --- a/eng/common/templates/jobs/codeql-build.yml +++ /dev/null @@ -1,31 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - -jobs: -- template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true - - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml deleted file mode 100644 index 289bb2396..000000000 --- a/eng/common/templates/jobs/jobs.yml +++ /dev/null @@ -1,97 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete - - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml deleted file mode 100644 index 3ec997108..000000000 --- a/eng/common/templates/jobs/source-build.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - - # If set to true and running on a non-public project, - # Internal nuget and blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - -jobs: - -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - enableInternalSources: ${{ parameters.enableInternalSources }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} - enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml deleted file mode 100644 index 173914f23..000000000 --- a/eng/common/templates/post-build/common-variables.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - - group: Publish-Build-Assets - - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - - name: runCodesignValidationInjection - value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml deleted file mode 100644 index c3b6a3012..000000000 --- a/eng/common/templates/post-build/post-build.yml +++ /dev/null @@ -1,283 +0,0 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - -stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(System.AccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml deleted file mode 100644 index 64b9abc68..000000000 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,70 +0,0 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - -steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: AzureCLI@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - . $(Build.SourcesDirectory)\eng\common\tools.ps1 - $darc = Get-Darc - $buildInfo = & $darc get-build ` - --id ${{ parameters.BARBuildId }} ` - --extended ` - --output-format json ` - --ci ` - | convertFrom-Json - - $BarId = ${{ parameters.BARBuildId }} - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml deleted file mode 100644 index da669030d..000000000 --- a/eng/common/templates/post-build/trigger-subscription.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Triggering subscriptions - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 - arguments: -SourceRepo $(Build.Repository.Uri) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml deleted file mode 100644 index f67a210d6..000000000 --- a/eng/common/templates/steps/add-build-to-channel.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/build-reason.yml b/eng/common/templates/steps/build-reason.yml deleted file mode 100644 index eba58109b..000000000 --- a/eng/common/templates/steps/build-reason.yml +++ /dev/null @@ -1,12 +0,0 @@ -# build-reason.yml -# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons -# to include steps (',' separated). -parameters: - conditions: '' - steps: [] - -steps: - - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: - - ${{ parameters.steps }} - - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml deleted file mode 100644 index cbba05967..000000000 --- a/eng/common/templates/steps/component-governance.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml deleted file mode 100644 index 54dc9416c..000000000 --- a/eng/common/templates/steps/enable-internal-runtimes.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' -# variable with the base64-encoded SAS token, by default - -parameters: -- name: federatedServiceConnection - type: string - default: 'dotnetbuilds-internal-read' -- name: outputVariableName - type: string - default: 'dotnetbuilds-internal-container-read-token-base64' -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: true - -steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/steps/get-delegation-sas.yml - parameters: - federatedServiceConnection: ${{ parameters.federatedServiceConnection }} - outputVariableName: ${{ parameters.outputVariableName }} - expiryInHours: ${{ parameters.expiryInHours }} - base64Encode: ${{ parameters.base64Encode }} - storageAccount: dotnetbuilds - container: internal - permissions: rl diff --git a/eng/common/templates/steps/execute-codeql.yml b/eng/common/templates/steps/execute-codeql.yml deleted file mode 100644 index 3930b1630..000000000 --- a/eng/common/templates/steps/execute-codeql.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - # Language that should be analyzed. Defaults to csharp - language: csharp - # Build Commands - buildCommands: '' - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - # optional: determines whether to continue the build if the step errors; - sdlContinueOnError: false - -steps: -- template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: '${{ parameters.additionalParameters }} - -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml deleted file mode 100644 index fe0ebf8c9..000000000 --- a/eng/common/templates/steps/execute-sdl.yml +++ /dev/null @@ -1,89 +0,0 @@ -parameters: - overrideGuardianVersion: '' - executeAllSdlToolsScript: '' - overrideParameters: '' - additionalParameters: '' - publishGuardianDirectoryToPipeline: false - sdlContinueOnError: false - condition: '' - -steps: -- task: NuGetAuthenticate@1 - -- task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian (Overridden) - -- ${{ if eq(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian - -- ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL (Overridden) - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - env: - GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - -- ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianCliLocation $(GuardianCliLocation) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - env: - GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - -- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() - - # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration - # with the "SARIF SAST Scans Tab" Azure DevOps extension - - task: CopyFiles@2 - displayName: Copy SARIF files - inputs: - flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ - contents: '**/*.sarif' - targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs - condition: succeededOrFailed() - - # Use PublishBuildArtifacts because the SARIF extension only checks this case - # see microsoft/sarif-azuredevops-extension#4 - - task: PublishBuildArtifacts@1 - displayName: Publish SARIF files to CodeAnalysisLogs container - inputs: - pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml deleted file mode 100644 index 2b21eae42..000000000 --- a/eng/common/templates/steps/generate-sbom.yml +++ /dev/null @@ -1,48 +0,0 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 8.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - -steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) - diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml deleted file mode 100644 index c690cc0a0..000000000 --- a/eng/common/templates/steps/get-delegation-sas.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: false -- name: storageAccount - type: string -- name: container - type: string -- name: permissions - type: string - default: 'rl' - -steps: -- task: AzureCLI@2 - displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - # Calculate the expiration of the SAS token and convert to UTC - $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") - - # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads - # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 - $sas = "" - do { - $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - } while($sas.IndexOf('/') -ne -1) - - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to generate SAS token." - exit 1 - } - - if ('${{ parameters.base64Encode }}' -eq 'true') { - $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) - } - - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml deleted file mode 100644 index 55e33bd38..000000000 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ /dev/null @@ -1,40 +0,0 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: stepName - type: string - default: 'getFederatedAccessToken' -- name: condition - type: string - default: '' -# Resource to get a token for. Common values include: -# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps -# - 'https://storage.azure.com/' for storage -# Defaults to Azure DevOps -- name: resource - type: string - default: '499b84ac-1321-427f-aa17-267ca6975798' -- name: isStepOutputVariable - type: boolean - default: false - -steps: -- task: AzureCLI@2 - displayName: 'Getting federated access token for feeds' - name: ${{ parameters.stepName }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" - exit 1 - } - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml deleted file mode 100644 index 88f238f36..000000000 --- a/eng/common/templates/steps/publish-logs.yml +++ /dev/null @@ -1,23 +0,0 @@ -parameters: - StageLabel: '' - JobLabel: '' - -steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml deleted file mode 100644 index 83d97a26a..000000000 --- a/eng/common/templates/steps/retain-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - -steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates/steps/run-on-unix.yml b/eng/common/templates/steps/run-on-unix.yml deleted file mode 100644 index e1733814f..000000000 --- a/eng/common/templates/steps/run-on-unix.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-on-windows.yml b/eng/common/templates/steps/run-on-windows.yml deleted file mode 100644 index 73e7e9c27..000000000 --- a/eng/common/templates/steps/run-on-windows.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-script-ifequalelse.yml b/eng/common/templates/steps/run-script-ifequalelse.yml deleted file mode 100644 index 3d1242f55..000000000 --- a/eng/common/templates/steps/run-script-ifequalelse.yml +++ /dev/null @@ -1,33 +0,0 @@ -parameters: - # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command - parameter1: '' - parameter2: '' - ifScript: '' - elseScript: '' - - # name of script step - name: Script - - # display name of script step - displayName: If-Equal-Else Script - - # environment - env: {} - - # conditional expression for step execution - condition: '' - -steps: -- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.ifScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} - -- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.elseScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml deleted file mode 100644 index 22f250130..000000000 --- a/eng/common/templates/steps/send-to-helix.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml deleted file mode 100644 index 41bbb9157..000000000 --- a/eng/common/templates/steps/source-build.yml +++ /dev/null @@ -1,129 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - -steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/source-build/self/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() - -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml deleted file mode 100644 index fadc04ca1..000000000 --- a/eng/common/templates/steps/telemetry-end.yml +++ /dev/null @@ -1,102 +0,0 @@ -parameters: - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- bash: | - if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then - errorCount=0 - else - errorCount=1 - fi - warningCount=0 - - curlStatus=1 - retryCount=0 - # retry loop to harden against spotty telemetry connections - # we don't retry successes and 4xx client errors - until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] - do - if [ $retryCount -gt 0 ]; then - echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." - sleep $RetryDelay - fi - - # create a temporary file for curl output - res=`mktemp` - - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ - --data-urlencode "errorCount=$errorCount" \ - --data-urlencode "warningCount=$warningCount"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult - fi - fi - - let retryCount++ - done - - if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Finish information after $retryCount retries" - vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" - echo "##$vstsLogOutput" - exit 1 - fi - displayName: Send Unix Build End Telemetry - env: - # defined via VSTS variables in start-job.sh - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) -- powershell: | - if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { - $ErrorCount = 0 - } else { - $ErrorCount = 1 - } - $WarningCount = 0 - - # Basic retry loop to harden against server flakiness - $retryCount = 0 - while ($retryCount -lt $env:MaxRetries) { - try { - Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } - break - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - if ($statusCode -ge 400 -and $statusCode -le 499) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" - Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message - exit 1 - } - Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." - $retryCount++ - sleep $env:RetryDelay - continue - } - } - - if ($retryCount -ge $env:MaxRetries) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." - exit 1 - } - displayName: Send Windows Build End Telemetry - env: - # defined via VSTS variables in start-job.ps1 - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml deleted file mode 100644 index 6abbcb33a..000000000 --- a/eng/common/templates/steps/telemetry-start.yml +++ /dev/null @@ -1,241 +0,0 @@ -parameters: - helixSource: 'undefined_defaulted_in_telemetry.yml' - helixType: 'undefined_defaulted_in_telemetry.yml' - buildConfig: '' - runAsPublic: false - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@2 - inputs: - azureSubscription: 'HelixProd_KeyVault' - KeyVaultName: HelixProdKV - SecretsFilter: 'HelixApiAccessToken' - condition: always() -- bash: | - # create a temporary file - jobInfo=`mktemp` - - # write job info content to temporary file - cat > $jobInfo < powershell invocations -# as dot sourcing isn't possible. -function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { - if (Test-Path variable:global:_DotNetInstallDir) { - return $global:_DotNetInstallDir - } - - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - $env:DOTNET_MULTILEVEL_LOOKUP=0 - - # Disable first run since we do not need all ASP.NET packages restored. - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - - # Disable telemetry on CI. - if ($ci) { - $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 - } - - # Source Build uses DotNetCoreSdkDir variable - if ($env:DotNetCoreSdkDir -ne $null) { - $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir - } - - # Find the first path on %PATH% that contains the dotnet.exe - if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $dotnetExecutable = GetExecutableFileName 'dotnet' - $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue - - if ($dotnetCmd -ne $null) { - $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent - } - } - - $dotnetSdkVersion = $GlobalJson.tools.dotnet - - # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, - # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { - $dotnetRoot = $env:DOTNET_INSTALL_DIR - } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' - - if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { - if ($install) { - InstallDotNetSdk $dotnetRoot $dotnetSdkVersion - } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" - ExitWithExitCode 1 - } - } - - $env:DOTNET_INSTALL_DIR = $dotnetRoot - } - - # Creates a temporary file under the toolset dir. - # The following code block is protecting against concurrent access so that this function can - # be called in parallel. - if ($createSdkLocationFile) { - do { - $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } - until (!(Test-Path $sdkCacheFileTemp)) - Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - - try { - Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt') - } catch { - # Somebody beat us - Remove-Item -Path $sdkCacheFileTemp - } - } - - # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom - # build steps from using anything other than what we've downloaded. - # It also ensures that VS msbuild will use the downloaded sdk targets. - $env:PATH = "$dotnetRoot;$env:PATH" - - # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build - Write-PipelinePrependPath -Path $dotnetRoot - - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' - Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' - - return $global:_DotNetInstallDir = $dotnetRoot -} - -function Retry($downloadBlock, $maxRetries = 5) { - $retries = 1 - - while($true) { - try { - & $downloadBlock - break - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." - break - } - - } -} - -function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' - if (!(Test-Path $installScript)) { - Create-Directory $dotnetRoot - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" - - Retry({ - Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript - }) - } - - return $installScript -} - -function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) { - InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath -} - -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = '', - [string] $runtime = '', - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '', - [switch] $noPath) { - - $dotnetVersionLabel = "'sdk v$version'" - - if ($runtime -ne '' -and $runtime -ne 'sdk') { - $runtimePath = $dotnetRoot - $runtimePath = $runtimePath + "\shared" - if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" } - if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" } - if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" } - $runtimePath = $runtimePath + "\" + $version - - $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'" - - if (Test-Path $runtimePath) { - Write-Host " Runtime toolset '$runtime/$architecture v$version' already installed." - $installSuccess = $true - Exit - } - } - - $installScript = GetDotNetInstallScript $dotnetRoot - $installParameters = @{ - Version = $version - InstallDir = $dotnetRoot - } - - if ($architecture) { $installParameters.Architecture = $architecture } - if ($runtime) { $installParameters.Runtime = $runtime } - if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } - if ($noPath) { $installParameters.NoPath = $True } - - $variations = @() - $variations += @($installParameters) - - $dotnetBuilds = $installParameters.Clone() - $dotnetbuilds.AzureFeed = "https://ci.dot.net/public" - $variations += @($dotnetBuilds) - - if ($runtimeSourceFeed) { - $runtimeSource = $installParameters.Clone() - $runtimeSource.AzureFeed = $runtimeSourceFeed - if ($runtimeSourceFeedKey) { - $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) - $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) - $runtimeSource.FeedCredential = $decodedString - } - $variations += @($runtimeSource) - } - - $installSuccess = $false - foreach ($variation in $variations) { - if ($variation | Get-Member AzureFeed) { - $location = $variation.AzureFeed - } else { - $location = "public location"; - } - Write-Host " Attempting to install $dotnetVersionLabel from $location." - try { - & $installScript @variation - $installSuccess = $true - break - } - catch { - Write-Host " Failed to install $dotnetVersionLabel from $location." - } - } - if (-not $installSuccess) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations." - ExitWithExitCode 1 - } -} - -# -# Locates Visual Studio MSBuild installation. -# The preference order for MSBuild to use is as follows: -# -# 1. MSBuild from an active VS command prompt -# 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package -# -# Returns full path to msbuild.exe. -# Throws on failure. -# -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { - if (-not (IsWindowsPlatform)) { - throw "Cannot initialize Visual Studio on non-Windows" - } - - if (Test-Path variable:global:_MSBuildExe) { - return $global:_MSBuildExe - } - - # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.12.0 - $defaultXCopyMSBuildVersion = '17.12.0' - - if (!$vsRequirements) { - if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { - $vsRequirements = $GlobalJson.tools.vs - } - else { - $vsRequirements = New-Object PSObject -Property @{ version = $vsMinVersionReqdStr } - } - } - $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } - $vsMinVersion = [Version]::new($vsMinVersionStr) - - # Try msbuild command available in the environment. - if ($env:VSINSTALLDIR -ne $null) { - $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue - if ($msbuildCmd -ne $null) { - # Workaround for https://github.com/dotnet/roslyn/issues/35793 - # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ - $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0]) - - if ($msbuildVersion -ge $vsMinVersion) { - return $global:_MSBuildExe = $msbuildCmd.Path - } - - # Report error - the developer environment is initialized with incompatible VS version. - throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window" - } - } - - # Locate Visual Studio installation or download x-copy msbuild. - $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null) { - # Ensure vsInstallDir has a trailing slash - $vsInstallDir = Join-Path $vsInfo.installationPath "\" - $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] - - InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion - } else { - - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } - } - - $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - - $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" - $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false } - if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { - $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" - } else { - $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" - } - - return $global:_MSBuildExe -} - -function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { - $env:VSINSTALLDIR = $vsInstallDir - Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") - - $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" - if (Test-Path $vsSdkInstallDir) { - Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir - $env:VSSDKInstall = $vsSdkInstallDir - } -} - -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'RoslynTools.MSBuild' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath - }) - - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - -# -# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. -# -# The following properties of tools.vs are recognized: -# "version": "{major}.{minor}" -# Two part minimal VS version, e.g. "15.9", "16.0", etc. -# "components": ["componentId1", "componentId2", ...] -# Array of ids of workload components that must be available in the VS instance. -# See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 -# -# Returns JSON describing the located VS instance (same format as returned by vswhere), -# or $null if no instance meeting the requirements is found on the machine. -# -function LocateVisualStudio([object]$vsRequirements = $null){ - if (-not (IsWindowsPlatform)) { - throw "Cannot run vswhere on non-Windows platforms." - } - - if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { - $vswhereVersion = $GlobalJson.tools.vswhere - } else { - $vswhereVersion = '2.5.2' - } - - $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" - $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe' - - if (!(Test-Path $vsWhereExe)) { - Create-Directory $vsWhereDir - Write-Host 'Downloading vswhere' - Retry({ - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe - }) - } - - if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') - - if (!$excludePrereleaseVS) { - $args += '-prerelease' - } - - if (Get-Member -InputObject $vsRequirements -Name 'version') { - $args += '-version' - $args += $vsRequirements.version - } - - if (Get-Member -InputObject $vsRequirements -Name 'components') { - foreach ($component in $vsRequirements.components) { - $args += '-requires' - $args += $component - } - } - - $vsInfo =& $vsWhereExe $args | ConvertFrom-Json - - if ($lastExitCode -ne 0) { - return $null - } - - # use first matching instance - return $vsInfo[0] -} - -function InitializeBuildTool() { - if (Test-Path variable:global:_BuildTool) { - # If the requested msbuild parameters do not match, clear the cached variables. - if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) { - Remove-Item variable:global:_BuildTool - Remove-Item variable:global:_MSBuildExe - } else { - return $global:_BuildTool - } - } - - if (-not $msbuildEngine) { - $msbuildEngine = GetDefaultMSBuildEngine - } - - # Initialize dotnet cli if listed in 'tools' - $dotnetRoot = $null - if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') { - $dotnetRoot = InitializeDotNetCli -install:$restore - } - - if ($msbuildEngine -eq 'dotnet') { - if (!$dotnetRoot) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'." - ExitWithExitCode 1 - } - $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - - # Use override if it exists - commonly set by source-build - if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { - $initializeBuildToolFramework="net8.0" - } else { - $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework - } - - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } - } elseif ($msbuildEngine -eq "vs") { - try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore - } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 - } - - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } - } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." - ExitWithExitCode 1 - } - - return $global:_BuildTool = $buildTool -} - -function GetDefaultMSBuildEngine() { - # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. - if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { - return 'vs' - } - - if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') { - return 'dotnet' - } - - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." - ExitWithExitCode 1 -} - -function GetNuGetPackageCachePath() { - if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build. - # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 - # use global cache in dev builds to avoid cost of downloading packages. - # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 - if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' - } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' - $env:RESTORENOCACHE = $true - } - } - - return $env:NUGET_PACKAGES -} - -# Returns a full path to an Arcade SDK task project file. -function GetSdkTaskProject([string]$taskName) { - return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" -} - -function InitializeNativeTools() { - if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) { - $nativeArgs= @{} - if ($ci) { - $nativeArgs = @{ - InstallDirectory = "$ToolsDir" - } - } - if ($env:NativeToolsOnMachine) { - Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." - $nativeArgs += @{ PathPromotion = $true } - } - & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs - } -} - -function Read-ArcadeSdkVersion() { - return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' -} - -function InitializeToolset() { - if (Test-Path variable:global:_ToolsetBuildProj) { - return $global:_ToolsetBuildProj - } - - $nugetCache = GetNuGetPackageCachePath - - $toolsetVersion = Read-ArcadeSdkVersion - $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" - - if (Test-Path $toolsetLocationFile) { - $path = Get-Content $toolsetLocationFile -TotalCount 1 - if (Test-Path $path) { - return $global:_ToolsetBuildProj = $path - } - } - - if (-not $restore) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored." - ExitWithExitCode 1 - } - - $buildTool = InitializeBuildTool - - $proj = Join-Path $ToolsetDir 'restore.proj' - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } - - '' | Set-Content $proj - - MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - - $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 - if (!(Test-Path $path)) { - throw "Invalid toolset path: $path" - } - - return $global:_ToolsetBuildProj = $path -} - -function ExitWithExitCode([int] $exitCode) { - if ($ci -and $prepareMachine) { - Stop-Processes - } - exit $exitCode -} - -# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for -# diagnostics, then exit the script with the $LASTEXITCODE. -function Exit-IfNZEC([string] $category = "General") { - Write-Host "Exit code $LASTEXITCODE" - if ($LASTEXITCODE -ne 0) { - $message = "Last command failed with exit code $LASTEXITCODE." - Write-PipelineTelemetryError -Force -Category $category -Message $message - ExitWithExitCode $LASTEXITCODE - } -} - -function Stop-Processes() { - Write-Host 'Killing running build processes...' - foreach ($processName in $processesToStopOnExit) { - Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process - } -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild() { - if ($pipelinesLog) { - $buildTool = InitializeBuildTool - - if ($ci -and $buildTool.Tool -eq 'dotnet') { - $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 - $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' - } - - Enable-Nuget-EnhancedRetry - - $toolsetBuildProject = InitializeToolset - $basePath = Split-Path -parent $toolsetBuildProject - $possiblePaths = @( - # new scripts need to work with old packages, so we need to look for the old names/versions - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll')) - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')), - (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')) - ) - $selectedPath = $null - foreach ($path in $possiblePaths) { - if (Test-Path $path -PathType Leaf) { - $selectedPath = $path - break - } - } - if (-not $selectedPath) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' - ExitWithExitCode 1 - } - $args += "/logger:$selectedPath" - } - - MSBuild-Core @args -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild-Core() { - if ($ci) { - if (!$binaryLog -and !$excludeCIBinarylog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' - ExitWithExitCode 1 - } - - if ($nodeReuse) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.' - ExitWithExitCode 1 - } - } - - Enable-Nuget-EnhancedRetry - - $buildTool = InitializeBuildTool - - $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - - if ($warnAsError) { - $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' - } - else { - $cmdArgs += ' /p:TreatWarningsAsErrors=false' - } - - foreach ($arg in $args) { - if ($null -ne $arg -and $arg.Trim() -ne "") { - if ($arg.EndsWith('\')) { - $arg = $arg + "\" - } - $cmdArgs += " `"$arg`"" - } - } - - $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" - - $exitCode = Exec-Process $buildTool.Path $cmdArgs - - if ($exitCode -ne 0) { - # We should not Write-PipelineTaskError here because that message shows up in the build summary - # The build already logged an error, that's the reason it failed. Producing an error here only adds noise. - Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red - - $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($null -ne $buildLog) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray - } - - # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { - Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." - # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error - # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error - ExitWithExitCode 0 - } else { - ExitWithExitCode $exitCode - } - } -} - -function GetMSBuildBinaryLogCommandLineArgument($arguments) { - foreach ($argument in $arguments) { - if ($argument -ne $null) { - $arg = $argument.Trim() - if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) { - return $arg.Substring('/bl:'.Length) - } - - if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) { - return $arg.Substring('/binaryLogger:'.Length) - } - } - } - - return $null -} - -function GetExecutableFileName($baseName) { - if (IsWindowsPlatform) { - return "$baseName.exe" - } - else { - return $baseName - } -} - -function IsWindowsPlatform() { - return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT -} - -function Get-Darc($version) { - $darcPath = "$TempDir\darc\$([guid]::NewGuid())" - if ($version -ne $null) { - & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host - } else { - & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host - } - return "$darcPath\darc.exe" -} - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\') -$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..') -$ArtifactsDir = Join-Path $RepoRoot 'artifacts' -$ToolsetDir = Join-Path $ArtifactsDir 'toolset' -$ToolsDir = Join-Path $RepoRoot '.tools' -$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration -$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration -$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json -# true if global.json contains a "runtimes" section -$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } - -Create-Directory $ToolsetDir -Create-Directory $TempDir -Create-Directory $LogDir - -Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir -Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir -Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir -Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir -Write-PipelineSetVariable -Name 'TMP' -Value $TempDir - -# Import custom tools configuration, if present in the repo. -# Note: Import in global scope so that the script set top-level variables without qualification. -if (!$disableConfigureToolsetImport) { - $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1' - if (Test-Path $configureToolsetScript) { - . $configureToolsetScript - if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) { - if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) { - Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code' - ExitWithExitCode $LastExitCode - } - } - } -} - -# -# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. -# -function Enable-Nuget-EnhancedRetry() { - if ($ci) { - Write-Host "Setting NUGET enhanced retry environment variables" - $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true' - $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6 - $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 - $env:NUGET_RETRY_HTTP_429 = 'true' - Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' - Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true' - } -} diff --git a/eng/common/tools.sh b/eng/common/tools.sh deleted file mode 100644 index 68db15430..000000000 --- a/eng/common/tools.sh +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env bash - -# Initialize variables if they aren't already defined. - -# CI mode - set to true on CI server for PR validation build or official build. -ci=${ci:-false} - -# Set to true to use the pipelines logger which will enable Azure logging output. -# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md -# This flag is meant as a temporary opt-opt for the feature while validate it across -# our consumers. It will be deleted in the future. -if [[ "$ci" == true ]]; then - pipelines_log=${pipelines_log:-true} -else - pipelines_log=${pipelines_log:-false} -fi - -# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. -configuration=${configuration:-'Debug'} - -# Set to true to opt out of outputting binary log while running in CI -exclude_ci_binary_log=${exclude_ci_binary_log:-false} - -if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then - binary_log_default=true -else - binary_log_default=false -fi - -# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -binary_log=${binary_log:-$binary_log_default} - -# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). -prepare_machine=${prepare_machine:-false} - -# True to restore toolsets and dependencies. -restore=${restore:-true} - -# Adjusts msbuild verbosity level. -verbosity=${verbosity:-'minimal'} - -# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. -if [[ "$ci" == true ]]; then - node_reuse=${node_reuse:-false} -else - node_reuse=${node_reuse:-true} -fi - -# Configures warning treatment in msbuild. -warn_as_error=${warn_as_error:-true} - -# True to attempt using .NET Core already that meets requirements specified in global.json -# installed on the machine instead of downloading one. -use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} - -# Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh -dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} - -# True to use global NuGet cache instead of restoring packages to repository-local directory. -if [[ "$ci" == true ]]; then - use_global_nuget_cache=${use_global_nuget_cache:-false} -else - use_global_nuget_cache=${use_global_nuget_cache:-true} -fi - -# Used when restoring .NET SDK from alternative feeds -runtime_source_feed=${runtime_source_feed:-''} -runtime_source_feed_key=${runtime_source_feed_key:-''} - -# Resolve any symlinks in the given path. -function ResolvePath { - local path=$1 - - while [[ -h $path ]]; do - local dir="$( cd -P "$( dirname "$path" )" && pwd )" - path="$(readlink "$path")" - - # if $path was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $path != /* ]] && path="$dir/$path" - done - - # return value - _ResolvePath="$path" -} - -# ReadVersionFromJson [json key] -function ReadGlobalVersion { - local key=$1 - - if command -v jq &> /dev/null; then - _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")" - elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then - _ReadGlobalVersion=${BASH_REMATCH[1]} - fi - - if [[ -z "$_ReadGlobalVersion" ]]; then - Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file" - ExitWithExitCode 1 - fi -} - -function InitializeDotNetCli { - if [[ -n "${_InitializeDotNetCli:-}" ]]; then - return - fi - - local install=$1 - - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism - export DOTNET_MULTILEVEL_LOOKUP=0 - - # Disable first run since we want to control all package sources - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - - # Disable telemetry on CI - if [[ $ci == true ]]; then - export DOTNET_CLI_TELEMETRY_OPTOUT=1 - fi - - # LTTNG is the logging infrastructure used by Core CLR. Need this variable set - # so it doesn't output warnings to the console. - export LTTNG_HOME="$HOME" - - # Source Build uses DotNetCoreSdkDir variable - if [[ -n "${DotNetCoreSdkDir:-}" ]]; then - export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" - fi - - # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then - local dotnet_path=`command -v dotnet` - if [[ -n "$dotnet_path" ]]; then - ResolvePath "$dotnet_path" - export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"` - fi - fi - - ReadGlobalVersion "dotnet" - local dotnet_sdk_version=$_ReadGlobalVersion - local dotnet_root="" - - # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, - # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then - dotnet_root="$DOTNET_INSTALL_DIR" - else - dotnet_root="$repo_root/.dotnet" - - export DOTNET_INSTALL_DIR="$dotnet_root" - - if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then - if [[ "$install" == true ]]; then - InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" - else - Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" - ExitWithExitCode 1 - fi - fi - fi - - # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom - # build steps from using anything other than what we've downloaded. - Write-PipelinePrependPath -path "$dotnet_root" - - Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" - Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" - - # return value - _InitializeDotNetCli="$dotnet_root" -} - -function InstallDotNetSdk { - local root=$1 - local version=$2 - local architecture="unset" - if [[ $# -ge 3 ]]; then - architecture=$3 - fi - InstallDotNet "$root" "$version" $architecture 'sdk' 'true' $runtime_source_feed $runtime_source_feed_key -} - -function InstallDotNet { - local root=$1 - local version=$2 - local runtime=$4 - - local dotnetVersionLabel="'$runtime v$version'" - if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - runtimePath="$root" - runtimePath="$runtimePath/shared" - case "$runtime" in - dotnet) - runtimePath="$runtimePath/Microsoft.NETCore.App" - ;; - aspnetcore) - runtimePath="$runtimePath/Microsoft.AspNetCore.App" - ;; - windowsdesktop) - runtimePath="$runtimePath/Microsoft.WindowsDesktop.App" - ;; - *) - ;; - esac - runtimePath="$runtimePath/$version" - - dotnetVersionLabel="runtime toolset '$runtime/$architecture v$version'" - - if [ -d "$runtimePath" ]; then - echo " Runtime toolset '$runtime/$architecture v$version' already installed." - local installSuccess=1 - return - fi - fi - - GetDotNetInstallScript "$root" - local install_script=$_GetDotNetInstallScript - - local installParameters=(--version $version --install-dir "$root") - - if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then - installParameters+=(--architecture $3) - fi - if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - installParameters+=(--runtime $4) - fi - if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then - installParameters+=(--skip-non-versioned-files) - fi - - local variations=() # list of variable names with parameter arrays in them - - local public_location=("${installParameters[@]}") - variations+=(public_location) - - local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://ci.dot.net/public") - variations+=(dotnetbuilds) - - if [[ -n "${6:-}" ]]; then - variations+=(private_feed) - local private_feed=("${installParameters[@]}" --azure-feed $6) - if [[ -n "${7:-}" ]]; then - # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' - # '-d'. To work around this, do a simple detection and switch the parameter - # accordingly. - decodeArg="--decode" - if base64 --help 2>&1 | grep -q "BusyBox"; then - decodeArg="-d" - fi - decodedFeedKey=`echo $7 | base64 $decodeArg` - private_feed+=(--feed-credential $decodedFeedKey) - fi - fi - - local installSuccess=0 - for variationName in "${variations[@]}"; do - local name="$variationName[@]" - local variation=("${!name}") - echo " Attempting to install $dotnetVersionLabel from $variationName." - bash "$install_script" "${variation[@]}" && installSuccess=1 - if [[ "$installSuccess" -eq 1 ]]; then - break - fi - - echo " Failed to install $dotnetVersionLabel from $variationName." - done - - if [[ "$installSuccess" -eq 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install $dotnetVersionLabel from any of the specified locations." - ExitWithExitCode 1 - fi -} - -function with_retries { - local maxRetries=5 - local retries=1 - echo "Trying to run '$@' for maximum of $maxRetries attempts." - while [[ $((retries++)) -le $maxRetries ]]; do - "$@" - - if [[ $? == 0 ]]; then - echo "Ran '$@' successfully." - return 0 - fi - - timeout=$((3**$retries-1)) - echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 - sleep $timeout - done - - echo "Failed to execute '$@' for $maxRetries times." 1>&2 - - return 1 -} - -function GetDotNetInstallScript { - local root=$1 - local install_script="$root/dotnet-install.sh" - local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" - - if [[ ! -a "$install_script" ]]; then - mkdir -p "$root" - - echo "Downloading '$install_script_url'" - - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - # first, try directly, if this fails we will retry with verbose logging - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - if command -v openssl &> /dev/null; then - echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" - echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 - fi - echo "Will now retry the same URL with verbose logging." - with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - } - else - with_retries wget -v -O "$install_script" "$install_script_url" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - fi - fi - # return value - _GetDotNetInstallScript="$install_script" -} - -function InitializeBuildTool { - if [[ -n "${_InitializeBuildTool:-}" ]]; then - return - fi - - InitializeDotNetCli $restore - - # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" - _InitializeBuildToolCommand="msbuild" - # use override if it exists - commonly set by source-build - if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then - _InitializeBuildToolFramework="net8.0" - else - _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" - fi -} - -# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 -function GetNuGetPackageCachePath { - if [[ -z ${NUGET_PACKAGES:-} ]]; then - if [[ "$use_global_nuget_cache" == true ]]; then - export NUGET_PACKAGES="$HOME/.nuget/packages" - else - export NUGET_PACKAGES="$repo_root/.packages" - export RESTORENOCACHE=true - fi - fi - - # return value - _GetNuGetPackageCachePath=$NUGET_PACKAGES -} - -function InitializeNativeTools() { - if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then - return - fi - if grep -Fq "native-tools" $global_json_file - then - local nativeArgs="" - if [[ "$ci" == true ]]; then - nativeArgs="--installDirectory $tools_dir" - fi - "$_script_dir/init-tools-native.sh" $nativeArgs - fi -} - -function InitializeToolset { - if [[ -n "${_InitializeToolset:-}" ]]; then - return - fi - - GetNuGetPackageCachePath - - ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" - - local toolset_version=$_ReadGlobalVersion - local toolset_location_file="$toolset_dir/$toolset_version.txt" - - if [[ -a "$toolset_location_file" ]]; then - local path=`cat "$toolset_location_file"` - if [[ -a "$path" ]]; then - # return value - _InitializeToolset="$path" - return - fi - fi - - if [[ "$restore" != true ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." - ExitWithExitCode 2 - fi - - local proj="$toolset_dir/restore.proj" - - local bl="" - if [[ "$binary_log" == true ]]; then - bl="/bl:$log_dir/ToolsetRestore.binlog" - fi - - echo '' > "$proj" - MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" - - local toolset_build_proj=`cat "$toolset_location_file"` - - if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" - ExitWithExitCode 3 - fi - - # return value - _InitializeToolset="$toolset_build_proj" -} - -function ExitWithExitCode { - if [[ "$ci" == true && "$prepare_machine" == true ]]; then - StopProcesses - fi - exit $1 -} - -function StopProcesses { - echo "Killing running build processes..." - pkill -9 "dotnet" || true - pkill -9 "vbcscompiler" || true - return 0 -} - -function MSBuild { - local args=$@ - if [[ "$pipelines_log" == true ]]; then - InitializeBuildTool - InitializeToolset - - if [[ "$ci" == true ]]; then - export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 - export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 - Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" - Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" - fi - - local toolset_dir="${_InitializeToolset%/*}" - # new scripts need to work with old packages, so we need to look for the old names/versions - local selectedPath= - local possiblePaths=() - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" ) - possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" ) - for path in "${possiblePaths[@]}"; do - if [[ -f $path ]]; then - selectedPath=$path - break - fi - done - if [[ -z "$selectedPath" ]]; then - Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." - ExitWithExitCode 1 - fi - args+=( "-logger:$selectedPath" ) - fi - - MSBuild-Core ${args[@]} -} - -function MSBuild-Core { - if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." - ExitWithExitCode 1 - fi - - if [[ "$node_reuse" == true ]]; then - Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build." - ExitWithExitCode 1 - fi - fi - - InitializeBuildTool - - local warnaserror_switch="" - if [[ $warn_as_error == true ]]; then - warnaserror_switch="/warnaserror" - fi - - function RunBuildTool { - export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" - - "$_InitializeBuildTool" "$@" || { - local exit_code=$? - # We should not Write-PipelineTaskError here because that message shows up in the build summary - # The build already logged an error, that's the reason it failed. Producing an error here only adds noise. - echo "Build failed with exit code $exit_code. Check errors above." - - # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then - Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." - # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error - # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error - ExitWithExitCode 0 - else - ExitWithExitCode $exit_code - fi - } - } - - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" -} - -function GetDarc { - darc_path="$temp_dir/darc" - version="$1" - - if [[ -n "$version" ]]; then - version="--darcversion $version" - fi - - "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version -} - -ResolvePath "${BASH_SOURCE[0]}" -_script_dir=`dirname "$_ResolvePath"` - -. "$_script_dir/pipeline-logging-functions.sh" - -eng_root=`cd -P "$_script_dir/.." && pwd` -repo_root=`cd -P "$_script_dir/../.." && pwd` -repo_root="${repo_root}/" -artifacts_dir="${repo_root}artifacts" -toolset_dir="$artifacts_dir/toolset" -tools_dir="${repo_root}.tools" -log_dir="$artifacts_dir/log/$configuration" -temp_dir="$artifacts_dir/tmp/$configuration" - -global_json_file="${repo_root}global.json" -# determine if global.json contains a "runtimes" entry -global_json_has_runtimes=false -if command -v jq &> /dev/null; then - if jq -e '.tools | has("runtimes")' "$global_json_file" &> /dev/null; then - global_json_has_runtimes=true - fi -elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then - global_json_has_runtimes=true -fi - -# HOME may not be defined in some scenarios, but it is required by NuGet -if [[ -z $HOME ]]; then - export HOME="${repo_root}artifacts/.home/" - mkdir -p "$HOME" -fi - -mkdir -p "$toolset_dir" -mkdir -p "$temp_dir" -mkdir -p "$log_dir" - -Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" -Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" -Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" -Write-PipelineSetVariable -name "Temp" -value "$temp_dir" -Write-PipelineSetVariable -name "TMP" -value "$temp_dir" - -# Import custom tools configuration, if present in the repo. -if [ -z "${disable_configure_toolset_import:-}" ]; then - configure_toolset_script="$eng_root/configure-toolset.sh" - if [[ -a "$configure_toolset_script" ]]; then - . "$configure_toolset_script" - fi -fi - -# TODO: https://github.com/dotnet/arcade/issues/1468 -# Temporary workaround to avoid breaking change. -# Remove once repos are updated. -if [[ -n "${useInstalledDotNetCli:-}" ]]; then - use_installed_dotnet_cli="$useInstalledDotNetCli" -fi diff --git a/eng/targets/Settings.props b/eng/targets/Settings.props index c38bf4fd4..c9f5ac661 100644 --- a/eng/targets/Settings.props +++ b/eng/targets/Settings.props @@ -4,7 +4,6 @@ MIT Microsoft dotnet try - $(ArtifactsBinDir) - $(CopyrightMicrosoft) + Copyright (c) Microsoft. All rights reserved. diff --git a/eng/targets/Versions.targets b/eng/targets/Versions.targets index 2a81f2358..faf2349ba 100644 --- a/eng/targets/Versions.targets +++ b/eng/targets/Versions.targets @@ -1,7 +1,3 @@ - - - - diff --git a/eng/templates/build-and-test-job-linux-templates.yml b/eng/templates/build-and-test-job-linux-templates.yml deleted file mode 100644 index b6760104d..000000000 --- a/eng/templates/build-and-test-job-linux-templates.yml +++ /dev/null @@ -1,41 +0,0 @@ -# build-and-test-job-linux-template.yml -parameters: - buildConfig: 'Release' - buildArgs: '' - skipTests: false - additionalArgs: '' # Additional arguments to pass to the test command - -steps: -- template: /eng/templates/setup-template.yml@self - -- script: | - mkdir -p "$(Build.SourcesDirectory)/artifacts" - cp eng/resources/* "$(Build.SourcesDirectory)/artifacts" - displayName: Prevent test directory crawling - -- pwsh: | - $testArg = if ($env:SKIPTESTS -ne "true") { "--test" } else { "" } - Write-Host "##vso[task.setvariable variable=_TestArgs]$testArg" - displayName: Promote variables - -- script: $(Build.SourcesDirectory)/eng/cibuild.sh - --configuration ${{ parameters.buildConfig }} - --prepareMachine - $(_TestArgs) - displayName: Build - env: - POCKETLOGGER_LOG_PATH: $(PocketLoggerLogPath) - TRYDOTNET_PREBUILDS_PATH: $(TryDotNetPrebuildsPath) - -- script: | - dotnet test -l trx --no-build --blame-hang-timeout 15m --blame-hang-dump-type full -c ${{ parameters.buildConfig }} --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.buildConfig }} ${{ parameters.additionalArgs }} - displayName: Test / Blame - workingDirectory: $(Build.SourcesDirectory) - condition: ne('${{ parameters.skipTests }}', 'true') - env: - RunIntegrationTests: $(RunIntegrationTests) - -- template: /eng/templates/delete-dump-files-template.yml@self - parameters: - buildConfig: ${{ parameters.buildConfig }} - skipTests: ${{ parameters.skipTests }} \ No newline at end of file diff --git a/eng/templates/build-and-test-job-windows-templates.yml b/eng/templates/build-and-test-job-windows-templates.yml deleted file mode 100644 index 625ffc101..000000000 --- a/eng/templates/build-and-test-job-windows-templates.yml +++ /dev/null @@ -1,45 +0,0 @@ -# build-and-test-job-windows-template.yml -parameters: - buildConfig: 'Release' # Pass build configuration as a parameter - buildArgs: '' # Pass additional arguments to the build script - skipTests: false # Default is not skipping tests - additionalArgs: '' # Additional arguments to pass to the test command - -steps: -- template: /eng/templates/setup-template.yml@self - -- script: | - robocopy "eng\resources" "$(Build.SourcesDirectory)\artifacts" - :: robocopy return codes are terrible; 1 means files were copied - if "%errorlevel%" == "1" exit /b 0 - exit /b 1 - displayName: Prevent test directory crawling - -- pwsh: | - $testArg = if ($env:SKIPTESTS -ne "true") { "-test" } else { "" } - Write-Host "##vso[task.setvariable variable=_TestArgs]$testArg" - displayName: Promote variables - -- script: $(Build.SourcesDirectory)\eng\CIBuild.cmd - -configuration ${{ parameters.buildConfig }} - -prepareMachine - -sign - ${{ parameters.buildArgs }} - $(_TestArgs) - displayName: Build - env: - POCKETLOGGER_LOG_PATH: $(PocketLoggerLogPath) - TRYDOTNET_PREBUILDS_PATH: $(TryDotNetPrebuildsPath) - -- script: | - dotnet test -l trx --no-build --blame-hang-timeout 15m --blame-hang-dump-type full -c ${{ parameters.buildConfig }} --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.buildConfig }} ${{ parameters.additionalArgs }} - displayName: Test / Blame - workingDirectory: $(Build.SourcesDirectory) - condition: ne('${{ parameters.skipTests }}', 'true') - env: - RunIntegrationTests: $(RunIntegrationTests) - -- template: /eng/templates/delete-dump-files-template.yml@self - parameters: - buildConfig: ${{ parameters.buildConfig }} - skipTests: ${{ parameters.skipTests }} \ No newline at end of file diff --git a/eng/templates/delete-dump-files-template.yml b/eng/templates/delete-dump-files-template.yml deleted file mode 100644 index 89ebc2c54..000000000 --- a/eng/templates/delete-dump-files-template.yml +++ /dev/null @@ -1,10 +0,0 @@ -# delete-dump-files-template.yml -parameters: - buildConfig: '' - skipTests: false - -steps: -- pwsh: Get-ChildItem *.dmp -Recurse | Remove-Item - displayName: Delete dump files - workingDirectory: $(Build.SourcesDirectory)/artifacts/TestResults/${{ parameters.buildConfig }} - condition: and(ne(variables['KeepDumps'], 'true'), ne('${{ parameters.skipTests }}', 'true')) \ No newline at end of file diff --git a/eng/templates/es-metadata.yml b/eng/templates/es-metadata.yml deleted file mode 100644 index 97cac8c05..000000000 --- a/eng/templates/es-metadata.yml +++ /dev/null @@ -1,8 +0,0 @@ -schemaVersion: 0.0.1 -isProduction: true -accountableOwners: - service: ab4d58de-beb3-4714-866c-1b46a48ea002 -routing: - defaultAreaPath: - org: devdiv - path: DevDiv\NET Tools\TryDotNET\trydotnet diff --git a/eng/templates/setup-template.yml b/eng/templates/setup-template.yml deleted file mode 100644 index a08b4e8aa..000000000 --- a/eng/templates/setup-template.yml +++ /dev/null @@ -1,19 +0,0 @@ -# setup-template.yml -steps: -- script: git config --global core.longpaths true - displayName: Enable `git clean` to handle long paths - -- checkout: self - clean: true - -- task: NodeTool@0 - displayName: Add NodeJS/npm - inputs: - versionSpec: $(NodeJSVersion) - -- task: UseDotNet@2 - displayName: Add dotnet - inputs: - packageType: sdk - version: $(DotNetSdkVersion) - installationPath: $(Agent.ToolsDirectory)/dotnet diff --git a/eng/templates/variables-template.yml b/eng/templates/variables-template.yml deleted file mode 100644 index 72ec112f9..000000000 --- a/eng/templates/variables-template.yml +++ /dev/null @@ -1,17 +0,0 @@ -# variables-template.yml - -variables: - - name: _TeamName - value: DotNetTry - - name: _BuildConfig - value: Release - - name: _PublishUsingPipelines - value: true - - name: _DotNetArtifactsCategory - value: .NETCore - - name: DotNetSdkVersion - value: '8.0.204' - - name: NodeJSVersion - value: '16.13.0' - - name: Codeql.Enabled - value: true \ No newline at end of file diff --git a/global.json b/global.json index 61dd3ac63..d10248f63 100644 --- a/global.json +++ b/global.json @@ -7,8 +7,5 @@ "tools": { "dotnet": "10.0.302", "rollForward": "latestMinor" - }, - "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26371.5" } } \ No newline at end of file diff --git a/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj b/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj index 2d64d056f..55dc89431 100644 --- a/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj +++ b/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj @@ -37,7 +37,7 @@ - + @@ -68,7 +68,7 @@ public static class BuildProperties - + diff --git a/src/Microsoft.TryDotNet.Tests/Microsoft.TryDotNet.Tests.csproj b/src/Microsoft.TryDotNet.Tests/Microsoft.TryDotNet.Tests.csproj index 917c99fc4..0675ba00c 100644 --- a/src/Microsoft.TryDotNet.Tests/Microsoft.TryDotNet.Tests.csproj +++ b/src/Microsoft.TryDotNet.Tests/Microsoft.TryDotNet.Tests.csproj @@ -36,7 +36,7 @@ - + From 305d974026776d9d1a1d740543f9e45b29ffd1d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:58:32 +0000 Subject: [PATCH 2/4] fix: restore dotnet-tools NuGet feed; prevent integration-test assembly race Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com> --- .github/workflows/Build-Test-And-Deploy.yaml | 2 +- NuGet.config | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Build-Test-And-Deploy.yaml b/.github/workflows/Build-Test-And-Deploy.yaml index c8a5036c0..76ae02356 100644 --- a/.github/workflows/Build-Test-And-Deploy.yaml +++ b/.github/workflows/Build-Test-And-Deploy.yaml @@ -156,7 +156,7 @@ jobs: - name: Run .NET Integration Tests id: run-dotnet-integration-tests - run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults + run: dotnet test --no-build --configuration Release --max-parallel-test-assemblies 1 --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults env: RunIntegrationTests: true POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log diff --git a/NuGet.config b/NuGet.config index a6909550f..1ece74a9d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,6 +5,8 @@ + + From bc699f448eedd8f1138d850fd032de7f67f89f59 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:31:23 +0000 Subject: [PATCH 3/4] Remove unsupported --max-parallel-test-assemblies flag from dotnet test command Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com> --- .github/workflows/Build-Test-And-Deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Build-Test-And-Deploy.yaml b/.github/workflows/Build-Test-And-Deploy.yaml index 76ae02356..c8a5036c0 100644 --- a/.github/workflows/Build-Test-And-Deploy.yaml +++ b/.github/workflows/Build-Test-And-Deploy.yaml @@ -156,7 +156,7 @@ jobs: - name: Run .NET Integration Tests id: run-dotnet-integration-tests - run: dotnet test --no-build --configuration Release --max-parallel-test-assemblies 1 --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults + run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults env: RunIntegrationTests: true POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log From 269e9ca724c0e2c5c966b61a1c93fe7bc6fe3e39 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:07:03 +0000 Subject: [PATCH 4/4] Reorder NuGet sources and make PublishTool incremental Co-authored-by: BenjaminMichaelis <22186029+BenjaminMichaelis@users.noreply.github.com> --- NuGet.config | 2 +- .../Microsoft.TryDotNet.IntegrationTests.csproj | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NuGet.config b/NuGet.config index 1ece74a9d..9c19a8388 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,8 +5,8 @@ + - diff --git a/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj b/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj index 55dc89431..05add5010 100644 --- a/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj +++ b/src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj @@ -68,7 +68,9 @@ public static class BuildProperties - +