From 69ad5852e4603024fdb57a78cfffd5ba9b8c0521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20De=20Francesco?= Date: Sun, 15 Nov 2026 23:23:42 -0300 Subject: [PATCH] feat(deploy): fail Service Fabric deploys that do not stay healthy Add verify-servicefabric-stable: after a deployment the application must reach Ok health and keep it for 120 s with the same replicas and processes. An Error or a restart rolls back to the previous version when one is given. The shared deploy_service_fabric.ps1 (fetched from main by its consumers) now runs it. Co-Authored-By: Claude Opus 5.5 --- .../Assert-ServiceFabricApplicationStable.ps1 | 196 ++++++++++++++++++ .../verify-servicefabric-stable/action.yml | 45 ++++ .../tests/Test-ServiceFabricStability.ps1 | 39 ++++ .../servicefabric-stability-tests.yml | 28 +++ scripts/deploy_service_fabric.ps1 | 41 +++- 5 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 .github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1 create mode 100644 .github/actions/verify-servicefabric-stable/action.yml create mode 100644 .github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1 create mode 100644 .github/workflows/servicefabric-stability-tests.yml diff --git a/.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1 b/.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1 new file mode 100644 index 0000000..5e4241d --- /dev/null +++ b/.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1 @@ -0,0 +1,196 @@ +<# +.SYNOPSIS + Fails a Service Fabric deployment unless the application stays healthy without restarts. + +.DESCRIPTION + Waits until the application's aggregated health is Ok, then watches it for StableSeconds. Any health state other + than Ok, or any replica, instance or code package process that changes during that window (a restart, a crash loop + or a move), fails the check. When RollbackApplicationTypeVersion is set and the failure is an Error or a restart, + the application is upgraded back to that version with its current parameters before the check fails. + + Runs in Windows PowerShell with the ServiceFabric module. Parameter values are never printed. + Dot-source the script without arguments to load the functions only (used by the tests). +#> +[CmdletBinding()] +param( + [string]$ApplicationName, + [int]$StableSeconds = 120, + [int]$ReadyTimeoutSeconds = 300, + [int]$PollSeconds = 10, + [string]$RollbackApplicationTypeVersion, + [string]$ConnectionEndpoint = 'servicefabric.eternet.cc:19000', + [switch]$SkipConnect +) + +function Get-SfHealthDetails { + param([object]$Health) + $details = @() + foreach ($evaluation in @($Health.UnhealthyEvaluations)) { + if ($null -eq $evaluation) { continue } + $text = if ($evaluation.PSObject.Properties['Description'] -and $evaluation.Description) { [string]$evaluation.Description } else { [string]$evaluation } + $details += ($text -split "`r?`n" | Where-Object { $_.Trim() } | Select-Object -First 6) + } + return @($details | Select-Object -First 12) +} + +function Get-SfStabilitySnapshot { + param([Parameter(Mandatory)][string]$Application) + + $health = Get-ServiceFabricApplicationHealth -ApplicationName $Application -ErrorAction Stop + $identities = @{} + $nodes = @{} + foreach ($service in @(Get-ServiceFabricService -ApplicationName $Application -ErrorAction Stop)) { + foreach ($partition in @(Get-ServiceFabricPartition -ServiceName $service.ServiceName -ErrorAction Stop)) { + foreach ($replica in @(Get-ServiceFabricReplica -PartitionId $partition.PartitionId -ErrorAction Stop)) { + $id = if ($replica.PSObject.Properties['InstanceId']) { $replica.InstanceId } else { $replica.ReplicaId } + $identities["replica $($service.ServiceName) $($partition.PartitionId) $($replica.NodeName)"] = "$id" + $nodes[[string]$replica.NodeName] = $true + } + } + } + foreach ($node in @($nodes.Keys)) { + foreach ($package in @(Get-ServiceFabricDeployedCodePackage -NodeName $node -ApplicationName $Application -ErrorAction Stop)) { + $entryPoint = $package.EntryPoint + if ($null -eq $entryPoint) { continue } + $activation = if ($package.PSObject.Properties['ServicePackageActivationId']) { $package.ServicePackageActivationId } else { '' } + $identities["process $node $($package.ServiceManifestName) $($package.CodePackageName) $activation"] = "$($entryPoint.InstanceId)/$($entryPoint.ProcessId)" + } + } + return [pscustomobject]@{ + Health = [string]$health.AggregatedHealthState + Details = Get-SfHealthDetails $health + Identities = $identities + } +} + +function Compare-SfIdentities { + param([hashtable]$Baseline, [hashtable]$Current) + $changes = @() + foreach ($key in $Baseline.Keys) { + if (-not $Current.ContainsKey($key)) { $changes += "$key is gone (was $($Baseline[$key]))" } + elseif ($Current[$key] -ne $Baseline[$key]) { $changes += "$key changed $($Baseline[$key]) -> $($Current[$key])" } + } + foreach ($key in $Current.Keys) { + if (-not $Baseline.ContainsKey($key)) { $changes += "$key appeared ($($Current[$key]))" } + } + return @($changes | Sort-Object) +} + +function Wait-SfApplicationStable { + <# + Returns a result object; never throws for an unstable application. SnapshotProvider and Sleep are injectable so + the decision logic can be tested without a cluster. + #> + param( + [Parameter(Mandatory)][string]$Application, + [int]$StableSeconds = 120, + [int]$ReadyTimeoutSeconds = 300, + [int]$PollSeconds = 10, + [scriptblock]$SnapshotProvider = { param($name) Get-SfStabilitySnapshot -Application $name }, + [scriptblock]$Sleep = { param($seconds) Start-Sleep -Seconds $seconds }, + [scriptblock]$Clock = { [DateTime]::UtcNow } + ) + + $readyDeadline = (& $Clock).AddSeconds($ReadyTimeoutSeconds) + $snapshot = & $SnapshotProvider $Application + while ($snapshot.Health -ne 'Ok') { + Write-Host "Waiting for $Application to report Ok health (now $($snapshot.Health))." + if ((& $Clock) -ge $readyDeadline) { + return [pscustomobject]@{ Stable = $false; Reason = 'NeverOk'; Health = $snapshot.Health; Details = $snapshot.Details + Message = "$Application did not reach Ok health within $ReadyTimeoutSeconds s (last state $($snapshot.Health))." } + } + & $Sleep $PollSeconds + $snapshot = & $SnapshotProvider $Application + } + + $baseline = $snapshot + $stableUntil = (& $Clock).AddSeconds($StableSeconds) + Write-Host "$Application is Ok with $($baseline.Identities.Count) replica/process identities; watching for $StableSeconds s." + while ((& $Clock) -lt $stableUntil) { + & $Sleep $PollSeconds + $snapshot = & $SnapshotProvider $Application + $changes = Compare-SfIdentities -Baseline $baseline.Identities -Current $snapshot.Identities + if ($changes.Count -gt 0) { + return [pscustomobject]@{ Stable = $false; Reason = 'Restarted'; Health = $snapshot.Health; Details = $changes + $snapshot.Details + Message = "$Application restarted or moved a replica/process during the $StableSeconds s stability window." } + } + if ($snapshot.Health -ne 'Ok') { + return [pscustomobject]@{ Stable = $false; Reason = "Health$($snapshot.Health)"; Health = $snapshot.Health; Details = $snapshot.Details + Message = "$Application health changed to $($snapshot.Health) during the $StableSeconds s stability window." } + } + } + return [pscustomobject]@{ Stable = $true; Reason = 'Stable'; Health = 'Ok'; Details = @() + Message = "$Application stayed Ok for $StableSeconds s without restarts." } +} + +function Test-SfRollbackWarranted { + param([Parameter(Mandatory)][object]$Result) + # A Warning alone (for example a slow health probe) does not justify replacing the deployed version. + return (-not $Result.Stable) -and ($Result.Reason -eq 'Restarted' -or $Result.Health -eq 'Error') +} + +function Invoke-SfApplicationRollback { + param( + [Parameter(Mandatory)][string]$Application, + [Parameter(Mandatory)][string]$TargetVersion, + [int]$TimeoutMinutes = 20 + ) + $current = Get-ServiceFabricApplication -ApplicationName $Application -ErrorAction Stop + $registered = @(Get-ServiceFabricApplicationType -ApplicationTypeName $current.ApplicationTypeName -ApplicationTypeVersion $TargetVersion -ErrorAction SilentlyContinue) + if ($registered.Count -eq 0) { throw "Cannot roll back: application type $($current.ApplicationTypeName) $TargetVersion is no longer registered." } + $parameters = @{} + foreach ($entry in @($current.ApplicationParameters)) { $parameters[$entry.Name] = $entry.Value } + Write-Host "Rolling $Application back from $($current.ApplicationTypeVersion) to $TargetVersion with its current $($parameters.Count) parameters (values not shown)." + Start-ServiceFabricApplicationUpgrade -ApplicationName $Application -ApplicationTypeVersion $TargetVersion -ApplicationParameter $parameters -UnmonitoredAuto -Force -ErrorAction Stop | Out-Null + $deadline = [DateTime]::UtcNow.AddMinutes($TimeoutMinutes) + do { + Start-Sleep -Seconds 10 + $upgrade = Get-ServiceFabricApplicationUpgrade -ApplicationName $Application -ErrorAction Stop + if ($upgrade.UpgradeState -eq 'RollingForwardCompleted' -and $upgrade.TargetApplicationTypeVersion -eq $TargetVersion) { return } + if ($upgrade.UpgradeState -in @('RollingBackCompleted', 'Failed')) { throw "Rollback upgrade to $TargetVersion ended in $($upgrade.UpgradeState)." } + } while ([DateTime]::UtcNow -lt $deadline) + throw "Rollback upgrade to $TargetVersion did not complete within $TimeoutMinutes minutes." +} + +function Assert-SfApplicationStable { + param( + [Parameter(Mandatory)][string]$Application, + [int]$StableSeconds = 120, + [int]$ReadyTimeoutSeconds = 300, + [int]$PollSeconds = 10, + [string]$RollbackVersion + ) + $result = Wait-SfApplicationStable -Application $Application -StableSeconds $StableSeconds -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds + if ($result.Stable) { + Write-Host $result.Message + return + } + foreach ($line in @($result.Details)) { Write-Host " $line" } + $message = "Post-deploy stability check failed: $($result.Message)" + if ($RollbackVersion -and (Test-SfRollbackWarranted $result)) { + try { + Invoke-SfApplicationRollback -Application $Application -TargetVersion $RollbackVersion + $after = Wait-SfApplicationStable -Application $Application -StableSeconds 30 -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds + $message += " Rolled back to $RollbackVersion; after rollback: $($after.Message)" + } catch { + $message += " Rollback to $RollbackVersion FAILED, manual action needed: $($_.Exception.Message)" + } + } elseif ($RollbackVersion) { + $message += " No rollback for a $($result.Health) state without restarts." + } else { + $message += ' No rollback version was given; the new version stays deployed.' + } + if ($env:GITHUB_ACTIONS -eq 'true') { Write-Host "::error title=Service Fabric deployment unstable::$message" } + throw $message +} + +if ($MyInvocation.InvocationName -ne '.') { + $ErrorActionPreference = 'Stop' + if (-not $ApplicationName -or $ApplicationName -notmatch '^fabric:/\S+$') { throw 'ApplicationName must be an exact fabric:/ application URI.' } + if ($StableSeconds -lt 30 -or $PollSeconds -lt 1 -or $ReadyTimeoutSeconds -lt 0) { throw 'StableSeconds must be at least 30 and PollSeconds at least 1.' } + if (-not $SkipConnect) { + Import-Module ServiceFabric -ErrorAction Stop + Connect-ServiceFabricCluster -ConnectionEndpoint $ConnectionEndpoint -TimeoutSec 30 | Out-Null + } + Assert-SfApplicationStable -Application $ApplicationName -StableSeconds $StableSeconds -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds -RollbackVersion $RollbackApplicationTypeVersion +} diff --git a/.github/actions/verify-servicefabric-stable/action.yml b/.github/actions/verify-servicefabric-stable/action.yml new file mode 100644 index 0000000..75e3e01 --- /dev/null +++ b/.github/actions/verify-servicefabric-stable/action.yml @@ -0,0 +1,45 @@ +name: Verify a Service Fabric application stays healthy +description: Fail the deployment unless the application reaches Ok health and keeps it, without restarts, for a stability window. Optionally roll back. +inputs: + application-name: + description: Exact application URI, for example fabric:/Eternet.Events. + required: true + stable-seconds: + description: Seconds the application must stay Ok with the same replicas and processes. + required: false + default: '120' + ready-timeout-seconds: + description: Seconds to wait for the first Ok health report after the deployment. + required: false + default: '300' + poll-seconds: + description: Seconds between health and replica checks. + required: false + default: '10' + rollback-version: + description: Application type version to return to on an Error or a restart. Empty keeps the new version and only fails. + required: false + default: '' + connection-endpoint: + description: Service Fabric client endpoint. + required: false + default: servicefabric.eternet.cc:19000 +runs: + using: composite + steps: + - shell: powershell + env: + SF_STABLE_APPLICATION: ${{ inputs.application-name }} + SF_STABLE_SECONDS: ${{ inputs.stable-seconds }} + SF_STABLE_READY_TIMEOUT: ${{ inputs.ready-timeout-seconds }} + SF_STABLE_POLL: ${{ inputs.poll-seconds }} + SF_STABLE_ROLLBACK: ${{ inputs.rollback-version }} + SF_STABLE_ENDPOINT: ${{ inputs.connection-endpoint }} + run: | + & "$env:GITHUB_ACTION_PATH\Assert-ServiceFabricApplicationStable.ps1" ` + -ApplicationName $env:SF_STABLE_APPLICATION ` + -StableSeconds ([int]$env:SF_STABLE_SECONDS) ` + -ReadyTimeoutSeconds ([int]$env:SF_STABLE_READY_TIMEOUT) ` + -PollSeconds ([int]$env:SF_STABLE_POLL) ` + -RollbackApplicationTypeVersion $env:SF_STABLE_ROLLBACK ` + -ConnectionEndpoint $env:SF_STABLE_ENDPOINT diff --git a/.github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1 b/.github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1 new file mode 100644 index 0000000..2627d7d --- /dev/null +++ b/.github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1 @@ -0,0 +1,39 @@ +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot/../Assert-ServiceFabricApplicationStable.ps1" + +$script:now = [DateTime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) +$clock = { $script:now } +$sleep = { param($seconds) $script:now = $script:now.AddSeconds($seconds) } + +function New-Snapshot([string]$Health, [string]$ProcessId = '100') { + [pscustomobject]@{ Health = $Health; Details = @("fixture $Health"); Identities = @{ 'replica svc p1 vm1' = '7'; 'process vm1 Pkg Code ' = "/$ProcessId" } } +} + +function Invoke-Scenario([object[]]$Snapshots, [int]$ReadyTimeoutSeconds = 60) { + $script:queue = [System.Collections.Generic.Queue[object]]::new() + foreach ($snapshot in $Snapshots) { $script:queue.Enqueue($snapshot) } + $script:last = $null + $provider = { param($name) if ($script:queue.Count) { $script:last = $script:queue.Dequeue() }; $script:last } + Wait-SfApplicationStable -Application 'fabric:/Fixture' -StableSeconds 30 -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds 10 -SnapshotProvider $provider -Sleep $sleep -Clock $clock +} + +function Assert-Result($Result, [bool]$Stable, [string]$Reason, [bool]$Rollback, [string]$Name) { + if ($Result.Stable -ne $Stable -or $Result.Reason -ne $Reason -or (Test-SfRollbackWarranted $Result) -ne $Rollback) { + throw "$Name expected Stable=$Stable Reason=$Reason Rollback=$Rollback, got Stable=$($Result.Stable) Reason=$($Result.Reason)." + } + Write-Host "PASS $Name" +} + +Assert-Result (Invoke-Scenario @((New-Snapshot Ok))) $true 'Stable' $false 'HealthyApplicationPasses' +Assert-Result (Invoke-Scenario @((New-Snapshot Error), (New-Snapshot Warning), (New-Snapshot Ok))) $true 'Stable' $false 'StartupErrorsBeforeFirstOkAreTolerated' +Assert-Result (Invoke-Scenario @((New-Snapshot Error)) -ReadyTimeoutSeconds 30) $false 'NeverOk' $true 'NeverOkFailsAndRollsBack' +Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Ok '200'))) $false 'Restarted' $true 'ProcessRestartWhileOkFails' +Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Ok), (New-Snapshot Error))) $false 'HealthError' $true 'ErrorInWindowFailsAndRollsBack' +Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Warning))) $false 'HealthWarning' $false 'WarningInWindowFailsWithoutRollback' +Assert-Result (Invoke-Scenario @((New-Snapshot Warning)) -ReadyTimeoutSeconds 20) $false 'NeverOk' $false 'PersistentWarningFailsWithoutRollback' + +$changes = Compare-SfIdentities -Baseline @{ a = '1'; b = '2' } -Current @{ a = '1'; c = '3' } +if ($changes.Count -ne 2) { throw 'Identity comparison must report removed and added replicas.' } +Write-Host 'PASS MovedReplicaIsReported' + +$global:LASTEXITCODE = 0 diff --git a/.github/workflows/servicefabric-stability-tests.yml b/.github/workflows/servicefabric-stability-tests.yml new file mode 100644 index 0000000..e7785d3 --- /dev/null +++ b/.github/workflows/servicefabric-stability-tests.yml @@ -0,0 +1,28 @@ +name: Service Fabric stability check tests +on: + pull_request: + paths: + - .github/actions/verify-servicefabric-stable/** + - scripts/deploy_service_fabric.ps1 + - .github/workflows/servicefabric-stability-tests.yml +permissions: + contents: read +jobs: + decision-fixtures: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: [self-hosted, Windows] + timeout-minutes: 5 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Check health and restart decisions without a cluster + shell: powershell + run: | + ./.github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1 + foreach ($file in @('scripts/deploy_service_fabric.ps1', '.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1')) { + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path $file), [ref]$null, [ref]$errors) | Out-Null + if ($errors) { throw "$file does not parse: $($errors[0].Message)" } + } diff --git a/scripts/deploy_service_fabric.ps1 b/scripts/deploy_service_fabric.ps1 index 04c31a6..cbe65f7 100644 --- a/scripts/deploy_service_fabric.ps1 +++ b/scripts/deploy_service_fabric.ps1 @@ -1,6 +1,8 @@ param ( [string]$serviceFabricProjectPath, - [string]$connectionEndpoint + [string]$connectionEndpoint, + # Seconds the application must stay Ok, without restarts, after the deployment. Minimum 30. + [int]$stableSeconds = 120 ) $configurationFile = "Cloud.xml" @@ -9,17 +11,52 @@ $ApplicationPackagePath = "$serviceFabricProjectPath\pkg\Release" $StartupServicesFile = "$serviceFabricProjectPath\StartupServices.xml" $deployScriptPath = "$serviceFabricProjectPath\Scripts\Deploy-FabricApplication.ps1" $OverwriteBehavior = 'Always' +$stabilityScriptUrl = 'https://raw.githubusercontent.com/Eternet/github/main/.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1' + +function Get-DeployedApplicationName { + [xml]$publishProfile = Get-Content -LiteralPath $PublishProfileFile + $parameterFile = [string]$publishProfile.PublishProfile.ApplicationParameterFile.Path + if (-not $parameterFile) { throw "$PublishProfileFile has no ApplicationParameterFile." } + $parameterPath = Join-Path (Split-Path -Parent $PublishProfileFile) $parameterFile + [xml]$parameters = Get-Content -LiteralPath $parameterPath + $name = [string]$parameters.Application.Name + if ($name -notmatch '^fabric:/\S+$') { throw "$parameterPath has no fabric:/ application name." } + return $name +} + +function Get-StabilityScript { + $local = Join-Path $PSScriptRoot '..\.github\actions\verify-servicefabric-stable\Assert-ServiceFabricApplicationStable.ps1' + if (Test-Path -LiteralPath $local) { return (Resolve-Path -LiteralPath $local).Path } + $downloaded = Join-Path ([IO.Path]::GetTempPath()) ('sf-stability-' + [guid]::NewGuid().ToString('N') + '.ps1') + Invoke-WebRequest -Uri $stabilityScriptUrl -OutFile $downloaded -UseBasicParsing + return $downloaded +} try { Import-Module ServiceFabric Connect-ServiceFabricCluster -ConnectionEndpoint $connectionEndpoint - + + $applicationName = Get-DeployedApplicationName + $previous = Get-ServiceFabricApplication -ApplicationName $applicationName -ErrorAction SilentlyContinue + $previousVersion = if ($previous) { [string]$previous.ApplicationTypeVersion } else { '' } + & $deployScriptPath ` -PublishProfileFile $PublishProfileFile ` -ApplicationPackagePath $ApplicationPackagePath ` -StartupServicesFile $StartupServicesFile ` -OverwriteBehavior $OverwriteBehavior ` -SkipPackageValidation + + # A completed upgrade does not prove the new code keeps running: check health and restarts in a separate + # Windows PowerShell process, and roll back to the previous version on an Error or a restart. + $deployed = Get-ServiceFabricApplication -ApplicationName $applicationName -ErrorAction Stop + $rollbackVersion = if ($previousVersion -and $previousVersion -ne [string]$deployed.ApplicationTypeVersion) { $previousVersion } else { '' } + Write-Host "Deployed $applicationName $($deployed.ApplicationTypeVersion) (previous: $(if ($previousVersion) { $previousVersion } else { 'none' }))." + $stabilityArguments = @('-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', (Get-StabilityScript), + '-ApplicationName', $applicationName, '-StableSeconds', $stableSeconds, '-ConnectionEndpoint', $connectionEndpoint) + if ($rollbackVersion) { $stabilityArguments += @('-RollbackApplicationTypeVersion', $rollbackVersion) } + & "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" @stabilityArguments + if ($LASTEXITCODE -ne 0) { throw "$applicationName did not stay healthy after the deployment (see the stability check above)." } } catch { Write-Error "Deployment failed: $_"