Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
<#
.SYNOPSIS
Fails a Service Fabric deployment unless the application stays healthy without restarts.

.DESCRIPTION
Waits until the application's aggregated health is Ok, then watches it for StableSeconds. Any health state other
than Ok, or any replica, instance or code package process that changes during that window (a restart, a crash loop
or a move), fails the check. When RollbackApplicationTypeVersion is set and the failure is an Error or a restart,
the application is upgraded back to that version with its current parameters before the check fails.

Runs in Windows PowerShell with the ServiceFabric module. Parameter values are never printed.
Dot-source the script without arguments to load the functions only (used by the tests).
#>
[CmdletBinding()]
param(
[string]$ApplicationName,
[int]$StableSeconds = 120,
[int]$ReadyTimeoutSeconds = 300,
[int]$PollSeconds = 10,
[string]$RollbackApplicationTypeVersion,
[string]$ConnectionEndpoint = 'servicefabric.eternet.cc:19000',
[switch]$SkipConnect
)

function Get-SfHealthDetails {
param([object]$Health)
$details = @()
foreach ($evaluation in @($Health.UnhealthyEvaluations)) {
if ($null -eq $evaluation) { continue }
$text = if ($evaluation.PSObject.Properties['Description'] -and $evaluation.Description) { [string]$evaluation.Description } else { [string]$evaluation }
$details += ($text -split "`r?`n" | Where-Object { $_.Trim() } | Select-Object -First 6)
}
return @($details | Select-Object -First 12)
}

function Get-SfStabilitySnapshot {
param([Parameter(Mandatory)][string]$Application)

$health = Get-ServiceFabricApplicationHealth -ApplicationName $Application -ErrorAction Stop
$identities = @{}
$nodes = @{}
foreach ($service in @(Get-ServiceFabricService -ApplicationName $Application -ErrorAction Stop)) {
foreach ($partition in @(Get-ServiceFabricPartition -ServiceName $service.ServiceName -ErrorAction Stop)) {
foreach ($replica in @(Get-ServiceFabricReplica -PartitionId $partition.PartitionId -ErrorAction Stop)) {
$id = if ($replica.PSObject.Properties['InstanceId']) { $replica.InstanceId } else { $replica.ReplicaId }
$identities["replica $($service.ServiceName) $($partition.PartitionId) $($replica.NodeName)"] = "$id"
$nodes[[string]$replica.NodeName] = $true
}
}
}
foreach ($node in @($nodes.Keys)) {
foreach ($package in @(Get-ServiceFabricDeployedCodePackage -NodeName $node -ApplicationName $Application -ErrorAction Stop)) {
$entryPoint = $package.EntryPoint
if ($null -eq $entryPoint) { continue }
$activation = if ($package.PSObject.Properties['ServicePackageActivationId']) { $package.ServicePackageActivationId } else { '' }
$identities["process $node $($package.ServiceManifestName) $($package.CodePackageName) $activation"] = "$($entryPoint.InstanceId)/$($entryPoint.ProcessId)"
}
}
return [pscustomobject]@{
Health = [string]$health.AggregatedHealthState
Details = Get-SfHealthDetails $health
Identities = $identities
}
}

function Compare-SfIdentities {
param([hashtable]$Baseline, [hashtable]$Current)
$changes = @()
foreach ($key in $Baseline.Keys) {
if (-not $Current.ContainsKey($key)) { $changes += "$key is gone (was $($Baseline[$key]))" }
elseif ($Current[$key] -ne $Baseline[$key]) { $changes += "$key changed $($Baseline[$key]) -> $($Current[$key])" }
}
foreach ($key in $Current.Keys) {
if (-not $Baseline.ContainsKey($key)) { $changes += "$key appeared ($($Current[$key]))" }
}
return @($changes | Sort-Object)
}

function Wait-SfApplicationStable {
<#
Returns a result object; never throws for an unstable application. SnapshotProvider and Sleep are injectable so
the decision logic can be tested without a cluster.
#>
param(
[Parameter(Mandatory)][string]$Application,
[int]$StableSeconds = 120,
[int]$ReadyTimeoutSeconds = 300,
[int]$PollSeconds = 10,
[scriptblock]$SnapshotProvider = { param($name) Get-SfStabilitySnapshot -Application $name },
[scriptblock]$Sleep = { param($seconds) Start-Sleep -Seconds $seconds },
[scriptblock]$Clock = { [DateTime]::UtcNow }
)

$readyDeadline = (& $Clock).AddSeconds($ReadyTimeoutSeconds)
$snapshot = & $SnapshotProvider $Application
while ($snapshot.Health -ne 'Ok') {
Write-Host "Waiting for $Application to report Ok health (now $($snapshot.Health))."
if ((& $Clock) -ge $readyDeadline) {
return [pscustomobject]@{ Stable = $false; Reason = 'NeverOk'; Health = $snapshot.Health; Details = $snapshot.Details
Message = "$Application did not reach Ok health within $ReadyTimeoutSeconds s (last state $($snapshot.Health))." }
}
& $Sleep $PollSeconds
$snapshot = & $SnapshotProvider $Application
}

$baseline = $snapshot
$stableUntil = (& $Clock).AddSeconds($StableSeconds)
Write-Host "$Application is Ok with $($baseline.Identities.Count) replica/process identities; watching for $StableSeconds s."
while ((& $Clock) -lt $stableUntil) {
& $Sleep $PollSeconds
$snapshot = & $SnapshotProvider $Application
$changes = Compare-SfIdentities -Baseline $baseline.Identities -Current $snapshot.Identities
if ($changes.Count -gt 0) {
return [pscustomobject]@{ Stable = $false; Reason = 'Restarted'; Health = $snapshot.Health; Details = $changes + $snapshot.Details
Message = "$Application restarted or moved a replica/process during the $StableSeconds s stability window." }
}
if ($snapshot.Health -ne 'Ok') {
return [pscustomobject]@{ Stable = $false; Reason = "Health$($snapshot.Health)"; Health = $snapshot.Health; Details = $snapshot.Details
Message = "$Application health changed to $($snapshot.Health) during the $StableSeconds s stability window." }
}
}
return [pscustomobject]@{ Stable = $true; Reason = 'Stable'; Health = 'Ok'; Details = @()
Message = "$Application stayed Ok for $StableSeconds s without restarts." }
}

function Test-SfRollbackWarranted {
param([Parameter(Mandatory)][object]$Result)
# A Warning alone (for example a slow health probe) does not justify replacing the deployed version.
return (-not $Result.Stable) -and ($Result.Reason -eq 'Restarted' -or $Result.Health -eq 'Error')
}

function Invoke-SfApplicationRollback {
param(
[Parameter(Mandatory)][string]$Application,
[Parameter(Mandatory)][string]$TargetVersion,
[int]$TimeoutMinutes = 20
)
$current = Get-ServiceFabricApplication -ApplicationName $Application -ErrorAction Stop
$registered = @(Get-ServiceFabricApplicationType -ApplicationTypeName $current.ApplicationTypeName -ApplicationTypeVersion $TargetVersion -ErrorAction SilentlyContinue)
if ($registered.Count -eq 0) { throw "Cannot roll back: application type $($current.ApplicationTypeName) $TargetVersion is no longer registered." }
$parameters = @{}
foreach ($entry in @($current.ApplicationParameters)) { $parameters[$entry.Name] = $entry.Value }
Write-Host "Rolling $Application back from $($current.ApplicationTypeVersion) to $TargetVersion with its current $($parameters.Count) parameters (values not shown)."
Start-ServiceFabricApplicationUpgrade -ApplicationName $Application -ApplicationTypeVersion $TargetVersion -ApplicationParameter $parameters -UnmonitoredAuto -Force -ErrorAction Stop | Out-Null
$deadline = [DateTime]::UtcNow.AddMinutes($TimeoutMinutes)
do {
Start-Sleep -Seconds 10
$upgrade = Get-ServiceFabricApplicationUpgrade -ApplicationName $Application -ErrorAction Stop
if ($upgrade.UpgradeState -eq 'RollingForwardCompleted' -and $upgrade.TargetApplicationTypeVersion -eq $TargetVersion) { return }
if ($upgrade.UpgradeState -in @('RollingBackCompleted', 'Failed')) { throw "Rollback upgrade to $TargetVersion ended in $($upgrade.UpgradeState)." }
} while ([DateTime]::UtcNow -lt $deadline)
throw "Rollback upgrade to $TargetVersion did not complete within $TimeoutMinutes minutes."
}

function Assert-SfApplicationStable {
param(
[Parameter(Mandatory)][string]$Application,
[int]$StableSeconds = 120,
[int]$ReadyTimeoutSeconds = 300,
[int]$PollSeconds = 10,
[string]$RollbackVersion
)
$result = Wait-SfApplicationStable -Application $Application -StableSeconds $StableSeconds -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds
if ($result.Stable) {
Write-Host $result.Message
return
}
foreach ($line in @($result.Details)) { Write-Host " $line" }
$message = "Post-deploy stability check failed: $($result.Message)"
if ($RollbackVersion -and (Test-SfRollbackWarranted $result)) {
try {
Invoke-SfApplicationRollback -Application $Application -TargetVersion $RollbackVersion
$after = Wait-SfApplicationStable -Application $Application -StableSeconds 30 -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds
$message += " Rolled back to $RollbackVersion; after rollback: $($after.Message)"
} catch {
$message += " Rollback to $RollbackVersion FAILED, manual action needed: $($_.Exception.Message)"
}
} elseif ($RollbackVersion) {
$message += " No rollback for a $($result.Health) state without restarts."
} else {
$message += ' No rollback version was given; the new version stays deployed.'
}
if ($env:GITHUB_ACTIONS -eq 'true') { Write-Host "::error title=Service Fabric deployment unstable::$message" }
throw $message
}

if ($MyInvocation.InvocationName -ne '.') {
$ErrorActionPreference = 'Stop'
if (-not $ApplicationName -or $ApplicationName -notmatch '^fabric:/\S+$') { throw 'ApplicationName must be an exact fabric:/ application URI.' }
if ($StableSeconds -lt 30 -or $PollSeconds -lt 1 -or $ReadyTimeoutSeconds -lt 0) { throw 'StableSeconds must be at least 30 and PollSeconds at least 1.' }
if (-not $SkipConnect) {
Import-Module ServiceFabric -ErrorAction Stop
Connect-ServiceFabricCluster -ConnectionEndpoint $ConnectionEndpoint -TimeoutSec 30 | Out-Null
}
Assert-SfApplicationStable -Application $ApplicationName -StableSeconds $StableSeconds -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds $PollSeconds -RollbackVersion $RollbackApplicationTypeVersion
}
45 changes: 45 additions & 0 deletions .github/actions/verify-servicefabric-stable/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: Verify a Service Fabric application stays healthy
description: Fail the deployment unless the application reaches Ok health and keeps it, without restarts, for a stability window. Optionally roll back.
inputs:
application-name:
description: Exact application URI, for example fabric:/Eternet.Events.
required: true
stable-seconds:
description: Seconds the application must stay Ok with the same replicas and processes.
required: false
default: '120'
ready-timeout-seconds:
description: Seconds to wait for the first Ok health report after the deployment.
required: false
default: '300'
poll-seconds:
description: Seconds between health and replica checks.
required: false
default: '10'
rollback-version:
description: Application type version to return to on an Error or a restart. Empty keeps the new version and only fails.
required: false
default: ''
connection-endpoint:
description: Service Fabric client endpoint.
required: false
default: servicefabric.eternet.cc:19000
runs:
using: composite
steps:
- shell: powershell
env:
SF_STABLE_APPLICATION: ${{ inputs.application-name }}
SF_STABLE_SECONDS: ${{ inputs.stable-seconds }}
SF_STABLE_READY_TIMEOUT: ${{ inputs.ready-timeout-seconds }}
SF_STABLE_POLL: ${{ inputs.poll-seconds }}
SF_STABLE_ROLLBACK: ${{ inputs.rollback-version }}
SF_STABLE_ENDPOINT: ${{ inputs.connection-endpoint }}
run: |
& "$env:GITHUB_ACTION_PATH\Assert-ServiceFabricApplicationStable.ps1" `
-ApplicationName $env:SF_STABLE_APPLICATION `
-StableSeconds ([int]$env:SF_STABLE_SECONDS) `
-ReadyTimeoutSeconds ([int]$env:SF_STABLE_READY_TIMEOUT) `
-PollSeconds ([int]$env:SF_STABLE_POLL) `
-RollbackApplicationTypeVersion $env:SF_STABLE_ROLLBACK `
-ConnectionEndpoint $env:SF_STABLE_ENDPOINT
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
$ErrorActionPreference = 'Stop'
. "$PSScriptRoot/../Assert-ServiceFabricApplicationStable.ps1"

$script:now = [DateTime]::new(2026, 1, 1, 0, 0, 0, [DateTimeKind]::Utc)
$clock = { $script:now }
$sleep = { param($seconds) $script:now = $script:now.AddSeconds($seconds) }

function New-Snapshot([string]$Health, [string]$ProcessId = '100') {
[pscustomobject]@{ Health = $Health; Details = @("fixture $Health"); Identities = @{ 'replica svc p1 vm1' = '7'; 'process vm1 Pkg Code ' = "/$ProcessId" } }
}

function Invoke-Scenario([object[]]$Snapshots, [int]$ReadyTimeoutSeconds = 60) {
$script:queue = [System.Collections.Generic.Queue[object]]::new()
foreach ($snapshot in $Snapshots) { $script:queue.Enqueue($snapshot) }
$script:last = $null
$provider = { param($name) if ($script:queue.Count) { $script:last = $script:queue.Dequeue() }; $script:last }
Wait-SfApplicationStable -Application 'fabric:/Fixture' -StableSeconds 30 -ReadyTimeoutSeconds $ReadyTimeoutSeconds -PollSeconds 10 -SnapshotProvider $provider -Sleep $sleep -Clock $clock
}

function Assert-Result($Result, [bool]$Stable, [string]$Reason, [bool]$Rollback, [string]$Name) {
if ($Result.Stable -ne $Stable -or $Result.Reason -ne $Reason -or (Test-SfRollbackWarranted $Result) -ne $Rollback) {
throw "$Name expected Stable=$Stable Reason=$Reason Rollback=$Rollback, got Stable=$($Result.Stable) Reason=$($Result.Reason)."
}
Write-Host "PASS $Name"
}

Assert-Result (Invoke-Scenario @((New-Snapshot Ok))) $true 'Stable' $false 'HealthyApplicationPasses'
Assert-Result (Invoke-Scenario @((New-Snapshot Error), (New-Snapshot Warning), (New-Snapshot Ok))) $true 'Stable' $false 'StartupErrorsBeforeFirstOkAreTolerated'
Assert-Result (Invoke-Scenario @((New-Snapshot Error)) -ReadyTimeoutSeconds 30) $false 'NeverOk' $true 'NeverOkFailsAndRollsBack'
Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Ok '200'))) $false 'Restarted' $true 'ProcessRestartWhileOkFails'
Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Ok), (New-Snapshot Error))) $false 'HealthError' $true 'ErrorInWindowFailsAndRollsBack'
Assert-Result (Invoke-Scenario @((New-Snapshot Ok), (New-Snapshot Warning))) $false 'HealthWarning' $false 'WarningInWindowFailsWithoutRollback'
Assert-Result (Invoke-Scenario @((New-Snapshot Warning)) -ReadyTimeoutSeconds 20) $false 'NeverOk' $false 'PersistentWarningFailsWithoutRollback'

$changes = Compare-SfIdentities -Baseline @{ a = '1'; b = '2' } -Current @{ a = '1'; c = '3' }
if ($changes.Count -ne 2) { throw 'Identity comparison must report removed and added replicas.' }
Write-Host 'PASS MovedReplicaIsReported'

$global:LASTEXITCODE = 0
28 changes: 28 additions & 0 deletions .github/workflows/servicefabric-stability-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: Service Fabric stability check tests
on:
pull_request:
paths:
- .github/actions/verify-servicefabric-stable/**
- scripts/deploy_service_fabric.ps1
- .github/workflows/servicefabric-stability-tests.yml
permissions:
contents: read
jobs:
decision-fixtures:
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: [self-hosted, Windows]
timeout-minutes: 5
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09
with:
ref: ${{ github.sha }}
persist-credentials: false
- name: Check health and restart decisions without a cluster
shell: powershell
run: |
./.github/actions/verify-servicefabric-stable/tests/Test-ServiceFabricStability.ps1
foreach ($file in @('scripts/deploy_service_fabric.ps1', '.github/actions/verify-servicefabric-stable/Assert-ServiceFabricApplicationStable.ps1')) {
$errors = $null
[System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path $file), [ref]$null, [ref]$errors) | Out-Null
if ($errors) { throw "$file does not parse: $($errors[0].Message)" }
}
Loading
Loading