From 2d34b985e0142b4d68624683502920f54291b605 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sat, 19 Sep 2026 13:57:45 -0400 Subject: [PATCH] bug: auto-restore rounds after a server restart - a server that is behind the backend (restart, map reload) can no longer be resumed or publish rounds; it holds the match and restores the latest usable round on its own once the roster is whole - a restore requested while short-handed is queued instead of dropped, and .resume during a recovery votes to restore with whoever is here - backups are validated before upload and before restore: right round, and players in both PlayersOnTeam sections - a restore is verified, teams are re-enforced after it, and the in-memory round list no longer keeps stale duplicates --- .../src/FiveStack.Commands/BackupRounds.cs | 2 +- .../src/FiveStack.Commands/Vote.cs | 11 +- .../src/FiveStack.Events/RoundEnd.cs | 14 +- .../src/FiveStack.Events/RoundStart.cs | 6 +- .../FiveStack.Services/GameBackUpRounds.cs | 574 +++++++++++++++--- .../src/FiveStack.Services/MatchManager.cs | 15 +- .../src/FiveStack.Services/TimeoutSystem.cs | 9 + .../src/FiveStack.Commands/BackupRounds.cs | 4 +- apps/swiftly/src/FiveStack.Commands/Vote.cs | 5 +- apps/swiftly/src/FiveStack.Events/RoundEnd.cs | 16 +- .../src/FiveStack.Events/RoundStart.cs | 6 +- .../FiveStack.Services/GameBackUpRounds.cs | 541 ++++++++++++++--- .../src/FiveStack.Services/MatchManager.cs | 13 +- .../src/FiveStack.Services/TimeoutSystem.cs | 9 + apps/swiftly/test/BackupRoundUtilityTests.cs | 163 +++++ .../FiveStack.Utilities/BackupRoundUtility.cs | 224 +++++++ 16 files changed, 1406 insertions(+), 206 deletions(-) create mode 100644 apps/swiftly/test/BackupRoundUtilityTests.cs create mode 100644 shared/dotnet/FiveStack.Utilities/BackupRoundUtility.cs diff --git a/apps/counterstrikesharp/src/FiveStack.Commands/BackupRounds.cs b/apps/counterstrikesharp/src/FiveStack.Commands/BackupRounds.cs index bfd510dc..74454081 100644 --- a/apps/counterstrikesharp/src/FiveStack.Commands/BackupRounds.cs +++ b/apps/counterstrikesharp/src/FiveStack.Commands/BackupRounds.cs @@ -50,7 +50,7 @@ public void OnApiResetRound(CCSPlayerController? player, CommandInfo command) return; } - _gameBackupRounds.RestoreRound(round); + _gameBackupRounds.RestoreRound(round, command.ArgByIndex(2) == "force"); } [ConsoleCommand("css_reset", "Restores to a previous round")] diff --git a/apps/counterstrikesharp/src/FiveStack.Commands/Vote.cs b/apps/counterstrikesharp/src/FiveStack.Commands/Vote.cs index 2356e260..cc8a1535 100644 --- a/apps/counterstrikesharp/src/FiveStack.Commands/Vote.cs +++ b/apps/counterstrikesharp/src/FiveStack.Commands/Vote.cs @@ -16,13 +16,10 @@ public void OnResetAnswer(CCSPlayerController? player, CommandInfo command) return; } - if (_gameBackupRounds.IsResettingRound()) - { - _gameBackupRounds.restoreRoundVote?.CastVote( - player, - command.GetCommandString == "css_y" - ); - } + _gameBackupRounds.restoreRoundVote?.CastVote( + player, + command.GetCommandString == "css_y" + ); if (_surrenderSystem.IsSurrendering()) { diff --git a/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs b/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs index 996b3aed..01724163 100644 --- a/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs +++ b/apps/counterstrikesharp/src/FiveStack.Events/RoundEnd.cs @@ -35,9 +35,9 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event, GameE match.UpdateMapStatus(eMapStatus.Overtime); } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { - _logger.LogInformation("OnRoundOfficiallyEnded skipping capture: restoring round"); + _logger.LogInformation("OnRoundOfficiallyEnded skipping capture: round restore pending or in progress"); return HookResult.Continue; } @@ -55,10 +55,10 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event, GameE [GameEventHandler] public HookResult OnRoundEnd(EventRoundEnd @event, GameEventInfo info) { - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { _logger.LogInformation( - $"OnRoundEnd ignored (restoring round): message={@event.Message}" + $"OnRoundEnd ignored (round restore pending or in progress): message={@event.Message}" ); return HookResult.Continue; } @@ -182,6 +182,12 @@ private void CaptureRoundResult(MatchManager match, MatchData matchData, MatchMa public void PublishPendingRound(bool SendBackupRound) { + if (_gameBackupRounds.BlocksPlay()) + { + _matchEvents.ClearPendingRoundResult(); + return; + } + MatchEvents.RoundResultSnapshot? snap = _matchEvents.PendingRoundResult; if (snap == null) { diff --git a/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs b/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs index f3bdff6c..3a0f4dc0 100644 --- a/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs +++ b/apps/counterstrikesharp/src/FiveStack.Events/RoundStart.cs @@ -35,9 +35,11 @@ public HookResult OnRoundStart(EventRoundStart @event, GameEventInfo info) return HookResult.Continue; } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { - _logger.LogInformation("OnRoundStart skipping publish: restoring round"); + _logger.LogInformation( + "OnRoundStart skipping publish: round restore pending or in progress" + ); return HookResult.Continue; } diff --git a/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs b/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs index 71d58932..72d16a05 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/GameBackUpRounds.cs @@ -1,18 +1,39 @@ using CounterStrikeSharp.API; using CounterStrikeSharp.API.Core; using CounterStrikeSharp.API.Modules.Cvars; +using CounterStrikeSharp.API.Modules.Timers; using CounterStrikeSharp.API.Modules.Utils; using FiveStack.Entities; +using FiveStack.Enums; using FiveStack.Utilities; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Localization; using Microsoft.Extensions.Logging; +using Timer = CounterStrikeSharp.API.Modules.Timers.Timer; namespace FiveStack; public class GameBackUpRounds { private int? _resetRound; + + // A restore that has to happen before play may continue: the backend knows + // rounds this game does not (fresh process, map reload), or an organizer + // asked for one while the roster was short. + private int? _pendingRestoreRound; + private int? _forcedRestoreRound; + private bool _recoveryNeedsOrganizer; + private bool _behindNeedsConfirmation; + private int _restoreAttempts; + private Guid? _syncedMapId; + private DateTime _lastRestoreRequestAt = DateTime.MinValue; + private Timer? _recoveryTimer; + + private const int RecoveryTickSeconds = 5; + private const int RestoreRequestRetrySeconds = 15; + private const int MaxRestoreAttempts = 3; + + private readonly MatchEvents _matchEvents; private readonly GameServer _gameServer; private readonly MatchService _matchService; @@ -52,6 +73,8 @@ IStringLocalizer localizer } } + private string BackupDirectory => Path.Join(Server.GameDirectory, "csgo"); + public void RemovePlayerVoteOnDisconnect(ulong steamId) { restoreRoundVote?.RemovePlayerVote(steamId); @@ -74,50 +97,316 @@ public bool IsResettingRound() return _resetRound != null; } + public void Reset() + { + _recoveryTimer?.Kill(); + _recoveryTimer = null; + _resetRound = null; + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _recoveryNeedsOrganizer = false; + _behindNeedsConfirmation = false; + _restoreAttempts = 0; + _syncedMapId = null; + restoreRoundVote = null; + } + + public bool IsRecoveryPending() + { + return _pendingRestoreRound != null || _recoveryNeedsOrganizer; + } + + // Nothing that advances the match -- resuming, capturing or publishing a + // round -- may run while the game state is not the backend's. + public bool BlocksPlay() + { + return IsResettingRound() || IsRecoveryPending(); + } + + private int MinPlayersPerTeam() + { + return BackupRoundUtility.MinPlayersPerTeam( + _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10 + ); + } + + // Runs on every match setup, not just the first: a map reload mid-match + // leaves a long-lived process exactly as far behind as a crash does. public void CheckForBackupRestore() { - MatchMap? matchMap = _matchService.GetCurrentMatch()?.GetCurrentMap(); + MatchManager? matchManager = _matchService.GetCurrentMatch(); + MatchData? match = matchManager?.GetMatchData(); + MatchMap? matchMap = matchManager?.GetCurrentMap(); - if (matchMap == null) + if (matchManager == null || match == null || matchMap == null || BlocksPlay()) { return; } - var availableRounds = matchMap.rounds.Where( - (backupRound) => - { - return backupRound.deleted_at == null; - } + eMapStatus backendStatus = MatchUtility.MapStatusStringToEnum(matchMap.status); + if ( + backendStatus != eMapStatus.Live + && backendStatus != eMapStatus.Paused + && backendStatus != eMapStatus.Overtime + ) + { + return; + } + + if (_environmentService.IsOfflineMode()) + { + LoadBackupRoundsFromDisk(match, matchMap); + } + + int highestRound = BackupRoundUtility.HighestRound(matchMap.rounds); + int totalRoundsPlayed = _gameServer.GetTotalRoundsPlayed(); + + _logger.LogInformation( + $"Highest recorded round: {highestRound}, and total rounds played is {totalRoundsPlayed}" ); - if (availableRounds.Count() == 0) + if (highestRound <= totalRoundsPlayed) { + _syncedMapId = matchMap.id; + _behindNeedsConfirmation = false; return; } - int highestNumber = availableRounds.Max( - (backupRound) => + // A process that was in step a moment ago only looks behind if this + // match data predates a restore it just ran. Believe it on a second, + // fresh read; a new process has nothing to be stale against. + if (_syncedMapId == matchMap.id && !_behindNeedsConfirmation) + { + _behindNeedsConfirmation = true; + _logger.LogWarning( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, confirming with a fresh match fetch" + ); + _matchService.GetMatchFromApi(); + return; + } + + _behindNeedsConfirmation = false; + + int restorableRound = BackupRoundUtility.HighestRestorableRound( + matchMap.rounds, + MinPlayersPerTeam() + ); + + if (restorableRound <= totalRoundsPlayed) + { + _logger.LogCritical( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, and no recorded round has a usable backup" + ); + _recoveryNeedsOrganizer = true; + } + else + { + if (restorableRound < highestRound) { - return backupRound.round; + _logger.LogCritical( + $"Rounds {restorableRound + 1}-{highestRound} have no usable backup, recovering to round {restorableRound}" + ); } + + _logger.LogWarning( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, holding the match until round {restorableRound} is restored" + ); + _pendingRestoreRound = restorableRound; + } + + _restoreAttempts = 0; + StartRecoveryTimer(); + } + + private void StartRecoveryTimer() + { + _recoveryTimer?.Kill(); + _recoveryTimer = TimerUtility.AddTimer( + RecoveryTickSeconds, + RecoveryTick, + TimerFlags.REPEAT ); + } - int totalRoundsPlayed = _gameServer.GetTotalRoundsPlayed(); + private void RecoveryTick() + { + if (!IsRecoveryPending()) + { + _recoveryTimer?.Kill(); + _recoveryTimer = null; + return; + } - _logger.LogInformation( - $"Highest Backup Round: {highestNumber}, and total rounds played is {totalRoundsPlayed}" + if (IsResettingRound()) + { + return; + } + + MatchManager? match = _matchService.GetCurrentMatch(); + if (match == null || !match.IsInPlay()) + { + return; + } + + // Held by CS2 as well as by the plugin: a pause a restart dropped + // would otherwise let rounds play out that can never count. + _gameServer.SendCommands(["mp_pause_match"]); + if (!match.IsPaused()) + { + match.PauseMatch(); + } + + if (_recoveryNeedsOrganizer) + { + _gameServer.Message( + HudDestination.Alert, + " No usable round backup. An organizer must run restore_round (0 restarts the map)." + ); + return; + } + + int round = _pendingRestoreRound!.Value; + + if (!IsRosterWhole(out int connected, out int expected) && _forcedRestoreRound != round) + { + _gameServer.Message( + HudDestination.Alert, + $" Round {round} will be restored once everyone is back ({connected}/{expected}). {CommandUtility.PublicChatTrigger}resume to vote to restore now." + ); + return; + } + + if ((DateTime.UtcNow - _lastRestoreRequestAt).TotalSeconds < RestoreRequestRetrySeconds) + { + return; + } + + RequestPendingRestore(); + } + + private void RequestPendingRestore() + { + if (_pendingRestoreRound == null) + { + return; + } + + _lastRestoreRequestAt = DateTime.UtcNow; + + if (_environmentService.IsOfflineMode()) + { + RestoreRound(_pendingRestoreRound.Value); + return; + } + + // Through the backend, not straight to CS2: it voids the stats of the + // round that was cut short before that round is played again. + SendRestoreRoundToBackend(_pendingRestoreRound.Value); + } + + // The way out when someone is not coming back: .resume during a recovery + // asks to restore with whoever is here rather than to unpause a game + // that must not be played. + public void RequestRecoveryNow(CCSPlayerController? player, bool isAdmin) + { + if (_recoveryNeedsOrganizer || _pendingRestoreRound == null) + { + _gameServer.Message( + HudDestination.Chat, + $" {ChatColors.Red}No usable round backup. An organizer must run restore_round .", + player + ); + return; + } + + int round = _pendingRestoreRound.Value; + + if (IsResettingRound()) + { + return; + } + + if (player == null || isAdmin || IsRosterWhole(out _, out _)) + { + restoreRoundVote?.CancelVote(); + _forcedRestoreRound = round; + RequestPendingRestore(); + return; + } + + if (restoreRoundVote != null) + { + restoreRoundVote.CastVote(player, true); + return; + } + + restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; + + if (restoreRoundVote == null) + { + return; + } + + restoreRoundVote.StartVote( + $"Restore round {round} without waiting for everyone", + new CsTeam[] { CsTeam.CounterTerrorist, CsTeam.Terrorist }, + () => + { + _logger.LogInformation("restore without full roster vote passed"); + restoreRoundVote = null; + _forcedRestoreRound = round; + RequestPendingRestore(); + }, + () => + { + _logger.LogInformation("restore without full roster vote failed"); + restoreRoundVote = null; + }, + true, + 30 ); - if (totalRoundsPlayed > 0 && totalRoundsPlayed >= highestNumber) + restoreRoundVote?.CastVote(player, true); + } + + // Offline matches have no backend to hold the rounds; the files CS2 wrote + // are all there is. Online they are never trusted past the backend: a + // file for a round whose score was never published would restore a round + // the backend has no record of. + private void LoadBackupRoundsFromDisk(MatchData match, MatchMap matchMap) + { + string csgoDir = BackupDirectory; + string prefix = MatchUtility.GetSafeMatchPrefix(match); + + if (!Directory.Exists(csgoDir)) { - // we are already live, do not restart the match accidently return; } - if (highestNumber > totalRoundsPlayed) + try { - _logger.LogInformation("Server restarted, requires a vote to restore round"); - RequestRestoreBackupRound(highestNumber, null, true); + foreach (string file in Directory.GetFiles(csgoDir, $"{prefix}_round*.txt")) + { + string name = Path.GetFileNameWithoutExtension(file); + int idx = name.LastIndexOf("_round", StringComparison.Ordinal); + if ( + idx < 0 + || !int.TryParse(name.Substring(idx + "_round".Length), out int round) + || matchMap.rounds.Any(backupRound => backupRound.round == round) + ) + { + continue; + } + + matchMap.rounds = BackupRoundUtility.Upsert( + matchMap.rounds, + new BackupRound { round = round, backup_file = File.ReadAllText(file) } + ); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed scanning for backup round files"); } } @@ -145,16 +434,16 @@ public void RequestRestoreBackupRound( return; } - BackupRound? backupRound = matchMap.rounds.FirstOrDefault( - (backupRound) => - { - return backupRound.round == round; - } - ); - - if (backupRound == null) + if ( + BackupRoundUtility.FindRestorable(matchMap.rounds, round, MinPlayersPerTeam()) == null + ) { - _logger.LogWarning($"missing backup round: {round}"); + _logger.LogWarning($"no usable backup for round: {round}"); + _gameServer.Message( + HudDestination.Chat, + $" {ChatColors.Red}Round {round} has no usable backup.", + player + ); return; } @@ -162,8 +451,6 @@ public void RequestRestoreBackupRound( if (player != null || vote == true) { - _resetRound = round; - restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; @@ -172,14 +459,17 @@ public void RequestRestoreBackupRound( return; } + _resetRound = round; + restoreRoundVote.StartVote( _localizer["backup.vote.restore_to", round], new CsTeam[] { CsTeam.CounterTerrorist, CsTeam.Terrorist }, () => { _logger.LogInformation("restore round vote passed"); - SendRestoreRoundToBackend(round); + restoreRoundVote = null; _resetRound = null; + SendRestoreRoundToBackend(round); }, () => { @@ -239,13 +529,28 @@ public void RequestRestoreBackupRound( string backupRoundFile = File.ReadAllText(backupRoundFilePath); + string? invalidReason = BackupRoundUtility.Validate( + backupRoundFile, + round, + MinPlayersPerTeam() + ); + + if (invalidReason != null) + { + _logger.LogCritical( + $"Not publishing backup round {round}: {invalidReason} ({backupRoundFilePath})" + ); + return null; + } + MatchMap? currentMap = _matchService.GetCurrentMatch()?.GetCurrentMap(); if (currentMap != null) { - currentMap.rounds = currentMap - .rounds.Append(new BackupRound { round = round, backup_file = backupRoundFile }) - .ToArray(); + currentMap.rounds = BackupRoundUtility.Upsert( + currentMap.rounds, + new BackupRound { round = round, backup_file = backupRoundFile } + ); } return backupRoundFile; @@ -275,7 +580,7 @@ public void SendRestoreRoundToBackend(int round) ); } - public void RestoreRound(int round) + public void RestoreRound(int round, bool force = false) { if (IsResettingRound()) { @@ -283,77 +588,206 @@ public void RestoreRound(int round) return; } - if (!CanRestoreRound(round)) + MatchManager? matchManager = _matchService.GetCurrentMatch(); + MatchData? match = matchManager?.GetMatchData(); + MatchMap? matchMap = matchManager?.GetCurrentMap(); + + if (matchManager == null || match == null || matchMap == null) { + _logger.LogWarning( + "RestoreRound({Round}) dropped: no current match/map (match={HasMatch} map={HasMap})", + round, + match != null, + matchMap != null + ); return; } - MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - MatchMap? matchMap = _matchService.GetCurrentMatch()?.GetCurrentMap(); - - if (match == null || matchMap == null) + if (round == 0) { + RestartMap(matchMap); return; } - BackupRound? backupRound = matchMap.rounds.FirstOrDefault( - (backupRound) => - { - return backupRound.round == round; - } + BackupRound? backupRound = BackupRoundUtility.FindRestorable( + matchMap.rounds, + round, + MinPlayersPerTeam() ); if (backupRound == null) { - _logger.LogWarning($"missing backup round: {round}"); + _logger.LogWarning( + "no usable backup for round: {Round} (known rounds: [{Rounds}])", + round, + string.Join(", ", matchMap.rounds.Select(r => r.round)) + ); + _gameServer.Message( + HudDestination.Alert, + $" Round {round} has no usable backup and was not restored." + ); + return; + } + + // Queued, never dropped: the backend has already voided the rounds + // past this one, so giving up here would strand the match between + // the two. + if (!force && _forcedRestoreRound != round && !IsRosterWhole(out _, out _)) + { + _logger.LogWarning($"Restore round {round} queued until the roster is whole"); + _recoveryNeedsOrganizer = false; + _pendingRestoreRound = round; + _restoreAttempts = 0; + _lastRestoreRequestAt = DateTime.UtcNow; + StartRecoveryTimer(); + RecoveryTick(); return; } string backupRoundFileName = $"restore-{MatchUtility.GetSafeMatchPrefix(match)}round{round.ToString().PadLeft(2, '0')}.txt"; string backupRoundFilePath = Path.Join( - Server.GameDirectory + "/csgo/", + BackupDirectory, backupRoundFileName ); - File.WriteAllText(backupRoundFilePath, backupRound.backup_file); + // Pending first: cancelling a vote runs its failure path, which + // resumes the match unless something is already holding it. + _pendingRestoreRound = round; + _recoveryNeedsOrganizer = false; + restoreRoundVote?.CancelVote(); + restoreRoundVote = null; _resetRound = round; + _restoreAttempts++; + + _matchEvents.ClearPendingRoundResult(); + matchMap.rounds = BackupRoundUtility.DropAbove(matchMap.rounds, round); _logger.LogInformation($"Loading backup round file {backupRoundFileName}"); - Server.NextFrame(async () => + string backupFileContents = backupRound.backup_file; + + try + { + File.WriteAllText(backupRoundFilePath, backupFileContents); + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed writing restore backup round file {File}", + backupRoundFileName + ); + Server.NextFrame(() => FinishRestore(round)); + return; + } + + Server.NextFrame(() => { _gameServer.SendCommands([$"mp_backup_restore_load_file {backupRoundFileName}"]); _matchService.GetCurrentMatch()?.PauseMatch(); - await Task.Delay(5 * 1000); + TimerUtility.AddTimer(5, () => FinishRestore(round)); + }); + } + + // The restore is only over once CS2 is actually at that round. Anything + // else -- an unwritable file, a load CS2 refused -- leaves the recovery + // pending so it is retried rather than played through. + private void FinishRestore(int round) + { + _resetRound = null; + + MatchManager? match = _matchService.GetCurrentMatch(); + int totalRoundsPlayed = _gameServer.GetTotalRoundsPlayed(); + + if (totalRoundsPlayed != round) + { + _logger.LogCritical( + $"Restore of round {round} did not take: game is at round {totalRoundsPlayed} (attempt {_restoreAttempts}/{MaxRestoreAttempts})" + ); + + if (_restoreAttempts >= MaxRestoreAttempts) + { + _pendingRestoreRound = null; + _recoveryNeedsOrganizer = true; + } + + _lastRestoreRequestAt = DateTime.UtcNow; + StartRecoveryTimer(); + return; + } - Server.NextFrame(() => + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _restoreAttempts = 0; + _behindNeedsConfirmation = false; + _syncedMapId = match?.GetCurrentMap()?.id; + + // CS2 seats players from the file; anyone it could not place is put + // back where the lineups say they belong. + if (match != null) + { + foreach (CCSPlayerController player in MatchUtility.Players()) { - _resetRound = null; - - _logger.LogInformation($"Sending Message for Round {round}"); - - _gameServer.Message( - HudDestination.Alert, - _localizer[ - "backup.round_restored", - ChatColors.Red, - round, - CommandUtility.PublicChatTrigger - ] - ); - }); - }); + match.EnforceMemberTeam(player); + } + } + + _logger.LogInformation($"Sending Message for Round {round}"); + + _gameServer.Message( + HudDestination.Alert, + _localizer[ + "backup.round_restored", + ChatColors.Red, + round, + CommandUtility.PublicChatTrigger + ] + ); } - private bool CanRestoreRound(int round) + // restore_round 0: the backend has voided every round, so the map starts + // over. The only way forward when no round has a usable backup. + private void RestartMap(MatchMap matchMap) + { + _logger.LogWarning("Restoring to round 0: restarting the map"); + + restoreRoundVote?.CancelVote(); + _matchEvents.ClearPendingRoundResult(); + matchMap.rounds = new BackupRound[0]; + + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _recoveryNeedsOrganizer = false; + _behindNeedsConfirmation = false; + _restoreAttempts = 0; + _syncedMapId = matchMap.id; + + _gameServer.SendCommands(["mp_restartgame 1"]); + _matchService.GetCurrentMatch()?.PauseMatch(); + } + + // Casters and admins never make a match whole. Placeholder lineups have + // no steam ids to match against, so there a head count is the best there is. + private bool IsRosterWhole(out int connected, out int expected) { - int connectedPlayers = MatchUtility.Players().Count; - int expectedPlayers = _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10; + MatchManager? match = _matchService.GetCurrentMatch(); + MatchData? matchData = match?.GetMatchData(); - if (connectedPlayers >= expectedPlayers) + expected = match?.GetExpectedPlayerCount() ?? 10; + connected = + matchData == null || MatchUtility.HasPlaceholderMembers(matchData) + ? MatchUtility.Players().Count + : MatchUtility.ConnectedRosterCount(matchData); + + return connected >= expected; + } + + private bool CanRestoreRound(int round) + { + if (IsRosterWhole(out int connectedPlayers, out int expectedPlayers)) { return true; } diff --git a/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs b/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs index ba2de22a..f7368397 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/MatchManager.cs @@ -273,7 +273,7 @@ public void PauseMatch( 3, () => { - if (!IsFreezePeriod() || !IsPaused() || _backUpManagement.IsResettingRound()) + if (!IsFreezePeriod() || !IsPaused() || _backUpManagement.BlocksPlay()) { return; } @@ -302,9 +302,9 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) return; } - if (_backUpManagement.IsResettingRound()) + if (_backUpManagement.BlocksPlay()) { - _logger.LogInformation("Resetting round, cannot resume match"); + _logger.LogInformation("Round restore pending or in progress, cannot resume match"); return; } @@ -376,11 +376,6 @@ public void UpdateMapStatus(eMapStatus status, Guid? winningLineupId = null) _logger.LogInformation($"Update Map Status {_currentMapStatus} -> {status}"); - if (_currentMapStatus == eMapStatus.Unknown) - { - _backUpManagement.CheckForBackupRestore(); - } - var currentMap = GetCurrentMap(); // TODO - this should only happen discord matches @@ -632,6 +627,9 @@ public void SetupMatch(MatchData match) ConVar.Find("mp_match_restart_delay")?.SetValue(matchRestartDelay); ConVar.Find("hostname")?.SetValue("5Stack.gg"); + _backUpManagement.Setup(); + _backUpManagement.CheckForBackupRestore(); + if (MatchUtility.MapStatusStringToEnum(_currentMap.status) != _currentMapStatus) { UpdateMapStatus(MatchUtility.MapStatusStringToEnum(_currentMap.status)); @@ -1473,5 +1471,6 @@ public void Reset() captainSystem.Reset(); knifeSystem.Reset(); _surrenderSystem.Reset(); + _backUpManagement.Reset(); } } diff --git a/apps/counterstrikesharp/src/FiveStack.Services/TimeoutSystem.cs b/apps/counterstrikesharp/src/FiveStack.Services/TimeoutSystem.cs index 0321e9b4..b84f4b48 100644 --- a/apps/counterstrikesharp/src/FiveStack.Services/TimeoutSystem.cs +++ b/apps/counterstrikesharp/src/FiveStack.Services/TimeoutSystem.cs @@ -241,6 +241,15 @@ public void RequestResume(CCSPlayerController? player) return; } + if (_backUpManagement.IsRecoveryPending()) + { + _backUpManagement.RequestRecoveryNow( + player, + player == null || IsAdminOrOrganizer(player, matchData) + ); + return; + } + string resumeMessage = _localizer["timeout.admin_resumed"]; // Refuse while a required camera is still down, on the same terms as the diff --git a/apps/swiftly/src/FiveStack.Commands/BackupRounds.cs b/apps/swiftly/src/FiveStack.Commands/BackupRounds.cs index 0a0c3072..19c916e2 100644 --- a/apps/swiftly/src/FiveStack.Commands/BackupRounds.cs +++ b/apps/swiftly/src/FiveStack.Commands/BackupRounds.cs @@ -53,7 +53,9 @@ public void OnApiResetRound(ICommandContext context) return; } - _gameBackupRounds.RestoreRound(round); + bool force = context.Args.Length > 1 && context.Args[1] == "force"; + + _gameBackupRounds.RestoreRound(round, force); } [Command("reset", registerRaw: false, permission: "")] diff --git a/apps/swiftly/src/FiveStack.Commands/Vote.cs b/apps/swiftly/src/FiveStack.Commands/Vote.cs index ae786650..307753a3 100644 --- a/apps/swiftly/src/FiveStack.Commands/Vote.cs +++ b/apps/swiftly/src/FiveStack.Commands/Vote.cs @@ -27,10 +27,7 @@ private void CastVoteAnswer(ICommandContext context, bool answer) return; } - if (_gameBackupRounds.IsResettingRound()) - { - _gameBackupRounds.restoreRoundVote?.CastVote(player, answer); - } + _gameBackupRounds.restoreRoundVote?.CastVote(player, answer); if (_surrenderSystem.IsSurrendering()) { diff --git a/apps/swiftly/src/FiveStack.Events/RoundEnd.cs b/apps/swiftly/src/FiveStack.Events/RoundEnd.cs index ea9fa87f..85dae0c6 100644 --- a/apps/swiftly/src/FiveStack.Events/RoundEnd.cs +++ b/apps/swiftly/src/FiveStack.Events/RoundEnd.cs @@ -36,9 +36,11 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event) match.UpdateMapStatus(eMapStatus.Overtime); } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { - _logger.LogInformation("OnRoundOfficiallyEnded skipping capture: restoring round"); + _logger.LogInformation( + "OnRoundOfficiallyEnded skipping capture: round restore pending or in progress" + ); return HookResult.Continue; } @@ -56,10 +58,10 @@ public HookResult OnRoundOfficiallyEnded(EventRoundOfficiallyEnded @event) [GameEventHandler(HookMode.Post)] public HookResult OnRoundEnd(EventRoundEnd @event) { - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { _logger.LogInformation( - $"OnRoundEnd ignored (restoring round): message={@event.Message}" + $"OnRoundEnd ignored (round restore pending or in progress): message={@event.Message}" ); return HookResult.Continue; } @@ -188,6 +190,12 @@ private void CaptureRoundResult(MatchManager match, MatchData matchData, MatchMa public void PublishPendingRound(bool SendBackupRound) { + if (_gameBackupRounds.BlocksPlay()) + { + _matchEvents.ClearPendingRoundResult(); + return; + } + MatchEvents.RoundResultSnapshot? snap = _matchEvents.PendingRoundResult; if (snap == null) { diff --git a/apps/swiftly/src/FiveStack.Events/RoundStart.cs b/apps/swiftly/src/FiveStack.Events/RoundStart.cs index d7fe0408..b594f4bd 100644 --- a/apps/swiftly/src/FiveStack.Events/RoundStart.cs +++ b/apps/swiftly/src/FiveStack.Events/RoundStart.cs @@ -36,9 +36,11 @@ public HookResult OnRoundStart(EventRoundStart @event) return HookResult.Continue; } - if (_gameBackupRounds.IsResettingRound()) + if (_gameBackupRounds.BlocksPlay()) { - _logger.LogInformation("OnRoundStart skipping publish: restoring round"); + _logger.LogInformation( + "OnRoundStart skipping publish: round restore pending or in progress" + ); return HookResult.Continue; } diff --git a/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs b/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs index 8336864b..1a4d3287 100644 --- a/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs +++ b/apps/swiftly/src/FiveStack.Services/GameBackUpRounds.cs @@ -1,4 +1,5 @@ using FiveStack.Entities; +using FiveStack.Enums; using FiveStack.Utilities; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -13,6 +14,23 @@ namespace FiveStack; public class GameBackUpRounds { private int? _resetRound; + + // A restore that has to happen before play may continue: the backend knows + // rounds this game does not (fresh process, map reload), or an organizer + // asked for one while the roster was short. + private int? _pendingRestoreRound; + private int? _forcedRestoreRound; + private bool _recoveryNeedsOrganizer; + private bool _behindNeedsConfirmation; + private int _restoreAttempts; + private Guid? _syncedMapId; + private DateTime _lastRestoreRequestAt = DateTime.MinValue; + private CancellationTokenSource? _recoveryTimer; + + private const int RecoveryTickSeconds = 5; + private const int RestoreRequestRetrySeconds = 15; + private const int MaxRestoreAttempts = 3; + private readonly ISwiftlyCore _core; private readonly MatchEvents _matchEvents; private readonly GameServer _gameServer; @@ -132,76 +150,286 @@ public bool IsResettingRound() return _resetRound != null; } + public void Reset() + { + TimerUtility.Kill(_recoveryTimer); + _recoveryTimer = null; + _resetRound = null; + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _recoveryNeedsOrganizer = false; + _behindNeedsConfirmation = false; + _restoreAttempts = 0; + _syncedMapId = null; + restoreRoundVote = null; + } + + public bool IsRecoveryPending() + { + return _pendingRestoreRound != null || _recoveryNeedsOrganizer; + } + + // Nothing that advances the match -- resuming, capturing or publishing a + // round -- may run while the game state is not the backend's. + public bool BlocksPlay() + { + return IsResettingRound() || IsRecoveryPending(); + } + + private int MinPlayersPerTeam() + { + return BackupRoundUtility.MinPlayersPerTeam( + _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10 + ); + } + + // Runs on every match setup, not just the first: a map reload mid-match + // leaves a long-lived process exactly as far behind as a crash does. public void CheckForBackupRestore() { MatchManager? matchManager = _matchService.GetCurrentMatch(); MatchData? match = matchManager?.GetMatchData(); MatchMap? matchMap = matchManager?.GetCurrentMap(); - if (match == null || matchMap == null) + if (matchManager == null || match == null || matchMap == null || BlocksPlay()) { return; } - // Detect from the backup files on disk (written by mp_backup_round_auto), - // not just the backend's recorded rounds — the files are the ground truth - // and exist even when the backend/offline match data has no rounds. - string csgoDir = BackupDirectory; - string prefix = MatchUtility.GetSafeMatchPrefix(match); + eMapStatus backendStatus = MatchUtility.MapStatusStringToEnum(matchMap.status); + if ( + backendStatus != eMapStatus.Live + && backendStatus != eMapStatus.Paused + && backendStatus != eMapStatus.Overtime + ) + { + return; + } + + if (_environmentService.IsOfflineMode()) + { + LoadBackupRoundsFromDisk(match, matchMap); + } - int highestNumber = GetHighestBackupRoundOnDisk(csgoDir, prefix); + int highestRound = BackupRoundUtility.HighestRound(matchMap.rounds); int totalRoundsPlayed = _gameServer.GetTotalRoundsPlayed(); _logger.LogInformation( - $"Highest Backup Round (disk): {highestNumber}, and total rounds played is {totalRoundsPlayed}" + $"Highest recorded round: {highestRound}, and total rounds played is {totalRoundsPlayed}" ); - // Nothing ahead of the current round — we are live where we should be. - if (highestNumber <= totalRoundsPlayed) + if (highestRound <= totalRoundsPlayed) { + _syncedMapId = matchMap.id; + _behindNeedsConfirmation = false; return; } - // A backup file exists ahead of the current round (server/match restarted). - // Load it so the restore flow has it, then prompt to restore. - string backupFilePath = Path.Join( - csgoDir, - $"{prefix}_round{highestNumber.ToString().PadLeft(2, '0')}.txt" + // A process that was in step a moment ago only looks behind if this + // match data predates a restore it just ran. Believe it on a second, + // fresh read; a new process has nothing to be stale against. + if (_syncedMapId == matchMap.id && !_behindNeedsConfirmation) + { + _behindNeedsConfirmation = true; + _logger.LogWarning( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, confirming with a fresh match fetch" + ); + _matchService.GetMatchFromApi(); + return; + } + + _behindNeedsConfirmation = false; + + int restorableRound = BackupRoundUtility.HighestRestorableRound( + matchMap.rounds, + MinPlayersPerTeam() ); - try + if (restorableRound <= totalRoundsPlayed) + { + _logger.LogCritical( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, and no recorded round has a usable backup" + ); + _recoveryNeedsOrganizer = true; + } + else { - if (!matchMap.rounds.Any(backupRound => backupRound.round == highestNumber)) + if (restorableRound < highestRound) { - string content = File.ReadAllText(backupFilePath); - matchMap.rounds = matchMap - .rounds.Append( - new BackupRound { round = highestNumber, backup_file = content } - ) - .ToArray(); + _logger.LogCritical( + $"Rounds {restorableRound + 1}-{highestRound} have no usable backup, recovering to round {restorableRound}" + ); } + + _logger.LogWarning( + $"Game is at round {totalRoundsPlayed} but round {highestRound} is recorded, holding the match until round {restorableRound} is restored" + ); + _pendingRestoreRound = restorableRound; } - catch (Exception ex) + + _restoreAttempts = 0; + StartRecoveryTimer(); + } + + private void StartRecoveryTimer() + { + TimerUtility.Kill(_recoveryTimer); + _recoveryTimer = TimerUtility.Repeat(RecoveryTickSeconds, RecoveryTick); + } + + private void RecoveryTick() + { + if (!IsRecoveryPending()) { - _logger.LogError(ex, $"Failed to read backup round file {backupFilePath}"); + TimerUtility.Kill(_recoveryTimer); + _recoveryTimer = null; return; } - _logger.LogInformation( - $"Backup round {highestNumber} is ahead of current round {totalRoundsPlayed}, prompting restore" + if (IsResettingRound()) + { + return; + } + + MatchManager? match = _matchService.GetCurrentMatch(); + if (match == null || !match.IsInPlay()) + { + return; + } + + // Held by CS2 as well as by the plugin: a pause a restart dropped + // would otherwise let rounds play out that can never count. + _gameServer.SendCommands(["mp_pause_match"]); + if (!match.IsPaused()) + { + match.PauseMatch(); + } + + if (_recoveryNeedsOrganizer) + { + _gameServer.Message( + MessageType.Alert, + " No usable round backup. An organizer must run restore_round (0 restarts the map)." + ); + return; + } + + int round = _pendingRestoreRound!.Value; + + if (!IsRosterWhole(out int connected, out int expected) && _forcedRestoreRound != round) + { + _gameServer.Message( + MessageType.Alert, + $" Round {round} will be restored once everyone is back ({connected}/{expected}). {CommandUtility.PublicChatTrigger}resume to vote to restore now." + ); + return; + } + + if ((DateTime.UtcNow - _lastRestoreRequestAt).TotalSeconds < RestoreRequestRetrySeconds) + { + return; + } + + RequestPendingRestore(); + } + + private void RequestPendingRestore() + { + if (_pendingRestoreRound == null) + { + return; + } + + _lastRestoreRequestAt = DateTime.UtcNow; + + if (_environmentService.IsOfflineMode()) + { + RestoreRound(_pendingRestoreRound.Value); + return; + } + + // Through the backend, not straight to CS2: it voids the stats of the + // round that was cut short before that round is played again. + SendRestoreRoundToBackend(_pendingRestoreRound.Value); + } + + // The way out when someone is not coming back: .resume during a recovery + // asks to restore with whoever is here rather than to unpause a game + // that must not be played. + public void RequestRecoveryNow(IPlayer? player, bool isAdmin) + { + if (_recoveryNeedsOrganizer || _pendingRestoreRound == null) + { + _gameServer.Message( + MessageType.Chat, + $" {ChatColors.Red}No usable round backup. An organizer must run restore_round .", + player + ); + return; + } + + int round = _pendingRestoreRound.Value; + + if (IsResettingRound()) + { + return; + } + + if (player == null || isAdmin || IsRosterWhole(out _, out _)) + { + restoreRoundVote?.CancelVote(); + _forcedRestoreRound = round; + RequestPendingRestore(); + return; + } + + if (restoreRoundVote != null) + { + restoreRoundVote.CastVote(player, true); + return; + } + + restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; + + if (restoreRoundVote == null) + { + return; + } + + restoreRoundVote.StartVote( + $"Restore round {round} without waiting for everyone", + new Team[] { Team.CT, Team.T }, + () => + { + _logger.LogInformation("restore without full roster vote passed"); + restoreRoundVote = null; + _forcedRestoreRound = round; + RequestPendingRestore(); + }, + () => + { + _logger.LogInformation("restore without full roster vote failed"); + restoreRoundVote = null; + }, + true, + 30 ); - RequestRestoreBackupRound(highestNumber, null, true); + + restoreRoundVote?.CastVote(player, true); } - // Scans the game dir for CS2 round backup files ({prefix}_round.txt) and - // returns the highest round number present (0 if none). - private int GetHighestBackupRoundOnDisk(string csgoDir, string prefix) + // Offline matches have no backend to hold the rounds; the files CS2 wrote + // are all there is. Online they are never trusted past the backend: a + // file for a round whose score was never published would restore a round + // the backend has no record of. + private void LoadBackupRoundsFromDisk(MatchData match, MatchMap matchMap) { - int highest = 0; + string csgoDir = BackupDirectory; + string prefix = MatchUtility.GetSafeMatchPrefix(match); if (!Directory.Exists(csgoDir)) { - return highest; + return; } try @@ -210,24 +438,25 @@ private int GetHighestBackupRoundOnDisk(string csgoDir, string prefix) { string name = Path.GetFileNameWithoutExtension(file); int idx = name.LastIndexOf("_round", StringComparison.Ordinal); - if (idx < 0) + if ( + idx < 0 + || !int.TryParse(name.Substring(idx + "_round".Length), out int round) + || matchMap.rounds.Any(backupRound => backupRound.round == round) + ) { continue; } - string numberPart = name.Substring(idx + "_round".Length); - if (int.TryParse(numberPart, out int round) && round > highest) - { - highest = round; - } + matchMap.rounds = BackupRoundUtility.Upsert( + matchMap.rounds, + new BackupRound { round = round, backup_file = File.ReadAllText(file) } + ); } } catch (Exception ex) { _logger.LogError(ex, "Failed scanning for backup round files"); } - - return highest; } // Diagnostic: lists every backup file for this match's prefix with size and @@ -297,16 +526,16 @@ public void RequestRestoreBackupRound( return; } - BackupRound? backupRound = matchMap.rounds.FirstOrDefault( - (backupRound) => - { - return backupRound.round == round; - } - ); - - if (backupRound == null) + if ( + BackupRoundUtility.FindRestorable(matchMap.rounds, round, MinPlayersPerTeam()) == null + ) { - _logger.LogWarning($"missing backup round: {round}"); + _logger.LogWarning($"no usable backup for round: {round}"); + _gameServer.Message( + MessageType.Chat, + $" {ChatColors.Red}Round {round} has no usable backup.", + player + ); return; } @@ -314,8 +543,6 @@ public void RequestRestoreBackupRound( if (player != null || vote == true) { - _resetRound = round; - restoreRoundVote = _serviceProvider.GetRequiredService(typeof(VoteSystem)) as VoteSystem; @@ -324,14 +551,17 @@ public void RequestRestoreBackupRound( return; } + _resetRound = round; + restoreRoundVote.StartVote( _localizer["backup.vote.restore_to", round], new Team[] { Team.CT, Team.T }, () => { _logger.LogInformation("restore round vote passed"); - SendRestoreRoundToBackend(round); + restoreRoundVote = null; _resetRound = null; + SendRestoreRoundToBackend(round); }, () => { @@ -413,13 +643,31 @@ public void RequestRestoreBackupRound( backupRoundFile.Length ); + string? invalidReason = BackupRoundUtility.Validate( + backupRoundFile, + round, + MinPlayersPerTeam() + ); + + if (invalidReason != null) + { + _logger.LogCritical( + "Not publishing backup round {Round}: {Reason} ({Path})", + round, + invalidReason, + backupRoundFilePath + ); + return null; + } + MatchMap? currentMap = _matchService.GetCurrentMatch()?.GetCurrentMap(); if (currentMap != null) { - currentMap.rounds = currentMap - .rounds.Append(new BackupRound { round = round, backup_file = backupRoundFile }) - .ToArray(); + currentMap.rounds = BackupRoundUtility.Upsert( + currentMap.rounds, + new BackupRound { round = round, backup_file = backupRoundFile } + ); } return backupRoundFile; @@ -462,7 +710,7 @@ public void SendRestoreRoundToBackend(int round) ); } - public void RestoreRound(int round) + public void RestoreRound(int round, bool force = false) { if (IsResettingRound()) { @@ -470,15 +718,11 @@ public void RestoreRound(int round) return; } - if (!CanRestoreRound(round)) - { - return; - } - - MatchData? match = _matchService.GetCurrentMatch()?.GetMatchData(); - MatchMap? matchMap = _matchService.GetCurrentMatch()?.GetCurrentMap(); + MatchManager? matchManager = _matchService.GetCurrentMatch(); + MatchData? match = matchManager?.GetMatchData(); + MatchMap? matchMap = matchManager?.GetCurrentMap(); - if (match == null || matchMap == null) + if (matchManager == null || match == null || matchMap == null) { _logger.LogWarning( "RestoreRound({Round}) dropped: no current match/map (match={HasMatch} map={HasMap})", @@ -489,20 +733,44 @@ public void RestoreRound(int round) return; } - BackupRound? backupRound = matchMap.rounds.FirstOrDefault( - (backupRound) => - { - return backupRound.round == round; - } + if (round == 0) + { + RestartMap(matchMap); + return; + } + + BackupRound? backupRound = BackupRoundUtility.FindRestorable( + matchMap.rounds, + round, + MinPlayersPerTeam() ); if (backupRound == null) { _logger.LogWarning( - "missing backup round: {Round} (known rounds: [{Rounds}])", + "no usable backup for round: {Round} (known rounds: [{Rounds}])", round, string.Join(", ", matchMap.rounds.Select(r => r.round)) ); + _gameServer.Message( + MessageType.Alert, + $" Round {round} has no usable backup and was not restored." + ); + return; + } + + // Queued, never dropped: the backend has already voided the rounds + // past this one, so giving up here would strand the match between + // the two. + if (!force && _forcedRestoreRound != round && !IsRosterWhole(out _, out _)) + { + _logger.LogWarning($"Restore round {round} queued until the roster is whole"); + _recoveryNeedsOrganizer = false; + _pendingRestoreRound = round; + _restoreAttempts = 0; + _lastRestoreRequestAt = DateTime.UtcNow; + StartRecoveryTimer(); + RecoveryTick(); return; } @@ -513,7 +781,18 @@ public void RestoreRound(int round) backupRoundFileName ); + // Pending first: cancelling a vote runs its failure path, which + // resumes the match unless something is already holding it. + _pendingRestoreRound = round; + _recoveryNeedsOrganizer = false; + restoreRoundVote?.CancelVote(); + restoreRoundVote = null; + _resetRound = round; + _restoreAttempts++; + + _matchEvents.ClearPendingRoundResult(); + matchMap.rounds = BackupRoundUtility.DropAbove(matchMap.rounds, round); _logger.LogInformation($"Loading backup round file {backupRoundFileName}"); @@ -532,7 +811,7 @@ public void RestoreRound(int round) "Failed writing restore backup round file {File}", backupRoundFileName ); - _core.Scheduler.NextTick(() => _resetRound = null); + _core.Scheduler.NextTick(() => FinishRestore(round)); return; } @@ -543,35 +822,107 @@ public void RestoreRound(int round) ); _matchService.GetCurrentMatch()?.PauseMatch(); - TimerUtility.AddTimer( - 5, - () => - { - _resetRound = null; - - _logger.LogInformation($"Sending Message for Round {round}"); - - _gameServer.Message( - MessageType.Alert, - _localizer[ - "backup.round_restored", - ChatColors.Red, - round, - CommandUtility.PublicChatTrigger - ] - ); - } - ); + TimerUtility.AddTimer(5, () => FinishRestore(round)); }); }); } - private bool CanRestoreRound(int round) + // The restore is only over once CS2 is actually at that round. Anything + // else -- an unwritable file, a load CS2 refused -- leaves the recovery + // pending so it is retried rather than played through. + private void FinishRestore(int round) { - int connectedPlayers = MatchUtility.PlayerCount(); - int expectedPlayers = _matchService.GetCurrentMatch()?.GetExpectedPlayerCount() ?? 10; + _resetRound = null; + + MatchManager? match = _matchService.GetCurrentMatch(); + int totalRoundsPlayed = _gameServer.GetTotalRoundsPlayed(); + + if (totalRoundsPlayed != round) + { + _logger.LogCritical( + $"Restore of round {round} did not take: game is at round {totalRoundsPlayed} (attempt {_restoreAttempts}/{MaxRestoreAttempts})" + ); - if (connectedPlayers >= expectedPlayers) + if (_restoreAttempts >= MaxRestoreAttempts) + { + _pendingRestoreRound = null; + _recoveryNeedsOrganizer = true; + } + + _lastRestoreRequestAt = DateTime.UtcNow; + StartRecoveryTimer(); + return; + } + + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _restoreAttempts = 0; + _behindNeedsConfirmation = false; + _syncedMapId = match?.GetCurrentMap()?.id; + + // CS2 seats players from the file; anyone it could not place is put + // back where the lineups say they belong. + if (match != null) + { + foreach (IPlayer player in MatchUtility.Players()) + { + match.EnforceMemberTeam(player); + } + } + + _logger.LogInformation($"Sending Message for Round {round}"); + + _gameServer.Message( + MessageType.Alert, + _localizer[ + "backup.round_restored", + ChatColors.Red, + round, + CommandUtility.PublicChatTrigger + ] + ); + } + + // restore_round 0: the backend has voided every round, so the map starts + // over. The only way forward when no round has a usable backup. + private void RestartMap(MatchMap matchMap) + { + _logger.LogWarning("Restoring to round 0: restarting the map"); + + restoreRoundVote?.CancelVote(); + _matchEvents.ClearPendingRoundResult(); + matchMap.rounds = new BackupRound[0]; + + _pendingRestoreRound = null; + _forcedRestoreRound = null; + _recoveryNeedsOrganizer = false; + _behindNeedsConfirmation = false; + _restoreAttempts = 0; + _syncedMapId = matchMap.id; + + _gameServer.SendCommands(["mp_restartgame 1"]); + _matchService.GetCurrentMatch()?.PauseMatch(); + } + + // Casters and admins never make a match whole. Placeholder lineups have + // no steam ids to match against, so there a head count is the best there is. + private bool IsRosterWhole(out int connected, out int expected) + { + MatchManager? match = _matchService.GetCurrentMatch(); + MatchData? matchData = match?.GetMatchData(); + + expected = match?.GetExpectedPlayerCount() ?? 10; + connected = + matchData == null || MatchUtility.HasPlaceholderMembers(matchData) + ? MatchUtility.PlayerCount() + : MatchUtility.ConnectedRosterCount(matchData); + + return connected >= expected; + } + + private bool CanRestoreRound(int round) + { + if (IsRosterWhole(out int connectedPlayers, out int expectedPlayers)) { return true; } diff --git a/apps/swiftly/src/FiveStack.Services/MatchManager.cs b/apps/swiftly/src/FiveStack.Services/MatchManager.cs index 1d11647f..d6223e6a 100644 --- a/apps/swiftly/src/FiveStack.Services/MatchManager.cs +++ b/apps/swiftly/src/FiveStack.Services/MatchManager.cs @@ -273,7 +273,7 @@ public void PauseMatch( 3, () => { - if (!IsFreezePeriod() || !IsPaused() || _backUpManagement.IsResettingRound()) + if (!IsFreezePeriod() || !IsPaused() || _backUpManagement.BlocksPlay()) { return; } @@ -301,9 +301,9 @@ public void ResumeMatch(string? message = null, bool skipUpdate = false) return; } - if (_backUpManagement.IsResettingRound()) + if (_backUpManagement.BlocksPlay()) { - _logger.LogInformation("Resetting round, cannot resume match"); + _logger.LogInformation("Round restore pending or in progress, cannot resume match"); return; } @@ -375,11 +375,6 @@ public void UpdateMapStatus(eMapStatus status, Guid? winningLineupId = null) _logger.LogInformation($"Update Map Status {_currentMapStatus} -> {status}"); - if (_currentMapStatus == eMapStatus.Unknown) - { - _backUpManagement.CheckForBackupRestore(); - } - var currentMap = GetCurrentMap(); switch (status) @@ -625,6 +620,7 @@ public void SetupMatch(MatchData match) // going live) — a plugin reload of a live/paused match never hits // StartLive, so the backup convars would otherwise never be set. _backUpManagement.Setup(); + _backUpManagement.CheckForBackupRestore(); if (MatchUtility.MapStatusStringToEnum(_currentMap.status) != _currentMapStatus) { @@ -1448,6 +1444,7 @@ public void Reset() captainSystem.Reset(); knifeSystem.Reset(); _surrenderSystem.Reset(); + _backUpManagement.Reset(); } private void SetConVar(string name, string value) diff --git a/apps/swiftly/src/FiveStack.Services/TimeoutSystem.cs b/apps/swiftly/src/FiveStack.Services/TimeoutSystem.cs index d7650061..8b69e9ec 100644 --- a/apps/swiftly/src/FiveStack.Services/TimeoutSystem.cs +++ b/apps/swiftly/src/FiveStack.Services/TimeoutSystem.cs @@ -245,6 +245,15 @@ public void RequestResume(IPlayer? player) return; } + if (_backUpManagement.IsRecoveryPending()) + { + _backUpManagement.RequestRecoveryNow( + player, + player == null || IsAdminOrOrganizer(player, matchData) + ); + return; + } + string resumeMessage = _localizer["timeout.admin_resumed"]; // Refuse while a required camera is still down, on the same terms as the diff --git a/apps/swiftly/test/BackupRoundUtilityTests.cs b/apps/swiftly/test/BackupRoundUtilityTests.cs new file mode 100644 index 00000000..5ea2c7b5 --- /dev/null +++ b/apps/swiftly/test/BackupRoundUtilityTests.cs @@ -0,0 +1,163 @@ +using FiveStack.Entities; +using FiveStack.Utilities; +using Xunit; + +public class BackupRoundUtilityTests +{ + private static string Backup(int round, int team1Players, int team2Players) + { + string Players(int count, int offset) => + string.Join( + "\n", + Enumerable + .Range(0, count) + .Select(i => + $"\t\t\"{874739096 + offset + i}\"\n\t\t{{\n\t\t\t\"name\"\t\t\"p{{{i}}}\"\n\t\t\t\"cash\"\t\t\"800\"\n\t\t\t\"Items\"\n\t\t\t{{\n\t\t\t\t\"weapon_ak47\"\t\t\"1\"\n\t\t\t}}\n\t\t}}" + ) + ); + + return "\"SaveFile\"\n{\n" + + $"\t\"timestamp\"\t\t\"2026-09-19 13:41:02\"\n\t\"map\"\t\t\"de_inferno\"\n\t\"round\"\t\t\"{round}\"\n" + + "\t\"FirstHalfScore\"\n\t{\n\t\t\"team1\"\t\t\"0\"\n\t\t\"team2\"\t\t\"1\"\n\t}\n" + + "\t\"Timeouts\"\n\t{\n\t\t\"team1\"\t\t\"3\"\n\t\t\"technical\"\n\t\t{\n\t\t\t\"team1\"\t\t\"0\"\n\t\t}\n\t}\n" + + $"\t\"PlayersOnTeam1\"\n\t{{\n{Players(team1Players, 0)}\n\t}}\n" + + $"\t\"PlayersOnTeam2\"\n\t{{\n{Players(team2Players, 100)}\n\t}}\n" + + "}\n"; + } + + // What CS2 wrote on 2026-09-19 for a round that ended with nobody seated: + // well-formed, right round number, and no player sections at all. + private const string PlayerlessBackup = + "\"SaveFile\"\n{\n\t\"timestamp\"\t\t\"2026-09-19 13:49:42\"\n\t\"map\"\t\t\"de_inferno\"\n\t\"round\"\t\t\"1\"\n" + + "\t\"FirstHalfScore\"\n\t{\n\t\t\"team1\"\t\t\"1\"\n\t\t\"team2\"\t\t\"0\"\n\t}\n" + + "\t\"Timeouts\"\n\t{\n\t\t\"team1\"\t\t\"3\"\n\t\t\"team2\"\t\t\"3\"\n\t\t\"technical\"\n\t\t{\n\t\t\t\"team1\"\t\t\"0\"\n\t\t\t\"team2\"\t\t\"0\"\n\t\t}\n\t}\n}\n"; + + [Theory] + [InlineData(10, 3)] + [InlineData(4, 1)] + [InlineData(2, 1)] + public void MinPlayersPerTeam_IsAMajorityOfOneSide(int expectedPlayers, int expected) + { + Assert.Equal(expected, BackupRoundUtility.MinPlayersPerTeam(expectedPlayers)); + } + + [Fact] + public void Validate_AcceptsAFullBackup() + { + Assert.Null(BackupRoundUtility.Validate(Backup(7, 5, 5), 7, 3)); + } + + [Fact] + public void Validate_RejectsABackupWithNoPlayers() + { + Assert.NotNull(BackupRoundUtility.Validate(PlayerlessBackup, 1, 3)); + } + + [Fact] + public void Validate_RejectsAOneSidedBackup() + { + Assert.NotNull(BackupRoundUtility.Validate(Backup(1, 5, 1), 1, 3)); + } + + [Fact] + public void Validate_RejectsTheWrongRound() + { + Assert.NotNull(BackupRoundUtility.Validate(Backup(3, 5, 5), 4, 3)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("not a backup")] + public void Validate_RejectsGarbage(string? backupFile) + { + Assert.NotNull(BackupRoundUtility.Validate(backupFile, 1, 3)); + } + + [Fact] + public void Validate_DoesNotCountNestedSectionsAsPlayers() + { + // Each player carries an Items block; 2 players must not read as 4. + Assert.NotNull(BackupRoundUtility.Validate(Backup(1, 2, 2), 1, 3)); + } + + [Fact] + public void FindRestorable_PrefersTheNewestEntryForARound() + { + BackupRound stale = new BackupRound { round = 2, backup_file = Backup(2, 5, 5) }; + BackupRound replayed = new BackupRound { round = 2, backup_file = Backup(2, 5, 5) }; + + Assert.Same( + replayed, + BackupRoundUtility.FindRestorable(new[] { stale, replayed }, 2, 3) + ); + } + + [Fact] + public void FindRestorable_SkipsDeletedAndInvalidRounds() + { + BackupRound[] rounds = + { + new BackupRound + { + round = 1, + backup_file = Backup(1, 5, 5), + deleted_at = "2026-09-19T13:53:00Z", + }, + new BackupRound { round = 2, backup_file = PlayerlessBackup }, + }; + + Assert.Null(BackupRoundUtility.FindRestorable(rounds, 1, 3)); + Assert.Null(BackupRoundUtility.FindRestorable(rounds, 2, 3)); + } + + [Fact] + public void HighestRestorableRound_FallsBackPastAnUnusableLatestRound() + { + BackupRound[] rounds = + { + new BackupRound { round = 1, backup_file = Backup(1, 5, 5) }, + new BackupRound { round = 2, backup_file = Backup(2, 5, 5) }, + new BackupRound { round = 3, backup_file = "" }, + }; + + Assert.Equal(3, BackupRoundUtility.HighestRound(rounds)); + Assert.Equal(2, BackupRoundUtility.HighestRestorableRound(rounds, 3)); + } + + [Fact] + public void HighestRound_IsZeroWithNothingRecorded() + { + Assert.Equal(0, BackupRoundUtility.HighestRound(new BackupRound[0])); + Assert.Equal(0, BackupRoundUtility.HighestRestorableRound(new BackupRound[0], 3)); + } + + [Fact] + public void Upsert_ReplacesRatherThanDuplicates() + { + BackupRound[] rounds = + { + new BackupRound { round = 1, backup_file = "old" }, + new BackupRound { round = 2, backup_file = "keep" }, + }; + + BackupRound[] updated = BackupRoundUtility.Upsert( + rounds, + new BackupRound { round = 1, backup_file = "new" } + ); + + Assert.Equal(2, updated.Length); + Assert.Equal("new", updated.Single(backupRound => backupRound.round == 1).backup_file); + } + + [Fact] + public void DropAbove_VoidsRoundsPastTheRestorePoint() + { + BackupRound[] rounds = Enumerable + .Range(1, 6) + .Select(round => new BackupRound { round = round }) + .ToArray(); + + Assert.Equal(new[] { 1, 2, 3 }, BackupRoundUtility.DropAbove(rounds, 3).Select(r => r.round)); + } +} diff --git a/shared/dotnet/FiveStack.Utilities/BackupRoundUtility.cs b/shared/dotnet/FiveStack.Utilities/BackupRoundUtility.cs new file mode 100644 index 00000000..eafa5af5 --- /dev/null +++ b/shared/dotnet/FiveStack.Utilities/BackupRoundUtility.cs @@ -0,0 +1,224 @@ +using FiveStack.Entities; + +namespace FiveStack.Utilities +{ + public static class BackupRoundUtility + { + // A backup only counts if CS2 can seat a team from it. A round that + // ended on a near-empty server still writes a well-formed file, just + // with no players in it -- restoring that leaves everyone unassigned. + public static int MinPlayersPerTeam(int expectedPlayers) + { + return Math.Max(1, (expectedPlayers / 2 + 1) / 2); + } + + // Null when the file is restorable, otherwise why it is not. + public static string? Validate(string? backupFile, int expectedRound, int minPlayersPerTeam) + { + if (string.IsNullOrWhiteSpace(backupFile)) + { + return "empty"; + } + + List tokens = Tokenize(backupFile); + + if (tokens.Count < 2 || tokens[0] != "SaveFile" || tokens[1] != "{") + { + return "not a SaveFile"; + } + + int? round = null; + Dictionary teamPlayers = new Dictionary(); + + int depth = 0; + for (int i = 1; i < tokens.Count; i++) + { + string token = tokens[i]; + + if (token == "{") + { + depth++; + continue; + } + + if (token == "}") + { + depth--; + continue; + } + + if (depth != 1 || i + 1 >= tokens.Count) + { + continue; + } + + if (token == "round" && tokens[i + 1] != "{") + { + if (int.TryParse(tokens[i + 1], out int parsed)) + { + round = parsed; + } + i++; + continue; + } + + if (token.StartsWith("PlayersOnTeam") && tokens[i + 1] == "{") + { + teamPlayers[token] = CountSections(tokens, i + 1); + } + } + + if (round == null) + { + return "missing round"; + } + + if (round != expectedRound) + { + return $"round {round} does not match expected round {expectedRound}"; + } + + foreach (string team in new[] { "PlayersOnTeam1", "PlayersOnTeam2" }) + { + int players = teamPlayers.GetValueOrDefault(team, 0); + if (players < minPlayersPerTeam) + { + return $"{team} has {players} player(s), need at least {minPlayersPerTeam}"; + } + } + + return null; + } + + public static bool IsDeleted(BackupRound backupRound) + { + return !string.IsNullOrEmpty(backupRound.deleted_at); + } + + // Last entry wins: a replayed round is newer than the one it replaced. + public static BackupRound? FindRestorable( + IEnumerable rounds, + int round, + int minPlayersPerTeam + ) + { + return rounds.LastOrDefault(backupRound => + backupRound.round == round + && !IsDeleted(backupRound) + && Validate(backupRound.backup_file, round, minPlayersPerTeam) == null + ); + } + + public static int HighestRound(IEnumerable rounds) + { + return rounds + .Where(backupRound => !IsDeleted(backupRound)) + .Select(backupRound => backupRound.round) + .DefaultIfEmpty(0) + .Max(); + } + + public static int HighestRestorableRound( + IEnumerable rounds, + int minPlayersPerTeam + ) + { + return rounds + .Where(backupRound => + !IsDeleted(backupRound) + && Validate(backupRound.backup_file, backupRound.round, minPlayersPerTeam) + == null + ) + .Select(backupRound => backupRound.round) + .DefaultIfEmpty(0) + .Max(); + } + + public static BackupRound[] Upsert(BackupRound[] rounds, BackupRound backupRound) + { + return rounds + .Where(existing => existing.round != backupRound.round) + .Append(backupRound) + .ToArray(); + } + + // Rounds past a restore point are void the moment the restore runs, + // whether or not the backend has been heard from yet. + public static BackupRound[] DropAbove(BackupRound[] rounds, int round) + { + return rounds.Where(existing => existing.round <= round).ToArray(); + } + + private static int CountSections(List tokens, int openIndex) + { + int count = 0; + int depth = 0; + + for (int i = openIndex; i < tokens.Count; i++) + { + if (tokens[i] == "{") + { + depth++; + if (depth == 2) + { + count++; + } + } + else if (tokens[i] == "}") + { + depth--; + if (depth == 0) + { + break; + } + } + } + + return count; + } + + // Braces inside a quoted value (a player name) are text, not structure. + private static List Tokenize(string text) + { + List tokens = new List(); + + int i = 0; + while (i < text.Length) + { + char c = text[i]; + + if (c == '{' || c == '}') + { + tokens.Add(c.ToString()); + i++; + continue; + } + + if (c != '"') + { + i++; + continue; + } + + System.Text.StringBuilder value = new System.Text.StringBuilder(); + i++; + while (i < text.Length && text[i] != '"') + { + if (text[i] == '\\' && i + 1 < text.Length) + { + i++; + } + value.Append(text[i]); + i++; + } + i++; + + // A quoted brace must not read as structure. + string token = value.ToString(); + tokens.Add(token == "{" || token == "}" ? $"\"{token}\"" : token); + } + + return tokens; + } + } +}