From af8d20c32cdc54cfb919e3de86cfbc98b012dc0f Mon Sep 17 00:00:00 2001 From: needs <624097+needs@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:04:18 +0200 Subject: [PATCH] Show the snapshot archiving process on the status page The status page listed every background job except the archiver, so the only way to tell whether snapshots were still draining to R2 was to open bullboard or query the database by hand. That gap mattered during the failures documented in 0b44cf7, where archive jobs were dying on connection loss and nothing surfaced it. Adds an "Archiving snapshots" row to the job list, plus an Archiving section with the retention window, the age of the oldest snapshot still in Postgres, whether that snapshot is past retention, and the queue's failed job count. Oldest snapshot is read with ORDER BY id ASC LIMIT 1, a primary key lookup. Counting rows past the retention cutoff would be a sequential scan, since GameServerSnapshot has no index on createdAt, and it would be slowest exactly when the table is backlogged and the page is being looked at. Oldest-snapshot age carries the same signal. Two supporting changes: - SNAPSHOT_RETENTION_HOURS moves from the worker to libs/teerank so the frontend renders the same number the worker enforces. Neither container sets it, so both still default to 48, but an override now has to be set on both. - removeOnComplete.age on the archive queue goes 10 minutes -> 6 hours. The job is scheduled every 10 minutes, so the completion record expired at roughly the schedule cadence and the last run would intermittently disappear, reading as Down while the archiver was healthy. The archiving row uses a 30 minute staleness threshold rather than the 10 minutes the other jobs use, because the job is scheduled every 10 minutes and is allowed to run for 5. Co-authored-by: Claude Opus 5 (1M context) --- apps/frontend/app/status/page.tsx | 96 ++++++++++++++++++- apps/worker/src/workers/archiveSnapshots.ts | 2 +- .../src/lib/bullmq/queueArchiveSnapshots.ts | 8 +- libs/teerank/src/lib/storage.ts | 3 +- 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/apps/frontend/app/status/page.tsx b/apps/frontend/app/status/page.tsx index 0dee11a..067908a 100644 --- a/apps/frontend/app/status/page.tsx +++ b/apps/frontend/app/status/page.tsx @@ -5,9 +5,17 @@ import { getLastGameTypeCountDate, getLastMapCountDate, getLastPollMasterServerDate, + getLastArchiveSnapshotsDate, + getArchiveSnapshotsFailedCount, + SNAPSHOT_RETENTION_HOURS, } from '@teerank/teerank'; import prisma from '../../utils/prisma'; -import { formatDistanceToNow, subMinutes } from 'date-fns'; +import { + formatDistanceStrict, + formatDistanceToNow, + subHours, + subMinutes, +} from 'date-fns'; export const metadata = { title: 'Status - Teerank', @@ -27,6 +35,9 @@ export default async function Index() { lastRankedSnapshotDate, lastGameTypeCountDate, lastMapCountDate, + lastArchiveSnapshotsDate, + archiveSnapshotsFailedCount, + oldestSnapshot, masterServers, unreferencedGameServersCount, ] = await Promise.all([ @@ -36,6 +47,16 @@ export default async function Index() { getLastRankPlayerDate(), getLastGameTypeCountDate(), getLastMapCountDate(), + getLastArchiveSnapshotsDate(), + getArchiveSnapshotsFailedCount(), + prisma.gameServerSnapshot.findFirst({ + orderBy: { + id: 'asc', + }, + select: { + createdAt: true, + }, + }), prisma.masterServer.findMany({ select: { address: true, @@ -67,36 +88,54 @@ export default async function Index() { { title: 'Polling Master Servers', date: lastPollMasterServerDate, + staleAfterMinutes: 10, }, { title: 'Polling Game Servers', date: lastPollGameServerDate, + staleAfterMinutes: 10, }, { title: 'Ranking', date: lastRankedSnapshotDate, + staleAfterMinutes: 10, }, { title: 'Playtiming', date: lastPlayTimedSnapshotDate, + staleAfterMinutes: 10, }, { title: 'Game type count', date: lastGameTypeCountDate, + staleAfterMinutes: 10, }, { title: 'Map count', date: lastMapCountDate, + staleAfterMinutes: 10, + }, + { + title: 'Archiving snapshots', + date: lastArchiveSnapshotsDate, + staleAfterMinutes: 30, }, ]; + const retentionCutoff = subHours(new Date(), SNAPSHOT_RETENTION_HOURS); + const archiveBacklog = + oldestSnapshot !== null && oldestSnapshot.createdAt < retentionCutoff + ? formatDistanceStrict(oldestSnapshot.createdAt, retentionCutoff) + : null; + return (

Teerank

{sections.map((section) => { const isOk = - section.date !== null && section.date > subMinutes(new Date(), 10); + section.date !== null && + section.date > subMinutes(new Date(), section.staleAfterMinutes); return (
@@ -121,6 +160,59 @@ export default async function Index() { })}
+

Archiving

+
+
+ Retention +
+ + {SNAPSHOT_RETENTION_HOURS} hours + +
+
+ +
+ Oldest snapshot +
+ + {oldestSnapshot === null + ? 'None' + : formatDistanceToNow(oldestSnapshot.createdAt, { + addSuffix: true, + })} + +
+
+ +
+ Backlog +
+ {archiveBacklog !== null && ( + + {archiveBacklog} past retention + + )} + {archiveBacklog === null ? ( + Up to date + ) : ( + Late + )} +
+
+ +
+ Failed jobs +
+ + {archiveSnapshotsFailedCount} + + {archiveSnapshotsFailedCount > 0 && ( + Failing + )} +
+
+
+

Teeworlds

{masterServers.map((masterServer) => ( diff --git a/apps/worker/src/workers/archiveSnapshots.ts b/apps/worker/src/workers/archiveSnapshots.ts index 60a866d..6a948ce 100644 --- a/apps/worker/src/workers/archiveSnapshots.ts +++ b/apps/worker/src/workers/archiveSnapshots.ts @@ -2,6 +2,7 @@ import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; import { ArchiveSnapshotsJobData, S3_BUCKET, + SNAPSHOT_RETENTION_HOURS, getEnvInt, getS3Client, processArchiveSnapshotsJobs, @@ -10,7 +11,6 @@ import { import { prisma } from "../prisma"; import { SnapshotArchiveRow, encodeSnapshotRowsToParquet } from "../parquet"; -const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48); const ARCHIVE_BATCH_SIZE = getEnvInt('ARCHIVE_BATCH_SIZE', 5000); const ARCHIVE_TIME_BUDGET_MS = getEnvInt('ARCHIVE_TIME_BUDGET_MS', 5 * 60 * 1000); const ARCHIVE_BATCH_PAUSE_MS = getEnvInt('ARCHIVE_BATCH_PAUSE_MS', 200); diff --git a/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts b/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts index e1f920e..3a60d57 100644 --- a/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts +++ b/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts @@ -1,7 +1,7 @@ import { Job, Queue, Worker } from "bullmq"; import { bullmqConnection, lastCompletedJobDate } from "./config"; import { z } from "zod"; -import { minutesToSeconds } from "date-fns"; +import { hoursToSeconds } from "date-fns"; let archiveSnapshotsQueue: Queue | null = null; @@ -35,7 +35,7 @@ export async function processArchiveSnapshotsJobs(processor: (data: ArchiveSnaps connection: bullmqConnection, concurrency: 1, removeOnComplete: { - age: minutesToSeconds(10), + age: hoursToSeconds(6), }, removeOnFail: { count: 1000, @@ -53,4 +53,8 @@ export async function getLastArchiveSnapshotsDate() { return lastCompletedJobDate(getQueueArchiveSnapshots()); } +export async function getArchiveSnapshotsFailedCount() { + return getQueueArchiveSnapshots().getFailedCount(); +} + export { getQueueArchiveSnapshots }; diff --git a/libs/teerank/src/lib/storage.ts b/libs/teerank/src/lib/storage.ts index 143191b..e28d67c 100644 --- a/libs/teerank/src/lib/storage.ts +++ b/libs/teerank/src/lib/storage.ts @@ -1,7 +1,8 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { getEnv } from "./utils"; +import { getEnv, getEnvInt } from "./utils"; export const S3_BUCKET = getEnv('S3_BUCKET', 'teerank-snapshots'); +export const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48); let s3Client: S3Client | null = null;