Show the snapshot archiving process on the status page

apps/frontend/app/status/page.tsx

@@ -5,9 +5,17 @@ import {
   getLastGameTypeCountDate,
   getLastMapCountDate,
   getLastPollMasterServerDate,
+  getLastArchiveSnapshotsDate,
+  getArchiveSnapshotsFailedCount,
+  SNAPSHOT_RETENTION_HOURS,
 } from '@teerank/teerank';
 import prisma from '../../utils/prisma';
-import { formatDistanceToNow, subMinutes } from 'date-fns';
+import {
+  formatDistanceStrict,
+  formatDistanceToNow,
+  subHours,
+  subMinutes,
+} from 'date-fns';
 
 export const metadata = {
   title: 'Status - Teerank',
@@ -27,6 +35,9 @@ export default async function Index() {
     lastRankedSnapshotDate,
     lastGameTypeCountDate,
     lastMapCountDate,
+    lastArchiveSnapshotsDate,
+    archiveSnapshotsFailedCount,
+    oldestSnapshot,
     masterServers,
     unreferencedGameServersCount,
   ] = await Promise.all([
@@ -36,6 +47,16 @@ export default async function Index() {
     getLastRankPlayerDate(),
     getLastGameTypeCountDate(),
     getLastMapCountDate(),
+    getLastArchiveSnapshotsDate(),
+    getArchiveSnapshotsFailedCount(),
+    prisma.gameServerSnapshot.findFirst({
+      orderBy: {
+        id: 'asc',
+      },
+      select: {
+        createdAt: true,
+      },
+    }),
     prisma.masterServer.findMany({
       select: {
         address: true,
@@ -67,36 +88,54 @@ export default async function Index() {
     {
       title: 'Polling Master Servers',
       date: lastPollMasterServerDate,
+      staleAfterMinutes: 10,
     },
     {
       title: 'Polling Game Servers',
       date: lastPollGameServerDate,
+      staleAfterMinutes: 10,
     },
     {
       title: 'Ranking',
       date: lastRankedSnapshotDate,
+      staleAfterMinutes: 10,
     },
     {
       title: 'Playtiming',
       date: lastPlayTimedSnapshotDate,
+      staleAfterMinutes: 10,
     },
     {
       title: 'Game type count',
       date: lastGameTypeCountDate,
+      staleAfterMinutes: 10,
     },
     {
       title: 'Map count',
       date: lastMapCountDate,
+      staleAfterMinutes: 10,
+    },
+    {
+      title: 'Archiving snapshots',
+      date: lastArchiveSnapshotsDate,
+      staleAfterMinutes: 30,
     },
   ];
 
+  const retentionCutoff = subHours(new Date(), SNAPSHOT_RETENTION_HOURS);
+  const archiveBacklog =
+    oldestSnapshot !== null && oldestSnapshot.createdAt < retentionCutoff
+      ? formatDistanceStrict(oldestSnapshot.createdAt, retentionCutoff)
+      : null;
+
   return (
     <main className="py-12 px-4 md:px-12 xl:px-20 text-[#666] flex flex-col gap-4">
       <h1 className="text-2xl font-bold clear-both">Teerank</h1>
       <div className="flex flex-col divide-y">
         {sections.map((section) => {
           const isOk =
-            section.date !== null && section.date > subMinutes(new Date(), 10);
+            section.date !== null &&
+            section.date > subMinutes(new Date(), section.staleAfterMinutes);
 
           return (
             <div key={section.title} className="flex flex-row items-center p-2">
@@ -121,6 +160,59 @@ export default async function Index() {
         })}
       </div>
 
+      <h1 className="text-2xl font-bold clear-both">Archiving</h1>
+      <div className="flex flex-col divide-y">
+        <div className="flex flex-row items-center p-2">
+          <span className="grow px-4">Retention</span>
+          <div className="flex flex-row divide-x">
+            <span className="text-sm text-[#aaa] px-4">
+              {SNAPSHOT_RETENTION_HOURS} hours
+            </span>
+          </div>
+        </div>
+
+        <div className="flex flex-row items-center p-2">
+          <span className="grow px-4">Oldest snapshot</span>
+          <div className="flex flex-row divide-x">
+            <span className="text-sm text-[#aaa] px-4">
+              {oldestSnapshot === null
+                ? 'None'
+                : formatDistanceToNow(oldestSnapshot.createdAt, {
+                    addSuffix: true,
+                  })}
+            </span>
+          </div>
+        </div>
+
+        <div className="flex flex-row items-center p-2">
+          <span className="grow px-4">Backlog</span>
+          <div className="flex flex-row divide-x">
+            {archiveBacklog !== null && (
+              <span className="text-sm text-[#aaa] px-4">
+                {archiveBacklog} past retention
+              </span>
+            )}
+            {archiveBacklog === null ? (
+              <span className="font-bold text-[#5a8d39] px-4">Up to date</span>
+            ) : (
+              <span className="font-bold text-[#b05656] px-4">Late</span>
+            )}
+          </div>
+        </div>
+
+        <div className="flex flex-row items-center p-2">
+          <span className="grow px-4">Failed jobs</span>
+          <div className="flex flex-row divide-x">
+            <span className="text-sm text-[#aaa] px-4">
+              {archiveSnapshotsFailedCount}
+            </span>
+            {archiveSnapshotsFailedCount > 0 && (
+              <span className="font-bold text-[#b05656] px-4">Failing</span>
+            )}
+          </div>
+        </div>
+      </div>
+
       <h1 className="text-2xl font-bold clear-both">Teeworlds</h1>
       <div className="flex flex-col divide-y">
         {masterServers.map((masterServer) => (

apps/worker/src/workers/archiveSnapshots.ts

@@ -2,6 +2,7 @@ import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
 import {
   ArchiveSnapshotsJobData,
   S3_BUCKET,
+  SNAPSHOT_RETENTION_HOURS,
   getEnvInt,
   getS3Client,
   processArchiveSnapshotsJobs,
@@ -10,7 +11,6 @@ import {
 import { prisma } from "../prisma";
 import { SnapshotArchiveRow, encodeSnapshotRowsToParquet } from "../parquet";
 
-const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48);
 const ARCHIVE_BATCH_SIZE = getEnvInt('ARCHIVE_BATCH_SIZE', 5000);
 const ARCHIVE_TIME_BUDGET_MS = getEnvInt('ARCHIVE_TIME_BUDGET_MS', 5 * 60 * 1000);
 const ARCHIVE_BATCH_PAUSE_MS = getEnvInt('ARCHIVE_BATCH_PAUSE_MS', 200);

libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts

@@ -1,7 +1,7 @@
 import { Job, Queue, Worker } from "bullmq";
 import { bullmqConnection, lastCompletedJobDate } from "./config";
 import { z } from "zod";
-import { minutesToSeconds } from "date-fns";
+import { hoursToSeconds } from "date-fns";
 
 let archiveSnapshotsQueue: Queue | null = null;
 
@@ -35,7 +35,7 @@ export async function processArchiveSnapshotsJobs(processor: (data: ArchiveSnaps
     connection: bullmqConnection,
     concurrency: 1,
     removeOnComplete: {
-      age: minutesToSeconds(10),
+      age: hoursToSeconds(6),
     },
     removeOnFail: {
       count: 1000,
@@ -53,4 +53,8 @@ export async function getLastArchiveSnapshotsDate() {
   return lastCompletedJobDate(getQueueArchiveSnapshots());
 }
 
+export async function getArchiveSnapshotsFailedCount() {
+  return getQueueArchiveSnapshots().getFailedCount();
+}
+
 export { getQueueArchiveSnapshots };

libs/teerank/src/lib/storage.ts

@@ -1,7 +1,8 @@
 import { S3Client } from "@aws-sdk/client-s3";
-import { getEnv } from "./utils";
+import { getEnv, getEnvInt } from "./utils";
 
 export const S3_BUCKET = getEnv('S3_BUCKET', 'teerank-snapshots');
+export const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48);
 
 let s3Client: S3Client | null = null;