diff --git a/apps/frontend/app/status/page.tsx b/apps/frontend/app/status/page.tsx index 0dee11a..067908a 100644 --- a/apps/frontend/app/status/page.tsx +++ b/apps/frontend/app/status/page.tsx @@ -5,9 +5,17 @@ import { getLastGameTypeCountDate, getLastMapCountDate, getLastPollMasterServerDate, + getLastArchiveSnapshotsDate, + getArchiveSnapshotsFailedCount, + SNAPSHOT_RETENTION_HOURS, } from '@teerank/teerank'; import prisma from '../../utils/prisma'; -import { formatDistanceToNow, subMinutes } from 'date-fns'; +import { + formatDistanceStrict, + formatDistanceToNow, + subHours, + subMinutes, +} from 'date-fns'; export const metadata = { title: 'Status - Teerank', @@ -27,6 +35,9 @@ export default async function Index() { lastRankedSnapshotDate, lastGameTypeCountDate, lastMapCountDate, + lastArchiveSnapshotsDate, + archiveSnapshotsFailedCount, + oldestSnapshot, masterServers, unreferencedGameServersCount, ] = await Promise.all([ @@ -36,6 +47,16 @@ export default async function Index() { getLastRankPlayerDate(), getLastGameTypeCountDate(), getLastMapCountDate(), + getLastArchiveSnapshotsDate(), + getArchiveSnapshotsFailedCount(), + prisma.gameServerSnapshot.findFirst({ + orderBy: { + id: 'asc', + }, + select: { + createdAt: true, + }, + }), prisma.masterServer.findMany({ select: { address: true, @@ -67,36 +88,54 @@ export default async function Index() { { title: 'Polling Master Servers', date: lastPollMasterServerDate, + staleAfterMinutes: 10, }, { title: 'Polling Game Servers', date: lastPollGameServerDate, + staleAfterMinutes: 10, }, { title: 'Ranking', date: lastRankedSnapshotDate, + staleAfterMinutes: 10, }, { title: 'Playtiming', date: lastPlayTimedSnapshotDate, + staleAfterMinutes: 10, }, { title: 'Game type count', date: lastGameTypeCountDate, + staleAfterMinutes: 10, }, { title: 'Map count', date: lastMapCountDate, + staleAfterMinutes: 10, + }, + { + title: 'Archiving snapshots', + date: lastArchiveSnapshotsDate, + staleAfterMinutes: 30, }, ]; + const retentionCutoff = subHours(new Date(), SNAPSHOT_RETENTION_HOURS); + const archiveBacklog = + oldestSnapshot !== null && oldestSnapshot.createdAt < retentionCutoff + ? formatDistanceStrict(oldestSnapshot.createdAt, retentionCutoff) + : null; + return (

Teerank

{sections.map((section) => { const isOk = - section.date !== null && section.date > subMinutes(new Date(), 10); + section.date !== null && + section.date > subMinutes(new Date(), section.staleAfterMinutes); return (
@@ -121,6 +160,59 @@ export default async function Index() { })}
+

Archiving

+
+
+ Retention +
+ + {SNAPSHOT_RETENTION_HOURS} hours + +
+
+ +
+ Oldest snapshot +
+ + {oldestSnapshot === null + ? 'None' + : formatDistanceToNow(oldestSnapshot.createdAt, { + addSuffix: true, + })} + +
+
+ +
+ Backlog +
+ {archiveBacklog !== null && ( + + {archiveBacklog} past retention + + )} + {archiveBacklog === null ? ( + Up to date + ) : ( + Late + )} +
+
+ +
+ Failed jobs +
+ + {archiveSnapshotsFailedCount} + + {archiveSnapshotsFailedCount > 0 && ( + Failing + )} +
+
+
+

Teeworlds

{masterServers.map((masterServer) => ( diff --git a/apps/worker/src/workers/archiveSnapshots.ts b/apps/worker/src/workers/archiveSnapshots.ts index 60a866d..6a948ce 100644 --- a/apps/worker/src/workers/archiveSnapshots.ts +++ b/apps/worker/src/workers/archiveSnapshots.ts @@ -2,6 +2,7 @@ import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; import { ArchiveSnapshotsJobData, S3_BUCKET, + SNAPSHOT_RETENTION_HOURS, getEnvInt, getS3Client, processArchiveSnapshotsJobs, @@ -10,7 +11,6 @@ import { import { prisma } from "../prisma"; import { SnapshotArchiveRow, encodeSnapshotRowsToParquet } from "../parquet"; -const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48); const ARCHIVE_BATCH_SIZE = getEnvInt('ARCHIVE_BATCH_SIZE', 5000); const ARCHIVE_TIME_BUDGET_MS = getEnvInt('ARCHIVE_TIME_BUDGET_MS', 5 * 60 * 1000); const ARCHIVE_BATCH_PAUSE_MS = getEnvInt('ARCHIVE_BATCH_PAUSE_MS', 200); diff --git a/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts b/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts index e1f920e..3a60d57 100644 --- a/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts +++ b/libs/teerank/src/lib/bullmq/queueArchiveSnapshots.ts @@ -1,7 +1,7 @@ import { Job, Queue, Worker } from "bullmq"; import { bullmqConnection, lastCompletedJobDate } from "./config"; import { z } from "zod"; -import { minutesToSeconds } from "date-fns"; +import { hoursToSeconds } from "date-fns"; let archiveSnapshotsQueue: Queue | null = null; @@ -35,7 +35,7 @@ export async function processArchiveSnapshotsJobs(processor: (data: ArchiveSnaps connection: bullmqConnection, concurrency: 1, removeOnComplete: { - age: minutesToSeconds(10), + age: hoursToSeconds(6), }, removeOnFail: { count: 1000, @@ -53,4 +53,8 @@ export async function getLastArchiveSnapshotsDate() { return lastCompletedJobDate(getQueueArchiveSnapshots()); } +export async function getArchiveSnapshotsFailedCount() { + return getQueueArchiveSnapshots().getFailedCount(); +} + export { getQueueArchiveSnapshots }; diff --git a/libs/teerank/src/lib/storage.ts b/libs/teerank/src/lib/storage.ts index 143191b..e28d67c 100644 --- a/libs/teerank/src/lib/storage.ts +++ b/libs/teerank/src/lib/storage.ts @@ -1,7 +1,8 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { getEnv } from "./utils"; +import { getEnv, getEnvInt } from "./utils"; export const S3_BUCKET = getEnv('S3_BUCKET', 'teerank-snapshots'); +export const SNAPSHOT_RETENTION_HOURS = getEnvInt('SNAPSHOT_RETENTION_HOURS', 48); let s3Client: S3Client | null = null;