From 9ee4c974a00164be069849c6073e9515beee65ba Mon Sep 17 00:00:00 2001 From: needs <624097+needs@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:20:25 +0200 Subject: [PATCH] Enable map counting with two grouped SQL statements Replace the never-enabled per-map count jobs (one BullMQ job per map, three queries each) with a single map-count job running one grouped UPDATE. The full recompute of playerCount/clanCount/gameServerCount scans PlayerInfoMap and ClanInfoMap once (~160s in production) and runs daily; a gameServerCount-only refresh (67ms) runs hourly. Both skip rows whose counts are unchanged. Co-Authored-By: Claude Fable 5 --- apps/scheduler/src/main.ts | 2 + apps/scheduler/src/schedulers/mapScheduler.ts | 43 +++--------------- apps/worker/src/workers/updateMapsCounts.ts | 45 +++---------------- libs/prisma/prisma/sql/updateMapsCounts.sql | 18 ++++++++ .../prisma/sql/updateMapsGameServerCounts.sql | 11 +++++ libs/teerank/src/lib/bullmq/queueMapCount.ts | 12 ++--- 6 files changed, 48 insertions(+), 83 deletions(-) create mode 100644 libs/prisma/prisma/sql/updateMapsCounts.sql create mode 100644 libs/prisma/prisma/sql/updateMapsGameServerCounts.sql diff --git a/apps/scheduler/src/main.ts b/apps/scheduler/src/main.ts index 321d83c..fe15433 100644 --- a/apps/scheduler/src/main.ts +++ b/apps/scheduler/src/main.ts @@ -1,6 +1,7 @@ import { masterServerScheduler } from './schedulers/masterServerScheduler'; import { gameServerScheduler } from './schedulers/gameServerScheduler'; import { gameTypeScheduler } from './schedulers/gameTypeScheduler'; +import { mapScheduler } from './schedulers/mapScheduler'; import { addDefaultGameTypes } from './addDefaultGameTypes'; import { addDefaultMasterServers } from './addDefaultMasterServers'; import { fillClanActivePlayerCountScheduler } from './schedulers/fillClanActivePlayerCountScheduler'; @@ -20,6 +21,7 @@ async function main() { masterServerScheduler(); gameServerScheduler(); gameTypeScheduler(); + mapScheduler(); fillClanActivePlayerCountScheduler(); updateGlobalCountsScheduler(); archiveSnapshotsScheduler(); diff --git a/apps/scheduler/src/schedulers/mapScheduler.ts b/apps/scheduler/src/schedulers/mapScheduler.ts index 34a4cee..b380846 100644 --- a/apps/scheduler/src/schedulers/mapScheduler.ts +++ b/apps/scheduler/src/schedulers/mapScheduler.ts @@ -1,42 +1,13 @@ import { scheduleMapCount } from "@teerank/teerank"; -import { hoursToMilliseconds, minutesToMilliseconds } from "date-fns"; -import { prisma } from "../prisma"; -import { schedule, scheduleWithSpread } from "../utils"; - -let lastId = 0; +import { hoursToMilliseconds } from "date-fns"; +import { schedule } from "../utils"; export async function mapScheduler() { - schedule(minutesToMilliseconds(5), async () => { - const maps = await prisma.map.findMany({ - where: { - id: { - gt: lastId, - }, - }, - select: { - gameTypeName: true, - name: true, - id: true, - }, - orderBy: { - id: 'asc', - }, - }); - - for (const map of maps) { - scheduleWithSpread(hoursToMilliseconds(24), async () => { - await scheduleMapCount({ - gameTypeName: map.gameTypeName, - mapName: map.name, - mapId: map.id, - }); - }); - } - - console.log(`Scheduled ${maps.length} new maps`); + schedule(hoursToMilliseconds(24), async () => { + await scheduleMapCount({ mode: 'full' }); + }); - if (maps.length > 0) { - lastId = maps[maps.length - 1].id; - } + schedule(hoursToMilliseconds(1), async () => { + await scheduleMapCount({ mode: 'gameServers' }); }); } diff --git a/apps/worker/src/workers/updateMapsCounts.ts b/apps/worker/src/workers/updateMapsCounts.ts index f5f13c8..4169c8a 100644 --- a/apps/worker/src/workers/updateMapsCounts.ts +++ b/apps/worker/src/workers/updateMapsCounts.ts @@ -1,46 +1,13 @@ +import { updateMapsCounts, updateMapsGameServerCounts } from "@prisma/client/sql"; import { prisma } from "../prisma"; import { MapCountJobData, processMapCountJobs } from "@teerank/teerank" export async function updateMapsCount(data: MapCountJobData) { - const map = await prisma.map.findUniqueOrThrow({ - select: { - _count: { - select: { - playerInfoMaps: true, - clanInfoMaps: true, - }, - }, - }, - where: { - name_gameTypeName: { - name: data.mapName, - gameTypeName: data.gameTypeName, - }, - }, - }); - - const gameServerCount = await prisma.gameServerState.count({ - where: { - map: { - name: data.mapName, - gameTypeName: data.gameTypeName, - }, - }, - }); - - await prisma.map.update({ - where: { - name_gameTypeName: { - name: data.mapName, - gameTypeName: data.gameTypeName, - }, - }, - data: { - playerCount: map._count.playerInfoMaps, - clanCount: map._count.clanInfoMaps, - gameServerCount, - }, - }); + if (data.mode === 'full') { + await prisma.$queryRawTyped(updateMapsCounts()); + } else { + await prisma.$queryRawTyped(updateMapsGameServerCounts()); + } } export async function startUpdateMapsCountsWorker() { diff --git a/libs/prisma/prisma/sql/updateMapsCounts.sql b/libs/prisma/prisma/sql/updateMapsCounts.sql new file mode 100644 index 0000000..3afce2c --- /dev/null +++ b/libs/prisma/prisma/sql/updateMapsCounts.sql @@ -0,0 +1,18 @@ +UPDATE "Map" SET + "playerCount" = counts."playerCount", + "clanCount" = counts."clanCount", + "gameServerCount" = counts."gameServerCount" +FROM ( + SELECT + m.id, + COALESCE(p.count, 0)::int4 AS "playerCount", + COALESCE(c.count, 0)::int4 AS "clanCount", + COALESCE(g.count, 0)::int4 AS "gameServerCount" + FROM "Map" m + LEFT JOIN (SELECT "mapId", count(*) AS count FROM "PlayerInfoMap" GROUP BY "mapId") p ON p."mapId" = m.id + LEFT JOIN (SELECT "mapId", count(*) AS count FROM "ClanInfoMap" GROUP BY "mapId") c ON c."mapId" = m.id + LEFT JOIN (SELECT "mapId", count(*) AS count FROM "GameServerState" GROUP BY "mapId") g ON g."mapId" = m.id +) counts +WHERE "Map".id = counts.id + AND ("Map"."playerCount", "Map"."clanCount", "Map"."gameServerCount") + IS DISTINCT FROM (counts."playerCount", counts."clanCount", counts."gameServerCount"); diff --git a/libs/prisma/prisma/sql/updateMapsGameServerCounts.sql b/libs/prisma/prisma/sql/updateMapsGameServerCounts.sql new file mode 100644 index 0000000..1deeb53 --- /dev/null +++ b/libs/prisma/prisma/sql/updateMapsGameServerCounts.sql @@ -0,0 +1,11 @@ +UPDATE "Map" SET + "gameServerCount" = counts."gameServerCount" +FROM ( + SELECT + m.id, + COALESCE(g.count, 0)::int4 AS "gameServerCount" + FROM "Map" m + LEFT JOIN (SELECT "mapId", count(*) AS count FROM "GameServerState" GROUP BY "mapId") g ON g."mapId" = m.id +) counts +WHERE "Map".id = counts.id + AND "Map"."gameServerCount" IS DISTINCT FROM counts."gameServerCount"; diff --git a/libs/teerank/src/lib/bullmq/queueMapCount.ts b/libs/teerank/src/lib/bullmq/queueMapCount.ts index ad4ce76..3ab0ca8 100644 --- a/libs/teerank/src/lib/bullmq/queueMapCount.ts +++ b/libs/teerank/src/lib/bullmq/queueMapCount.ts @@ -2,12 +2,10 @@ import { Job, Queue, Worker } from "bullmq"; import { bullmqConnection, lastCompletedJobDate } from "./config"; import { z } from "zod"; import { minutesToSeconds } from "date-fns"; -import { getEnvInt } from "../utils"; let mapCountQueue: Queue | null = null; const QUEUE_NAME_MAP_COUNT = 'map-count'; -const UPDATE_MAPS_COUNTS_CONCURRENCY = getEnvInt('UPDATE_MAPS_COUNTS_CONCURRENCY', 5); function getQueueMapCount() { mapCountQueue ??= new Queue(QUEUE_NAME_MAP_COUNT, { connection: bullmqConnection }); @@ -15,18 +13,16 @@ function getQueueMapCount() { } const schema = z.object({ - gameTypeName: z.string(), - mapName: z.string(), - mapId: z.number(), + mode: z.enum(['full', 'gameServers']), }); export type MapCountJobData = z.infer; export async function scheduleMapCount(data: MapCountJobData) { const queue = getQueueMapCount(); - await queue.add(`${data.gameTypeName} - ${data.mapName}`, data, { + await queue.add(data.mode, data, { deduplication: { - id: data.mapId.toString(), + id: data.mode, } }); } @@ -39,7 +35,7 @@ export async function processMapCountJobs(processor: (data: MapCountJobData) => return new Worker(QUEUE_NAME_MAP_COUNT, jobProcessor, { connection: bullmqConnection, - concurrency: UPDATE_MAPS_COUNTS_CONCURRENCY, + concurrency: 1, removeOnComplete: { age: minutesToSeconds(10), },