diff --git a/docs/occurrence-data-pipeline.md b/docs/occurrence-data-pipeline.md index 1225430c..589a26ea 100644 --- a/docs/occurrence-data-pipeline.md +++ b/docs/occurrence-data-pipeline.md @@ -43,16 +43,25 @@ generation: - Stale **with** a previous valid version: serve the stale file immediately with the `X-Occurrence-Data-Refresh: in-progress` header (the UI shows a toast) and spawn the OccurrenceGeoJob CLI in the background. -- Stale with **no** previous version (cold start): block until generation finishes. +- Stale with **no** previous version (cold start): wait for the lock, re-check + staleness (another replica may have just regenerated), then block until + generation finishes. Generation runs as an **awaited CLI child process** — the API spawns `node $OCC_GEO_JOB_DIR/dist/index.js` (default `src/OccurrenceGeoJob`) with `OCC_GEO_JOB_*`-prefixed env vars — never in-process, so the API event loop stays responsive (an earlier in-process BullMQ version blocked all endpoints -during the big `JSON.stringify`/brotli passes). A single in-process `inflight` -lock makes it a singleton. The job writes to a staging dir and atomically moves -files into the target folder, skipping files whose checksum is unchanged, and -writes the API-compatible `generation-ledger.json`. +during the big `JSON.stringify`/brotli passes). Singleton across **all** API +replicas (HA, shared PVC) via a Redis lock (`full-occurrence-data:generation-lock`, +`SET NX PX` 30 min, token-checked Lua release): a background refresh spawns +nothing when another replica holds the lock; `status()` reports `refreshing` +from the lock too. All Redis calls have a 3 s timeout guard so a Redis outage +cannot hang requests (generation triggering does depend on Redis, same as the +export/email queues). `ioredis` is pinned to `^5` because `typeorm@0.3.31` +peer-optionally requires `^5.0.4` (v6 breaks `npm ci` with ERESOLVE). +The job writes to a staging dir and atomically moves files into the target +folder (EXDEV copy fallback for cross-device moves), skipping files whose +checksum is unchanged, and writes the API-compatible `generation-ledger.json`. `OCC_GEO_JOB_DIR` is API config (`src/API/src/config/config.ts`); the API Dockerfile is built with context `src/` (`docker build -f src/API/Dockerfile src`) diff --git a/src/API/package-lock.json b/src/API/package-lock.json index e73e18e6..dd2f21a9 100644 --- a/src/API/package-lock.json +++ b/src/API/package-lock.json @@ -44,6 +44,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4", @@ -592,6 +593,7 @@ "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.2.tgz", "integrity": "sha512-1D2LpsU7y9xrqKjdIbsB7PlrRePw0xsVV8p+AKTlzITrWmscajryfJCdDJB/oGwvDI5HmRo04eMMADB67uwAwQ==", "license": "MIT", + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -653,6 +655,7 @@ "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.24.0.tgz", "integrity": "sha512-PpLsoDQ3AMmKZ0VU+0GrmqMxgp/sExjlVm4R+nLWngeoEGAzOIPVifaxKGU5gMv+nWELUoHfvrolWD+ZS/nFJg==", "license": "MIT", + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -856,6 +859,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -3710,6 +3714,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.4.22.tgz", "integrity": "sha512-fxJ4v85nDHaqT1PmfNCQ37b/jcv2OojtXTaK1P2uAXhzLf9qq6WNUOFvxBrV4fhQek1EQoT1o9oj5xAZmv3NRw==", "license": "MIT", + "peer": true, "dependencies": { "file-type": "20.4.1", "iterare": "1.2.1", @@ -3768,6 +3773,7 @@ "integrity": "sha512-6IX9+VwjiKtCjx+mXVPncpkQ5ZjKfmssOZPFexmT+6T9H9wZ3svpYACAo7+9e7Nr9DZSoRZw3pffkJP7Z0UjaA==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@nuxtjs/opencollective": "0.3.2", "fast-safe-stringify": "2.1.1", @@ -3805,6 +3811,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/graphql/-/graphql-12.2.2.tgz", "integrity": "sha512-lUDy/1uqbRA1kBKpXcmY0aHhcPbfeG52Wg5+9Jzd1d57dwSjCAmuO+mWy5jz9ugopVCZeK0S/kdAMvA+r9fNdA==", "license": "MIT", + "peer": true, "dependencies": { "@graphql-tools/merge": "9.0.11", "@graphql-tools/schema": "10.0.10", @@ -3997,6 +4004,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.22.tgz", "integrity": "sha512-ySSq7Py/DFozzZdNDH67m/vHoeVdphDniWBnl6q5QVoXldDdrZIHLXLRMPayTDh5A95nt7jjJzmD4qpTbNQ6tA==", "license": "MIT", + "peer": true, "dependencies": { "body-parser": "1.20.4", "cors": "2.8.5", @@ -5221,6 +5229,7 @@ "integrity": "sha512-mpT8LJJ4CMeeahobofYWIjFo0xonRS/HfxnVEPMPFSQdGUt1uHCnoPT7Zhb+sjDU2wz0oKV0OLUR0WzrHNgfeA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "jest-matcher-utils": "^27.0.0", "pretty-format": "^27.0.0" @@ -5282,6 +5291,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -5485,6 +5495,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -5935,6 +5946,7 @@ "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "devOptional": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5991,6 +6003,7 @@ "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -6809,6 +6822,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", @@ -6969,6 +6983,7 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -7256,6 +7271,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.11.23", "caniuse-lite": "^1.0.30001810", @@ -7360,6 +7376,7 @@ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.5.tgz", "integrity": "sha512-pW7pFc/X+9AEIx91TWTdIpZ1icfbJnA0unsyzD7VICjCUw8a3qedSjej5dxxn6LjfM1kr2jvalOCGbCmoA5Kxg==", "license": "MIT", + "peer": true, "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.11.1", @@ -7667,13 +7684,15 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/class-validator": { "version": "0.14.4", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.4.tgz", "integrity": "sha512-AwNusCCam51q703dW82x95tOqQp6oC9HNUl724KxJJOfnKscI8dOloXFgyez7LbTTKWuRBA37FScqVbJEoq8Yw==", "license": "MIT", + "peer": true, "dependencies": { "@types/validator": "^13.15.3", "libphonenumber-js": "^1.11.1", @@ -9071,8 +9090,7 @@ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/es-object-atoms": { "version": "1.1.2", @@ -9196,6 +9214,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -10855,6 +10874,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -11829,6 +11849,7 @@ "integrity": "sha512-uS+T5J3w5xyzd1KSJCGKhCo8WTJXbNl86f5SW11wgssbandJOVLRKKUxmhdFfmKxhPeksl1hHZ0HaA8VBzp7xA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^28.0.3", "import-local": "^3.0.2", @@ -14011,7 +14032,6 @@ "integrity": "sha512-2JVy2DY+iBtfRyrQB5cRM7lfurnJIi6jFTgLip/7FxDu0WxswrIRzr4fBySQ0X7HqM7BcwE28zm+HfQ5V6y4Cg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.31", "jest-worker": "^27.4.5", @@ -14085,7 +14105,6 @@ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -14103,7 +14122,6 @@ "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -14122,7 +14140,6 @@ "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "merge-stream": "^2.0.0", @@ -14138,7 +14155,6 @@ "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.15", "ajv": "^8.20.0", @@ -14159,7 +14175,6 @@ "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14921,6 +14936,7 @@ "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.10.1.tgz", "integrity": "sha512-Z+iLaBGVaSjbIzQ4pX6XV41HrooLsQ10ZWPUehGmuantvzWoDVBnmsdUcOIDM1t+yPor5pDhVlDESgOMEGxhHA==", "license": "MIT-0", + "peer": true, "engines": { "node": ">=6.0.0" } @@ -15378,6 +15394,7 @@ "resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz", "integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==", "license": "MIT", + "peer": true, "dependencies": { "passport-strategy": "1.x.x", "pause": "0.0.1", @@ -15515,6 +15532,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -15835,6 +15853,7 @@ "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -16327,6 +16346,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -17420,6 +17440,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -17475,8 +17496,7 @@ "version": "0.28.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/schema-utils": { "version": "3.3.0", @@ -17503,6 +17523,7 @@ "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -17536,6 +17557,7 @@ "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.12.0.tgz", "integrity": "sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==", "license": "MIT", + "peer": true, "dependencies": { "parseley": "~0.13.1" }, @@ -19030,6 +19052,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -19233,6 +19256,7 @@ "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.31.tgz", "integrity": "sha512-6u9EFtdLBgHjnPm78NStVeM+I/1MolTzKykDDcydzKUkh6E++YS6XViU/fePJbvDvEGU4Xq34KOM/CLeer9I2A==", "license": "MIT", + "peer": true, "dependencies": { "@sqltools/formatter": "^1.2.5", "ansis": "^4.3.1", @@ -19453,6 +19477,7 @@ "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -19965,7 +19990,6 @@ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19983,7 +20007,6 @@ "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -20002,7 +20025,6 @@ "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } @@ -20013,7 +20035,6 @@ "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.15", "ajv": "^8.20.0", @@ -20174,6 +20195,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", diff --git a/src/API/package.json b/src/API/package.json index 032fd9f4..9e0b82e9 100644 --- a/src/API/package.json +++ b/src/API/package.json @@ -61,6 +61,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4", diff --git a/src/API/src/full-occurrence-data/full-occurrence-data.service.ts b/src/API/src/full-occurrence-data/full-occurrence-data.service.ts index d72a10a8..c33b823b 100644 --- a/src/API/src/full-occurrence-data/full-occurrence-data.service.ts +++ b/src/API/src/full-occurrence-data/full-occurrence-data.service.ts @@ -1,9 +1,15 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { DataSource } from 'typeorm'; import { spawn } from 'child_process'; import * as crypto from 'crypto'; import * as fs from 'fs'; import * as path from 'path'; +import { Redis } from 'ioredis'; import config from 'src/config/config'; const LEDGER_FILE = 'generation-ledger.json'; @@ -13,6 +19,10 @@ const REQUIRED_FILES = [ 'presence.geojson', 'absence.geojson', ]; +// Redis key for the cross-replica generation lock (the API runs HA — e.g. 2 +// replicas — all sharing FULL_OCCURRENCE_DATA_FOLDER on a PVC) +const LOCK_KEY = 'full-occurrence-data:generation-lock'; +const LOCK_TTL_MS = 30 * 60 * 1000; export interface GenerationLedger { approvedDatasetId: string | null; @@ -31,12 +41,85 @@ export interface EnsureResult { } @Injectable() -export class FullOccurrenceDataService { +export class FullOccurrenceDataService + implements OnModuleInit, OnModuleDestroy +{ private readonly logger = new Logger(FullOccurrenceDataService.name); private inflight: Promise | null = null; + private redis: Redis; constructor(private readonly dataSource: DataSource) {} + onModuleInit() { + const password = process.env.REDIS_PASSWORD; + this.redis = new Redis({ + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379', 10), + ...(password ? { password } : {}), + connectTimeout: 5000, + maxRetriesPerRequest: 2, + }); + this.redis.on('error', (e) => + this.logger.warn(`redis (generation lock) error: ${e.message}`), + ); + } + + /** run a redis command but never hang the request if redis is down */ + private async redisOp(op: () => Promise, fallback: T): Promise { + try { + return await Promise.race([ + op(), + new Promise((r) => setTimeout(() => r(fallback), 3000)), + ]); + } catch { + return fallback; + } + } + + onModuleDestroy() { + void this.redis?.quit(); + } + + /** + * Try to acquire the cross-replica generation lock. With wait=true, poll + * until the lock is free (or timeoutMs elapses) — used on cold starts where + * the caller blocks until the files exist. + */ + private async acquireLock( + wait: boolean, + timeoutMs = LOCK_TTL_MS, + ): Promise { + const token = crypto.randomUUID(); + const deadline = Date.now() + timeoutMs; + for (;;) { + const ok = await this.redisOp( + () => this.redis.set(LOCK_KEY, token, 'PX', LOCK_TTL_MS, 'NX'), + null, + ); + if (ok === 'OK') return token; + if (!wait || Date.now() > deadline) return null; + await new Promise((r) => setTimeout(r, 2000)); + } + } + + private async releaseLock(token: string): Promise { + // only delete the lock if we still own it + await this.redisOp( + () => + this.redis.eval( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + 1, + LOCK_KEY, + token, + ), + 0, + ); + } + + private async lockHeldByOther(): Promise { + return (await this.redisOp(() => this.redis.exists(LOCK_KEY), 0)) === 1; + } + getFolder(): string { return config.get('fullOccurrenceDataFolder'); } @@ -125,57 +208,65 @@ export class FullOccurrenceDataService { return false; } - /** true while a generation run is in flight in this process */ - private jobInFlight(): boolean { - return this.inflight !== null; + /** true while a generation run is in flight (this process or another replica) */ + private async jobInFlight(): Promise { + if (this.inflight) return true; + try { + return await this.lockHeldByOther(); + } catch { + return false; // redis down — fall back to the local view only + } } /** - * Run the OccurrenceGeoJob CLI once and await it (singleton via in-process - * lock). The job runs in its own child process, so the API event loop stays - * responsive while it generates and compresses the payload files. + * Run the OccurrenceGeoJob CLI once and await it. Singleton across ALL API + * replicas via a Redis lock (they share FULL_OCCURRENCE_DATA_FOLDER on a + * PVC), plus an in-process lock. The job runs in its own child process, so + * the API event loop stays responsive while it generates and compresses. */ private kickOff(): Promise { if (this.inflight) return this.inflight; this.inflight = (async () => { - const jobDir = config.get('occurrenceGeoJobDir'); - const env = { - ...process.env, - // the job's pg Pool reads the standard PG* variables - PGHOST: process.env.POSTGRES_HOST, - PGPORT: process.env.PGPORT, - PGUSER: process.env.POSTGRES_USER, - PGPASSWORD: process.env.POSTGRES_PASSWORD, - PGDATABASE: process.env.POSTGRES_DB, - // job-specific settings use the OCC_GEO_JOB_ prefix - OCC_GEO_JOB_SQL_QUERIES_FOLDER: path.join(jobDir, 'sql'), - OCC_GEO_JOB_TARGET_DIRECTORY: this.getFolder(), - OCC_GEO_JOB_STAGING_DIRECTORY: path.join(jobDir, '.tmp/staging'), - OCC_GEO_JOB_CREATE_BACKUP: 'false', - }; - this.logger.log(`spawning OccurrenceGeoJob CLI in ${jobDir}`); - await new Promise((resolve, reject) => { - const child = spawn('node', ['dist/index.js'], { cwd: jobDir, env }); - child.stdout.on('data', (d) => - String(d) - .trim() - .split('\n') - .forEach((l) => l && this.logger.log(`[geojob] ${l}`)), - ); - child.stderr.on('data', (d) => - String(d) - .trim() - .split('\n') - .forEach((l) => l && this.logger.error(`[geojob] ${l}`)), - ); - child.on('error', reject); - child.on('close', (code) => - code === 0 - ? resolve() - : reject(new Error(`OccurrenceGeoJob exited with code ${code}`)), + const token = await this.acquireLock(false); // background refresh: don't wait + if (!token) { + this.logger.log( + 'generation already running on another replica, skipping', ); + return; + } + try { + await this.runJob(); + } finally { + await this.releaseLock(token); + } + })() + .catch((e) => { + this.logger.error(`generation failed: ${e?.message || e}`); + }) + .finally(() => { + this.inflight = null; }); - this.logger.log('OccurrenceGeoJob CLI finished'); + return this.inflight; + } + + /** Cold-start variant: block until the lock is free, then generate if still stale. */ + private kickOffBlocking(): Promise { + if (this.inflight) return this.inflight; + this.inflight = (async () => { + const token = await this.acquireLock(true); // wait for another replica if needed + if (!token) { + this.logger.error('timed out waiting for the generation lock'); + return; + } + try { + if (await this.isStale()) { + await this.runJob(); + } else { + this.logger.log('another replica already regenerated, skipping'); + } + } finally { + await this.releaseLock(token); + } })() .catch((e) => { this.logger.error(`generation failed: ${e?.message || e}`); @@ -186,6 +277,47 @@ export class FullOccurrenceDataService { return this.inflight; } + private async runJob(): Promise { + const jobDir = config.get('occurrenceGeoJobDir'); + const env = { + ...process.env, + // the job's pg Pool reads the standard PG* variables + PGHOST: process.env.POSTGRES_HOST, + PGPORT: process.env.PGPORT, + PGUSER: process.env.POSTGRES_USER, + PGPASSWORD: process.env.POSTGRES_PASSWORD, + PGDATABASE: process.env.POSTGRES_DB, + // job-specific settings use the OCC_GEO_JOB_ prefix + OCC_GEO_JOB_SQL_QUERIES_FOLDER: path.join(jobDir, 'sql'), + OCC_GEO_JOB_TARGET_DIRECTORY: this.getFolder(), + OCC_GEO_JOB_STAGING_DIRECTORY: path.join(jobDir, '.tmp/staging'), + OCC_GEO_JOB_CREATE_BACKUP: 'false', + }; + this.logger.log(`spawning OccurrenceGeoJob CLI in ${jobDir}`); + await new Promise((resolve, reject) => { + const child = spawn('node', ['dist/index.js'], { cwd: jobDir, env }); + child.stdout.on('data', (d) => + String(d) + .trim() + .split('\n') + .forEach((l) => l && this.logger.log(`[geojob] ${l}`)), + ); + child.stderr.on('data', (d) => + String(d) + .trim() + .split('\n') + .forEach((l) => l && this.logger.error(`[geojob] ${l}`)), + ); + child.on('error', reject); + child.on('close', (code) => + code === 0 + ? resolve() + : reject(new Error(`OccurrenceGeoJob exited with code ${code}`)), + ); + }); + this.logger.log('OccurrenceGeoJob CLI finished'); + } + /** * Ensure the files exist and are fresh. * - not stale -> nothing to do @@ -204,7 +336,7 @@ export class FullOccurrenceDataService { void this.kickOff(); // fire and forget; serve stale now return { refreshing: true }; } - await this.kickOff(); // cold: wait for the first generation + await this.kickOffBlocking(); // cold: wait for the first generation return { refreshing: false }; } diff --git a/src/OccurrenceGeoJob/src/config/atomic.ts b/src/OccurrenceGeoJob/src/config/atomic.ts index aca24313..8d14af5e 100644 --- a/src/OccurrenceGeoJob/src/config/atomic.ts +++ b/src/OccurrenceGeoJob/src/config/atomic.ts @@ -126,8 +126,20 @@ async function atomicReplaceFile( skipped: boolean; }> { try { - // Verify staging file exists - const stats = await fs.stat(stagingPath); + // Verify staging file exists — if it's missing (e.g. resume with a stale + // ledger whose files were already cleaned up), skip instead of failing hard + let stats; + try { + stats = await fs.stat(stagingPath); + } catch (error: unknown) { + if ((error as { code?: string }).code === 'ENOENT') { + logger.warn( + `Staging file missing, skipping replacement: ${stagingPath}`, + ); + return { success: true, checksum: undefined, skipped: true }; + } + throw error; + } if (!stats.isFile()) { throw new Error(`Staging file is not a regular file: ${stagingPath}`); } @@ -251,9 +263,11 @@ export async function atomicReplaceAll( await fs.unlink(stagingPath); logger.debug(`Cleaned up staging file (skipped): ${stagingPath}`); } catch (error) { - logger.warn( - `Failed to clean up staging file ${stagingPath}: ${error}`, - ); + if ((error as { code?: string }).code !== "ENOENT") { + logger.warn( + `Failed to clean up staging file ${stagingPath}: ${error}`, + ); + } } } } else { @@ -268,9 +282,11 @@ export async function atomicReplaceAll( await fs.unlink(stagingPath); logger.debug(`Cleaned up staging file: ${stagingPath}`); } catch (error) { - logger.warn( - `Failed to clean up staging file ${stagingPath}: ${error}`, - ); + if ((error as { code?: string }).code !== "ENOENT") { + logger.warn( + `Failed to clean up staging file ${stagingPath}: ${error}`, + ); + } } } }