From 83d61415717bebaf4e7d74636d0eaec821b1fc20 Mon Sep 17 00:00:00 2001 From: Declan Joseph Date: Mon, 9 Nov 2026 15:01:42 +0300 Subject: [PATCH 1/2] fix(occurrence-data): cross-replica generation lock via Redis; tolerate missing staging files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HA sync: the API runs with 2 replicas sharing FULL_OCCURRENCE_DATA_FOLDER on a PVC, and the in-process inflight lock did not stop a second replica from spawning its own OccurrenceGeoJob. kickOff now acquires a Redis lock (SET NX PX, token-checked release via Lua) before spawning the CLI child; background refreshes skip if another replica holds the lock, cold starts wait for it and then re-check staleness. status() reports refreshing from the lock too. All Redis calls are wrapped in a 3s-timeout guard so a Redis outage can never hang the endpoint. ioredis added as a direct dependency. Verified locally: with the lock key held manually, a stale request served in 87ms with X-Occurrence-Data-Refresh: in-progress and spawned no geojob (status showed refreshing:true); after deleting the key the next request spawned the job, which completed and released the lock. Geojob: atomicReplaceFile now checks the staging file exists and skips (with a warning) instead of failing hard on ENOENT, and the staging-cleanup unlink no longer warns on ENOENT (moveFile already removed the file). Reproduced empty/missing target directory runs — both complete successfully. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- src/API/package-lock.json | 98 ++++++-- src/API/package.json | 1 + .../full-occurrence-data.service.ts | 222 ++++++++++++++---- src/OccurrenceGeoJob/src/config/atomic.ts | 32 ++- 4 files changed, 276 insertions(+), 77 deletions(-) diff --git a/src/API/package-lock.json b/src/API/package-lock.json index e73e18e67..0f0f583df 100644 --- a/src/API/package-lock.json +++ b/src/API/package-lock.json @@ -44,6 +44,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", + "ioredis": "^6.0.0", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4", @@ -592,6 +593,7 @@ "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.2.tgz", "integrity": "sha512-1D2LpsU7y9xrqKjdIbsB7PlrRePw0xsVV8p+AKTlzITrWmscajryfJCdDJB/oGwvDI5HmRo04eMMADB67uwAwQ==", "license": "MIT", + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -653,6 +655,7 @@ "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.24.0.tgz", "integrity": "sha512-PpLsoDQ3AMmKZ0VU+0GrmqMxgp/sExjlVm4R+nLWngeoEGAzOIPVifaxKGU5gMv+nWELUoHfvrolWD+ZS/nFJg==", "license": "MIT", + "peer": true, "dependencies": { "@azure/abort-controller": "^2.1.2", "@azure/core-auth": "^1.10.0", @@ -856,6 +859,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -2670,9 +2674,9 @@ } }, "node_modules/@ioredis/commands": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", - "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-2.0.0.tgz", + "integrity": "sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==", "license": "MIT" }, "node_modules/@isaacs/cliui": { @@ -3710,6 +3714,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-10.4.22.tgz", "integrity": "sha512-fxJ4v85nDHaqT1PmfNCQ37b/jcv2OojtXTaK1P2uAXhzLf9qq6WNUOFvxBrV4fhQek1EQoT1o9oj5xAZmv3NRw==", "license": "MIT", + "peer": true, "dependencies": { "file-type": "20.4.1", "iterare": "1.2.1", @@ -3768,6 +3773,7 @@ "integrity": "sha512-6IX9+VwjiKtCjx+mXVPncpkQ5ZjKfmssOZPFexmT+6T9H9wZ3svpYACAo7+9e7Nr9DZSoRZw3pffkJP7Z0UjaA==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "@nuxtjs/opencollective": "0.3.2", "fast-safe-stringify": "2.1.1", @@ -3805,6 +3811,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/graphql/-/graphql-12.2.2.tgz", "integrity": "sha512-lUDy/1uqbRA1kBKpXcmY0aHhcPbfeG52Wg5+9Jzd1d57dwSjCAmuO+mWy5jz9ugopVCZeK0S/kdAMvA+r9fNdA==", "license": "MIT", + "peer": true, "dependencies": { "@graphql-tools/merge": "9.0.11", "@graphql-tools/schema": "10.0.10", @@ -3997,6 +4004,7 @@ "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.22.tgz", "integrity": "sha512-ySSq7Py/DFozzZdNDH67m/vHoeVdphDniWBnl6q5QVoXldDdrZIHLXLRMPayTDh5A95nt7jjJzmD4qpTbNQ6tA==", "license": "MIT", + "peer": true, "dependencies": { "body-parser": "1.20.4", "cors": "2.8.5", @@ -5221,6 +5229,7 @@ "integrity": "sha512-mpT8LJJ4CMeeahobofYWIjFo0xonRS/HfxnVEPMPFSQdGUt1uHCnoPT7Zhb+sjDU2wz0oKV0OLUR0WzrHNgfeA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "jest-matcher-utils": "^27.0.0", "pretty-format": "^27.0.0" @@ -5282,6 +5291,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~5.26.4" } @@ -5485,6 +5495,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -5935,6 +5946,7 @@ "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "devOptional": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5991,6 +6003,7 @@ "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -6809,6 +6822,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", @@ -6969,6 +6983,7 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -7256,6 +7271,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.11.23", "caniuse-lite": "^1.0.30001810", @@ -7360,6 +7376,7 @@ "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.81.5.tgz", "integrity": "sha512-pW7pFc/X+9AEIx91TWTdIpZ1icfbJnA0unsyzD7VICjCUw8a3qedSjej5dxxn6LjfM1kr2jvalOCGbCmoA5Kxg==", "license": "MIT", + "peer": true, "dependencies": { "cron-parser": "4.9.0", "ioredis": "5.11.1", @@ -7380,6 +7397,34 @@ } } }, + "node_modules/bullmq/node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "license": "MIT" + }, + "node_modules/bullmq/node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -7667,13 +7712,15 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/class-validator": { "version": "0.14.4", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.4.tgz", "integrity": "sha512-AwNusCCam51q703dW82x95tOqQp6oC9HNUl724KxJJOfnKscI8dOloXFgyez7LbTTKWuRBA37FScqVbJEoq8Yw==", "license": "MIT", + "peer": true, "dependencies": { "@types/validator": "^13.15.3", "libphonenumber-js": "^1.11.1", @@ -9071,8 +9118,7 @@ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/es-object-atoms": { "version": "1.1.2", @@ -9196,6 +9242,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -10855,6 +10902,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -11371,21 +11419,21 @@ } }, "node_modules/ioredis": { - "version": "5.11.1", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", - "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-6.0.0.tgz", + "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==", "license": "MIT", + "peer": true, "dependencies": { - "@ioredis/commands": "1.10.0", + "@ioredis/commands": "2.0.0", "cluster-key-slot": "1.1.1", "debug": "4.4.3", "denque": "2.1.0", "redis-errors": "1.2.0", - "redis-parser": "3.0.0", "standard-as-callback": "2.1.0" }, "engines": { - "node": ">=12.22.0" + "node": ">=20.0.0" }, "funding": { "type": "opencollective", @@ -11829,6 +11877,7 @@ "integrity": "sha512-uS+T5J3w5xyzd1KSJCGKhCo8WTJXbNl86f5SW11wgssbandJOVLRKKUxmhdFfmKxhPeksl1hHZ0HaA8VBzp7xA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^28.0.3", "import-local": "^3.0.2", @@ -14011,7 +14060,6 @@ "integrity": "sha512-2JVy2DY+iBtfRyrQB5cRM7lfurnJIi6jFTgLip/7FxDu0WxswrIRzr4fBySQ0X7HqM7BcwE28zm+HfQ5V6y4Cg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.31", "jest-worker": "^27.4.5", @@ -14085,7 +14133,6 @@ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -14103,7 +14150,6 @@ "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -14122,7 +14168,6 @@ "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "merge-stream": "^2.0.0", @@ -14138,7 +14183,6 @@ "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.15", "ajv": "^8.20.0", @@ -14159,7 +14203,6 @@ "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14921,6 +14964,7 @@ "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.10.1.tgz", "integrity": "sha512-Z+iLaBGVaSjbIzQ4pX6XV41HrooLsQ10ZWPUehGmuantvzWoDVBnmsdUcOIDM1t+yPor5pDhVlDESgOMEGxhHA==", "license": "MIT-0", + "peer": true, "engines": { "node": ">=6.0.0" } @@ -15378,6 +15422,7 @@ "resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz", "integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==", "license": "MIT", + "peer": true, "dependencies": { "passport-strategy": "1.x.x", "pause": "0.0.1", @@ -15515,6 +15560,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -15835,6 +15881,7 @@ "integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin-prettier.js" }, @@ -16327,6 +16374,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -17420,6 +17468,7 @@ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", "license": "Apache-2.0", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -17475,8 +17524,7 @@ "version": "0.28.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/schema-utils": { "version": "3.3.0", @@ -17503,6 +17551,7 @@ "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -17536,6 +17585,7 @@ "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.12.0.tgz", "integrity": "sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==", "license": "MIT", + "peer": true, "dependencies": { "parseley": "~0.13.1" }, @@ -19030,6 +19080,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -19233,6 +19284,7 @@ "resolved": "https://registry.npmjs.org/typeorm/-/typeorm-0.3.31.tgz", "integrity": "sha512-6u9EFtdLBgHjnPm78NStVeM+I/1MolTzKykDDcydzKUkh6E++YS6XViU/fePJbvDvEGU4Xq34KOM/CLeer9I2A==", "license": "MIT", + "peer": true, "dependencies": { "@sqltools/formatter": "^1.2.5", "ansis": "^4.3.1", @@ -19453,6 +19505,7 @@ "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -19965,7 +20018,6 @@ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19983,7 +20035,6 @@ "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ajv": "^8.0.0" }, @@ -20002,7 +20053,6 @@ "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } @@ -20013,7 +20063,6 @@ "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/json-schema": "^7.0.15", "ajv": "^8.20.0", @@ -20174,6 +20223,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", diff --git a/src/API/package.json b/src/API/package.json index 032fd9f49..35cfec7f4 100644 --- a/src/API/package.json +++ b/src/API/package.json @@ -61,6 +61,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", + "ioredis": "^6.0.0", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4", diff --git a/src/API/src/full-occurrence-data/full-occurrence-data.service.ts b/src/API/src/full-occurrence-data/full-occurrence-data.service.ts index d72a10a87..c33b823b0 100644 --- a/src/API/src/full-occurrence-data/full-occurrence-data.service.ts +++ b/src/API/src/full-occurrence-data/full-occurrence-data.service.ts @@ -1,9 +1,15 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { DataSource } from 'typeorm'; import { spawn } from 'child_process'; import * as crypto from 'crypto'; import * as fs from 'fs'; import * as path from 'path'; +import { Redis } from 'ioredis'; import config from 'src/config/config'; const LEDGER_FILE = 'generation-ledger.json'; @@ -13,6 +19,10 @@ const REQUIRED_FILES = [ 'presence.geojson', 'absence.geojson', ]; +// Redis key for the cross-replica generation lock (the API runs HA — e.g. 2 +// replicas — all sharing FULL_OCCURRENCE_DATA_FOLDER on a PVC) +const LOCK_KEY = 'full-occurrence-data:generation-lock'; +const LOCK_TTL_MS = 30 * 60 * 1000; export interface GenerationLedger { approvedDatasetId: string | null; @@ -31,12 +41,85 @@ export interface EnsureResult { } @Injectable() -export class FullOccurrenceDataService { +export class FullOccurrenceDataService + implements OnModuleInit, OnModuleDestroy +{ private readonly logger = new Logger(FullOccurrenceDataService.name); private inflight: Promise | null = null; + private redis: Redis; constructor(private readonly dataSource: DataSource) {} + onModuleInit() { + const password = process.env.REDIS_PASSWORD; + this.redis = new Redis({ + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379', 10), + ...(password ? { password } : {}), + connectTimeout: 5000, + maxRetriesPerRequest: 2, + }); + this.redis.on('error', (e) => + this.logger.warn(`redis (generation lock) error: ${e.message}`), + ); + } + + /** run a redis command but never hang the request if redis is down */ + private async redisOp(op: () => Promise, fallback: T): Promise { + try { + return await Promise.race([ + op(), + new Promise((r) => setTimeout(() => r(fallback), 3000)), + ]); + } catch { + return fallback; + } + } + + onModuleDestroy() { + void this.redis?.quit(); + } + + /** + * Try to acquire the cross-replica generation lock. With wait=true, poll + * until the lock is free (or timeoutMs elapses) — used on cold starts where + * the caller blocks until the files exist. + */ + private async acquireLock( + wait: boolean, + timeoutMs = LOCK_TTL_MS, + ): Promise { + const token = crypto.randomUUID(); + const deadline = Date.now() + timeoutMs; + for (;;) { + const ok = await this.redisOp( + () => this.redis.set(LOCK_KEY, token, 'PX', LOCK_TTL_MS, 'NX'), + null, + ); + if (ok === 'OK') return token; + if (!wait || Date.now() > deadline) return null; + await new Promise((r) => setTimeout(r, 2000)); + } + } + + private async releaseLock(token: string): Promise { + // only delete the lock if we still own it + await this.redisOp( + () => + this.redis.eval( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + 1, + LOCK_KEY, + token, + ), + 0, + ); + } + + private async lockHeldByOther(): Promise { + return (await this.redisOp(() => this.redis.exists(LOCK_KEY), 0)) === 1; + } + getFolder(): string { return config.get('fullOccurrenceDataFolder'); } @@ -125,57 +208,65 @@ export class FullOccurrenceDataService { return false; } - /** true while a generation run is in flight in this process */ - private jobInFlight(): boolean { - return this.inflight !== null; + /** true while a generation run is in flight (this process or another replica) */ + private async jobInFlight(): Promise { + if (this.inflight) return true; + try { + return await this.lockHeldByOther(); + } catch { + return false; // redis down — fall back to the local view only + } } /** - * Run the OccurrenceGeoJob CLI once and await it (singleton via in-process - * lock). The job runs in its own child process, so the API event loop stays - * responsive while it generates and compresses the payload files. + * Run the OccurrenceGeoJob CLI once and await it. Singleton across ALL API + * replicas via a Redis lock (they share FULL_OCCURRENCE_DATA_FOLDER on a + * PVC), plus an in-process lock. The job runs in its own child process, so + * the API event loop stays responsive while it generates and compresses. */ private kickOff(): Promise { if (this.inflight) return this.inflight; this.inflight = (async () => { - const jobDir = config.get('occurrenceGeoJobDir'); - const env = { - ...process.env, - // the job's pg Pool reads the standard PG* variables - PGHOST: process.env.POSTGRES_HOST, - PGPORT: process.env.PGPORT, - PGUSER: process.env.POSTGRES_USER, - PGPASSWORD: process.env.POSTGRES_PASSWORD, - PGDATABASE: process.env.POSTGRES_DB, - // job-specific settings use the OCC_GEO_JOB_ prefix - OCC_GEO_JOB_SQL_QUERIES_FOLDER: path.join(jobDir, 'sql'), - OCC_GEO_JOB_TARGET_DIRECTORY: this.getFolder(), - OCC_GEO_JOB_STAGING_DIRECTORY: path.join(jobDir, '.tmp/staging'), - OCC_GEO_JOB_CREATE_BACKUP: 'false', - }; - this.logger.log(`spawning OccurrenceGeoJob CLI in ${jobDir}`); - await new Promise((resolve, reject) => { - const child = spawn('node', ['dist/index.js'], { cwd: jobDir, env }); - child.stdout.on('data', (d) => - String(d) - .trim() - .split('\n') - .forEach((l) => l && this.logger.log(`[geojob] ${l}`)), - ); - child.stderr.on('data', (d) => - String(d) - .trim() - .split('\n') - .forEach((l) => l && this.logger.error(`[geojob] ${l}`)), - ); - child.on('error', reject); - child.on('close', (code) => - code === 0 - ? resolve() - : reject(new Error(`OccurrenceGeoJob exited with code ${code}`)), + const token = await this.acquireLock(false); // background refresh: don't wait + if (!token) { + this.logger.log( + 'generation already running on another replica, skipping', ); + return; + } + try { + await this.runJob(); + } finally { + await this.releaseLock(token); + } + })() + .catch((e) => { + this.logger.error(`generation failed: ${e?.message || e}`); + }) + .finally(() => { + this.inflight = null; }); - this.logger.log('OccurrenceGeoJob CLI finished'); + return this.inflight; + } + + /** Cold-start variant: block until the lock is free, then generate if still stale. */ + private kickOffBlocking(): Promise { + if (this.inflight) return this.inflight; + this.inflight = (async () => { + const token = await this.acquireLock(true); // wait for another replica if needed + if (!token) { + this.logger.error('timed out waiting for the generation lock'); + return; + } + try { + if (await this.isStale()) { + await this.runJob(); + } else { + this.logger.log('another replica already regenerated, skipping'); + } + } finally { + await this.releaseLock(token); + } })() .catch((e) => { this.logger.error(`generation failed: ${e?.message || e}`); @@ -186,6 +277,47 @@ export class FullOccurrenceDataService { return this.inflight; } + private async runJob(): Promise { + const jobDir = config.get('occurrenceGeoJobDir'); + const env = { + ...process.env, + // the job's pg Pool reads the standard PG* variables + PGHOST: process.env.POSTGRES_HOST, + PGPORT: process.env.PGPORT, + PGUSER: process.env.POSTGRES_USER, + PGPASSWORD: process.env.POSTGRES_PASSWORD, + PGDATABASE: process.env.POSTGRES_DB, + // job-specific settings use the OCC_GEO_JOB_ prefix + OCC_GEO_JOB_SQL_QUERIES_FOLDER: path.join(jobDir, 'sql'), + OCC_GEO_JOB_TARGET_DIRECTORY: this.getFolder(), + OCC_GEO_JOB_STAGING_DIRECTORY: path.join(jobDir, '.tmp/staging'), + OCC_GEO_JOB_CREATE_BACKUP: 'false', + }; + this.logger.log(`spawning OccurrenceGeoJob CLI in ${jobDir}`); + await new Promise((resolve, reject) => { + const child = spawn('node', ['dist/index.js'], { cwd: jobDir, env }); + child.stdout.on('data', (d) => + String(d) + .trim() + .split('\n') + .forEach((l) => l && this.logger.log(`[geojob] ${l}`)), + ); + child.stderr.on('data', (d) => + String(d) + .trim() + .split('\n') + .forEach((l) => l && this.logger.error(`[geojob] ${l}`)), + ); + child.on('error', reject); + child.on('close', (code) => + code === 0 + ? resolve() + : reject(new Error(`OccurrenceGeoJob exited with code ${code}`)), + ); + }); + this.logger.log('OccurrenceGeoJob CLI finished'); + } + /** * Ensure the files exist and are fresh. * - not stale -> nothing to do @@ -204,7 +336,7 @@ export class FullOccurrenceDataService { void this.kickOff(); // fire and forget; serve stale now return { refreshing: true }; } - await this.kickOff(); // cold: wait for the first generation + await this.kickOffBlocking(); // cold: wait for the first generation return { refreshing: false }; } diff --git a/src/OccurrenceGeoJob/src/config/atomic.ts b/src/OccurrenceGeoJob/src/config/atomic.ts index aca24313c..8d14af5e5 100644 --- a/src/OccurrenceGeoJob/src/config/atomic.ts +++ b/src/OccurrenceGeoJob/src/config/atomic.ts @@ -126,8 +126,20 @@ async function atomicReplaceFile( skipped: boolean; }> { try { - // Verify staging file exists - const stats = await fs.stat(stagingPath); + // Verify staging file exists — if it's missing (e.g. resume with a stale + // ledger whose files were already cleaned up), skip instead of failing hard + let stats; + try { + stats = await fs.stat(stagingPath); + } catch (error: unknown) { + if ((error as { code?: string }).code === 'ENOENT') { + logger.warn( + `Staging file missing, skipping replacement: ${stagingPath}`, + ); + return { success: true, checksum: undefined, skipped: true }; + } + throw error; + } if (!stats.isFile()) { throw new Error(`Staging file is not a regular file: ${stagingPath}`); } @@ -251,9 +263,11 @@ export async function atomicReplaceAll( await fs.unlink(stagingPath); logger.debug(`Cleaned up staging file (skipped): ${stagingPath}`); } catch (error) { - logger.warn( - `Failed to clean up staging file ${stagingPath}: ${error}`, - ); + if ((error as { code?: string }).code !== "ENOENT") { + logger.warn( + `Failed to clean up staging file ${stagingPath}: ${error}`, + ); + } } } } else { @@ -268,9 +282,11 @@ export async function atomicReplaceAll( await fs.unlink(stagingPath); logger.debug(`Cleaned up staging file: ${stagingPath}`); } catch (error) { - logger.warn( - `Failed to clean up staging file ${stagingPath}: ${error}`, - ); + if ((error as { code?: string }).code !== "ENOENT") { + logger.warn( + `Failed to clean up staging file ${stagingPath}: ${error}`, + ); + } } } } From 31302dc68ced7597d80c368f7135abc78b63c0ab Mon Sep 17 00:00:00 2001 From: Declan Joseph Date: Mon, 9 Nov 2026 16:22:12 +0300 Subject: [PATCH 2/2] fix(api): pin ioredis to ^5 for typeorm peer compatibility; document the Redis lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI npm ci failed with ERESOLVE: typeorm@0.3.31 declares peerOptional ioredis@^5.0.4 while the lock had ioredis@6. Downgrade to ^5.11.1 (the lock API used — set NX PX / eval / exists / quit — is identical in v5); npm ci now resolves cleanly and nest build passes. Verified at runtime: with the generation lock key held, status reports refreshing:true in ~13ms and a stale request serves with X-Occurrence-Data-Refresh: in-progress while logging 'generation already running on another replica, skipping'; after releasing the key the next request spawns the job and completes cleanly. docs/occurrence-data-pipeline.md updated with the cross-replica Redis lock semantics (background skip vs cold-start wait, lock TTL, 3s Redis guard). Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- docs/occurrence-data-pipeline.md | 19 +++++++++---- src/API/package-lock.json | 48 +++++++------------------------- src/API/package.json | 2 +- 3 files changed, 25 insertions(+), 44 deletions(-) diff --git a/docs/occurrence-data-pipeline.md b/docs/occurrence-data-pipeline.md index 1225430c5..589a26eaa 100644 --- a/docs/occurrence-data-pipeline.md +++ b/docs/occurrence-data-pipeline.md @@ -43,16 +43,25 @@ generation: - Stale **with** a previous valid version: serve the stale file immediately with the `X-Occurrence-Data-Refresh: in-progress` header (the UI shows a toast) and spawn the OccurrenceGeoJob CLI in the background. -- Stale with **no** previous version (cold start): block until generation finishes. +- Stale with **no** previous version (cold start): wait for the lock, re-check + staleness (another replica may have just regenerated), then block until + generation finishes. Generation runs as an **awaited CLI child process** — the API spawns `node $OCC_GEO_JOB_DIR/dist/index.js` (default `src/OccurrenceGeoJob`) with `OCC_GEO_JOB_*`-prefixed env vars — never in-process, so the API event loop stays responsive (an earlier in-process BullMQ version blocked all endpoints -during the big `JSON.stringify`/brotli passes). A single in-process `inflight` -lock makes it a singleton. The job writes to a staging dir and atomically moves -files into the target folder, skipping files whose checksum is unchanged, and -writes the API-compatible `generation-ledger.json`. +during the big `JSON.stringify`/brotli passes). Singleton across **all** API +replicas (HA, shared PVC) via a Redis lock (`full-occurrence-data:generation-lock`, +`SET NX PX` 30 min, token-checked Lua release): a background refresh spawns +nothing when another replica holds the lock; `status()` reports `refreshing` +from the lock too. All Redis calls have a 3 s timeout guard so a Redis outage +cannot hang requests (generation triggering does depend on Redis, same as the +export/email queues). `ioredis` is pinned to `^5` because `typeorm@0.3.31` +peer-optionally requires `^5.0.4` (v6 breaks `npm ci` with ERESOLVE). +The job writes to a staging dir and atomically moves files into the target +folder (EXDEV copy fallback for cross-device moves), skipping files whose +checksum is unchanged, and writes the API-compatible `generation-ledger.json`. `OCC_GEO_JOB_DIR` is API config (`src/API/src/config/config.ts`); the API Dockerfile is built with context `src/` (`docker build -f src/API/Dockerfile src`) diff --git a/src/API/package-lock.json b/src/API/package-lock.json index 0f0f583df..dd2f21a9b 100644 --- a/src/API/package-lock.json +++ b/src/API/package-lock.json @@ -44,7 +44,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", - "ioredis": "^6.0.0", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4", @@ -2674,9 +2674,9 @@ } }, "node_modules/@ioredis/commands": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-2.0.0.tgz", - "integrity": "sha512-vrx0AE/T0h7cRZwfo1M39Cr+ZhZrkf0V8mQN75wucKCxCLD9l/VX6no3gFvrLqD1IlG/1LtzWovqEw3t0Vr9zg==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", "license": "MIT" }, "node_modules/@isaacs/cliui": { @@ -7397,34 +7397,6 @@ } } }, - "node_modules/bullmq/node_modules/@ioredis/commands": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", - "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", - "license": "MIT" - }, - "node_modules/bullmq/node_modules/ioredis": { - "version": "5.11.1", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", - "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", - "license": "MIT", - "dependencies": { - "@ioredis/commands": "1.10.0", - "cluster-key-slot": "1.1.1", - "debug": "4.4.3", - "denque": "2.1.0", - "redis-errors": "1.2.0", - "redis-parser": "3.0.0", - "standard-as-callback": "2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/bundle-name": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", @@ -11419,21 +11391,21 @@ } }, "node_modules/ioredis": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-6.0.0.tgz", - "integrity": "sha512-f+Dtubxfpf6KYFq7WVXJoOLn0bk4TJrMrN9SzeE+jrWrCWj7XX3fA6vkryafhADX+GMymRxgDJDOI33COkJc0w==", + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", "license": "MIT", - "peer": true, "dependencies": { - "@ioredis/commands": "2.0.0", + "@ioredis/commands": "1.10.0", "cluster-key-slot": "1.1.1", "debug": "4.4.3", "denque": "2.1.0", "redis-errors": "1.2.0", + "redis-parser": "3.0.0", "standard-as-callback": "2.1.0" }, "engines": { - "node": ">=20.0.0" + "node": ">=12.22.0" }, "funding": { "type": "opencollective", diff --git a/src/API/package.json b/src/API/package.json index 35cfec7f4..9e0b82e9a 100644 --- a/src/API/package.json +++ b/src/API/package.json @@ -61,7 +61,7 @@ "graphql-type-json": "^0.3.2", "handlebars": "^4.7.8", "imapflow": "^1.0.165", - "ioredis": "^6.0.0", + "ioredis": "^5.11.1", "jszip": "^3.10.1", "jwks-rsa": "^2.1.4", "mailgun.js": "^10.2.4",