diff --git a/.github/workflows/attest-mirror-leg.yml b/.github/workflows/attest-mirror-leg.yml index 3fb0c839..4a9774d3 100644 --- a/.github/workflows/attest-mirror-leg.yml +++ b/.github/workflows/attest-mirror-leg.yml @@ -110,7 +110,7 @@ jobs: # on some images, v22.23.3 on others, the latter failing the consensus pin). - uses: actions/setup-node@v4 with: - node-version: '22.22.3' + node-version: '22.23.4' - name: Start headless MariaDB (for the image-building install) run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14f6c5e9..84212cc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" cache: npm - name: Install dependencies @@ -66,7 +66,7 @@ jobs: uses: XChain-Platform/.github/.github/workflows/ci-reusable.yml@fd9160695d677f0538e959460ba5d8dfbb7f3c8e # pin: XChain-Platform/.github @ master 2026-08-31; bump deliberately with: # The fleet's consensus runtime; a floating "22" now resolves 22.23.3, which the VM runtime gate refuses. - node-version: "22.22.3" + node-version: "22.23.4" # Cross-repo drift guards: this repo vendors the canonical # coin registry from xchain-hub, and per-repo CI cannot see sibling repos, so @@ -93,7 +93,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" - name: Coin-registry byte-identity vs canonical run: xchain-hub/bin/sync-coins.sh --check --only xchain-node @@ -141,7 +141,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" cache: npm - name: Install dependencies diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index e3b3e29f..4ca853e7 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -190,7 +190,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" # WHY THE COIN LIST IS EVENT-DEPENDENT. A scheduled run carries NO inputs, so # without this it would silently test bitcoin alone and the nightly would be @@ -361,7 +361,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" - name: Reclaim runner disk (coin daemon + per-service images are large) run: | @@ -463,12 +463,6 @@ jobs: echo "HUB_NETWORK=regtest" echo "ORACLE_MIN_SUBMISSIONS=1" } >> "$GITHUB_ENV" - # An identity alone does not finalize a bridge transfer: the engine signs - # against the cross_chain capability set, which is EMPTY on a fresh regtest - # until something STAKEs, and an empty set fails closed. This host-env - # passthrough (ConfigService hubPassthroughVars) makes the hub seed itself - # as the single validator on an empty set; the hub honours it on regtest - # only. Non-bitcoin legs only, so the bitcoin opt-in keeps its regime. # The hub's cross-chain engines resolve each chain's indexer URL ONCE, at # engine start, from env or from the configs table, and skip a chain with # none on every later tick. On this runner the hub boots before either @@ -487,7 +481,6 @@ jobs: # regtest rail pins them; the hub clamps them back up to the default off # regtest, so the same three names are inert on any real network. if [ "$COIN" != "bitcoin" ]; then - echo "XDEX_SEED_LOCAL_VALIDATOR=1" >> "$GITHUB_ENV" code=$(case "$COIN" in litecoin) echo LTC ;; dogecoin) echo DOGE ;; esac) { echo "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" @@ -594,6 +587,13 @@ jobs: node src/index.js install "$STACK_REF" all bitcoin regtest fi + - name: Enable the regtest validator seed after both stacks are ready + if: env.COIN != 'bitcoin' + run: | + export XDEX_SEED_LOCAL_VALIDATOR=1 + echo "XDEX_SEED_LOCAL_VALIDATOR=1" >> "$GITHUB_ENV" + node src/index.js recreate xchain-hub + - name: Select this shard's action suites # The plan job owns the split; this only reads this shard's entry, so the # pattern run here is the one the aggregate later checks it against. @@ -777,7 +777,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: "22.22.3" + node-version: "22.23.4" # A shard that never uploaded simply has no artifact here; the merge names # it as missing, so a failed download is not allowed to stop the job first. diff --git a/.github/workflows/rail-leg.yml b/.github/workflows/rail-leg.yml index 1c9dd5c1..8ab5ec99 100644 --- a/.github/workflows/rail-leg.yml +++ b/.github/workflows/rail-leg.yml @@ -126,7 +126,7 @@ jobs: # on some images, v22.23.3 on others, the latter failing the consensus pin). - uses: actions/setup-node@v4 with: - node-version: '22.22.3' + node-version: '22.23.4' - name: Resolve the anchor arm height id: anchor-arms diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f76495f..7b2329a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.27.0] - 2026-11-01 + +### Added +- Added coordinated hub checkpoint reset, and reset can now run on a named MariaDB container without the hub push. + +### Changed +- Reset can use a root password variable or the stack's own database logins, and moved service ports are now persisted. + +### Fixed +- Lifecycle commands now exit non-zero when the in-container command fails or matches nothing, and read-only commands no longer install or restart the hub. +- A hub answer carrying an error now counts as a failed config push, and the push is capped at three minutes. +- A failed reset database step is reported with the services left down, and the command lock file is published whole. +- Autoheal now restarts only running containers, backs off repeated restarts, and keeps going when a container vanishes mid-pass. +- The bootstrap health gate now ignores halts an operator has cleared and counts a sync halt held only in sync state. +- Module, explorer and config sidecar handling and the command lock were corrected, and status reports more accurately. + +### Fixed +- Kept bootstrap publisher capture logs on its configured temporary work volume. +- Published the command lock file complete so a contender never reclaims a live creator's empty lock. +- Made clear-reorg-halt name an unknown chain or network instead of reporting no decoder container installed. +- Made a reset whose database step fails after the stop report the failure and name the services it left down. +- Stamped each autoheal restart when it finishes so the cooldown and recovery probation no longer start early. +- Made a stop, start or other service command reject an unknown chain or network instead of acting on the shared explorer. +- Capped the hub config push at three minutes of wall time so a hung hub no longer holds the command lock for an hour. +- Made a hub config push that the hub answers with an error count as failed instead of delivered. +- Stopped the read-only ps, tail, logs and monitor commands from installing the hub and restarting one the operator stopped. +- Made exec exit non-zero with a short reason when the in-container command fails or matches no installed service. +- Made start, stop and restart exit non-zero when they match no installed service. +- Kept the shared explorer out of a start, stop, restart or uninstall scoped to one chain. + ## [0.26.3] - 2026-10-23 ### Changed @@ -64,6 +94,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Started the bundled MariaDB with a 1G InnoDB buffer pool by default, settable with `XCHAIN_NODE_DB_BUFFER_POOL_SIZE`. +## [0.23.3] - 2026-10-22 + +### Changed +- Pinned the v0.23.3 base-image and Brotli-guard patch train in the release manifest. + ## [0.23.0] - 2026-10-11 ### Added diff --git a/README.md b/README.md index f25b7d71..713f76c1 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ # XChain Platform Node

- Version - Tests + Version + Tests Node License

@@ -77,6 +77,10 @@ xchain-node stop all bitcoin regtest xchain-node start all bitcoin regtest ``` +The `start`, `stop`, `restart`, and `uninstall` commands leave the shared explorer +out when `all` is scoped to one chain, because one explorer serves every chain. Name it directly +when needed, for example `xchain-node restart explorer`. + ### Run a validator By default the bundled `xchain-hub` runs as a standalone config oracle. To run it @@ -170,19 +174,19 @@ Turn it off with any of: `--no-telemetry` on any command (sticks for future runs | Command | Description | |---|---| -| `npm test` | Unit tests (2,416 tests) | -| `npm run test:integration` | Integration tests (105 tests) | +| `npm test` | Unit tests (3,562 tests) | +| `npm run test:integration` | Integration tests (152 tests) | | `npm run test:smoke` | Smoke tests (159 tests) | -| `npm run test:boundary` | Boundary condition tests (57 tests) | -| `npm run test:security` | Security tests (149 tests) | -| `npm run test:e2e` | End-to-end tests (57 tests) | +| `npm run test:boundary` | Boundary condition tests (61 tests) | +| `npm run test:security` | Security tests (202 tests) | +| `npm run test:e2e` | End-to-end tests (58 tests) | | `npm run test:fuzz` | Fuzz tests (264 tests) | | `npm run test:chaos` | Chaos engineering tests (121 tests) | | `npm run test:regression` | Regression tests (59 tests) | | `npm run test:regression:p0` | Regression P0: critical gate (33 tests) | | `npm run test:regression:p0p1` | Regression P0+P1: standard gate (52 tests) | | `npm run test:mutation` | Mutation testing (Stryker Mutator) | -| `npm run test:all` | All tests (~3,181 tests; excludes security/boundary) | +| `npm run test:all` | All tests (~4,375 tests; excludes security/boundary) | | `npm run benchmark` | Performance benchmarks (5 scenarios) | | `npm run benchmark:quick` | Quick benchmarks | diff --git a/SECURITY.md b/SECURITY.md index 07f42524..e65499c7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,7 +25,7 @@ Email **security@dankest.llc** with: - The affected version (see `CHANGELOG.md` and the version badge in `README.md`) and the environment (host OS, Docker version, which services were installed). - Any patches or mitigations you'd like considered. -For sensitive reports, encrypt the email body to our PGP key. The fingerprint will be published alongside the first signed release artifact; until then, the email channel is acceptable for first contact and we will coordinate an encrypted exchange before you share proof-of-concept details. +For sensitive reports, the private advisory channel above is the confidential path we recommend: it is private until we publish it and needs no key exchange. If you would rather use email, send a first message without sensitive details and we will coordinate an encrypted exchange before you share them. We do not publish a PGP key for this address. Published release signing keys are provided only to verify downloads; do not encrypt a report to them. We do not currently offer a paid bug bounty. We do offer public credit in release notes and the advisory itself, unless you prefer to remain anonymous. @@ -92,4 +92,4 @@ We ship security fixes against the latest release on `master`. Older releases ar --- -Last reviewed: 2026-06-16. +Last reviewed: 2026-10-30. diff --git a/package-lock.json b/package-lock.json index bc86d599..54dab145 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-node", - "version": "0.26.3", + "version": "0.27.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-node", - "version": "0.26.3", + "version": "0.27.0", "license": "AGPL-3.0-or-later", "dependencies": { "@dankest-llc/xchain-sdk": "^0.18.0", @@ -33,7 +33,7 @@ "sinon": "^21.0.3" }, "engines": { - "node": ">=22.0.0" + "node": ">=22.0.0 <23" } }, "node_modules/@babel/code-frame": { @@ -2005,18 +2005,18 @@ "license": "MIT" }, "node_modules/body-parser": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", - "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.4.0.tgz", + "integrity": "sha512-Ifk1KcLrxaGt1occQagQNFNNOqRt1LFuehEnHHzB8Z6cRgIxRU1TBaiIsUiCdA+gy5PL7Y1IX31DNKkRmxtXjQ==", "license": "MIT", "dependencies": { "bytes": "^3.1.2", - "content-type": "^2.0.0", + "content-type": "^2.1.0", "debug": "^4.4.3", "http-errors": "^2.0.1", - "iconv-lite": "^0.7.2", + "iconv-lite": "^0.7.3", "on-finished": "^2.4.1", - "qs": "^6.15.2", + "qs": "^6.16.1", "raw-body": "^3.0.2", "type-is": "^2.1.0" }, @@ -3407,9 +3407,9 @@ "license": "ISC" }, "node_modules/follow-redirects": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.1.tgz", - "integrity": "sha512-FNvFGzoMLWmE6Yj9spb/zjd7yiNCHiAW9/Tg9CXrQ8wuu32HtlJOwWO11OJafl5FfY3DxTdQ0vj42zU1kvv5jg==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.2.tgz", + "integrity": "sha512-/kVP2U8z596Ia0gwFrwyIeo74ta5aVmSeU1FfAUMAmcEP5EP7zydmag3SLiN64NaRKef3lxsq7i/g7ltJQDGow==", "funding": [ { "type": "individual", @@ -6395,9 +6395,9 @@ } }, "node_modules/zod": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", - "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 45f4e33f..81e9b3fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xchain-node", - "version": "0.26.3", + "version": "0.27.0", "private": true, "description": "xchain-node allows users to install, configure and run XChain platform nodes.", "license": "AGPL-3.0-or-later", @@ -66,6 +66,7 @@ "js-yaml": "^4.3.1", "form-data": "^4.0.6", "ip-address": "^10.3.1", + "zod": "^4.6.6", "decimal.js": "10.4.3", "@modelcontextprotocol/sdk": "^1.31.0", "bitcoinjs-message": { @@ -77,6 +78,6 @@ "xchain-node": "./src/index.js" }, "engines": { - "node": ">=22.0.0" + "node": ">=22.0.0 <23" } } diff --git a/scripts/publish-bootstraps.sh b/scripts/publish-bootstraps.sh index 9932ddf3..01550625 100755 --- a/scripts/publish-bootstraps.sh +++ b/scripts/publish-bootstraps.sh @@ -351,6 +351,8 @@ log "publish plan (${#SELECTED[@]}): ${SELECTED[*]}" [ "$WITH_TRACKERS" = 1 ] && log "NOTE: tracker creates STOP the tracker container (downtime) for the compress." if [ "$DRY_RUN" = 1 ]; then log "dry-run: nothing executed."; exit 0; fi +mkdir -p "$TMP_DIR" + # ── Single-instance lock (cron-safe) ────────────────────────────────────── exec 9>"$LOCK_FILE" flock -n 9 || die "another publish run holds $LOCK_FILE - exiting." @@ -373,7 +375,7 @@ for c in "${SELECTED[@]}"; do # CLI does not also wait and multiply the budget. lock_deadline=$(( $(date +%s) + LOCK_WAIT_MIN * 60 )) while : ; do - create_log="$(mktemp)" + create_log="$(mktemp "$TMP_DIR/publish-create.XXXXXX")" create_rc=0 XCHAIN_NODE_BOOTSTRAP_SIGNING_KEY="$SIGNING_KEY" \ XCHAIN_NODE_DATA_DIR="$STAGE_DIR" \ diff --git a/src/cli/commands.js b/src/cli/commands.js index a8a71e83..f1843421 100644 --- a/src/cli/commands.js +++ b/src/cli/commands.js @@ -214,7 +214,7 @@ function registerUninstall(program, deps) { .argument('[network]', '(mainnet, testnet, regtest, all)') .option('--include-shared', 'Also uninstall shared services (xchain-explorer, xchain-sync, xchain-hub) once no chain/network is still installed; the database is never removed and must be removed manually') .action(async (service, chain, network, options) => { - const serviceList = filterCommandParameters(null, service, chain, network) + const serviceList = filterCommandParameters(null, service, chain, network, { includeSharedExplorer: false }) // A module that failed to uninstall must not be printed and forgotten, // leaving the command exiting 0 with containers still running. The // remaining modules are still attempted (uninstallModules finishes the @@ -329,8 +329,9 @@ function registerExec(program, deps) { .argument('', 'The shell command to execute') .action(async (service, chain, network, command) => { const serviceList = filterCommandParameters(null, service, chain, network) - await execModules(serviceList, command) - return process.exit(0) + // execModules has already printed the reason; scripts chaining on exec need the status. + const ok = await execModules(serviceList, command) + return process.exit(ok === true ? 0 : 1) }) } @@ -350,6 +351,9 @@ function registerClearReorgHalt(program, deps) { console.log("clear-reorg-halt takes one chain and one network; 'all' is invalid") return process.exit(1) } + // Refuse a typo here, before it reaches the container lookup and gets reported as "no container installed". + const bad = [['chain', chain, config.Coin], ['network', network, config.Network]].find(([, v, known]) => !Object.values(known).includes(v)) + if (bad) { console.log(`clear-reorg-halt: unknown ${bad[0]} '${bad[1]}'; expected one of ${Object.values(bad[2]).join(', ')}`); return process.exit(1) } const serviceList = filterCommandParameters(null, 'xchain-decoder', chain, network) const ok = await clearDecoderReorgHalt(serviceList, { reason: options.reason, force: !!options.force, dryRun: !!options.dryRun diff --git a/src/cli/dispatch.js b/src/cli/dispatch.js index fd12cf7b..417233fe 100644 --- a/src/cli/dispatch.js +++ b/src/cli/dispatch.js @@ -76,6 +76,14 @@ function skipsPreAction(actionCommand) { return commandName === 'bootstrap-republish-due' } +// A reset aimed at a stack this CLI did not install (--db-container) gets only the local docker checks. The bundled +// DB provisioning, the module scan, the hub install and the config push all act on this CLI's own stack, and the +// push would carry its DB hosts over the other stack's entries on a shared hub and explorer. +function preCheckOptionsFor(commandName, actionCommand) { + const opts = typeof actionCommand.opts === 'function' ? actionCommand.opts() : null + return commandName === 'reset' && opts && opts.dbContainer ? [{ localOnly: true }] : [] +} + // Name the lock for a validator subcommand that must serialize, or null for one // that need not. `init` writes the signing key, settings, capabilities and wallets // one file at a time, and a concurrent `install`/`update xchain-hub` mounts them. @@ -209,7 +217,8 @@ async function beforeAction(thisCommand, actionCommand, settings, deps) { settings.commandsNeedingVersions.includes(commandName), !settings.readOnlyCommands.includes(commandName), refForPreCheck(commandName, actionCommand), - commandRepairsHub(commandName, actionCommand) + commandRepairsHub(commandName, actionCommand), + ...preCheckOptionsFor(commandName, actionCommand) ) } finally { // Non-mutating commands hand the lock back as soon as provisioning is diff --git a/src/cli/errors.js b/src/cli/errors.js index d3589f4f..d39df72a 100644 --- a/src/cli/errors.js +++ b/src/cli/errors.js @@ -14,6 +14,199 @@ * XChain Node - CLI * Commander setup and command definitions ********************************************************************/ +const { execFile } = require('child_process') +const { promisify } = require('util') +const readline = require('readline') +const config = require('../config') +const dockerService = require('../services/docker_service') +const { stopModuleContainer } = require('../services/stop_budget_service') +const { statusChanged } = require('../services/status_service') +const { registerReset } = require('./reset_command') + +const HUB_CHECKPOINT_TABLES = Object.freeze([ + 'state_checkpoints', 'snapshot_checkpoints', 'anchor_published_checkpoints' +]) +const MIRROR_CHECKPOINT_TABLE = 'state_checkpoints' +const execFileAsync = promisify(execFile) +const hubCheckpointDefaults = { + execFileAsync, + readline, + stopContainer: dockerService.stopContainer, + startContainer: dockerService.startContainer, + getContainerStopSettings: dockerService.getContainerStopSettings, + stopModuleContainer, + statusChanged, + EXTERNAL_DB: config.EXTERNAL_DB, + HUB_MODULE_NAME: config.HUB_MODULE_NAME, + INDEXER_MODULE_NAME: config.XChainService.XCHAIN_INDEXER +} + +function commandOutput(result) { + return String(result && typeof result === 'object' && 'stdout' in result ? result.stdout || '' : result || '') +} +function dockerName(container) { + return String(container.Names || container.Name || '').replace(/^\//, '') +} +function running(container) { + return String(container.State || '').toLowerCase() === 'running' +} +function safeDatabaseName(value, label) { + const name = String(value || '') + if (!/^[A-Za-z0-9_$]+$/.test(name)) throw new Error(`${label} is not a safe database identifier`) + return name +} +function parseContainerList(result) { + return commandOutput(result).split(/\r?\n/).filter(line => line.trim()).map(line => JSON.parse(line)) +} +function railStacks(containers, deps) { + const suffix = '-' + deps.HUB_MODULE_NAME + const stacks = containers.filter(container => dockerName(container).endsWith(suffix)).map((hub) => { + const prefix = dockerName(hub).slice(0, -suffix.length) + const database = containers.find(container => dockerName(container) === prefix + '-database') + if (!database) throw new Error(`reset hub-checkpoints: no database container found for ${dockerName(hub)}`) + const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const indexer = new RegExp('^' + escaped + + '-(?:bitcoin|litecoin|dogecoin)-(?:mainnet|testnet|regtest)-xchain-indexer$') + const mirrors = containers.filter(container => indexer.test(dockerName(container)) + || dockerName(container) === prefix + '-xchain-explorer') + return { hub, database, mirrors } + }) + if (!stacks.length) throw new Error('reset hub-checkpoints: no rail hub containers were found') + return stacks +} +async function containerEnvironment(container, deps) { + const id = container.ID || dockerName(container) + const result = await deps.execFileAsync('docker', [ + 'inspect', '--type', 'container', '--format', '{{json .Config.Env}}', id + ]) + const entries = JSON.parse(commandOutput(result).trim()) + if (!Array.isArray(entries)) throw new Error(`container ${id} returned no environment array`) + return Object.fromEntries(entries.map((entry) => { + const separator = String(entry).indexOf('=') + return separator < 1 ? null : [String(entry).slice(0, separator), String(entry).slice(separator + 1)] + }).filter(Boolean)) +} +async function databaseTool(stack, script, database, table, deps) { + const id = stack.database.ID || dockerName(stack.database) + return deps.execFileAsync('docker', ['exec', id, 'sh', '-c', script, 'sh', database, table]) +} +const SHOW_TABLE = 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mariadb-show -u root "$1" "$2"' +const REBUILD_TABLE = 'f=$(mktemp) || exit 1; trap \'rm -f "$f"\' EXIT; ' + + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD"; export MYSQL_PWD; ' + + 'mariadb-dump -u root --no-data "$1" "$2" >"$f" && mariadb -u root "$1" <"$f"' + +async function tablePresent(stack, database, table, deps) { + try { + const result = await databaseTool(stack, SHOW_TABLE, database, table, deps) + return commandOutput(result).split(/\s+/).includes(table) + } catch { + return false + } +} + +async function prepareCheckpointTargets(stacks, deps) { + for (const stack of stacks) { + const hubEnv = await containerEnvironment(stack.hub, deps) + stack.hubDatabase = safeDatabaseName(hubEnv.HUB_DB_NAME, `${dockerName(stack.hub)} HUB_DB_NAME`) + for (const table of HUB_CHECKPOINT_TABLES) { + if (!(await tablePresent(stack, stack.hubDatabase, table, deps))) { + throw new Error(`reset hub-checkpoints: ${stack.hubDatabase}.${table} does not exist`) + } + } + const candidates = new Set() + for (const mirror of stack.mirrors.filter(item => dockerName(item).endsWith('-xchain-indexer'))) { + const env = await containerEnvironment(mirror, deps) + const database = safeDatabaseName(env.INDEXER_DB_NAME, `${dockerName(mirror)} INDEXER_DB_NAME`) + candidates.add(database) + candidates.add(database + '_HubMirror') + } + stack.mirrorDatabases = [] + for (const database of candidates) { + if (await tablePresent(stack, database, MIRROR_CHECKPOINT_TABLE, deps)) stack.mirrorDatabases.push(database) + } + } +} + +function serviceFor(container, deps) { + if (dockerName(container).endsWith('-xchain-indexer')) return deps.INDEXER_MODULE_NAME + if (dockerName(container).endsWith('-xchain-explorer')) return 'xchain-explorer' + return deps.HUB_MODULE_NAME +} + +async function stopContainers(containers, stopped, deps) { + for (const container of containers.filter(running)) { + const id = container.ID || dockerName(container) + await deps.stopModuleContainer(deps.stopContainer, serviceFor(container, deps), '', '', id, + undefined, deps.getContainerStopSettings) + stopped.push(container) + } +} + +async function startContainers(containers, failures, deps) { + for (const container of containers) { + try { + await deps.startContainer(container.ID || dockerName(container)) + } catch (err) { + failures.push(`${dockerName(container)} (${(err && err.message) || String(err)})`) + } + } +} + +async function confirmHubCheckpointReset(stacks, deps) { + if (!process.stdin.isTTY) throw new Error('reset hub-checkpoints requires --yes on a non-interactive terminal') + console.warn('\nWARNING: this will IRREVERSIBLY replace checkpoint lineage on every discovered rail hub.') + console.warn(` Rail hubs: ${stacks.map(stack => dockerName(stack.hub)).join(', ')}`) + console.warn(` Hub tables: ${HUB_CHECKPOINT_TABLES.join(', ')}`) + console.warn(` Mirror table: ${MIRROR_CHECKPOINT_TABLE}\n`) + const prompt = deps.readline.createInterface({ input: process.stdin, output: process.stdout }) + const answer = await new Promise(resolve => prompt.question('Type "yes" to confirm: ', resolve)) + prompt.close() + return answer.trim().toLowerCase() === 'yes' +} + +async function rebuildCheckpointTables(stacks, deps) { + for (const stack of stacks) { + for (const table of HUB_CHECKPOINT_TABLES) { + await databaseTool(stack, REBUILD_TABLE, stack.hubDatabase, table, deps) + } + } + for (const stack of stacks) { + for (const database of stack.mirrorDatabases) { + await databaseTool(stack, REBUILD_TABLE, database, MIRROR_CHECKPOINT_TABLE, deps) + } + } +} + +async function resetHubCheckpoints(force, overrides = {}) { + const deps = { ...hubCheckpointDefaults, ...overrides } + if (deps.EXTERNAL_DB) throw new Error('reset hub-checkpoints requires Docker-managed rail databases') + const listed = await deps.execFileAsync('docker', ['ps', '-a', '--no-trunc', '--format', '{{json .}}']) + const stacks = railStacks(parseContainerList(listed), deps) + await prepareCheckpointTargets(stacks, deps) + if (!force && !(await confirmHubCheckpointReset(stacks, deps))) return false + const hubs = [], mirrors = [], restartFailures = [] + try { + await stopContainers(stacks.map(stack => stack.hub), hubs, deps) + } catch (err) { + await startContainers(hubs, restartFailures, deps) + console.log(`Aborted: a rail hub failed to stop (${(err && err.message) || String(err)}). No data was touched.`) + return false + } + try { + await stopContainers(stacks.flatMap(stack => stack.mirrors), mirrors, deps) + } catch (err) { + await startContainers(mirrors.concat(hubs), restartFailures, deps) + console.log(`Aborted: a checkpoint mirror failed to stop (${(err && err.message) || String(err)}). No data was touched.`) + return false + } + let failure = null + try { await rebuildCheckpointTables(stacks, deps) } catch (err) { failure = err } + await startContainers(hubs.concat(mirrors), restartFailures, deps) + await deps.statusChanged() + if (failure) throw failure + if (restartFailures.length) throw new Error('Services failed to restart: ' + restartFailures.join(', ')) + return true +} // Commander's action handlers are async, but program.parse() is synchronous: // anything an action rejects with escapes as an unhandled rejection, which Node // prints as an ERR_UNHANDLED_REJECTION stack. Several services reject with a @@ -73,24 +266,6 @@ function registerE2eTest(program, deps) { } -function registerReset(program, deps) { - const { resetModules } = deps - program - .command('reset') - .description('Reset data for a specific service or all services of a coin/network') - .argument('', '(node, xchain-utxo-tracker, xchain-decoder, xchain-indexer, all)') - .argument('', '(bitcoin, litecoin, dogecoin)') - .argument('', '(mainnet, testnet, regtest)') - .option('--yes', 'Skip the destructive-reset confirmation prompt (for CI/scripted resets)') - .option('--with-indexer', 'Reset xchain-indexer alongside xchain-decoder; the pair is only coherent when both move together') - .action(async (service, chain, network, options) => { - const confirmed = await resetModules(service, chain, network, - !!(options && options.yes), !!(options && options.withIndexer)) - return process.exit(confirmed ? 0 : 1) - }) - -} - function registerRollback(program) { program .command('rollback') @@ -178,7 +353,7 @@ Notes: function registerPreRecoveryCommands(program, deps) { registerE2eTest(program, deps) - registerReset(program, deps) + registerReset(program, deps, resetHubCheckpoints) } function registerRecoveryCommands(program, deps) { @@ -190,5 +365,6 @@ module.exports = { installUnhandledRejectionHandler, installUncaughtExceptionHandler, registerPreRecoveryCommands, - registerRecoveryCommands + registerRecoveryCommands, + resetHubCheckpoints } diff --git a/src/cli/options.js b/src/cli/options.js index 4d2db95e..8de37b14 100644 --- a/src/cli/options.js +++ b/src/cli/options.js @@ -55,9 +55,11 @@ function registerStart(program, deps) { .argument('[chain]', '(bitcoin, litecoin, dogecoin, all)') .argument('[network]', '(mainnet, testnet, regtest, all)') .action(async (service, chain, network) => { - const serviceList = filterCommandParameters(null, service, chain, network) // Every selected module is still attempted; a failed one only changes the exit status. + // An unknown chain or network, or a selection with nothing installed, also exits 1 with its reason. try { + const serviceList = filterCommandParameters(null, service, chain, network, + { includeSharedExplorer: false }) await startModules(serviceList) } catch (err) { console.error('start failed: ' + redactSecrets(err && err.message ? err.message : err)) @@ -77,9 +79,11 @@ function registerStop(program, deps) { .argument('[chain]', '(bitcoin, litecoin, dogecoin, all)') .argument('[network]', '(mainnet, testnet, regtest, all)') .action(async (service, chain, network) => { - const serviceList = filterCommandParameters(null, service, chain, network) // Every selected module is still attempted; a failed one only changes the exit status. + // An unknown chain or network, or a selection with nothing installed, also exits 1 with its reason. try { + const serviceList = filterCommandParameters(null, service, chain, network, + { includeSharedExplorer: false }) await stopModules(serviceList) } catch (err) { console.error('stop failed: ' + redactSecrets(err && err.message ? err.message : err)) @@ -99,9 +103,11 @@ function registerRestart(program, deps) { .argument('[chain]', '(bitcoin, litecoin, dogecoin, all)') .argument('[network]', '(mainnet, testnet, regtest, all)') .action(async (service, chain, network) => { - const serviceList = filterCommandParameters(null, service, chain, network) // Every selected module is still attempted; a failed one only changes the exit status. + // An unknown chain or network, or a selection with nothing installed, also exits 1 with its reason. try { + const serviceList = filterCommandParameters(null, service, chain, network, + { includeSharedExplorer: false }) await restartModules(serviceList) } catch (err) { console.error('restart failed: ' + redactSecrets(err && err.message ? err.message : err)) @@ -118,9 +124,13 @@ function registerAutoheal(program, deps) { .command('autoheal') .description('Restart containers stuck in the Docker "unhealthy" state (opt-in per service); one-shot, cron/timer safe') .option('--dry-run', 'report restart candidates without acting') + .option('--resume-stopped', 'retry starting a container that an earlier autoheal restart left stopped') .action(async (options) => { const { runAutoheal, autohealExitCode } = loadModule('./services/autoheal_service') - const result = await runAutoheal({ dryRun: options.dryRun ?? false }) + const result = await runAutoheal({ + dryRun: options.dryRun ?? false, + resumeStopped: options.resumeStopped ?? false + }) // Exit 1 when a restart was attempted and failed, 2 when a container // outlived two restarts this episode (the escalation), and 0 otherwise, // so a timer unit alerts on both without paging on "nothing to do" passes. diff --git a/src/cli/reset_command.js b/src/cli/reset_command.js new file mode 100644 index 00000000..391da927 --- /dev/null +++ b/src/cli/reset_command.js @@ -0,0 +1,61 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - CLI + * The `reset` command: per-chain service resets and hub-checkpoints + ********************************************************************/ + +// resetHubCheckpoints is handed in by cli/errors.js, which defines it and registers this command. +function registerReset(program, deps, resetHubCheckpoints) { + const { resetModules } = deps + program + .command('reset') + .description('Reset data for a specific service or all services of a coin/network') + .argument('', '(node, xchain-utxo-tracker, xchain-decoder, xchain-indexer, all, hub-checkpoints)') + .argument('[chain]', '(bitcoin, litecoin, dogecoin); omit for hub-checkpoints') + .argument('[network]', '(mainnet, testnet, regtest); omit for hub-checkpoints') + .option('--yes', 'Skip the destructive-reset confirmation prompt (for CI/scripted resets)') + .option('--with-indexer', 'Reset xchain-indexer alongside xchain-decoder; the pair is only coherent when both move together') + .option('--db-container ', 'Run the database steps on this MariaDB container, for a stack this CLI did not install; finds its services by container name and skips the hub and explorer config push (xchain-decoder and xchain-indexer only)') + .option('--check', 'With --db-container: check the server, its root login (or, without one, the stack\'s own database users and their grants) and the target databases, then stop before anything is stopped or changed') + .action(async (service, chain, network, options) => { + if (options && options.check && !options.dbContainer) { + process.stderr.write('reset: --check needs --db-container\n') + return process.exit(1) + } + if (service === 'hub-checkpoints') { + if (options && options.dbContainer) { + process.stderr.write('reset: --db-container does not apply to hub-checkpoints\n') + return process.exit(1) + } + if (chain || network) { + process.stderr.write('reset: hub-checkpoints does not take a chain or network\n') + return process.exit(1) + } + const completed = await resetHubCheckpoints(!!(options && options.yes), deps.hubCheckpointDependencies) + return process.exit(completed ? 0 : 1) + } + if (service !== 'hub-checkpoints' && (!chain || !network)) { + process.stderr.write('reset: chain and network are required unless service is hub-checkpoints\n') + return process.exit(1) + } + // Passed only when given, so a reset without the flag calls resetModules exactly as before. + const overrideArgs = options && options.dbContainer ? [{ dbContainer: options.dbContainer, check: !!options.check }] : [] + const confirmed = await resetModules(service, chain, network, + !!(options && options.yes), !!(options && options.withIndexer), ...overrideArgs) + return process.exit(confirmed ? 0 : 1) + }) + +} + +module.exports = { registerReset } diff --git a/src/config/index.js b/src/config/index.js index c41642ed..edc576ba 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -243,6 +243,7 @@ module.exports = { get XCHAIN_NODE_BOOTSTRAP_SKIP_HEALTH_GATE() { return process.env.XCHAIN_NODE_BOOTSTRAP_SKIP_HEALTH_GATE || '' }, get XCHAIN_NODE_CONTAINERD_ROOT() { return process.env.XCHAIN_NODE_CONTAINERD_ROOT || '/var/lib/containerd' }, get XCHAIN_NODE_DATA_DIR() { return process.env.XCHAIN_NODE_DATA_DIR }, + get XCHAIN_NODE_DB_CONTAINER_ROOT_PASSWORD() { return process.env.XCHAIN_NODE_DB_CONTAINER_ROOT_PASSWORD }, get XCHAIN_NODE_DB_DATA_DIR() { return process.env.XCHAIN_NODE_DB_DATA_DIR }, get XCHAIN_NODE_DB_MAX_CONNECTIONS() { return process.env.XCHAIN_NODE_DB_MAX_CONNECTIONS }, get XCHAIN_NODE_DB_ROOT_PASSWORD() { return process.env.XCHAIN_NODE_DB_ROOT_PASSWORD }, diff --git a/src/db/halt_markers.js b/src/db/halt_markers.js index 53cfb01d..17f46f3f 100644 --- a/src/db/halt_markers.js +++ b/src/db/halt_markers.js @@ -80,6 +80,17 @@ function pendingSyncHaltCountSql(name) { `AND CHAR_LENGTH(COALESCE(state_value, '')) > 0;` } +/** + * Saved rollback guards, encoded for the caller to judge with xchain-sync's + * loadRollbackGuardState rule. LEFT avoids treating '_' as a wildcard. + */ +function rollbackGuardStateSql(name) { + return `SELECT COUNT(*), COALESCE(GROUP_CONCAT(` + + `IF(state_value IS NULL, 'N', CONCAT('V', HEX(state_value))) ` + + `ORDER BY state_key SEPARATOR ','), '') FROM \`${name}\`.sync_state ` + + `WHERE LEFT(state_key, 15) = 'rollback_guard:';` +} + /** * The dump-window watermark for the events table. * @@ -122,6 +133,7 @@ module.exports = { newestReorgClearSql, liveSyncHaltCountSql, pendingSyncHaltCountSql, + rollbackGuardStateSql, eventsWatermarkSql, syncHaltWatermarkSql, reorgHaltsSinceSql, diff --git a/src/db/information_schema.js b/src/db/information_schema.js index b7affe67..07a70046 100644 --- a/src/db/information_schema.js +++ b/src/db/information_schema.js @@ -90,6 +90,9 @@ function createTimeSql(tableCount) { /** The session pin the create-time read above depends on. Run it on the same connection, first. */ const SESSION_UTC_SQL = "SET SESSION time_zone = '+00:00'" +/** The grants of the account this session logged in as, one GRANT line each; any user may read its own. */ +const CURRENT_USER_GRANTS_SQL = 'SHOW GRANTS FOR CURRENT_USER()' + module.exports = { schemaExistsSql, tableCountSql, @@ -97,5 +100,6 @@ module.exports = { tablesExistSql, schemaSizeSql, createTimeSql, - SESSION_UTC_SQL + SESSION_UTC_SQL, + CURRENT_USER_GRANTS_SQL } diff --git a/src/db/modules.js b/src/db/modules.js index 48ca7166..64f65d82 100644 --- a/src/db/modules.js +++ b/src/db/modules.js @@ -167,6 +167,21 @@ const modulesMixin = { } }, + async deleteModuleContainerIfMatches(module, coin, network, containerId) { + if (!this.pool) return false + try { + const result = await this.pool.query( + `DELETE FROM ${MODULES_TABLE} + WHERE module = ? AND coin = ? AND network = ? AND container_id = ?`, + [module, coin || '', network || '', containerId] + ) + return result.affectedRows > 0 + } catch (err) { + logger.info(err) + return false + } + }, + async getModuleCount() { if (!this.pool) return 0 const rows = await this.pool.query(`SELECT COUNT(*) AS cnt FROM ${MODULES_TABLE}`) diff --git a/src/operations/module_operations.js b/src/operations/module_operations.js index 45187028..6d99062a 100644 --- a/src/operations/module_operations.js +++ b/src/operations/module_operations.js @@ -30,7 +30,7 @@ const { sleep } = require('../utils/helpers') const { getDockerContainerImageName, getUtxoTrackerVolumeName, getDockerNetwork } = require('../services/config_service') const { createDockerNetwork, probeContainerPresenceByName, stopContainer, stopContainerByName, startContainer, restartContainer, execContainer, shellContainer, logContainer, startDockerMonitor, waitContainer, saveContainerLogs, getContainerBindMounts, getContainerStopSettings, removeContainer } = require('../services/docker_service') const { stopModuleContainer, restartModuleContainer } = require('../services/stop_budget_service') -const { buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, pingExternalDatabase, askMariadbRootPassword } = require('../services/database_service') +const { buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, resolveNamedDatabaseTarget, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, pingExternalDatabase, askMariadbRootPassword } = require('../services/database_service') const { getModuleBranch, installModule, uninstallModule } = require('../services/module_service') const { assertHubNotBehind } = require('../services/skew_guard_service') const { assertRequiredMigrationsApplied } = require('../services/migration_precondition_service') @@ -162,7 +162,7 @@ const dependencies = { shellContainer, logContainer, startDockerMonitor, waitContainer, saveContainerLogs, getContainerBindMounts, getContainerStopSettings, removeContainer, stopModuleContainer, restartModuleContainer, buildDatabaseModule, resetDatabases, - clearHubPriceIngestWatermark, purgeHubCrossChainRows, + clearHubPriceIngestWatermark, resolveNamedDatabaseTarget, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, askMariadbRootPassword, pingExternalDatabase, getModuleBranch, installModule: installModuleWithProgress, uninstallModule, diff --git a/src/operations/module_operations/module_controls.js b/src/operations/module_operations/module_controls.js index 5e62e1b1..a26b8141 100644 --- a/src/operations/module_operations/module_controls.js +++ b/src/operations/module_operations/module_controls.js @@ -85,14 +85,23 @@ async function readRegisteredContainer(target, failures) { // Run one docker action on every selected module that is installed, attempting all of them, and // throw one error naming each module that failed so the command can exit non-zero. +// A selection in which nothing is installed is a failure too: a typo'd service name, or a real one +// that is not installed on this chain and network, would otherwise act on nothing and report success, +// and a scripted `stop` would believe a service was down while it kept running. async function controlModules(verb, servicesList, act) { const failures = [] + const unmatched = [] + let matched = 0 for (const coin in servicesList) { for (const network in servicesList[coin]) { for (const module of servicesList[coin][network]) { const target = { module, coin, network } const containerId = await readRegisteredContainer(target, failures) - if (!containerId) continue + if (!containerId) { + unmatched.push(target) + continue + } + matched++ try { await act(target, containerId) } catch (err) { @@ -108,6 +117,12 @@ async function controlModules(verb, servicesList, act) { err.failures = failures throw err } + if (matched === 0) { + const asked = unmatched.map(t => `${t.module} (${t.coin || 'shared'} ${t.network || 'shared'})`).join(', ') + const err = new Error(`${verb} matched no installed service: ${asked || 'nothing was selected'}; nothing was changed`) + err.failures = unmatched.map(t => ({ ...t, reason: 'not installed' })) + throw err + } return true } @@ -214,23 +229,52 @@ async function clearDecoderReorgHalt(servicesList, { reason, force = false, dryR return ok } +// Print why one in-container command failed: whatever it wrote, then its exit code (or the +// spawn error when it never ran) on stderr, never the raw error object with its stack. +function reportExecFailure(target, err) { + const out = err && typeof err.stdout === 'string' ? err.stdout.trim() : '' + if (out) console.log(out) + const why = err && typeof err.code === 'number' + ? 'exit code ' + err.code + : String((err && err.message) || err).split('\n')[0] + console.error(`exec failed in ${target.module} (${target.coin} ${target.network}): ${why}`) + const errText = err && typeof err.stderr === 'string' ? err.stderr.trim() : '' + if (errText) console.error(errText) +} + +// Run one command in every selected container that is installed. Returns true only when at +// least one container matched and every run exited 0, so the CLI can exit non-zero when the +// command failed anywhere or the selection (a typo'd service, one not installed here) ran nothing. async function execModules(servicesList, command) { const commandArgs = command.split(/\s+/) - for (const nextCoin in servicesList) { - for (const nextNetwork in servicesList[nextCoin]) { - for (const nextModule of servicesList[nextCoin][nextNetwork]) { - const containerId = await db.getModuleContainer(nextModule, nextCoin, nextNetwork) - if (!containerId) continue + const unmatched = [] + let matched = 0 + let ok = true + for (const coin in servicesList) { + for (const network in servicesList[coin]) { + for (const module of servicesList[coin][network]) { + const target = { module, coin, network } + const containerId = await db.getModuleContainer(module, coin, network) + if (!containerId) { + unmatched.push(target) + continue + } + matched++ try { - const execStdOut = await execContainer(containerId, commandArgs) - console.log(execStdOut) + console.log(await execContainer(containerId, commandArgs)) } catch (err) { - console.log(err) + reportExecFailure(target, err) + ok = false } } } } - return true + if (matched === 0) { + const asked = unmatched.map(t => `${t.module} (${t.coin} ${t.network})`).join(', ') + console.error(`exec matched no installed service: ${asked || 'nothing was selected'}; nothing was run`) + return false + } + return ok } async function shellModule(servicesList) { @@ -251,8 +295,14 @@ async function shellModule(servicesList) { return true } +// The reset's lookup finds each container: by name under --db-container, through the registry otherwise. +function resetContainerLookup(context) { + return context.findContainer || ((module, coin, network) => db.getModuleContainerStrict(module, coin, network)) +} + async function restartResetModules(context) { const { coin, modulesToStop, network } = context + const findContainer = resetContainerLookup(context) console.log(`Restarting ${coin} ${network} services...`) // Track restart failures instead of swallowing them: a silent skip here left a wiped stack DOWN (node never restarted, every dependent service crash-looped) while `reset` still reported success. // "Not installed" (registry miss) stays a legitimate skip; a failed docker start gets one retry, then is reported loudly at the end. @@ -260,7 +310,7 @@ async function restartResetModules(context) { for (const module of modulesToStop) { let containerId = null try { - containerId = await db.getModuleContainerStrict(module, coin, network) + containerId = await findContainer(module, coin, network) } catch (err) { // Strict, like the stop loop: the swallowing read answered null on a SQL error too, so a registry blip here left a just-wiped service DOWN and still reported a clean reset // (uuid:846cc40d). Nothing can be undone at this point, so report it with the other start failures rather than aborting. @@ -292,6 +342,7 @@ async function bounceResetModules(context) { // empirically deterministic and costs ~5s on the happy path. const bounceCandidates = [XChainService.XCHAIN_DECODER, XChainService.XCHAIN_INDEXER] .filter((m) => modulesToStop.includes(m)) + const findContainer = resetContainerLookup(context) if (bounceCandidates.length > 0) { await sleep(5000) for (const module of bounceCandidates) { @@ -299,7 +350,7 @@ async function bounceResetModules(context) { if (startFailures.some((f) => f.module === module)) continue let containerId = null try { - containerId = await db.getModuleContainerStrict(module, coin, network) + containerId = await findContainer(module, coin, network) } catch (err) { // A failed read is not "not installed": the bounce did not happen, so report it like a start failure. startFailures.push({ module, error: `registry lookup failed before the post-reset bounce (${failureReason(err)})` }) @@ -324,7 +375,13 @@ async function bounceResetModules(context) { } } - await statusChanged() + // A --db-container stack is not in this CLI's config, so pushing that config would overwrite the hub's and + // explorer's view of it with the wrong DB hosts. + if (context.dbContainer) { + console.log('Skipped the hub and explorer config push: --db-container names a stack outside this CLI\'s config.') + } else { + await statusChanged() + } if (startFailures.length > 0) { const detail = startFailures.map((f) => `${f.module}: ${f.error}`).join('; ') diff --git a/src/operations/module_operations/repo_refs.js b/src/operations/module_operations/repo_refs.js index 43ddefaf..76e2eb7b 100644 --- a/src/operations/module_operations/repo_refs.js +++ b/src/operations/module_operations/repo_refs.js @@ -110,11 +110,13 @@ function failureReason(err) { // // Reads the registry strictly: getModuleContainer answers null on a SQL error as well as on a miss, so a rollback run during the very registry outage that caused the abort restarted nothing and still // reported zero failures (uuid:846cc40d). A lookup that fails now lands in `failed` and is named in the STILL DOWN line. -async function restartStoppedModules(modules, coin, network) { +// `findContainer` is the reset's own lookup (by container name under --db-container); the registry otherwise. +async function restartStoppedModules(modules, coin, network, + findContainer = (module, lookupCoin, lookupNetwork) => db.getModuleContainerStrict(module, lookupCoin, lookupNetwork)) { const failed = [] for (const module of modules) { try { - const containerId = await db.getModuleContainerStrict(module, coin, network) + const containerId = await findContainer(module, coin, network) if (!containerId) continue await startContainer(containerId) } catch { diff --git a/src/operations/module_operations/reset_db_override.js b/src/operations/module_operations/reset_db_override.js new file mode 100644 index 00000000..b56e60cb --- /dev/null +++ b/src/operations/module_operations/reset_db_override.js @@ -0,0 +1,111 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Reset against a stack's own DB server (--db-container) + * Which resets may take the override, and how its services are found + ********************************************************************/ + +'use strict' + +// Say why a reset may not take --db-container, or null when it may. A stack run by hand outside this CLI keeps +// set-aside chain dirs beside its live datadir, so only the database resets are allowed: no node wipe (`all`, +// `node`), and no tracker volume, which is not a database step. +function dbOverrideRefusal(service, { externalDb, nodeModuleName, XChainService }) { + if (externalDb) return '--db-container cannot be combined with the external-DB mode' + if (service === 'all' || service === nodeModuleName) { + return `--db-container never wipes a node datadir, so it does not take '${service}'` + } + if (service !== XChainService.XCHAIN_DECODER && service !== XChainService.XCHAIN_INDEXER) { + return `--db-container applies only to ${XChainService.XCHAIN_DECODER} and ${XChainService.XCHAIN_INDEXER}` + } + return null +} + +// Find a service container by the name the stack gives it, not through this CLI's registry: a stack this CLI did +// not install is not in it, and a recreated container leaves a stale id there. Only docker saying "no such +// container" is absence; any other failure throws, so the reset refuses instead of wiping under a live service. +function containerByNameLookup(execFileAsync, getDockerContainerImageName) { + return async (module, coin, network) => { + const name = getDockerContainerImageName(module, coin, network) + let stdout + try { + ({ stdout } = await execFileAsync('docker', ['inspect', '--type', 'container', '--format', '{{.Id}}', name])) + } catch (err) { + const text = String((err && (err.stderr || '')) + ' ' + ((err && err.message) || err)) + if (/no such (object|container)/i.test(text)) return null + throw new Error(`docker inspect ${name} failed (${(err && err.message) || err})`) + } + const id = String(stdout).trim() + if (!/^[a-f0-9]{64}$/.test(id)) throw new Error(`docker inspect ${name} returned no container id`) + return id + } +} + +// Read --db-container off a reset's options: why it is refused (or null), and how this reset finds its containers. +// `deps` carries the reset's own config constants, registry and docker exec. +function dbOverrideFor(service, options, deps) { + const dbContainer = (options && options.dbContainer) || null + const refusal = dbContainer + ? dbOverrideRefusal(service, { externalDb: deps.EXTERNAL_DB, nodeModuleName: deps.NODE_MODULE_NAME, XChainService: deps.XChainService }) + : null + const findContainer = dbContainer + ? containerByNameLookup(deps.execFileAsync, deps.getDockerContainerImageName) + : (module, coin, network) => deps.db.getModuleContainerStrict(module, coin, network) + return { refusal, dbContainer, findContainer, check: !!(dbContainer && options.check) } +} + +// The databases a reset drops and recreates, decoder first. +function dbModulesFor({ resetDecoder, resetIndexer }, XChainService) { + return [ + ...(resetDecoder ? [XChainService.XCHAIN_DECODER] : []), + ...(resetIndexer ? [XChainService.XCHAIN_INDEXER] : []), + ] +} + +// The trailing argument a DB call takes for a --db-container reset; none otherwise, so the default calls are unchanged. +function namedTargetArgs(context) { + return context.namedDbTarget ? [context.namedDbTarget] : [] +} + +// Resolve the named server while nothing is stopped: its id, a root or the stack's own database users that log in, +// and the target databases already on it. The stack's services are found by the same name lookup the stop loop +// uses, so their env can supply those users. Answers null after printing the refusal, so the caller returns early. +async function namedTargetBeforeStop(context, resolveNamedDatabaseTarget, failureReason, XChainService) { + const { coin, dbContainer, network } = context + const findServiceContainer = (module) => context.findContainer(module, coin, network) + let target + try { + target = await resolveNamedDatabaseTarget(dbContainer, coin, network, dbModulesFor(context, XChainService), findServiceContainer) + } catch (err) { + console.log(`Aborted: cannot use the MariaDB container ${dbContainer} (${failureReason(err)}). No data was touched.`) + return null + } + console.log(`Database steps run on ${dbContainer} (${target.databases.join(', ')}); the hub and explorer config push is skipped.`) + // Users and the grants that allowed them, never a password. A root run says so only under --check, so its output is unchanged. + if (target.authReport) { + for (const line of target.authReport) console.log(line) + } else if (context.check) { + console.log(`Logs in to ${dbContainer} as root.`) + } + return target +} + +// Where a --db-container reset ends while nothing is stopped: a refused server (null, already reported) fails, and a +// --check run (which skips the confirmation, since it changes nothing) passes here instead of reaching the stop loop. +function endBeforeStop(target) { + if (!target) return false + console.log('Check passed: the reset would run as above. Nothing was stopped or changed.') + return true +} + +module.exports = { dbOverrideRefusal, containerByNameLookup, dbOverrideFor, dbModulesFor, endBeforeStop, namedTargetArgs, namedTargetBeforeStop } diff --git a/src/operations/module_operations/reset_modules.js b/src/operations/module_operations/reset_modules.js index 69f063e3..89dfb6c0 100644 --- a/src/operations/module_operations/reset_modules.js +++ b/src/operations/module_operations/reset_modules.js @@ -1,11 +1,12 @@ 'use strict' -let askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer, stopModuleContainer, getContainerStopSettings +let askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, resolveNamedDatabaseTarget, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer, stopModuleContainer, getContainerStopSettings let RESETTABLE_SERVICES -const { WIPE_IMAGE, ensureWipeImage, reportPartialWipe } = require('./reset_wipe_safety') +const { WIPE_IMAGE, ensureWipeImage, reportPartialWipe, reportDatabaseWipeFailure } = require('./reset_wipe_safety') +const { dbOverrideFor, dbModulesFor, endBeforeStop, namedTargetArgs, namedTargetBeforeStop } = require('./reset_db_override') function configure(dependencies) { - ({ askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer, stopModuleContainer, getContainerStopSettings } = dependencies) + ({ askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, resolveNamedDatabaseTarget, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer, stopModuleContainer, getContainerStopSettings } = dependencies) RESETTABLE_SERVICES = [ 'all', NODE_MODULE_NAME, @@ -19,7 +20,8 @@ function configure(dependencies) { // an unrecognised service (a typo, or a non-resettable module like xchain-encoder) leaves every reset flag false, so `targets` is empty, no branch fires, and resetModules returns true - the CLI exits // 0 reporting success after resetting nothing. Fail loud instead, matching resolveArgs/rollback and the "fail fast BEFORE any destructive wipe" convention this file already follows. -async function resetModules(service, coin, network, force = false, withIndexer = false) { +// `options.dbContainer` (--db-container) is a stack's own MariaDB container; see reset_db_override.js. +async function resetModules(service, coin, network, force = false, withIndexer = false, options = {}) { if (!RESETTABLE_SERVICES.includes(service)) { throw new Error("reset: unknown service '" + service + "'; expected one of " + RESETTABLE_SERVICES.join(', ')) @@ -32,12 +34,14 @@ async function resetModules(service, coin, network, force = false, withIndexer = throw new Error("reset: unknown network '" + network + "'; expected one of " + Object.values(Network).join(', ')) } + const { refusal, dbContainer, findContainer, check } = dbOverrideFor(service, options, { EXTERNAL_DB, NODE_MODULE_NAME, XChainService, db, execFileAsync, getDockerContainerImageName }) + if (refusal) { console.log(`Aborted: ${refusal}. No data was touched.`); return false } const resetAll = service === 'all' const resetNode = resetAll || service === NODE_MODULE_NAME const resetUtxoTracker = resetAll || service === XChainService.XCHAIN_UTXO_TRACKER const resetDecoder = resetAll || service === XChainService.XCHAIN_DECODER const resetIndexer = resetAll || service === XChainService.XCHAIN_INDEXER || (withIndexer && resetDecoder) - return validateDecoderPair({ service, coin, network, force, resetAll, resetNode, resetUtxoTracker, resetDecoder, resetIndexer }) + return validateDecoderPair({ service, coin, network, force, resetAll, resetNode, resetUtxoTracker, resetDecoder, resetIndexer, dbContainer, findContainer, check }) } async function validateDecoderPair(context) { @@ -50,7 +54,7 @@ async function validateDecoderPair(context) { try { // Strict read: getModuleContainer answers null on a SQL error and on an unopened pool as well as on a genuine miss, so a registry blip read as "no indexer installed" and waved through the // one wipe this guard exists to stop (uuid:7cbafa08). A lookup that FAILS is not evidence of absence. Only an empty result set still means "not installed", which stays allowed. - indexerInstalled = await db.getModuleContainerStrict(XChainService.XCHAIN_INDEXER, coin, network) + indexerInstalled = await context.findContainer(XChainService.XCHAIN_INDEXER, coin, network) } catch (err) { // abortBeforeAnyWipe is declared further down this function and nothing has been stopped yet, so the plain refusal is the correct shape here. console.log(`Aborted: cannot read the ${XChainService.XCHAIN_INDEXER} registry row ` @@ -125,7 +129,7 @@ async function resolveResetPaths(context) { async function confirmReset(context) { const { blocksDir, blocksHostPath, coin, force, network, nodeDataPath, resetDecoder, resetIndexer, resetNode, resetUtxoTracker, txindexHostPath } = context - if (!force) { + if (!force && !context.check) { const targets = [] if (resetNode) { if (nodeDataPath) targets.push(`node datadir (${nodeDataPath})`) @@ -163,8 +167,12 @@ async function checkResetDatabase(context) { // on a partial env, and NOTHING probed for it (uuid:41887889). Both modes are probed here. It sits ahead of the stop loop, not after it: this abort returns before the restart pass, so probing // later left every already-stopped service DOWN while still reporting that no data was touched (uuid:bb190060). const dbResetNeeded = resetDecoder || resetIndexer + let namedDbTarget = null if (dbResetNeeded) { - if (EXTERNAL_DB) { + if (context.dbContainer) { + namedDbTarget = await namedTargetBeforeStop(context, resolveNamedDatabaseTarget, failureReason, XChainService) + if (!namedDbTarget || context.check) return endBeforeStop(namedDbTarget) + } else if (EXTERNAL_DB) { const probe = await pingExternalDatabase() if (!probe.ok) { console.log(`Aborted: cannot reach the external MariaDB at ${probe.host}:${probe.port}` @@ -198,7 +206,7 @@ async function checkResetDatabase(context) { if (resetDecoder) modulesToStop.push(XChainService.XCHAIN_DECODER) if (resetIndexer) modulesToStop.push(XChainService.XCHAIN_INDEXER) if (resetAll) modulesToStop.push(XChainService.XCHAIN_REGTEST_MINER) - return stopResetModules({ ...context, modulesToStop }) + return stopResetModules({ ...context, modulesToStop, namedDbTarget }) } async function stopResetModules(context) { @@ -210,7 +218,7 @@ async function stopResetModules(context) { const stoppedModules = [] // Refuse the whole reset, put back whatever this run stopped, and report it. Only reachable while nothing has been wiped yet, which is why it may still promise that no data was touched. const abortBeforeAnyWipe = async (reason) => { - const restartFailures = await restartStoppedModules(stoppedModules, coin, network) + const restartFailures = await restartStoppedModules(stoppedModules, coin, network, context.findContainer) console.log(`Aborted: ${reason}. No data was touched.`) if (stoppedModules.length > 0) { console.log(` Restarted ${stoppedModules.length - restartFailures.length} of ` @@ -224,7 +232,7 @@ async function stopResetModules(context) { for (const module of modulesToStop) { let containerId = null try { - containerId = await db.getModuleContainerStrict(module, coin, network) + containerId = await context.findContainer(module, coin, network) } catch (err) { // A registry read that FAILED is not evidence the module is absent. The swallowing read this replaced answered null on any SQL error, so a blip after the reachability precheck made a live // indexer look uninstalled: the loop skipped stopping it and the wipes below, resetDatabases included, ran underneath it (uuid:846cc40d). @@ -273,7 +281,7 @@ async function classifyResetVolume(context) { } async function wipeResetStores(context) { - const { blocksHostPath, coin, network, nodeDataPath, resetDecoder, resetIndexer, resetNode, resetUtxoTracker, txindexHostPath, utxoVolumeName, utxoVolumePresent } = context + const { blocksHostPath, coin, network, nodeDataPath, resetNode, resetUtxoTracker, txindexHostPath, utxoVolumeName, utxoVolumePresent } = context // The node paths already cleared, so a later abort reports the stack's real state instead of promising an untouched one. const nodeCleared = [] // A failed node wipe may have deleted part of its target, so it is reported as possibly partial with services left down, never sent to abortBeforeAnyWipe (see reportPartialWipe). @@ -309,12 +317,9 @@ async function wipeResetStores(context) { } } - const dbModulesToReset = [ - ...(resetDecoder ? [XChainService.XCHAIN_DECODER] : []), - ...(resetIndexer ? [XChainService.XCHAIN_INDEXER] : []), - ] + const dbModulesToReset = dbModulesFor(context, XChainService) if (dbModulesToReset.length > 0) { - await resetDatabases(coin, network, dbModulesToReset) + try { await resetDatabases(coin, network, dbModulesToReset, ...namedTargetArgs(context)) } catch (err) { return reportDatabaseWipeFailure(context, { err, failureReason, nodeCleared, trackerCleared: Boolean(resetUtxoTracker && utxoVolumePresent) }) } } return cleanResetMetadata(context) @@ -327,7 +332,7 @@ async function cleanResetMetadata(context) { // the restart pass and leave the stack down. It is reported loudly instead, with the statement to run by hand. if (resetIndexer) { try { - await clearHubPriceIngestWatermark(coin, network) + await clearHubPriceIngestWatermark(coin, network, ...namedTargetArgs(context)) } catch (err) { console.warn('WARNING: clearing the hub price ingest fence failed: ' + (err && err.message ? err.message : err)) console.warn(" Run on the hub DB before the indexer catches up:") @@ -392,5 +397,4 @@ function recordResetReindex(context) { return restartResetModules(context) } - module.exports = { configure, resetModules } diff --git a/src/operations/module_operations/reset_wipe_safety.js b/src/operations/module_operations/reset_wipe_safety.js index a504e198..c94b84bb 100644 --- a/src/operations/module_operations/reset_wipe_safety.js +++ b/src/operations/module_operations/reset_wipe_safety.js @@ -40,21 +40,27 @@ async function ensureWipeImage(execFileAsync, failureReason) { // Report a wipe that started and failed, then end the reset with every stopped service LEFT DOWN. A non-zero exit does not prove nothing was deleted: `find -delete` removes every entry it can // before it exits non-zero on a per-entry error, and a daemon can die mid-run, so the target may be partly cleared. Restarting a service over it, or dropping the decoder/indexer databases around // it, would run a store that no longer describes the chain, so nothing restarts and nothing else is wiped until the operator re-runs the same reset. Returns false for the caller to return. -function reportPartialWipe(context, { target, err, failureReason, nodeCleared = [], trackerUntouched = false }) { +function reportPartialWipe(context, { target, err, failureReason, nodeCleared = [], trackerUntouched = false, trackerCleared = false, databasesAttempted = false }) { const { resetDecoder, resetIndexer, stoppedModules = [], utxoVolumeName } = context console.log(`Aborted: clearing ${target} failed (${failureReason(err)}).`) console.log(` ${target} may be PARTLY cleared.`) if (nodeCleared.length > 0) { console.log(` The node data for this stack WAS already cleared: ${nodeCleared.join(', ')}.`) } - // Name only the stores this reset was going to wipe and never reached, so the operator knows what still holds its old data. + if (trackerCleared) console.log(` The Docker volume ${utxoVolumeName} WAS already cleared.`) + // Name only the stores this reset was going to wipe and never reached, so the operator knows what still holds its old data. The database step itself is not "untouched" once it was attempted. const untouched = [] if (trackerUntouched) untouched.push(`the Docker volume ${utxoVolumeName}`) - if (resetDecoder || resetIndexer) untouched.push('the decoder/indexer databases') + if (!databasesAttempted && (resetDecoder || resetIndexer)) untouched.push('the decoder/indexer databases') if (untouched.length > 0) console.log(` NOT touched: ${untouched.join(', ')}.`) console.log(` The stopped services are left down: ${stoppedModules.length > 0 ? stoppedModules.join(', ') : 'none'}.`) console.log(' Fix the problem and re-run the same reset command.') return false } -module.exports = { WIPE_IMAGE, ensureWipeImage, reportPartialWipe } +// The database step is the last wipe and the services are already stopped, so its failure is reported like any other started wipe: a DROP may have run, nothing restarts, the stopped services are named. +function reportDatabaseWipeFailure(context, { err, failureReason, nodeCleared = [], trackerCleared = false }) { + return reportPartialWipe(context, { target: 'the decoder/indexer databases', err, failureReason, nodeCleared, trackerCleared, databasesAttempted: true }) +} + +module.exports = { WIPE_IMAGE, ensureWipeImage, reportPartialWipe, reportDatabaseWipeFailure } diff --git a/src/precheck.js b/src/precheck.js index a9e1d807..79280e23 100644 --- a/src/precheck.js +++ b/src/precheck.js @@ -280,13 +280,17 @@ async function pushConfigToHubAndExplorer(syncHubConfig, repairsHub) { // `repairsHub` says the command about to run rebuilds, recreates, updates or // removes the hub container (cli.js commandRepairsHub decides). It only ever // relaxes the config push below, and only while the hub is not answering. -async function preCheck(checkVersions = false, syncHubConfig = true, moduleRef = null, repairsHub = false) { +// +// `localOnly` stops after the docker and directory checks, for a reset aimed at a stack this CLI did not install +// (`reset --db-container`): no bundled DB provisioning, no module scan, no hub install, no hub or explorer push. +async function preCheck(checkVersions = false, syncHubConfig = true, moduleRef = null, repairsHub = false, { localOnly = false } = {}) { await requireDocker() await warnIfContainerdStoreOnRoot() await warnIfMemoryLimitsUnsupported() if (isVerbose()) console.log("Checking/Creating directories") createDirectories() + if (localOnly) return true await createBaseNetwork() await installDatabaseContainer() @@ -310,7 +314,12 @@ async function preCheck(checkVersions = false, syncHubConfig = true, moduleRef = // this never mints (a host with no sidecar stays keyless exactly as before). await applyHubApiKeyFromSidecar(hostEnv()) - await installHubAtRef(moduleRef) + // Provisioning the hub is state-changing: installHubModule ends in + // useInstalledHub, which restarts a hub whose State is 'exited'. A read-only + // command (syncHubConfig=false: ps, tail, logs, monitor) must not change any + // container's run state, or `ps` starts a hub the operator stopped and then + // prints its stale 'exited' status. Mutating commands still install here. + if (syncHubConfig) await installHubAtRef(moduleRef) await pushConfigToHubAndExplorer(syncHubConfig, repairsHub) diff --git a/src/release-manifest.json b/src/release-manifest.json index 481f7e4c..3acc8b5b 100644 --- a/src/release-manifest.json +++ b/src/release-manifest.json @@ -1,9 +1,9 @@ { "_comment": [ - "Pinned component set for XChain Platform v0.26.3.", + "Pinned component set for XChain Platform v0.27.0.", "Generated by bin/write-release-manifest.js from the ACTUAL tagged master merge commits.", "xchain-node is the carrier and is not listed: checking out its tag IS this manifest.", - "A train tags only the repos it touches. This one moves xchain-indexer.", + "A train tags only the repos it touches. This one moves xchain-vm, xchain-decoder, xchain-indexer, xchain-hub, xchain-sync, xchain-encoder, xchain-utxo-tracker, xchain-explorer and xchain-sdk.", "Every other component is unchanged and keeps the tag it already carries, which is", "what section 4 means by a version being the platform version at which a component", "last changed. A gap is unchanged, not skipped.", @@ -12,44 +12,44 @@ "repo's origin/master, and is a GPG-signed annotated tag verifying against the", "platform release key (fingerprint 1DA7C4896F56EA22CF491EDF4361611A82F90B70)." ], - "platform_version": "0.26.3", - "released": "2026-10-23", + "platform_version": "0.27.0", + "released": "2026-11-01", "components": { "xchain-vm": { - "tag": "v0.26.2", - "commit": "ec24d865aa98ee775256ac32a99ec8d05ba1f1dd" + "tag": "v0.27.0", + "commit": "a246047cef0489765f6aba7937a0f4b7db9c1220" }, "xchain-decoder": { - "tag": "v0.26.2", - "commit": "e7938b84a9bf53ae2dbffe48fa4dbd53cd8f0af9" + "tag": "v0.27.0", + "commit": "544bc68ed224e8059a7eadf3ffec0eb041be9e62" }, "xchain-indexer": { - "tag": "v0.26.3", - "commit": "aee467626fd0aa9ee08f2939d454bdaaea9bfb2c" + "tag": "v0.27.0", + "commit": "bec4a86e069eae20360fadfe1f53a813483ad341" }, "xchain-hub": { - "tag": "v0.26.2", - "commit": "7b95cb6bd0b010d822675cf7333eb15944a35331" + "tag": "v0.27.0", + "commit": "2456445a7e885161d636a8af5055ed03cdc521c1" }, "xchain-sync": { - "tag": "v0.26.2", - "commit": "ed65168cf45be46654b808dd9fd0efdd66de773f" + "tag": "v0.27.0", + "commit": "5bc106c7e4cfaa9a916a5b9e6cdb33a8a13e5db3" }, "xchain-encoder": { - "tag": "v0.26.2", - "commit": "45fa6a20d432a8e5021a6e7a2d47b9399084cddb" + "tag": "v0.27.0", + "commit": "3e56a93976c84e43570033c84ac1778fa5c5a728" }, "xchain-utxo-tracker": { - "tag": "v0.26.2", - "commit": "4797fdf5e559172182e9f3135dec86bfa61f732c" + "tag": "v0.27.0", + "commit": "e057c2f0eacf41fdae43d2a97ae5b95872a2468d" }, "xchain-explorer": { - "tag": "v0.26.2", - "commit": "7178a40fb4f63bb4f909009dacf5e1621e28856b" + "tag": "v0.27.0", + "commit": "61ec55a790546f00416b5fae387c49e1c9d66ae5" }, "xchain-sdk": { - "tag": "v0.26.2", - "commit": "589ff5af02f90279b23dd78b80a16e34281c6bf4" + "tag": "v0.27.0", + "commit": "4bff77b57b5e0fc4095df051b9f7ca19b5ad563a" }, "xchain-e2e-test": { "tag": "v0.21.0", diff --git a/src/services/autoheal_service.js b/src/services/autoheal_service.js index 29ba924f..12e7dbf9 100644 --- a/src/services/autoheal_service.js +++ b/src/services/autoheal_service.js @@ -21,29 +21,17 @@ * exits. An alive-but-stalled service therefore stays wedged forever, * visible only in `docker ps`. * - * `xchain-node autoheal` is a ONE-SHOT pass meant to run from cron or - * a systemd timer (it never prompts and never daemonizes): it walks - * the module registry, inspects each container whose service opted in - * via `autoheal: true` in SERVICE_HEALTHCHECK, and restarts the ones - * whose Docker health status is `unhealthy` AND which have been + * `xchain-node autoheal` is a one-shot cron or systemd timer pass. It walks + * opted-in module containers and restarts the ones whose Docker status is unhealthy and * continuously unhealthy past a grace window. An on-disk state file * (~/.xchain-node/autoheal-state.json) throttles restarts of the same * container: the first retry waits the base cooldown, and each further * restart of a container that never recovers DOUBLES the wait, up to a * ceiling. So a wedge a restart does not clear costs one restart per * cooldown, then per 2x, 4x, 8x... rather than one per cooldown forever. - * The counter is dropped on evidence the container RECOVERED: an observed - * `healthy`, or a retained passing probe that landed more than the restart - * probation window after the last autoheal restart. A pass inside that - * window is the restart's own artifact and is not recovery. So a transient - * wedge always starts again from the base cooldown, including when the - * recovery fell entirely between two passes and only the probe log ever - * witnessed it. - * - * Deliberately no attempt CAP and no terminal suppression: this is a - * watchdog on node containers, and a cap that stops retrying can leave a - * RECOVERABLE service down through an outage that would have self-cleared. - * Backing off ends the churn; giving up would trade it for an outage. + * The counter is dropped only on evidence the container recovered, including + * a retained passing probe beyond the restart probation window. + * There is deliberately no attempt cap: backoff ends churn without leaving a recoverable service down forever. * The escalation path instead: once a container has been restarted twice * in one episode without recovering, every pass that sees it logs an * "investigate" line at warn (stderr) and the command exits 2, so a @@ -112,8 +100,39 @@ async function inspectCandidate(candidate, result) { } } -// Heal only what is actually RUNNING. Docker freezes State.Health.Status -// at its last value the moment a container stops (the probe goroutine +async function handleAutohealStopped(candidate, state, result, dryRun, resumeStopped, now) { + const { module, coin, network, containerId, label, status } = candidate + if (!state.stoppedByAutoheal[containerId]) return null + if (status && status.State && status.State.Status === 'running') { + delete state.stoppedByAutoheal[containerId] + return { handled: false, changed: true } + } + result.candidates.push({ module, coin, network, containerId }) + if (!resumeStopped) { + result.escalated.push({ module, coin, network, containerId, attempts: state.restartCount[containerId] || 0 }) + logger.warn(`autoheal: ${label} remains stopped after a failed restart; investigate or rerun with --resume-stopped`) + return { handled: true, changed: false } + } + if (dryRun) { + logger.info(`autoheal: DRY RUN, would retry starting ${label}`) + return { handled: true, changed: false } + } + try { + await startContainer(containerId) + delete state.stoppedByAutoheal[containerId] + state.restarts[containerId] = now + state.restartCount[containerId] = (state.restartCount[containerId] || 0) + 1 + result.restarted.push({ module, coin, network, containerId }) + logger.info(`autoheal: started ${label} after an earlier restart left it stopped`) + return { handled: true, changed: true } + } catch (err) { + result.failed.push({ module, coin, network, containerId, reason: String(err) }) + logger.error(`autoheal: FAILED to start ${label} after an earlier restart left it stopped: ${err}`) + return { handled: true, changed: false } + } +} + +// Heal only what is running. Docker freezes State.Health.Status when a container stops (the probe goroutine // runs only while the container is up), so a container that happened to // be unhealthy when an operator stopped it keeps reporting `unhealthy` // while State.Status is `exited` - and a restart (stop, then start) of a @@ -230,7 +249,7 @@ function establishEpisode(candidate, health, state, now, probationMs, result) { return { since: episode.since, changed: episode.onsetReseeded || restartCountCleared } } -async function restartEligibleCandidate(candidate, since, timing, state, result, dryRun, now) { +async function restartEligibleCandidate(candidate, since, timing, state, result, dryRun, now, wallStart) { const { module, coin, network, containerId, label } = candidate if (now - since < timing.graceMs) { result.skipped.push({ module, coin, network, containerId, reason: 'inside grace window' }) @@ -269,13 +288,20 @@ async function restartEligibleCandidate(candidate, since, timing, state, result, // With the stop budget `stop` uses, so the restart gives a block-bound drain its time await restartModuleContainer(stopContainer, startContainer, module, coin, network, containerId, undefined, getContainerStopSettings) - state.restarts[containerId] = now + // Stamp the moment the restart finished, not the pass start: the cooldown + // and the recovery probation both run "after a restart", and a drain that + // hits its kill budget (120s for a decoder) plus earlier restarts in the + // same pass would otherwise shorten both windows. `now` may be injected, + // so advance it by the real time the restart consumed. + state.restarts[containerId] = now + Math.max(0, Date.now() - wallStart) state.restartCount[containerId] = attempts + 1 result.restarted.push({ module, coin, network, containerId }) logger.info(`autoheal: restarted ${label} (unhealthy for ${now - since}ms, restart #${attempts + 1} this episode)`) } catch (err) { + if (err && err.containerStopped) state.stoppedByAutoheal[containerId] = true result.failed.push({ module, coin, network, containerId, reason: String(err) }) logger.error(`autoheal: FAILED to restart ${label}: ${err}`) + return Boolean(err && err.containerStopped) } } @@ -292,14 +318,15 @@ function persistState(stateFile, state, modules) { for (const id of Object.keys(state.restartCount)) { if (!known.has(id)) delete state.restartCount[id] } + for (const id of Object.keys(state.stoppedByAutoheal)) { + if (!known.has(id)) delete state.stoppedByAutoheal[id] + } writeState(stateFile, state) } -// One autoheal pass over the module registry. Never throws for a single bad -// container (a vanished container id must not abort the whole sweep). -// Returns { candidates, restarted, failed, skipped, escalated } where each array holds -// { module, coin, network, containerId, reason? }; escalated entries carry attempts instead. -async function runAutoheal({ dryRun = false, now = Date.now() } = {}) { +// One pass. A vanished container does not abort the sweep; failures and escalation reach the CLI. +async function runAutoheal({ dryRun = false, resumeStopped = false, now = Date.now() } = {}) { + const wallStart = Date.now() const timing = autohealTiming() const stateFile = getStateFilePath() const state = readState(stateFile) @@ -321,6 +348,12 @@ async function runAutoheal({ dryRun = false, now = Date.now() } = {}) { if (!candidate) continue if (!await inspectCandidate(candidate, result)) continue + const startRetry = await handleAutohealStopped(candidate, state, result, dryRun, resumeStopped, now) + if (startRetry) { + if (startRetry.changed) onsetChanged = true + if (startRetry.handled) continue + } + const stoppedStateChanged = skipNonRunning(candidate, state, result) if (stoppedStateChanged !== null) { if (stoppedStateChanged) onsetChanged = true @@ -337,7 +370,9 @@ async function runAutoheal({ dryRun = false, now = Date.now() } = {}) { const episode = establishEpisode(candidate, health, state, now, timing.probationMs, result) if (!episode) continue if (episode.changed) onsetChanged = true - await restartEligibleCandidate(candidate, episode.since, timing, state, result, dryRun, now) + if (await restartEligibleCandidate(candidate, episode.since, timing, state, result, dryRun, now, wallStart)) { + onsetChanged = true + } } if (!dryRun && (onsetChanged || result.restarted.length > 0)) persistState(stateFile, state, modules) diff --git a/src/services/autoheal_service/restart_state.js b/src/services/autoheal_service/restart_state.js index 228dcf5d..a2aabf4c 100644 --- a/src/services/autoheal_service/restart_state.js +++ b/src/services/autoheal_service/restart_state.js @@ -73,7 +73,8 @@ function readState(stateFile) { // { // restarts: { : }, // unhealthySince: { : }, - // restartCount: { : } + // restartCount: { : }, + // stoppedByAutoheal: { : true when a stopped container still needs starting } // } try { const parsed = JSON.parse(fs.readFileSync(stateFile, 'utf8')) @@ -88,13 +89,16 @@ function readState(stateFile) { if (typeof parsed.restartCount !== 'object' || parsed.restartCount === null) { parsed.restartCount = {} } + if (typeof parsed.stoppedByAutoheal !== 'object' || parsed.stoppedByAutoheal === null) { + parsed.stoppedByAutoheal = {} + } return parsed } } catch { // Missing or corrupt state file: start clean. Worst case a container // gets one extra restart after a state loss, which is acceptable. } - return { restarts: {}, unhealthySince: {}, restartCount: {} } + return { restarts: {}, unhealthySince: {}, restartCount: {}, stoppedByAutoheal: {} } } function writeState(stateFile, state) { diff --git a/src/services/bootstrap_archive_meta.js b/src/services/bootstrap_archive_meta.js index c3ea045e..6b36046f 100644 --- a/src/services/bootstrap_archive_meta.js +++ b/src/services/bootstrap_archive_meta.js @@ -172,8 +172,7 @@ async function readBootstrapArchiveMeta(archivePath) { const ARCHIVE_IDENTITY_FIELDS = ['module', 'coin', 'network'] // An identity value trimmed and lowercased, or null when it is absent or blank: the same normalization the tracker's -// restore_validation.js applies to bootstrap.json. Only the normalization is shared: a target lacking a value for a -// declared field is a mismatch here but unchecked in the tracker, so the two readers can reach different verdicts. +// restore_validation.js applies to bootstrap.json. Both readers treat a target lacking a declared field as a mismatch. function identityValue(value) { if (typeof value !== 'string') return null const v = value.trim().toLowerCase() diff --git a/src/services/bootstrap_health_gate.js b/src/services/bootstrap_health_gate.js index ac1e28af..7007a83b 100644 --- a/src/services/bootstrap_health_gate.js +++ b/src/services/bootstrap_health_gate.js @@ -65,7 +65,9 @@ const logger = getLogger(); const { DEFAULT_MAX_LAG_BLOCKS, evaluateContainerState, evaluateStatusPayload } = require('./bootstrap_health_gate/status_evaluation.js') -const { parseCountTokens, readHaltMarkers } = require('./bootstrap_health_gate/halt_markers.js') +const { + parseCountTokens, readHaltMarkers, pendingSyncHaltReason +} = require('./bootstrap_health_gate/halt_markers.js') // Which env key carries the container-internal API port for each module. // Mirrors ModuleService's SERVICE_HEALTHCHECK portKeys, deliberately: the probe @@ -101,8 +103,8 @@ class BootstrapSourceUnhealthyError extends Error { 'A bootstrap archive becomes the newest (and therefore default) recovery source for the whole ' + 'fleet, so publishing an unverified one is worse than publishing nothing. Fix the service, then ' + 're-run: a durable halt marker takes the recovery its line above names (a decoder REORG_HALT ' + - 'also clears in place with `xchain-node clear-reorg-halt` once verified; any other marker means ' + - 'a full resync from a known-good snapshot). To snapshot ' + + 'also clears in place once verified, and an invalid rollback_guard row clears through xchain-sync; ' + + 'a divergence marker requires a full resync from a known-good snapshot). To snapshot ' + 'a known-bad database deliberately, set XCHAIN_NODE_BOOTSTRAP_SKIP_HEALTH_GATE=1.' ) this.name = 'BootstrapSourceUnhealthyError' @@ -296,14 +298,6 @@ async function healthSurfaceReasons(containerId, module, coin, network, runner) return reasons } -// Refusal text for an xchain-sync halt_pending copy found in `where`. -function pendingSyncHaltReason(where) { - return `${where} carries an xchain-sync halt_pending copy in sync_state (a divergence halt whose ` + - 'sync_halt write never landed): xchain-sync re-installs that halt on every replica that restores ' + - 'this archive. Clear the halt through xchain-sync (POST /halt/clear///), which drops ' + - 'the copy, then re-run the publish.' -} - // Read the durable halt markers and turn them into refusal reasons. Returns the // reasons and the watermark the caller hands back (null when the probe failed). async function haltMarkerReasons(coin, network, module, dbDeps, since) { @@ -331,6 +325,10 @@ async function haltMarkerReasons(coin, network, module, dbDeps, since) { // re-installs it on every replica that restores the archive. if (markers.pendingSyncHalt > 0) reasons.push(pendingSyncHaltReason('the database')) + if (markers.rollbackGuardInvalid > 0) + reasons.push('the database carries an invalid xchain-sync rollback_guard row in sync_state. ' + + 'xchain-sync halts on this row at startup on every replica that restores the archive; clear ' + + 'the halt through xchain-sync, then re-run the publish.') // Refuse on the paired decoder's markers too: REORG_HALT lives only there, and an // indexer frozen behind a halted decoder is exactly as unfit to publish. if (markers.upstream && markers.upstream.reorgHalt > 0) @@ -349,6 +347,10 @@ async function haltMarkerReasons(coin, network, module, dbDeps, since) { 'derived from it are known to diverge from the source of truth.') if (markers.upstream && markers.upstream.pendingSyncHalt > 0) reasons.push(pendingSyncHaltReason(`the paired decoder database ${markers.upstream.dbName}`)) + if (markers.upstream && markers.upstream.rollbackGuardInvalid > 0) + reasons.push(`the paired decoder database ${markers.upstream.dbName} carries an invalid xchain-sync ` + + 'rollback_guard row in sync_state. xchain-sync halts on it at startup on every replica that ' + + 'restores the archive; clear the halt through xchain-sync, then re-run the publish.') } catch (err) { reasons.push(`could not read the halt markers from the ${module} database: ${err && err.message}`) } diff --git a/src/services/bootstrap_health_gate/halt_markers.js b/src/services/bootstrap_health_gate/halt_markers.js index 31d89121..1d534818 100644 --- a/src/services/bootstrap_health_gate/halt_markers.js +++ b/src/services/bootstrap_health_gate/halt_markers.js @@ -16,7 +16,7 @@ * * Reads the durable halt markers (a live decoder REORG_HALT row, one no newer * audited REORG_HALT_CLEARED row supersedes, an uncleared sync_halt row, and an - * xchain-sync halt_pending copy in sync_state) and the + * xchain-sync halt_pending copy and invalid rollback_guard state in sync_state) and the * dump-window watermarks straight out of the database * about to be dumped, and for an indexer source out of its paired decoder * database as well. Every unreadable answer throws, and the gate turns a throw @@ -30,6 +30,7 @@ const { dockerMariadbArgs, mariadbEnv } = require('../../utils/docker_mariadb') const { configuredDatabaseName } = require('../../utils/module_database_name') const { markerTablesSql, liveReorgHaltCountSql, newestReorgClearSql, liveSyncHaltCountSql, pendingSyncHaltCountSql, + rollbackGuardStateSql, eventsWatermarkSql, syncHaltWatermarkSql, reorgHaltsSinceSql, syncHaltsSinceSql } = require('../../db/halt_markers') @@ -88,6 +89,34 @@ function parseNewestClear(raw, what) { return isAuditedClear(payload) ? 0 : 1 } +// Same validity rule as xchain-sync ClientSync.loadRollbackGuardState. +function isValidRollbackGuardState(raw) { + try { + const state = JSON.parse(raw) + return Number.isSafeInteger(state.peak) && Number.isSafeInteger(state.low) && + state.peak >= 0 && state.low >= 0 && state.low <= state.peak + } catch (_) { + return false + } +} + +function parseRollbackGuardRows(raw, what) { + const text = String(raw == null ? '' : raw).replace(/\r?\n$/, '') + const fields = text.split('\t') + const count = /^\d+$/.test(fields[0] || '') ? Number(fields[0]) : NaN + const encoded = fields[1] === 'NULL' ? '' : (fields[1] || '') + const tokens = encoded === '' ? [] : encoded.split(',') + const bad = fields.length !== 2 || !Number.isSafeInteger(count) || count < 0 || + tokens.length !== count || tokens.some(token => !/^(?:N|V(?:[0-9A-F]{2})*)$/.test(token)) + if (bad) + throw new Error(`the ${what} returned unreadable output: ${JSON.stringify(text.slice(0, 80))}`) + return tokens.reduce((invalid, token) => { + if (token === 'N') return invalid + const value = Buffer.from(token.slice(1), 'hex').toString('utf8') + return invalid + (isValidRollbackGuardState(value) ? 0 : 1) + }, 0) +} + // A watermark is interpolated into SQL below, and it comes back through a caller // chain rather than straight from parseCountTokens, so assert the shape rather // than trust it. null means "nothing recorded", which the caller reads as @@ -169,6 +198,34 @@ async function probeDatabase(run, readCount, name, since) { return found } +// Count the live halt rows in each optional marker table of `name`, for readLiveMarkers. +async function readHaltCounts(run, readCount, name, { hasSyncHalt, hasSyncState }) { + const found = { reorgHalt: 0, syncHalt: 0, pendingSyncHalt: 0, rollbackGuardInvalid: 0 } + // LIVE halts only: a REORG_HALT row older than the newest REORG_HALT_CLEARED + // row was cleared by an operator through `xchain-node clear-reorg-halt` + // (audited in the decoder's events table) and no longer disqualifies the + // database. The halt row itself is never deleted, so a plain count would + // refuse a cleared database forever. + found.reorgHalt = await readCount(liveReorgHaltCountSql(name), 'REORG_HALT marker') + // A newest clear row without an audited reason lifts nothing: the decoder reads it as + // a live halt, so it counts as one here too. + found.reorgHalt += parseNewestClear(await run(newestReorgClearSql(name)), + `REORG_HALT_CLEARED audit probe for ${name}`) + // sync_halt IS optional: xchain-sync provisions it only where it runs, so an + // absent table here is a genuine "no such marker", not an unread database. + if (hasSyncHalt > 0) + found.syncHalt = await readCount(liveSyncHaltCountSql(name), 'sync_halt marker') + // A halt whose sync_halt write failed lives only as xchain-sync's sync_state copy, + // which a restore re-installs as a halt; sync_state is optional for the same reason. + // It has no id sequence, so it is read live at each reading, never by watermark. + if (hasSyncState > 0) { + found.pendingSyncHalt = await readCount(pendingSyncHaltCountSql(name), 'sync_state halt_pending marker') + found.rollbackGuardInvalid = parseRollbackGuardRows(await run(rollbackGuardStateSql(name)), + `sync_state rollback_guard probe for ${name}`) + } + return found +} + // Read which marker tables exist in `name`, how many live halt rows each holds, // and the id watermark of each, for probeDatabase. async function readLiveMarkers(run, readCount, name) { @@ -195,26 +252,7 @@ async function readLiveMarkers(run, readCount, name) { if (hasEvents === 0) throw new Error(`${name} reports no events table, so the REORG_HALT marker could not be read`) - const found = { reorgHalt: 0, syncHalt: 0, pendingSyncHalt: 0 } - // LIVE halts only: a REORG_HALT row older than the newest REORG_HALT_CLEARED - // row was cleared by an operator through `xchain-node clear-reorg-halt` - // (audited in the decoder's events table) and no longer disqualifies the - // database. The halt row itself is never deleted, so a plain count would - // refuse a cleared database forever. - found.reorgHalt = await readCount(liveReorgHaltCountSql(name), 'REORG_HALT marker') - // A newest clear row without an audited reason lifts nothing: the decoder reads it as - // a live halt, so it counts as one here too. - found.reorgHalt += parseNewestClear(await run(newestReorgClearSql(name)), - `REORG_HALT_CLEARED audit probe for ${name}`) - // sync_halt IS optional: xchain-sync provisions it only where it runs, so an - // absent table here is a genuine "no such marker", not an unread database. - if (hasSyncHalt > 0) - found.syncHalt = await readCount(liveSyncHaltCountSql(name), 'sync_halt marker') - // A halt whose sync_halt write failed lives only as xchain-sync's sync_state copy, - // which a restore re-installs as a halt; sync_state is optional for the same reason. - // It has no id sequence, so it is read live at each reading, never by watermark. - if (hasSyncState > 0) - found.pendingSyncHalt = await readCount(pendingSyncHaltCountSql(name), 'sync_state halt_pending marker') + const found = await readHaltCounts(run, readCount, name, { hasSyncHalt, hasSyncState }) // The dump window watermark. Both marker tables are append-only and // id-ordered (decoder events: AUTO_INCREMENT, a clear is a NEW row, halt rows @@ -307,9 +345,20 @@ async function readHaltMarkers(coin, network, module, deps, since) { return markers } +// Refusal text for an xchain-sync halt_pending copy found in `where`. +function pendingSyncHaltReason(where) { + return `${where} carries an xchain-sync halt_pending copy in sync_state (a divergence halt whose ` + + 'sync_halt write never landed): xchain-sync re-installs that halt on every replica that restores ' + + 'this archive. Clear the halt through xchain-sync (POST /halt/clear///), which drops ' + + 'the copy, then re-run the publish.' +} + module.exports = { + pendingSyncHaltReason, parseCountTokens, parseNewestClear, + parseRollbackGuardRows, + isValidRollbackGuardState, isAuditedClear, readHaltMarkers } diff --git a/src/services/bootstrap_health_gate/status_evaluation.js b/src/services/bootstrap_health_gate/status_evaluation.js index f86dffc8..b8047d5d 100644 --- a/src/services/bootstrap_health_gate/status_evaluation.js +++ b/src/services/bootstrap_health_gate/status_evaluation.js @@ -129,12 +129,12 @@ function statusFlagReasons(payload) { return reasons } -// Refusal reasons from the indexer's wedged-counter verdict and a train-activation halt. +// Refusal reasons from the indexer's wedged-counter verdict and durable halts. function stallReasons(payload) { const reasons = [] // The indexer's own single-field verdict on its block counter: 'none' | - // 'future_block_wait' | 'bridge_proof_wait' | 'train_activation_halt' | 'price_landing_wait' | - // 'barrier_defer' | 'wedged', plus the published refinements 'upstream_halted' (only beside + // 'future_block_wait' | 'bridge_proof_wait' | 'train_activation_halt' | 'operator_action_hold' | + // 'price_landing_wait' | 'barrier_defer' | 'wedged', plus the published refinements 'upstream_halted' (only beside // decoderReorgHalted: true, refused above) | 'rollcall_proof_wait' | 'rollcall_proof_stuck' | // 'capability_proof_wait' | 'capability_proof_stuck'. The set is defined in xchain-indexer // src/XChainIndexer/stall_health.js (stallClassOf, publishedStallClass). 'wedged' and the @@ -151,6 +151,9 @@ function stallReasons(payload) { if (payload.stallClass === 'wedged') reasons.push('the service reports its block counter WEDGED (stallClass "wedged": no commit for longer ' + 'than its stall grace window)' + (payload.stallReason ? `: ${payload.stallReason}` : '')) + if (payload.stallClass === 'operator_action_hold') + reasons.push('the service reports its block counter held until an operator acts (stallClass ' + + '"operator_action_hold": a restart cannot clear it)' + (payload.stallReason ? `: ${payload.stallReason}` : '')) // Refuse a proof wait past its hold ceiling: the indexer wedges on it only later, above the ceiling. if (payload.stallClass === 'rollcall_proof_stuck' || payload.stallClass === 'capability_proof_stuck') reasons.push(`the service reports its block counter held on a proof past its hold ceiling (stallClass ` + diff --git a/src/services/bootstrap_node_tip_guard.js b/src/services/bootstrap_node_tip_guard.js index c85b5b89..b6de3d55 100644 --- a/src/services/bootstrap_node_tip_guard.js +++ b/src/services/bootstrap_node_tip_guard.js @@ -55,12 +55,15 @@ const { execFile } = require('child_process') const { promisify } = require('util') +const fs = require('fs') +const path = require('path') const execFileAsync = promisify(execFile) const { XChainService, NODE_MODULE_NAME } = require('../config') const gate = require('./bootstrap_health_gate'); const archiveMeta = require('./bootstrap_archive_meta'); const config = require('../config'); +const configService = require('./config_service') const { getLogger } = require('../observability/logger'); const logger = getLogger(); @@ -299,6 +302,83 @@ async function assessNodeTipForRestore({ coin, network, module, archivePath }, d return result } +function movedPortTarget(environmentVariables, constants) { + const ticker = String(environmentVariables.INDEXER_COIN || '') + const network = String(environmentVariables.INDEXER_NETWORK || '') + const coin = Object.entries(constants.CoinTickerSymbol || {}) + .find(([, value]) => String(value) === ticker)?.[0] + return coin && network ? { coin, network } : null +} + +function configuredPortKeys(...fileContents) { + const keys = new Set() + for (const contents of fileContents) { + for (const line of String(contents || '').split(/\r?\n/)) { + const at = line.indexOf('=') + if (at > 0) keys.add(line.substring(0, at)) + } + } + return keys +} + +function createMovedPortClaim(baseClaimFreeHostPorts, deps = {}) { + return function claimFreeHostPortsWithFollowups(conflicts, portArgs, requested, published, claim) { + if (!claim || conflicts.length === 0) { + return baseClaimFreeHostPorts(conflicts, portArgs, requested, published, claim) + } + + const constants = deps.constants || config + const target = movedPortTarget(claim.environmentVariables, constants) + const configDir = deps.configDir || constants.configDir + const configPath = target && configDir + ? path.resolve(configDir, `${target.coin}-${target.network}`) + : null + const readConfigFile = deps.readConfigFile || ((file) => { + try { return fs.readFileSync(file, 'utf8') } catch { return '' } + }) + const explicitKeys = configuredPortKeys( + configPath ? readConfigFile(configPath) : '', + configPath ? readConfigFile(configPath + '.local') : '') + const ports = claim.registry?.[claim.module]?.docker?.ports || [] + const originalConflicts = conflicts.slice() + const movable = conflicts.filter(({ hostPort }) => ports.some((entry) => + !entry.always && !explicitKeys.has(entry.host) && + String(claim.environmentVariables[entry.host]) === hostPort)) + const candidates = new Set(movable) + const before = new Map(ports.map(entry => [entry.host, claim.environmentVariables[entry.host]])) + + baseClaimFreeHostPorts(movable, portArgs, requested, published, claim) + const unresolved = new Set(movable) + conflicts.splice(0, conflicts.length, + ...originalConflicts.filter(conflict => !candidates.has(conflict) || unresolved.has(conflict))) + + const movedValues = {} + for (const entry of ports) { + if (before.get(entry.host) !== claim.environmentVariables[entry.host]) { + movedValues[entry.host] = claim.environmentVariables[entry.host] + } + } + if (target && Object.keys(movedValues).length > 0) { + const service = deps.configService || configService + service.upsertSidecarValues(configPath + '.local', movedValues) + } + return conflicts + } +} + +function installMovedPortClaim(hostPortClaim, deps = {}) { + const current = hostPortClaim.claimFreeHostPorts + const base = current.baseClaimFreeHostPorts || current + if (!current.baseClaimFreeHostPorts) { + hostPortClaim.claimFreeHostPorts = createMovedPortClaim(base, deps) + hostPortClaim.claimFreeHostPorts.baseClaimFreeHostPorts = base + } + return hostPortClaim.claimFreeHostPorts +} + +const hostPortClaim = require('./module_service/host_port_claim') +installMovedPortClaim(hostPortClaim) + module.exports = { VERDICT, NODE_CLI_ARGS, diff --git a/src/services/config_service/filters.js b/src/services/config_service/filters.js index ffcbf2ce..bbc13329 100644 --- a/src/services/config_service/filters.js +++ b/src/services/config_service/filters.js @@ -48,17 +48,29 @@ function serviceAliases() { // box, had not even started. Every other command that expands `all` // (update, start, stop, uninstall) tolerates either order. // Explicitly-named shared service → emit under the empty ""/"" key only. -function filterCommandParameters(branch, modules, coins, networks) { +function filterCommandParameters(branch, modules, coins, networks, { includeSharedExplorer = true } = {}) { const servicesList = {} let addExplorer = false const aliases = serviceAliases() if (modules && aliases[modules]) modules = aliases[modules] - coins = coins && coins !== "all" ? [coins] : Object.values(Coin) + // An unknown chain or network must fail here, before any service list is + // built: otherwise `stop all ` still yields the shared explorer entry, + // stops it, matches no chain container and exits 0. + if (coins && coins !== "all" && !Object.values(Coin).includes(coins)) { + throw new Error(`unknown coin: ${coins}`) + } + if (networks && networks !== "all" && !Object.values(Network).includes(networks)) { + throw new Error(`unknown network: ${networks}`) + } + // The explorer serves every chain, so a lifecycle verb that opts out of it + // still carries it when no single chain was named. + const allCoins = !coins || coins === "all" + coins = allCoins ? Object.values(Coin) : [coins] networks = networks && networks !== "all" ? [networks] : Object.values(Network) const sharedServices = [HUB_MODULE_NAME, EXPLORER_MODULE_NAME, DB_MODULE_NAME, SYNC_MODULE_NAME] if (modules === "all") { modules = [NODE_MODULE_NAME, ...Object.values(XChainService).filter(m => m !== XChainService.XCHAIN_E2E_TEST)] - addExplorer = true + addExplorer = includeSharedExplorer || allCoins } else if (modules === "explorer") { addExplorer = true coins = [] diff --git a/src/services/config_service/sidecars.js b/src/services/config_service/sidecars.js index fbd5640c..9800fe46 100644 --- a/src/services/config_service/sidecars.js +++ b/src/services/config_service/sidecars.js @@ -25,8 +25,25 @@ function configure(dependencies) { } +// Replace a file via a flushed temp file and rename (as credentials.json), so a crash +// never leaves a torn sidecar whose missing keys the next load re-mints; symlinks survive. +function writeFileAtomic(filePath, data, mode) { + let target = filePath + try { target = fs.realpathSync(filePath) } catch { /* not created yet */ } + const tmpName = '.' + path.basename(target) + '.' + process.pid + '.' + crypto.randomBytes(6).toString('hex') + '.tmp' + const tmpPath = path.join(path.dirname(target), tmpName) + try { + fs.writeFileSync(tmpPath, data, { mode, flag: 'wx', flush: true }) + fs.renameSync(tmpPath, target) + } catch (err) { + try { fs.unlinkSync(tmpPath) } catch { /* never created */ } + throw err + } + try { fs.chmodSync(target, mode) } catch { /* best effort, as credentials.json */ } +} + // Persist RPC credentials to the untracked -.local sidecar. A fresh sidecar -// is created with writeFileSync; an existing one is appended to, unless overwrite is set +// is created; new keys are added after an existing one's content, unless overwrite is set // (used by the legacy-migration path to replace it outright). The sidecar holds the live // node RPC user/password, so it is forced to 0600 the same way credentials.json is: without // an explicit mode the file lands at the process umask (commonly 0644), leaving the RPC @@ -36,30 +53,12 @@ function persistSidecarCreds(localFilePath, creds, { overwrite = false } = {}) { if (overwrite || !fs.existsSync(localFilePath)) { const dir = path.dirname(localFilePath) if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) - fs.writeFileSync(localFilePath, body, { mode: 0o600 }) + writeFileAtomic(localFilePath, body, 0o600) } else { - let needsLeadingNewline = false - try { - const size = fs.statSync(localFilePath).size - if (size > 0) { - const fd = fs.openSync(localFilePath, 'r') - try { - const buf = Buffer.alloc(1) - fs.readSync(fd, buf, 0, 1, size - 1) - needsLeadingNewline = buf.toString('utf8') !== '\n' - } finally { - fs.closeSync(fd) - } - } - } catch { - needsLeadingNewline = false - } - fs.appendFileSync(localFilePath, needsLeadingNewline ? '\n' + body : body) + const existing = String(fs.readFileSync(localFilePath, 'utf8')) + const needsLeadingNewline = existing.length > 0 && !existing.endsWith('\n') + writeFileAtomic(localFilePath, existing + (needsLeadingNewline ? '\n' : '') + body, 0o600) } - // chmod unconditionally: writeFileSync's mode only applies on create, and an - // already-existing sidecar (append path, or one written before this fix) keeps its - // old permissions otherwise. - try { fs.chmodSync(localFilePath, 0o600) } catch {} } // Update specific KEY=VALUE entries in a sidecar while PRESERVING all other keys. Unlike @@ -214,9 +213,9 @@ async function applyHubApiKeyFromSidecar(target) { // would fire on what is really just the ordinary sidecar-wins precedence. // Credentials from the sidecar take precedence over anything in the main file. // One-time migration for legacy installs: older versions appended NODE_USER / -// NODE_PASSWORD into the main config file alongside non-secret settings. Move the -// credentials into the sidecar and strip them from the main file so the two never -// share a file again. Existing creds keep working; they are simply relocated. +// NODE_PASSWORD into the main config file alongside non-secret settings. Preserve +// every existing sidecar value, then rewrite only the main file's own lines without +// credentials. Sidecar values keep precedence over stale values in the main file. // Generate and persist to the sidecar whichever RPC credential is missing, // evaluated PER KEY. The old both-absent (&&) guard meant a partial sidecar (one // key present, one absent) generated nothing, and the missing half then silently @@ -230,21 +229,30 @@ function coinConfigPaths(coin, network) { return { configFilePath, localFilePath: configFilePath + ".local" } } +function peelGluedCredentials(value) { + const credentials = {} + let cleanValue = value + for (const credKey of ["NODE_PASSWORD", "NODE_USER"]) { + const at = cleanValue.indexOf(credKey + "=") + if (at >= 0) { + credentials[credKey] = cleanValue.substring(at + credKey.length + 1) + cleanValue = cleanValue.substring(0, at) + } + } + return { cleanValue, credentials } +} + function readMainConfigLine(defaultConfig, line) { const eqIndex = line.indexOf("=") if (eqIndex <= 0) return false const key = line.substring(0, eqIndex) - let value = line.substring(eqIndex + 1) + const peeled = peelGluedCredentials(line.substring(eqIndex + 1)) let hasCredentials = false - for (const credKey of ["NODE_PASSWORD", "NODE_USER"]) { - const at = value.indexOf(credKey + "=") - if (at >= 0) { - defaultConfig[credKey] = value.substring(at + credKey.length + 1) - value = value.substring(0, at) - hasCredentials = true - } + for (const credKey of Object.keys(peeled.credentials)) { + defaultConfig[credKey] = peeled.credentials[credKey] + hasCredentials = true } - defaultConfig[key] = value + defaultConfig[key] = peeled.cleanValue return hasCredentials || key === "NODE_USER" || key === "NODE_PASSWORD" || key === "NODE_SECRET" } @@ -267,12 +275,21 @@ function migrateMainCredentials(defaultConfig, configFilePath, localFilePath) { const creds = {} if ("NODE_USER" in defaultConfig) creds["NODE_USER"] = defaultConfig["NODE_USER"] if ("NODE_PASSWORD" in defaultConfig) creds["NODE_PASSWORD"] = defaultConfig["NODE_PASSWORD"] - persistSidecarCreds(localFilePath, creds, { overwrite: true }) - const remaining = [] - for (const key in defaultConfig) { - if (key !== "NODE_USER" && key !== "NODE_PASSWORD") remaining.push(`${key}=${defaultConfig[key]}`) - } - fs.writeFileSync(configFilePath, remaining.length ? remaining.join("\n") + "\n" : "") + upsertSidecarValues(localFilePath, creds) + const lines = String(fs.readFileSync(configFilePath, "utf8")).split(/\r?\n/) + if (lines[lines.length - 1] === "") lines.pop() + const remaining = lines.flatMap((line) => { + const eqIndex = line.indexOf("=") + if (eqIndex <= 0) return [line] + const key = line.substring(0, eqIndex) + if (["NODE_USER", "NODE_PASSWORD", "NODE_SECRET"].includes(key)) return [] + const peeled = peelGluedCredentials(line.substring(eqIndex + 1)) + return [`${key}=${peeled.cleanValue}`] + }) + // Keep the main config's own permissions across the replace (it is not a 0600 file). + let mode = 0o644 + try { mode = fs.statSync(configFilePath).mode & 0o777 } catch { /* not created yet */ } + writeFileAtomic(configFilePath, remaining.length ? remaining.join("\n") + "\n" : "", mode) } function generateRpcCredentials(defaultConfig, localFilePath) { diff --git a/src/services/database_service.js b/src/services/database_service.js index d34f86e7..38c317ff 100644 --- a/src/services/database_service.js +++ b/src/services/database_service.js @@ -61,15 +61,17 @@ const { const { XCHAIN_NODE_DB_HOST, XCHAIN_NODE_DB_DEFAULT_PORT, getDatabaseContainerId, getDatabaseContainerPresence, getDatabaseHostPort, checkIfDatabaseModuleExists, checkIfDatabaseIsReady, configureDependencies: configureContainerAccess } = require('./database_service/container_access') const { getExternalDbConfig, pingExternalDatabase, configureDependencies: configureExternalDb } = require('./database_service/external_db') -const { askMariadbRootPassword, executeDockerMariaDbCommand, executeNativeMariaDbCommand, configureDependencies: configureMariadbExec } = require('./database_service/mariadb_exec') +const { askMariadbRootPassword, executeDockerMariaDbCommand, executeDockerMariaDbCommandAs, executeNativeMariaDbCommand, configureDependencies: configureMariadbExec } = require('./database_service/mariadb_exec') const { configureDependencies: configurePersistedDatadir } = require('./database_service/persisted_datadir') const { addUserPasswordToDatabase, configureDependencies: configureUserProvisioning } = require('./database_service/user_provisioning') const { setDatabaseParameters, setHubDatabaseParameters, configureDependencies: configureDbParameters } = require('./database_service/db_parameters') -const { resetDatabases, clearHubPriceIngestWatermark, configureDependencies: configureResetDatabases } = require('./database_service/reset_databases') +const { resetDatabases, clearHubPriceIngestWatermark, resolveResetTargets, configureDependencies: configureResetDatabases } = require('./database_service/reset_databases') +const { configureDependencies: configureServiceDbLogins } = require('./database_service/service_db_logins') +const { resolveNamedDatabaseTarget, configureDependencies: configureNamedDbTarget } = require('./database_service/named_db_target') const { purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, configureDependencies: configureHubCrossChainPurge } = require('./database_service/hub_cross_chain_purge') const { buildDatabaseModule, ensureXchainNodeAccess, configureDependencies: configureDatabaseModule } = require('./database_service/database_module') -const databaseServiceDependencies = { execFile, spawn, promisify, execFileAsync, mariadb, Password, Input, NumberPrompt, DB_MODULE_NAME, HUB_MODULE_NAME, XChainService, SEP, CoinTickerSymbol, EXTERNAL_DB, EXTERNAL_DB_HOST, EXTERNAL_DB_PORT, EXTERNAL_DB_ROOT_USER, DEPENDENCY_HEALTH_START_PERIOD, db, getDbRootPassword, setDbRootPassword, sleep, redactSecrets, assertSafeDbIdentifier, escapeSqlStringLiteral, dockerMariadbArgs, mariadbEnv, PING_SQL, schemaExistsSql, tableExistsSql, tablesExistSql, PRICE_FENCE_TABLE, FENCE_NETWORK_COLUMN, clearChainFenceSql, clearNetworkFenceSql, manualClearStatement, CROSS_CHAIN_MATCH_TABLE, CROSS_CHAIN_CALL_TABLE, CAPABILITY_SNAPSHOT_TABLE, purgeAllMatchesSql, purgeAllCallsSql, purgeChainMatchesSql, purgeChainCallsSql, purgeCapabilitySnapshotsSql, foreignNetworkMatchCountSql, manualPurgeStatements, manualSnapshotPurgeStatement, getDefaultConfig, getDockerContainerImageName, getDockerNetwork, getModuleDatabaseName, validatePort, getStatusFromContainer, getDockerNetworkInspect, addContainerToNetwork, forceRemoveContainerByName, probeContainerPresenceByName, assertNoDbCredentialDrift, assertNoHubDbCredentialDrift, isDbCredentialDriftError, statusChanged, config, statusService, peers, getLogger, logger, XCHAIN_NODE_DB, getOsUserDbName, generatePassword, hasCredentials, loadCredentials, saveCredentials, hasExternalDbConfig, loadExternalDbConfig, saveExternalDbConfig, loadDbRootPassword, saveDbRootPassword } +const databaseServiceDependencies = { execFile, spawn, promisify, execFileAsync, mariadb, Password, Input, NumberPrompt, DB_MODULE_NAME, HUB_MODULE_NAME, XChainService, SEP, CoinTickerSymbol, EXTERNAL_DB, EXTERNAL_DB_HOST, EXTERNAL_DB_PORT, EXTERNAL_DB_ROOT_USER, DEPENDENCY_HEALTH_START_PERIOD, db, getDbRootPassword, setDbRootPassword, sleep, redactSecrets, assertSafeDbIdentifier, escapeSqlStringLiteral, dockerMariadbArgs, mariadbEnv, PING_SQL, schemaExistsSql, tableExistsSql, tablesExistSql, PRICE_FENCE_TABLE, FENCE_NETWORK_COLUMN, clearChainFenceSql, clearNetworkFenceSql, manualClearStatement, CROSS_CHAIN_MATCH_TABLE, CROSS_CHAIN_CALL_TABLE, CAPABILITY_SNAPSHOT_TABLE, purgeAllMatchesSql, purgeAllCallsSql, purgeChainMatchesSql, purgeChainCallsSql, purgeCapabilitySnapshotsSql, foreignNetworkMatchCountSql, manualPurgeStatements, manualSnapshotPurgeStatement, getDefaultConfig, getDockerContainerImageName, getDockerNetwork, getModuleDatabaseName, validatePort, getStatusFromContainer, getDockerNetworkInspect, addContainerToNetwork, forceRemoveContainerByName, probeContainerPresenceByName, assertNoDbCredentialDrift, assertNoHubDbCredentialDrift, isDbCredentialDriftError, statusChanged, config, statusService, peers, getLogger, logger, XCHAIN_NODE_DB, getOsUserDbName, generatePassword, hasCredentials, loadCredentials, saveCredentials, hasExternalDbConfig, loadExternalDbConfig, saveExternalDbConfig, loadDbRootPassword, saveDbRootPassword, executeDockerMariaDbCommand, executeDockerMariaDbCommandAs, resolveResetTargets } configureContainerAccess(databaseServiceDependencies) configureExternalDb(databaseServiceDependencies) configureMariadbExec(databaseServiceDependencies) @@ -78,6 +80,8 @@ configureUserProvisioning(databaseServiceDependencies) configureDbParameters(databaseServiceDependencies) configureResetDatabases(databaseServiceDependencies) configureHubCrossChainPurge(databaseServiceDependencies) +configureServiceDbLogins(databaseServiceDependencies) +configureNamedDbTarget(databaseServiceDependencies) configureDatabaseModule(databaseServiceDependencies) // Open the shared MariaDB connection pool if it isn't already open. @@ -128,6 +132,7 @@ module.exports = { buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, + resolveNamedDatabaseTarget, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, diff --git a/src/services/database_service/container_access.js b/src/services/database_service/container_access.js index c386ccd0..b83ba490 100644 --- a/src/services/database_service/container_access.js +++ b/src/services/database_service/container_access.js @@ -97,23 +97,43 @@ async function checkIfDatabaseModuleExists(coin, network) { // answer (ER_ACCESS_DENIED, unknown database) is not transient, so it must be able // to buy a shorter budget instead of paying ~100s to learn it. Same reasoning the // env-override ping and the provisioning precheck already act on above. +// +// The container id is resolved on every attempt. Right after `docker run` the +// inspect can miss the new container, and an id read once before the loop made +// every retry exec against nothing, so the whole budget ran out on a container +// that came up seconds later. An attempt with no container counts as not ready. async function checkIfDatabaseIsReady(user, userPassword, database = null, { tries = 10, retryDelay = 10000 } = {}) { - const mariadbContainerId = await getDatabaseContainerId() - let remaining = tries while (remaining > 0) { - try { - const args = dockerMariadbArgs(mariadbContainerId, ['mariadb', '-u', user], { interactive: true }) - if (database) args.push('-D', database) - args.push('-e', PING_SQL) - await execFileAsync('docker', args, { env: mariadbEnv(userPassword) }) - return true - } catch { - remaining-- - if (remaining > 0) await sleep(retryDelay) + const mariadbContainerId = await getDatabaseContainerId() + if (mariadbContainerId) { + try { + const args = dockerMariadbArgs(mariadbContainerId, ['mariadb', '-u', user], { interactive: true }) + if (database) args.push('-D', database) + args.push('-e', PING_SQL) + await execFileAsync('docker', args, { env: mariadbEnv(userPassword) }) + return true + } catch { /* not accepting connections yet */ } } + remaining-- + if (remaining > 0) await sleep(retryDelay) } return false } -module.exports = { XCHAIN_NODE_DB_HOST, XCHAIN_NODE_DB_DEFAULT_PORT, getDatabaseContainerId, getDatabaseContainerPresence, getDatabaseHostPort, checkIfDatabaseModuleExists, checkIfDatabaseIsReady, configureDependencies } +// How long a FRESH datadir gets to accept its first root login: 60 x 5 s. The +// image initialises the datadir and restarts its server before it listens, which +// on a busy disk ran past the 10 x 10 s wait of the first caller to touch it, and +// install stopped on "MariaDB is not responding" against a database seconds away +// from ready. Waited once, straight after `docker run`, so every later caller +// meets a running server. +const FIRST_BOOT_READY = { tries: 60, retryDelay: 5000 } + +async function waitForFirstBoot(containerName, mariadbRootPassword) { + if (await checkIfDatabaseIsReady("root", mariadbRootPassword, null, FIRST_BOOT_READY)) return + const waitedMin = Math.round(FIRST_BOOT_READY.tries * FIRST_BOOT_READY.retryDelay / 60000) + throw new Error("MariaDB container " + containerName + " did not accept connections within " + waitedMin + + " min of starting; check `docker logs " + containerName + "` and re-run install") +} + +module.exports = { XCHAIN_NODE_DB_HOST, XCHAIN_NODE_DB_DEFAULT_PORT, FIRST_BOOT_READY, getDatabaseContainerId, getDatabaseContainerPresence, getDatabaseHostPort, checkIfDatabaseModuleExists, checkIfDatabaseIsReady, waitForFirstBoot, configureDependencies } diff --git a/src/services/database_service/database_module.js b/src/services/database_service/database_module.js index 48a9f183..67ea1d8e 100644 --- a/src/services/database_service/database_module.js +++ b/src/services/database_service/database_module.js @@ -41,7 +41,7 @@ let { hasCredentials, loadCredentials, saveCredentials } = require('../credentials_service') -let { XCHAIN_NODE_DB_HOST, XCHAIN_NODE_DB_DEFAULT_PORT, getDatabaseContainerId, checkIfDatabaseModuleExists, checkIfDatabaseIsReady } = require('./container_access') +let { XCHAIN_NODE_DB_HOST, XCHAIN_NODE_DB_DEFAULT_PORT, getDatabaseContainerId, checkIfDatabaseModuleExists, checkIfDatabaseIsReady, waitForFirstBoot } = require('./container_access') let { getExternalDbConfig, pingMariaDb } = require('./external_db') let { askMariadbRootPassword, executeNativeMariaDbCommand, executeDockerMariaDbCommand } = require('./mariadb_exec') let { dbDatadirState, confirmPersistedDatadirRootPassword } = require('./persisted_datadir') @@ -246,8 +246,8 @@ async function createDatabaseModule(coin, network) { // Read before the start: a new container initialises an empty datadir, which then holds mysql/. const persisted = dbDatadirState() === 'initialised' const containerId = await startDatabaseContainer(runArgs, mariadbRootPassword) - // The image ignored MYSQL_ROOT_PASSWORD over a datadir that already held a database, so store one only once it authenticates. - if (persisted) await confirmPersistedDatadirRootPassword(containerId, mariadbRootPassword) + // A held datadir ignored MYSQL_ROOT_PASSWORD, so store one only once it authenticates; a fresh one waits out its first boot. + await (persisted ? confirmPersistedDatadirRootPassword(containerId, mariadbRootPassword) : waitForFirstBoot(containerPrefix, mariadbRootPassword)) return containerId } diff --git a/src/services/database_service/mariadb_exec.js b/src/services/database_service/mariadb_exec.js index 39368cf4..fea9df97 100644 --- a/src/services/database_service/mariadb_exec.js +++ b/src/services/database_service/mariadb_exec.js @@ -290,6 +290,12 @@ async function askMariadbRootPassword(coin, network) { } async function executeDockerMariaDbCommand(mariadbContainerId, mariadbRootPassword, command, commandOptions = "") { + return executeDockerMariaDbCommandAs(mariadbContainerId, 'root', mariadbRootPassword, command, commandOptions) +} + +// The same client run as another MariaDB user, for a server whose root is not available to this CLI. The caller +// checks the user name first: it reaches argv, and a name starting with a dash would read as a client option. +async function executeDockerMariaDbCommandAs(mariadbContainerId, user, password, command, commandOptions = "") { return new Promise((resolve, reject) => { // The SQL is fed to the mariadb client over STDIN, never as an // `-e ` argv entry. User-creation statements embed a secret @@ -298,14 +304,14 @@ async function executeDockerMariaDbCommand(mariadbContainerId, mariadbRootPasswo // entirely, closing the transient exposure that a `-e ` argv left // open during the exec. The mariadb client reads statements from stdin // when no -e is given; `docker exec -i` (interactive) pipes our stdin - // through to it. The root password still travels via MYSQL_PWD env (see + // through to it. The password still travels via MYSQL_PWD env (see // dockerMariadbArgs), never argv. - const args = dockerMariadbArgs(mariadbContainerId, ['mariadb', '-u', 'root'], { interactive: true }) + const args = dockerMariadbArgs(mariadbContainerId, ['mariadb', '-u', user], { interactive: true }) if (commandOptions) { args.push(...commandOptions.trim().split(/\s+/)) } - const child = spawn('docker', args, { env: mariadbEnv(mariadbRootPassword) }) + const child = spawn('docker', args, { env: mariadbEnv(password) }) let stdout = '' let stderr = '' child.stdout.on('data', (chunk) => { stdout += chunk }) @@ -331,4 +337,4 @@ async function executeDockerMariaDbCommand(mariadbContainerId, mariadbRootPasswo }) } -module.exports = { parseMariaDbClientOptions, executeNativeMariaDbCommand, scrubSqlFromError, askMariadbRootPassword, executeDockerMariaDbCommand, configureDependencies } +module.exports = { parseMariaDbClientOptions, executeNativeMariaDbCommand, scrubSqlFromError, askMariadbRootPassword, executeDockerMariaDbCommand, executeDockerMariaDbCommandAs, configureDependencies } diff --git a/src/services/database_service/named_db_target.js b/src/services/database_service/named_db_target.js new file mode 100644 index 00000000..11568bd2 --- /dev/null +++ b/src/services/database_service/named_db_target.js @@ -0,0 +1,136 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Database Service + * A MariaDB container named by the operator, for a stack that runs its own + ********************************************************************/ + +let { execFile } = require('child_process') +const { promisify } = require('util') +let execFileAsync = promisify(execFile) +const config = require('../../config') +let { assertSafeDbIdentifier } = require('../../utils/sql_safety') +let { schemaExistsSql } = require('../../db/information_schema') +let { dockerMariadbArgs, mariadbEnv } = require('../../utils/docker_mariadb') +let { executeDockerMariaDbCommand } = require('./mariadb_exec') +let { resolveResetTargets } = require('./reset_databases') +let { resolveServiceLogins, serviceAuthReport, namedTargetRunner } = require('./service_db_logins') + +const nativeExecFile = execFile +function configureDependencies(dependencies) { + if (dependencies.execFile === nativeExecFile) return + ;({ execFile, execFileAsync, assertSafeDbIdentifier, schemaExistsSql, dockerMariadbArgs, mariadbEnv, executeDockerMariaDbCommand, resolveResetTargets } = dependencies) +} + +// Docker's own container-name grammar, checked before the name reaches any argv. +const CONTAINER_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/ + +// The env names the official mariadb and mysql images take their root password from. +const ROOT_PASSWORD_VARS = ['MARIADB_ROOT_PASSWORD', 'MYSQL_ROOT_PASSWORD'] + +async function findNamedContainerId(containerName) { + if (!CONTAINER_NAME.test(String(containerName || ''))) { + throw new Error(`'${containerName}' is not a valid docker container name`) + } + let stdout + try { + ({ stdout } = await execFileAsync('docker', ['inspect', '--type', 'container', '--format', '{{.Id}}', containerName])) + } catch (err) { + throw new Error(`docker inspect ${containerName} failed (${(err && err.message) || err})`) + } + const id = String(stdout).trim() + if (!/^[a-f0-9]{64}$/.test(id)) throw new Error(`docker inspect ${containerName} returned no container id`) + return id +} + +// A server whose datadir was moved in from another install keeps that install's root and was started without the +// env, so the operator can hand the root to this run in this variable instead. Read only after the container's own. +const ROOT_PASSWORD_OVERRIDE_VAR = 'XCHAIN_NODE_DB_CONTAINER_ROOT_PASSWORD' + +async function rootLogsIn(containerId, value) { + const ping = await execFileAsync('docker', dockerMariadbArgs(containerId, ['mariadb-admin', '-u', 'root', 'ping']), { env: mariadbEnv(value) }) + return String(ping.stdout).includes('mysqld is alive') +} + +// Read the root password the container was started with, else the override variable, and keep it only if it logs +// in. It is never cached or saved: the CLI's stored root belongs to its bundled server, and this one is another's. +async function rootPasswordForContainer(containerId, containerName) { + for (const name of ROOT_PASSWORD_VARS) { + try { + const { stdout } = await execFileAsync('docker', ['exec', containerId, 'printenv', name]) + const value = String(stdout).replace(/\r?\n$/, '') + if (value && await rootLogsIn(containerId, value)) return value + } catch { /* unset in this container, or it did not log in: try the next name */ } + } + const override = config[ROOT_PASSWORD_OVERRIDE_VAR] + if (override) { + try { + if (await rootLogsIn(containerId, override)) return override + } catch { /* refused below, naming the variable */ } + throw new Error(`${ROOT_PASSWORD_OVERRIDE_VAR} does not log in to ${containerName} as root`) + } + throw new Error(`no root password in ${containerName}'s environment (${ROOT_PASSWORD_VARS.join(', ')}) logs in to it,` + + ` and ${ROOT_PASSWORD_OVERRIDE_VAR} is not set`) +} + +// Each target database must already exist on the named server: a name that differs from the stack's own would +// otherwise DROP nothing and CREATE an empty database beside the live one, and the reset would report success. +async function checkTargetDatabasesExist(target) { + for (const dbName of target.databases) { + const found = await namedTargetRunner(target, dbName)(schemaExistsSql(dbName), '-B -N') + if (parseInt(String(found).trim(), 10) !== 1) { + throw new Error(`${target.containerName} holds no database ${dbName}, so this is not the server that stack uses`) + } + } +} + +// No root logs in, so each step runs as the database user the stack's own services use, read from their env. Every +// login and grant is checked here; a refusal names both why root failed and which user or grant fell short. +async function resolveServiceTarget(containerName, containerId, coin, network, modules, findServiceContainer, rootErr) { + const databases = await resolveResetTargets(coin, network, modules) + for (const dbName of databases) assertSafeDbIdentifier(dbName, 'database name') + let resolved + try { + resolved = await resolveServiceLogins({ containerName, containerId, modules, databases, findServiceContainer }) + } catch (err) { + throw new Error(`${rootErr.message}; the stack's own database users cannot stand in: ${err.message}`) + } + const target = { containerName, containerId, databases, authMode: 'service', rootRefusal: rootErr.message, ...resolved } + await checkTargetDatabasesExist(target) + target.authReport = serviceAuthReport(target) + return target +} + +// Resolve everything the reset's DB steps need from the named server, before anything is stopped. A root that logs +// in is used as before; without one, `findServiceContainer(module)` finds the stack's services whose env carries +// their own database logins, and the reset runs as those users instead. +async function resolveNamedDatabaseTarget(containerName, coin, network, modules, findServiceContainer = async () => null) { + const containerId = await findNamedContainerId(containerName) + let rootPassword + try { + rootPassword = await rootPasswordForContainer(containerId, containerName) + } catch (rootErr) { + return resolveServiceTarget(containerName, containerId, coin, network, modules, findServiceContainer, rootErr) + } + const databases = await resolveResetTargets(coin, network, modules) + for (const dbName of databases) { + assertSafeDbIdentifier(dbName, 'database name') + const found = await executeDockerMariaDbCommand(containerId, rootPassword, schemaExistsSql(dbName), '-B -N') + if (parseInt(String(found).trim(), 10) !== 1) { + throw new Error(`${containerName} holds no database ${dbName}, so this is not the server that stack uses`) + } + } + return { containerName, containerId, rootPassword, databases } +} + +module.exports = { resolveNamedDatabaseTarget, configureDependencies } diff --git a/src/services/database_service/reset_databases.js b/src/services/database_service/reset_databases.js index bab213f3..08fb6d3a 100644 --- a/src/services/database_service/reset_databases.js +++ b/src/services/database_service/reset_databases.js @@ -29,6 +29,7 @@ let logger = getLogger() let { getDatabaseContainerId } = require('./container_access') let { getExternalDbConfig } = require('./external_db') let { executeNativeMariaDbCommand, askMariadbRootPassword, executeDockerMariaDbCommand } = require('./mariadb_exec') +const { namedTargetRunner } = require('./service_db_logins') const nativeExecFile = execFile function configureDependencies(dependencies) { @@ -63,8 +64,19 @@ async function resolveResetTargets(coin, network, modules) { return resetTargets } -async function resetDatabases(coin, network, modules = [XChainService.XCHAIN_DECODER, XChainService.XCHAIN_INDEXER]) { +// `namedTarget` is the server the operator named with --db-container (resolveNamedDatabaseTarget); without it +// the bundled container or the external DB is used, as before. +async function resetDatabases(coin, network, modules = [XChainService.XCHAIN_DECODER, XChainService.XCHAIN_INDEXER], namedTarget = null) { const resetTargets = await resolveResetTargets(coin, network, modules) + // A stack with its own DB server: its id and its root, or each database's own user, were resolved and checked + // before anything was stopped. A database user with DROP and CREATE on its own database can recreate it. + if (namedTarget) { + for (const dbName of resetTargets) { + await namedTargetRunner(namedTarget, dbName)(`DROP DATABASE IF EXISTS ${dbName}; CREATE DATABASE ${dbName}`) + logger.info(`Database ${dbName} reset on ${namedTarget.containerName}!`) + } + return + } // External (host-native) MariaDB: there is no database container to exec // into it (`docker exec ... null` failed here and aborted the reset mid-way, // leaving data wiped, DBs stale, services stopped). Use the driver-based @@ -165,7 +177,7 @@ async function clearPriceFence(runner, hubDbName, ticker, fenceNetwork) { // manual statement is printed. Only the calling chain's row on the calling // network is touched: every other chain's fence, and every other network's fence // for this chain, is still protecting its live ingest. -async function clearHubPriceIngestWatermark(coin, network) { +async function clearHubPriceIngestWatermark(coin, network, namedTarget = null) { const ticker = CoinTickerSymbol[coin] if (!ticker) { throw new Error("clearHubPriceIngestWatermark: unknown coin '" + coin + "'") @@ -184,7 +196,11 @@ async function clearHubPriceIngestWatermark(coin, network) { assertSafeDbIdentifier(hubDbName, 'database name') let runner - if (EXTERNAL_DB) { + // A stack with its own DB server keeps its hub's fence row there; the probe below says so when it does not. + // Without a root it runs as the hub's own user, whose DELETE grant was checked before anything was stopped. + if (namedTarget) { + runner = namedTargetRunner(namedTarget, null) + } else if (EXTERNAL_DB) { const externalCfg = await getExternalDbConfig() runner = (sql, options) => executeNativeMariaDbCommand(externalCfg, sql, options) } else { diff --git a/src/services/database_service/service_db_logins.js b/src/services/database_service/service_db_logins.js new file mode 100644 index 00000000..e66d5291 --- /dev/null +++ b/src/services/database_service/service_db_logins.js @@ -0,0 +1,207 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Database Service + * A named server's reset run as the stack's own database users, when no root login is available for it + ********************************************************************/ + +let { execFile } = require('child_process') +const { promisify } = require('util') +let execFileAsync = promisify(execFile) +let { XChainService, HUB_MODULE_NAME } = require('../../config') +let { getDefaultConfig } = require('../config_service') +let { assertSafeDbIdentifier } = require('../../utils/sql_safety') +let { executeDockerMariaDbCommand, executeDockerMariaDbCommandAs } = require('./mariadb_exec') +const { CURRENT_USER_GRANTS_SQL } = require('../../db/information_schema') +const { PRICE_FENCE_TABLE } = require('../../db/price_fence') + +const nativeExecFile = execFile +function configureDependencies(dependencies) { + if (dependencies.execFile === nativeExecFile) return + ;({ execFile, execFileAsync, XChainService, HUB_MODULE_NAME, getDefaultConfig, assertSafeDbIdentifier, executeDockerMariaDbCommand, executeDockerMariaDbCommandAs } = dependencies) +} + +// The env prefix each reset database's login sits under in the stack's service containers. +function loginPrefixFor(module) { + if (module === XChainService.XCHAIN_DECODER) return 'DECODER' + if (module === XChainService.XCHAIN_INDEXER) return 'INDEXER' + throw new Error(`${module} has no database login this reset knows`) +} + +// MariaDB's own user-name characters. The name reaches the client's argv, so a leading dash is never accepted. +const DB_USER_NAME = /^[A-Za-z0-9_][A-Za-z0-9_.$-]{0,79}$/ + +// A login read from a service container. The password is not enumerable, so logging or serialising the login +// object shows the user and never the password. +function makeLogin(prefix, source, user, password, envDatabase) { + const login = { prefix, source, user, envDatabase, database: null, grants: [] } + Object.defineProperty(login, 'password', { value: password, enumerable: false }) + return login +} + +// One variable from a service container's environment, or null when it is unset or the container cannot answer. +// Read from the container's config, not `docker exec`, because a reset runs with the stack's services stopped. +async function readServiceEnv(containerId, name) { + try { + const { stdout } = await execFileAsync('docker', ['inspect', '--type', 'container', '--format', '{{json .Config.Env}}', containerId]) + const entry = JSON.parse(String(stdout)).find((pair) => String(pair).startsWith(name + '=')) + return entry ? entry.slice(name.length + 1) || null : null + } catch { + return null + } +} + +// The containers whose env carries the stack's logins, indexer first, then decoder. A failed lookup throws (the +// finder's own rule), so the reset refuses rather than guess. +async function serviceEnvSources(findServiceContainer) { + const sources = [] + for (const module of [XChainService.XCHAIN_INDEXER, XChainService.XCHAIN_DECODER]) { + const id = await findServiceContainer(module) + if (id) sources.push({ module, id }) + } + if (!sources.length) throw new Error('no indexer or decoder container was found to read the database logins from') + return sources +} + +// Read _DB_USER and _DB_PASS from the first container that carries both, plus its _DB_NAME when set. +// The values are held in memory for this run only: never printed, cached or saved. +async function readLogin(sources, prefix) { + for (const source of sources) { + const user = await readServiceEnv(source.id, `${prefix}_DB_USER`) + const password = user ? await readServiceEnv(source.id, `${prefix}_DB_PASS`) : null + if (!user || !password) continue + // Refuse a name the client could misread, without echoing it. + if (!DB_USER_NAME.test(user)) throw new Error(`${prefix}_DB_USER in the ${source.module} container is not a MariaDB user name`) + return makeLogin(prefix, source.module, user, password, await readServiceEnv(source.id, `${prefix}_DB_NAME`)) + } + throw new Error(`no indexer or decoder container's environment carries ${prefix}_DB_USER and ${prefix}_DB_PASS`) +} + +// One SHOW GRANTS line: its privilege list and the scope after ON. Role and PROXY grants carry no database scope. +const GRANT_LINE = /^GRANT\s+(.+?)\s+ON\s+(?:TABLE\s+)?(\S+)\s+TO\s/i + +function parseGrant(line) { + const match = GRANT_LINE.exec(String(line).trim()) + if (!match) return null + return { privileges: match[1].split(',').map((p) => p.trim().toUpperCase()), scope: match[2], text: `${match[1]} ON ${match[2]}` } +} + +// A quoted name as SHOW GRANTS prints it: doubled backticks are one, and the backslash MariaDB puts before a +// wildcard character in a database name (doubled again in batch output) is dropped, leaving the literal name. +function unquoteGrantName(name) { + return name.replace(/``/g, '`').replace(/\\+([_%])/g, '$1') +} + +// Whether a grant scope covers the whole database (table null), or that one table in it. *.* covers everything; +// a scope naming another database, or a pattern that is not this exact name, does not count. +function scopeCovers(scope, database, table) { + if (scope === '*.*') return true + const match = /^`((?:[^`]|``)+)`\.(\*|`((?:[^`]|``)+)`)$/.exec(scope) + if (!match || unquoteGrantName(match[1]) !== database) return false + return match[2] === '*' || (table !== null && unquoteGrantName(match[3]) === table) +} + +// Which of the needed privileges the grant lines leave missing on that scope, and the lines that gave the rest. +// ALL PRIVILEGES gives every one; CREATE VIEW and the like are other privileges and do not stand in for CREATE. +function grantedPrivileges(lines, needed, database, table = null) { + const found = new Set() + const matched = [] + for (const grant of lines.map(parseGrant).filter(Boolean)) { + if (!scopeCovers(grant.scope, database, table)) continue + const all = grant.privileges.includes('ALL PRIVILEGES') || grant.privileges.includes('ALL') + const hits = needed.filter((p) => all || grant.privileges.includes(p)) + if (!hits.length) continue + matched.push(grant.text) + for (const p of hits) found.add(p) + } + return { missing: needed.filter((p) => !found.has(p)), matched } +} + +// The stack's own config must name the database this reset targets, or its login guards some other database. +function checkLoginNamesDatabase(login, database) { + if (login.envDatabase && login.envDatabase !== database) { + throw new Error(`${login.prefix}_DB_NAME in the ${login.source} container names ${login.envDatabase}, not ${database}`) + } +} + +// Log in as the user and read its own grants: a failed login, or a grant short of what the step needs, refuses, +// naming the user and the privilege. The password is cut out of any client error before it is quoted. +async function checkServiceLogin(target, login, needed, database, table = null) { + let text + try { + text = await executeDockerMariaDbCommandAs(target.containerId, login.user, login.password, CURRENT_USER_GRANTS_SQL, '-B -N') + } catch (err) { + const detail = String((err && err.message) || err).split(login.password).join('') + throw new Error(`${login.prefix}_DB_USER ${login.user} does not log in to ${target.containerName} (${detail})`) + } + const { missing, matched } = grantedPrivileges(String(text).split(/\r?\n/), needed, database, table) + const on = table ? `${database}.${table}` : database + if (missing.length) throw new Error(`${login.prefix}_DB_USER ${login.user} has no ${missing.join(' or ')} grant on ${on}`) + login.database = on + login.grants = matched + return login +} + +// The hub user the price fence clear runs as. The clear reads HUB_DB_NAME from this CLI's hub config and runs a +// DELETE on its fence table, so that is the grant checked. Like the clear itself, a hub config with no name, or a +// stack whose hub uses another database, skips the clear (which prints the hand-run statement) rather than refuse. +async function resolveHubLogin(target, sources) { + const cfg = await getDefaultConfig(HUB_MODULE_NAME, null, null) + const hubDbName = cfg && cfg['HUB_DB_NAME'] + if (!hubDbName) return { hubLogin: null, hubNote: 'the hub configuration carries no HUB_DB_NAME' } + assertSafeDbIdentifier(hubDbName, 'database name') + const login = await readLogin(sources, 'HUB') + if (login.envDatabase && login.envDatabase !== hubDbName) { + return { hubLogin: null, hubNote: `HUB_DB_NAME in the ${login.source} container names ${login.envDatabase}, not ${hubDbName}` } + } + return { hubLogin: await checkServiceLogin(target, login, ['DELETE'], hubDbName, PRICE_FENCE_TABLE), hubNote: null } +} + +// Resolve every login a service-mode reset needs, before anything is stopped: each target database's own user, +// which must log in and hold DROP and CREATE on it, and the hub user when the indexer reset will clear the fence. +// `databases` lines up with `modules`, as resolveResetTargets builds it. +async function resolveServiceLogins({ containerName, containerId, modules, databases, findServiceContainer }) { + const target = { containerName, containerId } + const sources = await serviceEnvSources(findServiceContainer) + const logins = {} + for (let i = 0; i < modules.length; i++) { + const login = await readLogin(sources, loginPrefixFor(modules[i])) + checkLoginNamesDatabase(login, databases[i]) + logins[databases[i]] = await checkServiceLogin(target, login, ['DROP', 'CREATE'], databases[i]) + } + const hub = modules.includes(XChainService.XCHAIN_INDEXER) ? await resolveHubLogin(target, sources) : { hubLogin: null, hubNote: null } + return { logins, ...hub } +} + +// What a service-mode target logs in as and the grants that allowed it: users and privileges, never a password. +function serviceAuthReport(target) { + const lines = [`No root login works on ${target.containerName} (${target.rootRefusal}), so each step runs as the stack's own database user:`] + for (const login of [...Object.values(target.logins), ...(target.hubLogin ? [target.hubLogin] : [])]) { + lines.push(` ${login.database}: ${login.user} (${login.prefix}_DB_USER from the ${login.source} container), granted ${login.grants.join('; ')}`) + } + if (target.hubNote) lines.push(` hub price fence: not cleared by this reset (${target.hubNote}); it prints the statement to run by hand`) + return lines +} + +// The client call for one statement on the named server: as root when a root logged in, else as the login that +// owns `database` (null for the hub fence clear). A missing login throws rather than fall back to another user. +function namedTargetRunner(namedTarget, database) { + if (!namedTarget.logins) { + return (sql, options) => executeDockerMariaDbCommand(namedTarget.containerId, namedTarget.rootPassword, sql, options) + } + const login = database === null ? namedTarget.hubLogin : namedTarget.logins[database] + if (!login) throw new Error(`no ${database === null ? 'hub' : database} login was resolved on ${namedTarget.containerName}`) + return (sql, options) => executeDockerMariaDbCommandAs(namedTarget.containerId, login.user, login.password, sql, options) +} + +module.exports = { resolveServiceLogins, serviceAuthReport, namedTargetRunner, configureDependencies } diff --git a/src/services/explorer_service.js b/src/services/explorer_service.js index b819f89a..f280da6a 100644 --- a/src/services/explorer_service.js +++ b/src/services/explorer_service.js @@ -22,7 +22,7 @@ const { db, getLastStatus, isStatusUpdated } = require('../state') const { sleep, redactSecrets } = require('../utils/helpers') const { getDefaultConfig, getDockerNetwork } = require('./config_service') const { statusChanged, getStatus, getInstalledCoinsAndNetworks } = require('./status_service') -const { addContainerToNetwork, killContainer, removeContainer } = require('./docker_service') +const { addContainerToNetwork } = require('./docker_service') const { cloneGit, buildAndUp } = require('./module_service') const ExplorerConnector = require('./explorer_connector.js') // Destructured where it is used, so each call reads the export at that moment. @@ -85,18 +85,6 @@ async function updateExplorer() { return true } -async function removeExistingExplorer() { - // Force-rebuild: tear down any existing explorer container + DB row so - // the fresh clone + buildAndUp below doesn't collide with stale state. - const existingContainerId = await db.getModuleContainer(EXPLORER_MODULE_NAME, "", "") - if (existingContainerId) { - logger.info("Force rebuild: removing existing xchain-explorer container") - try { await killContainer(existingContainerId) } catch { /* may already be exited */ } - try { await removeContainer(existingContainerId) } catch { /* may already be gone */ } - try { await db.deleteModuleContainer(EXPLORER_MODULE_NAME, "", "") } catch { /* row may already be gone */ } - } -} - async function waitForInstalledExplorer(explorerConnector) { // A healthy explorer is one holding at least one DB pool, and its pools come // from the COIN stacks. So on a host with no coin installed yet there is no @@ -161,6 +149,9 @@ async function installExplorerModule(force = false, branch = null) { logger.info("Checking if xchain-explorer module is running") const explorerConnector = new ExplorerConnector(defaultConfig["EXPLORER_HOST"], defaultConfig["EXPLORER_PORT"]) + // A forced rebuild hands the running explorer to buildAndUp, which stops and + // replaces it only after the image builds, so a failed clone or build keeps it. + let overwriteContainerId = null if (!force) { const pingExplorer = await explorerConnector.ping() if (pingExplorer) return true @@ -172,7 +163,7 @@ async function installExplorerModule(force = false, branch = null) { if (explorerStatus !== undefined) return true } } else { - await removeExistingExplorer() + overwriteContainerId = await db.getModuleContainer(EXPLORER_MODULE_NAME, "", "") || null } logger.info("Downloading xchain-explorer...") @@ -180,7 +171,7 @@ async function installExplorerModule(force = false, branch = null) { const explorerPin = resolveComponentRef(EXPLORER_MODULE_NAME, branch) await cloneGit(EXPLORER_MODULE_NAME, true, false, explorerPin.ref, explorerPin.commit) logger.info("Installing xchain-explorer module...") - await buildAndUp(EXPLORER_MODULE_NAME, null, null) + await buildAndUp(EXPLORER_MODULE_NAME, null, null, overwriteContainerId) await getStatus(null, null, false) logger.info("Waiting for the xchain-explorer to respond") return waitForInstalledExplorer(explorerConnector) diff --git a/src/services/hub_connector.js b/src/services/hub_connector.js index f930b011..c435ce85 100644 --- a/src/services/hub_connector.js +++ b/src/services/hub_connector.js @@ -113,6 +113,16 @@ class HubConnector { params: { config: newConfigJson }, id: 1 }, 60000); // config pushes on multi-coin nodes can exceed a 10s ceiling (prod-validated) + // The hub reports a refused push (e.g. PBFT "Not the leader for seq N") + // as a normal JSON-RPC result of the form {error}, not as a transport + // failure. Treat it as a failed push and record the text so the retry + // loop retries and finally surfaces the real cause. + if(result && typeof result === 'object' && result.error){ + const e = result.error; + const text = typeof e === 'string' ? e : (e && e.message) || JSON.stringify(e); + this.lastFailures.push('hub refused config push → ' + text); + return false; + } return result !== null; } } diff --git a/src/services/hub_service/update_hub_or_explorer.js b/src/services/hub_service/update_hub_or_explorer.js index 67d086db..47a9c902 100644 --- a/src/services/hub_service/update_hub_or_explorer.js +++ b/src/services/hub_service/update_hub_or_explorer.js @@ -152,10 +152,22 @@ async function writeExplorerConfig(module, jsonConfig, dependencies) { } } +// A recreated hub takes 40 to 60 s to open its API (measured on the regtest rail, +// 2026-10-24), so the push keeps trying for three minutes before it gives up. +const HUB_CONFIG_RETRY_MS = 3000 +const HUB_CONFIG_ATTEMPTS = 60 +// The attempt count assumes fast connection refusals. A paused or hung hub accepts +// the connection and answers nothing, so each attempt burns the connector's 60 s +// timeout and 60 attempts would hold the command lock for about an hour. The +// wall-clock budget bounds the loop whatever each attempt costs. +const HUB_CONFIG_WALL_MS = 180000 + async function pushHubConfig(module, moduleConnector, jsonConfig, dependencies) { const { sleep, redactSecrets, logger } = dependencies + const now = dependencies.now || Date.now + const startedAt = now() let hubUpdated = false - let tries = 10 + let tries = HUB_CONFIG_ATTEMPTS // Keep the last failure. updateConfig is an HTTP call to the module's own // API, so when the container is crash-looping every attempt fails with a // connection error and this loop reports only "there was a problem" - which @@ -181,17 +193,16 @@ async function pushHubConfig(module, moduleConnector, jsonConfig, dependencies) lastErr = moduleConnector.lastFailures.join('; ') } + if (hubUpdated) break tries-- // Out of retries: give up, naming the last concrete error instead of a generic failure. - if (tries <= 0) { + if (tries <= 0 || now() - startedAt >= HUB_CONFIG_WALL_MS) { throw "There was a problem trying to update a config in the " + module + " module" + (lastErr ? " (last error: " + redactSecrets(lastErr) + "; if this is a connection failure, check `docker logs` for the module - the service is not starting)" : "") } - if (!hubUpdated) { - logger.info("There was a problem trying to update a config in the " + module + " module" + - (lastErr ? " (" + redactSecrets(lastErr) + ")" : "") + ". Trying again in 3 seconds...") - await sleep(3000) - } + logger.info("There was a problem trying to update a config in the " + module + " module" + + (lastErr ? " (" + redactSecrets(lastErr) + ")" : "") + ". Trying again in 3 seconds...") + await sleep(HUB_CONFIG_RETRY_MS) } } diff --git a/src/services/module_service.js b/src/services/module_service.js index b826c0a8..802e5154 100644 --- a/src/services/module_service.js +++ b/src/services/module_service.js @@ -32,7 +32,10 @@ const { getDockerContainerImageName, getUtxoTrackerVolumeName, getDockerNetwork, getDefaultConfig, validatePort } = require('./config_service') const { statusChanged, getStatus } = require('./status_service') -const { stopContainerByName, removeContainer, getPublishedHostPorts, forceRemoveContainerByName, addContainerToNetwork, checkBuildKitAvailable } = require('./docker_service') +const { + stopContainerByName, removeContainer, getPublishedHostPorts, forceRemoveContainerByName, + addContainerToNetwork, checkBuildKitAvailable, probeContainerPresenceByName +} = require('./docker_service') const { stopModuleContainer, stopTimeoutArgs } = require('./stop_budget_service') const { setDatabaseParameters, setHubDatabaseParameters } = require('./database_service') const { redactSecrets, sleep } = require('../utils/helpers') @@ -71,25 +74,6 @@ for (const part of ['git_checkout', 'clone_and_refs', 'docker_args', 'container_ delete require.cache[require.resolve('./module_service/' + part + '.js')] } -// Singleton modules share one coin/network-independent container name (see -// getDockerContainerImageNamePrefix). DB and EXPLORER have dedicated install -// branches that are already idempotent; HUB and SYNC fall through to the -// generic branch whose "already built?" check is keyed per coin/network, which -// can't see the one shared container, so installing across multiple networks -// would re-run `docker run` with a duplicate name and crash. -const SINGLETON_MODULES = [HUB_MODULE_NAME, SYNC_MODULE_NAME] - -// True if a container with this exact name already exists (running or stopped). -// Mirrors getDatabaseContainerId's inspect-by-name probe. -async function containerExistsByName(name) { - try { - const { stdout } = await execFileAsync('docker', ['inspect', '--type', 'container', '--format', '{{.Id}}', name]) - return /^[a-f0-9]{64}$/.test(stdout.trim()) - } catch { - return false - } -} - async function installNodeModule(coin, network, remoteUpdate, overwriteContainerId = null) { const { getLocalNodeVersion, checkRemoteNodeVersion } = versionService const { getRemoteModuleVersions, getLastStatus } = stateModule @@ -140,11 +124,6 @@ async function installExplorer(branch, remoteUpdate) { return true } -async function singletonAlreadyInstalled(module, coin, network, remoteUpdate) { - if (!SINGLETON_MODULES.includes(module) || remoteUpdate) return false - return containerExistsByName(getDockerContainerImageName(module, coin, network)) -} - // Refuse credential rotations while the working container is still intact. // The post-build guards run after replacement and can no longer keep a sibling // on its deployed password, so the module being replaced is the only exclusion. @@ -229,12 +208,50 @@ async function finishServiceInstall(context, freshness) { return containerId } +// The services a bootstrap archive restores. Only these take the forced restore. +const BOOTSTRAP_RESTORED_MODULES = [XChainService.XCHAIN_UTXO_TRACKER, XChainService.XCHAIN_DECODER, XChainService.XCHAIN_INDEXER] + +// An installed module normally makes install a no-op. XCHAIN_NODE_FORCE_BOOTSTRAP +// is the one reason to go on for a bootstrap-restored service: every restore +// failure prints "re-run install with XCHAIN_NODE_FORCE_BOOTSTRAP=1", and this early +// return made that remedy a silent no-op. Any other module still returns here, so +// a flag left in the environment never rebuilds the hub or the encoder. +function forcedBootstrapReinstall(module, presence, remoteUpdate) { + if (presence !== 'exists' || remoteUpdate) return false + return BOOTSTRAP_RESTORED_MODULES.includes(module) && bootstrapService.forceBootstrapRequested() +} + +// The forced reinstall hands the running container over by id, so buildAndUp stops +// it on its SIGTERM budget instead of force-removing it by name mid-block. +async function forcedBootstrapContext(context) { + const { module, coin, network } = context + logger.info(`XCHAIN_NODE_FORCE_BOOTSTRAP is set: reinstalling the installed ${module} ${coin}/${network} to take the bootstrap restore again.`) + if (context.overwriteContainerId) return context + const containerId = await db.getModuleContainerStrict(module, coin, network) + if (!containerId) { + throw new Error(`Cannot reinstall ${module} ${coin}/${network}: its container exists but its module registry row is missing`) + } + return { ...context, overwriteContainerId: containerId } +} + +async function guardIdempotentInstall(context) { + const { module, coin, network, remoteUpdate, overwriteContainerId } = context + if (remoteUpdate || overwriteContainerId) return context + const name = getDockerContainerImageName(module, coin, network) + const presence = await probeContainerPresenceByName(name) + if (forcedBootstrapReinstall(module, presence, remoteUpdate)) return forcedBootstrapContext(context) + if (presence === 'exists') return false + if (presence === 'unknown') { + throw new Error(`Cannot install ${module} ${coin}/${network}: Docker could not confirm whether ${name} exists; nothing was changed`) + } + return context +} + async function installServiceModule(context) { + const guardedContext = await guardIdempotentInstall(context) + if (guardedContext === false) return false + context = guardedContext const { module, coin, network, remoteUpdate, onlyExecution, branch } = context - if (await singletonAlreadyInstalled(module, coin, network, remoteUpdate)) return false - const { getLastStatus } = stateModule - const containerVersion = getLastStatus()?.[coin ?? ""]?.[network ?? ""]?.[module]?.["container_version"] ?? null - if (containerVersion && !remoteUpdate) return false const { getLocalModuleVersion } = versionService let localModuleVersion = null try { diff --git a/src/services/module_service/build_and_up.js b/src/services/module_service/build_and_up.js index 7ac75be3..0b234269 100644 --- a/src/services/module_service/build_and_up.js +++ b/src/services/module_service/build_and_up.js @@ -387,7 +387,7 @@ async function buildAndUp(module, coin, network, overwriteContainerId = null, on ;({ coin, network } = dockerOptions) const memoryOptions = await resolveMemoryOptions(module, coin, network, onlyExecution) validatePortArgs(dockerOptions.portArgs) - await assertNoHostPortConflicts(dockerOptions.portArgs, containerPrefix) + await assertNoHostPortConflicts(dockerOptions.portArgs, containerPrefix, { module, environmentVariables }) await assertReusableImage(reuseImage, containerPrefix, module, coin, network) const sourceMetadata = resolveSourceMetadata(reuseImage, dir) const containerEnvironment = resolveContainerEnvironment(environmentVariables, onlyExecution ? null : module) diff --git a/src/services/module_service/docker_args.js b/src/services/module_service/docker_args.js index 5d98d40a..eebc0d5d 100644 --- a/src/services/module_service/docker_args.js +++ b/src/services/module_service/docker_args.js @@ -21,6 +21,7 @@ let { XChainService, SERVICE_REGISTRY, HUB_MODULE_NAME, EXPLORER_MODULE_NAME, SY let { getUtxoTrackerVolumeName } = require('../config_service') let { getPublishedHostPorts } = require('../docker_service') const { healthcheckCommand } = require('./healthcheck_command') +const { claimFreeHostPorts } = require('./host_port_claim') const requiredGetUtxoTrackerVolumeName = getUtxoTrackerVolumeName const requiredGetPublishedHostPorts = getPublishedHostPorts let config = require('../../config'); @@ -60,8 +61,8 @@ function parsePortSpec(pair) { // already allocated" AFTER the image build wastes minutes. `selfName` is the // container we're (re)creating (excluded so re-installs/updates of the same // service don't flag themselves; the old container is already killed+removed -// before this runs, but the name-exclusion is belt-and-suspenders). -async function assertNoHostPortConflicts(portArgs, selfName) { +// before this runs, but the name-exclusion is belt-and-suspenders). `claim` lets buildAndUp move a busy on-request port (host_port_claim.js). +async function assertNoHostPortConflicts(portArgs, selfName, claim = null) { const requested = [] for (let i = 0; i < portArgs.length; i++) { if (portArgs[i] === '-p') { @@ -79,7 +80,7 @@ async function assertNoHostPortConflicts(portArgs, selfName) { const others = [...holders].filter(n => n !== selfName) if (others.length > 0) conflicts.push({ hostPort, holders: others }) } - if (conflicts.length > 0) { + if (claimFreeHostPorts(conflicts, portArgs, requested, published, claim && { ...claim, registry: SERVICE_REGISTRY, logger }).length > 0) { const lines = conflicts.map(c => ` host port ${c.hostPort} is already published by: ${c.holders.join(', ')}`) throw new Error( 'Host port conflict: cannot publish the following port(s):\n' + diff --git a/src/services/module_service/host_port_claim.js b/src/services/module_service/host_port_claim.js new file mode 100644 index 00000000..52018d90 --- /dev/null +++ b/src/services/module_service/host_port_claim.js @@ -0,0 +1,70 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Module Service part: free host ports for a second install + * + * Every coin/network install defaults its per-service host ports to the same + * numbers (utxo-tracker 3001, decoder 3002, ...), so the second install on a + * host (testnet beside mainnet, a second coin) asked for ports the first one + * already published and stopped on a host port conflict. A port the registry + * publishes only on request (no `always`) is reached by other containers on the + * docker network, never through the host, so moving its host side is safe: + * this picks the same port 100 higher until one is free (3001, 3101, 3201 ...), + * which keeps a second install's ports out of the first install's 30xx band. + * The hub and explorer ports (`always`) are what the CLI itself dials on + * 127.0.0.1, so a conflict on one of them still stops the install. + ********************************************************************/ + +'use strict' + +const PORT_STEP = 100 +const MAX_PORT = 65535 + +// The first port at `from + k * PORT_STEP` (k >= 1) that `taken` does not hold. +function nextFreeHostPort(from, taken) { + for (let port = from + PORT_STEP; port <= MAX_PORT; port += PORT_STEP) { + if (!taken.has(String(port))) return port + } + return null +} + +// Move each conflicting on-request port of `claim.module` to a free host port: +// rewrites its `-p` value in `portArgs` and its host key in +// `claim.environmentVariables`, and drops it from `conflicts`. Returns +// `conflicts` with only the ports it could not move. With no `claim` it changes +// nothing, so a caller that does not opt in keeps the plain conflict error. +function claimFreeHostPorts(conflicts, portArgs, requested, published, claim) { + if (!claim || conflicts.length === 0) return conflicts + const { module, environmentVariables, registry, logger } = claim + const facet = registry[module] && registry[module].docker + const ports = (facet && facet.ports) || [] + const taken = new Set([...published.keys(), ...requested].map(String)) + for (let c = conflicts.length - 1; c >= 0; c--) { + const { hostPort, holders } = conflicts[c] + const entry = ports.find(p => !p.always && String(environmentVariables[p.host]) === hostPort) + if (!entry) continue + const at = portArgs.indexOf(`${environmentVariables[entry.host]}:${environmentVariables[entry.container]}`) + if (at < 1 || portArgs[at - 1] !== '-p') continue + const free = nextFreeHostPort(Number(hostPort), taken) + if (free === null) continue + taken.add(String(free)) + environmentVariables[entry.host] = free + portArgs[at] = `${free}:${environmentVariables[entry.container]}` + logger.info(`Host port ${hostPort} (${entry.host}) is already published by ${holders.join(', ')}; ` + + `publishing ${module} on host port ${free} instead. Set ${entry.host} in config to choose another.`) + conflicts.splice(c, 1) + } + return conflicts +} + +module.exports = { PORT_STEP, nextFreeHostPort, claimFreeHostPorts } diff --git a/src/services/status_service/get_status.js b/src/services/status_service/get_status.js index d1980baa..bae5646c 100644 --- a/src/services/status_service/get_status.js +++ b/src/services/status_service/get_status.js @@ -225,7 +225,7 @@ function readPublishedPorts(containerStatus) { return portParts.length > 0 ? portParts.join(", ") : "-" } -async function recordContainerFailure(err, nextModule, nextCoin, nextCoinNetwork, moduleStatus, toRemove, rows) { +async function recordContainerFailure(err, nextModule, nextCoin, nextCoinNetwork, containerId, moduleStatus, toRemove, rows) { if (isContainerGoneError(err)) { // Docker confirms the container no longer exists: // reconcile the persistent registry too, not only @@ -236,7 +236,7 @@ async function recordContainerFailure(err, nextModule, nextCoin, nextCoinNetwork // happens even if the registry delete fails. toRemove.push(nextModule) try { - await db.deleteModuleContainer(nextModule, nextCoin, nextCoinNetwork) + await db.deleteModuleContainerIfMatches(nextModule, nextCoin, nextCoinNetwork, containerId) } catch { /* registry cleanup is best-effort */ } return } @@ -266,7 +266,7 @@ async function inspectModule(nextModule, nextCoin, nextCoinNetwork, nextCoinNetw rows.push({ name: nextModule, coin: nextCoin || "-", network: nextCoinNetwork || "-", branch, commit, state: display.state, isChurning: display.isChurning, ports: readPublishedPorts(containerStatus) }) } catch (err) { - await recordContainerFailure(err, nextModule, nextCoin, nextCoinNetwork, moduleStatus, toRemove, rows) + await recordContainerFailure(err, nextModule, nextCoin, nextCoinNetwork, containerId, moduleStatus, toRemove, rows) } } diff --git a/src/services/status_service/status_reducers.js b/src/services/status_service/status_reducers.js index 24c685a1..dc710f2d 100644 --- a/src/services/status_service/status_reducers.js +++ b/src/services/status_service/status_reducers.js @@ -264,6 +264,13 @@ const INDEXER_STALL_HINTS = { "the signed release manifest names a rule set this build does not implement; `xchain-node update xchain-indexer " + coin + " " + network + "`.", reorg_rollback: () => "a reorg rollback is in progress; it resumes on its own once the rolled-back range is re-parsed.", + retroactive_replay_needs_go: (coin, network) => + "the replay exceeded its block cap and needs an operator go row." + + " `xchain-node logs xchain-indexer " + coin + " " + network + "` prints the exact go command.", + retroactive_replay_diverged: (coin, network) => + "a replay witness did not reproduce its recorded result, so no block applies." + + " Neither a restart nor a hub resync clears this data fault; fix the input named by" + + " `xchain-node logs xchain-indexer " + coin + " " + network + "`, then run the rewitness command for the target.", bridge_proof_barrier: (coin, network) => "it is waiting for a bridge escrow proof and defers the block rather than mint unproven. This is not a hub-mirror fault," + " and neither a hub resync nor a restart shortens it: it clears once this node holds a quorum-established checkpoint" diff --git a/src/services/stop_budget_service.js b/src/services/stop_budget_service.js index c4556d8b..91328044 100644 --- a/src/services/stop_budget_service.js +++ b/src/services/stop_budget_service.js @@ -271,7 +271,9 @@ async function restartModuleContainer(stopContainerById, startContainerById, mod } catch (err) { // Say the service is now down, since the stop half already happened const reason = err && err.message ? err.message : String(err) - throw new Error(`${module} was stopped but failed to start again (${reason})`) + const restartError = new Error(`${module} was stopped but failed to start again (${reason})`) + restartError.containerStopped = true + throw restartError } return outcome } diff --git a/src/utils/command_lock.js b/src/utils/command_lock.js index 61960a22..5029f107 100644 --- a/src/utils/command_lock.js +++ b/src/utils/command_lock.js @@ -130,7 +130,7 @@ function breakAbandonedReclaimMarker(reclaimFile) { function reclaimStaleLock(lockFile, reclaimFile, observed, pid, nonce) { const marker = JSON.stringify({ pid, nonce, startedAt: new Date().toISOString() }) + '\n' try { - fs.writeFileSync(reclaimFile, marker, { flag: 'wx', mode: 0o600 }) + publishLockAtomically(reclaimFile, marker, nonce) } catch (err) { if (err.code !== 'EEXIST') throw err // Break-then-loop, never break-and-reclaim in the same pass: reclaiming on @@ -151,7 +151,28 @@ function reclaimStaleLock(lockFile, reclaimFile, observed, pid, nonce) { if (unlinkErr.code !== 'ENOENT') throw unlinkErr } } finally { - try { fs.unlinkSync(reclaimFile) } catch { /* already gone */ } + unlinkIfOwned(reclaimFile, pid, nonce) + } +} + +// Create the lock or reclaim marker so its path never exists empty. An O_EXCL +// writeFileSync opens and writes in two steps; a contender reading between them +// sees an empty file, cannot tell it from a crashed creator's, and removes a LIVE one. +// Write the full payload to a private temp file first, then link() it to the lock +// path: link fails with EEXIST like O_EXCL, and the path appears already complete. +function publishLockAtomically(lockFile, payload, nonce) { + const tmpFile = `${lockFile}.${process.pid}.${nonce}.tmp` + fs.writeFileSync(tmpFile, payload, { flag: 'wx', mode: 0o600 }) + try { + try { + fs.linkSync(tmpFile, lockFile) + } catch (err) { + // A filesystem without hard links: fall back to the exclusive create. + if (!['EPERM', 'ENOSYS', 'ENOTSUP', 'EOPNOTSUPP', 'EXDEV'].includes(err.code)) throw err + fs.writeFileSync(lockFile, payload, { flag: 'wx', mode: 0o600 }) + } + } finally { + try { fs.unlinkSync(tmpFile) } catch { /* already gone */ } } } @@ -162,7 +183,7 @@ function tryAcquireOnce(lockFile, payload, pid, nonce) { const reclaimFile = lockFile + RECLAIM_FILE_SUFFIX for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) { try { - fs.writeFileSync(lockFile, payload, { flag: 'wx', mode: 0o600 }) + publishLockAtomically(lockFile, payload, nonce) return function release() { releaseCommandLock(lockFile, pid, nonce) } } catch (err) { if (err.code !== 'EEXIST') throw err @@ -247,10 +268,13 @@ function releaseCommandLock(lockFile, pid, nonce) { // Only delete a lock we still own; never clobber a successor's lock. The nonce // is what makes ownership provable: a successor that happens to run under a // RECYCLED pid matches on pid alone, and would be clobbered on our exit. - const holder = readLockHolder(lockFile) - if (!holder || holder.pid !== pid) return - if (holder.nonce !== nonce) return - try { fs.unlinkSync(lockFile) } catch { /* already gone */ } + unlinkIfOwned(lockFile, pid, nonce) +} + +function unlinkIfOwned(file, pid, nonce) { + const holder = readLockHolder(file) + if (!holder || holder.pid !== pid || holder.nonce !== nonce) return + try { fs.unlinkSync(file) } catch { /* already gone */ } } module.exports = { acquireCommandLock, adoptCommandLock, getLockFilePath, isPidAlive } diff --git a/test/boundary/boundary.test.js b/test/boundary/boundary.test.js index 0b564a9f..29541e9e 100644 --- a/test/boundary/boundary.test.js +++ b/test/boundary/boundary.test.js @@ -32,6 +32,7 @@ function withInertFsWrites(fsStub) { return { readFileSync: sinon.stub().returns(''), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), appendFileSync: sinon.stub(), chmodSync: sinon.stub(), ...fsStub @@ -64,6 +65,7 @@ describeBoundaryTests('ConfigService: config file parsing', function () { createReadStream: sinon.stub().callsFake(() => streamFromString('NODE_PASSWORD=p@ss=word=123\n')), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } @@ -166,6 +168,7 @@ describeBoundaryTests('ConfigService: config file parsing', function () { existsSync: sinon.stub().returns(false), createReadStream: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/boundary/boundary.test/filter_command_parameters_boundaries.test.js b/test/boundary/boundary.test/filter_command_parameters_boundaries.test.js index 3e0b7421..12e31b57 100644 --- a/test/boundary/boundary.test/filter_command_parameters_boundaries.test.js +++ b/test/boundary/boundary.test/filter_command_parameters_boundaries.test.js @@ -82,3 +82,32 @@ describeBoundaryTests('ConfigService: filterCommandParameters boundaries', funct expect(result['bitcoin']).to.be.undefined }) }) + +// 3b. unknown chain or network is rejected, never passed through +describeBoundaryTests('ConfigService: filterCommandParameters unknown chain or network', function () { + it('rejects an unknown chain for service=all instead of returning the shared explorer', function () { + let result + expect(() => { result = filterCommandParameters(null, 'all', 'litecion', 'regtest') }) + .to.throw(/unknown coin.*litecion/i) + expect(result).to.be.undefined + }) + + it('rejects an unknown chain for a named service', function () { + expect(() => filterCommandParameters(null, 'xchain-decoder', 'litecion', 'regtest')) + .to.throw(/unknown coin.*litecion/i) + }) + + it('rejects an unknown network with a valid chain', function () { + expect(() => filterCommandParameters(null, 'all', 'litecoin', 'regtset')) + .to.throw(/unknown network.*regtset/i) + }) + + it('still expands all with a valid chain and network or all/all; lifecycle policy leaves the explorer out', function () { + const one = filterCommandParameters(null, 'all', 'litecoin', 'regtest', + { includeSharedExplorer: false }) + expect(one).to.not.have.property('') + expect(one['litecoin']['regtest']).to.include('xchain-decoder') + const every = filterCommandParameters(null, 'all', 'all', 'all') + expect(every['']['']).to.include('xchain-explorer') + }) +}) diff --git a/test/chaos/config_resilience_chaos.test.js b/test/chaos/config_resilience_chaos.test.js index a1a6a6a4..0dc5ec3d 100644 --- a/test/chaos/config_resilience_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test.js @@ -49,6 +49,7 @@ describe('Chaos: Config Resilience', function () { existsSync: sinon.stub().returns(false), createReadStream: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } @@ -69,6 +70,7 @@ describe('Chaos: Config Resilience', function () { existsSync: sinon.stub().returns(false), createReadStream: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js b/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js index 2d8c539c..5f5e1d08 100644 --- a/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js @@ -92,6 +92,7 @@ describe('Chaos: Config Resilience', function () { existsSync: sinon.stub().returns(false), createReadStream: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js b/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js index cca10dd5..a73c73a9 100644 --- a/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js @@ -21,6 +21,7 @@ function withSidecarCalls(fsStub) { return { readFileSync: sinon.stub().returns(''), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), chmodSync: sinon.stub(), ...fsStub } @@ -120,6 +121,7 @@ describe('Chaos: Config Resilience', function () { createReadStream: sinon.stub().callsFake(() => Readable.from(configContent + '\n')), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } @@ -146,6 +148,7 @@ describe('Chaos: Config Resilience', function () { createReadStream: sinon.stub().callsFake(() => Readable.from(configContent + '\n')), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/e2e/helpers/e2e-env.js b/test/e2e/helpers/e2e-env.js index 965cb345..d0e27096 100644 --- a/test/e2e/helpers/e2e-env.js +++ b/test/e2e/helpers/e2e-env.js @@ -194,13 +194,18 @@ class E2EEnv extends TestEnv { */ setupDefaultRoutes() { const self = this + const databaseContainerId = 'd'.repeat(64) + const containerIdsByName = new Map([['xchain-node-database', databaseContainerId]]) // docker build always succeeds this.capture.when(/docker build/).returns({ stdout: '' }) // docker run returns a unique container ID each time - this.capture.when(/docker run/).respondsWith(() => { - return { stdout: self.nextContainerId() + '\n' } + this.capture.when(/docker run/).respondsWith((command) => { + const containerId = self.nextContainerId() + const name = command.match(/(?:^|\s)--name\s+(\S+)/)?.[1] + if (name) containerIdsByName.set(name, containerId) + return { stdout: containerId + '\n' } }) // docker kill/rm/stop/start/restart echo back the container ID @@ -229,13 +234,15 @@ class E2EEnv extends TestEnv { // setup inspect always succeeds so createDockerNetwork skips creation) this.capture.when(/docker network create/).returns({ stdout: '' }) - // docker inspect --type container --format {{.Id}}: the DB fail-fast - // precheck (getDatabaseContainerId, 32224ab) expects a bare 64-hex id; - // the generic inspect route below would feed it a JSON blob and the - // precheck would abort installs with "MariaDB container not found". - // Must be registered BEFORE the generic route (first match wins). - this.capture.when(/docker inspect --type container --format/).returns({ - stdout: 'd'.repeat(64) + '\n' + // Named presence checks see only containers created in this environment. + this.capture.when(/docker inspect --type container --format \{\{\.Id\}\}/).respondsWith((command) => { + const name = command.trim().split(/\s+/).pop() + const containerId = containerIdsByName.get(name) + if (containerId) return { stdout: containerId + '\n' } + const message = `Error: No such object: ${name}` + const error = new Error(message) + error.stderr = message + return { error, stderr: message } }) // docker inspect (container) returns running status diff --git a/test/e2e/scenarios/error_handling.test.js b/test/e2e/scenarios/error_handling.test.js index 17a1ad1b..d3f0f7c2 100644 --- a/test/e2e/scenarios/error_handling.test.js +++ b/test/e2e/scenarios/error_handling.test.js @@ -223,13 +223,14 @@ describe('E2E: Error Handling (Scenario 4.10)', function () { // E2E-064: Start with no installed modules describe('E2E-064: Start with empty LevelDB', function () { - it('startModules succeeds gracefully with no installed modules', async function () { + it('startModules fails with a named no-installed-service error when nothing is installed', async function () { env.setupDefaultRoutes() cli = env.createCLI() + // Nothing selected is installed, so the command fails with a reason instead of reporting success. const serviceList = filterCommandParameters(null, 'all', 'bitcoin', 'regtest') - const result = await cli.moduleOps.startModules(serviceList) - expect(result).to.be.true + const err = await cli.moduleOps.startModules(serviceList).then(() => null, e => e) + expect(err && err.message).to.match(/start matched no installed service/) // No docker start commands should have been issued const startCmds = env.capture.findCommands(/docker start/) @@ -245,13 +246,13 @@ describe('E2E: Error Handling (Scenario 4.10)', function () { // E2E-065: Stop with no installed modules describe('E2E-065: Stop with empty LevelDB', function () { - it('stopModules succeeds gracefully with no installed modules', async function () { + it('stopModules fails with a named no-installed-service error when nothing is installed', async function () { env.setupDefaultRoutes() cli = env.createCLI() const serviceList = filterCommandParameters(null, 'all', 'bitcoin', 'regtest') - const result = await cli.moduleOps.stopModules(serviceList) - expect(result).to.be.true + const err = await cli.moduleOps.stopModules(serviceList).then(() => null, e => e) + expect(err && err.message).to.match(/stop matched no installed service/) const stopCmds = env.capture.findCommands(/docker stop/) expect(stopCmds).to.have.lengthOf(0) diff --git a/test/e2e/scenarios/exec_logs.test.js b/test/e2e/scenarios/exec_logs.test.js index f837b75d..4fe081bd 100644 --- a/test/e2e/scenarios/exec_logs.test.js +++ b/test/e2e/scenarios/exec_logs.test.js @@ -71,13 +71,14 @@ describe('E2E: Exec and Logs Commands (Scenarios 4.11, 4.12)', function () { expect(hasCommand, 'exec includes the user command').to.be.true }) - it('skips exec when module has no container in LevelDB', async function () { + it('reports failure and runs nothing when module has no container in LevelDB', async function () { env.setupDefaultRoutes() cli = env.createCLI() const serviceList = filterCommandParameters(null, 'xchain-decoder', 'bitcoin', 'regtest') + // A selection that matches no installed container is a failure, so the exec CLI exits 1. const result = await cli.moduleOps.execModules(serviceList, 'ls /app') - expect(result).to.be.true + expect(result).to.equal(false) const execCmds = env.capture.findCommands(/docker exec/) expect(execCmds).to.have.lengthOf(0) diff --git a/test/fuzz/config_parsing.fuzz.js b/test/fuzz/config_parsing.fuzz.js index a2e7cc5c..6476bb78 100644 --- a/test/fuzz/config_parsing.fuzz.js +++ b/test/fuzz/config_parsing.fuzz.js @@ -37,6 +37,7 @@ function makeServiceWithConfig(configContent) { existsSync: sinon.stub().returns(true), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } @@ -200,6 +201,7 @@ describe('Fuzz: Config File Parsing', function () { createReadStream: sinon.stub(), existsSync: sinon.stub().returns(false), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/fuzz/filter_params.fuzz.js b/test/fuzz/filter_params.fuzz.js index e2598902..d5c01a25 100644 --- a/test/fuzz/filter_params.fuzz.js +++ b/test/fuzz/filter_params.fuzz.js @@ -60,10 +60,18 @@ describe('Fuzz: filterCommandParameters()', function () { 'bitcoin; echo pwned', ] + // An empty value means "all"; any other unknown chain is refused by name rather than + // expanded, so a typo can never act on the shared services. for (const coin of invalidCoins) { - it(`does not crash with invalid coin: ${JSON.stringify(coin)}`, function () { - const result = filterCommandParameters(null, 'xchain-encoder', coin, 'mainnet') - expect(result).to.be.an('object') + if (coin === '') { + it('treats an empty coin as all and does not crash', function () { + expect(filterCommandParameters(null, 'xchain-encoder', coin, 'mainnet')).to.be.an('object') + }) + continue + } + it(`refuses invalid coin ${JSON.stringify(coin)} with a named error`, function () { + expect(() => filterCommandParameters(null, 'xchain-encoder', coin, 'mainnet')) + .to.throw(Error, 'unknown coin: ' + coin) }) } }) @@ -79,9 +87,15 @@ describe('Fuzz: filterCommandParameters()', function () { ] for (const net of invalidNetworks) { - it(`does not crash with invalid network: ${JSON.stringify(net)}`, function () { - const result = filterCommandParameters(null, 'xchain-encoder', 'bitcoin', net) - expect(result).to.be.an('object') + if (net === '') { + it('treats an empty network as all and does not crash', function () { + expect(filterCommandParameters(null, 'xchain-encoder', 'bitcoin', net)).to.be.an('object') + }) + continue + } + it(`refuses invalid network ${JSON.stringify(net)} with a named error`, function () { + expect(() => filterCommandParameters(null, 'xchain-encoder', 'bitcoin', net)) + .to.throw(Error, 'unknown network: ' + net) }) } }) diff --git a/test/integration/database_setup.test/check_if_database_is_ready.test.js b/test/integration/database_setup.test/check_if_database_is_ready.test.js index 93f31040..315991e5 100644 --- a/test/integration/database_setup.test/check_if_database_is_ready.test.js +++ b/test/integration/database_setup.test/check_if_database_is_ready.test.js @@ -22,8 +22,10 @@ databaseSuite('checkIfDatabaseIsReady', function (fixture) { const dbContainerId = fakeContainerId('d') await env.insertModule(DB_MODULE_NAME, '', '', dbContainerId) + // The container id lookup answers at once; the mariadb ping fails twice first. let callCount = 0 const execFileAsyncStub = async (command, args) => { + if (args[0] === 'inspect') return { stdout: dbContainerId + '\n', stderr: '' } callCount++ if (callCount < 3) throw new Error('Connection refused') return { stdout: '1', stderr: '' } diff --git a/test/integration/database_setup.test/support/fixture.js b/test/integration/database_setup.test/support/fixture.js index cb72122d..7f44b569 100644 --- a/test/integration/database_setup.test/support/fixture.js +++ b/test/integration/database_setup.test/support/fixture.js @@ -63,7 +63,9 @@ function makeMariadbSpawnStub(cmdCapture) { function configureCapture(capture, options, dbContainerId) { capture.when(/docker pull/).returns({ stdout: '' }) capture.when(/docker tag/).returns({ stdout: '' }) - capture.when(/docker run/).returns({ stdout: dbContainerId + '\n' }) + // Like real docker, the container exists once `docker run` has made it. + let started = Boolean(options.containerExists) + capture.when(/docker run/).respondsWith(() => { started = true; return { stdout: dbContainerId + '\n' } }) capture.when(/docker exec.*SELECT 1/).returns({ stdout: '1' }) capture.when(/docker exec.*SELECT COUNT\(SCHEMA/).returns({ stdout: '0' }) capture.when(/docker exec.*SELECT COUNT\(\*\).*mysql.user/).returns({ stdout: '0' }) @@ -88,10 +90,10 @@ function configureCapture(capture, options, dbContainerId) { // regardless of options.dbContainerId. options.containerExists // toggles "database already installed" (used by the reuse/grant // tests) vs "fresh install" (the default, used by the first-install - // test) semantics. - capture.when(/docker inspect --type container --format/).returns( - options.containerExists ? { stdout: dbContainerId + '\n' } : { stdout: '' } - ) + // test) semantics; a fresh install finds the container after its run. + capture.when(/docker inspect --type container --format/).respondsWith(() => ( + started ? { stdout: dbContainerId + '\n' } : { stdout: '' } + )) capture.when(/docker inspect/).returns({ stdout: JSON.stringify([{ State: { Status: 'running' }, diff --git a/test/integration/halt_marker_sql_mariadb.test.js b/test/integration/halt_marker_sql_mariadb.test.js index ce2e9dda..848f0ea5 100644 --- a/test/integration/halt_marker_sql_mariadb.test.js +++ b/test/integration/halt_marker_sql_mariadb.test.js @@ -82,14 +82,15 @@ const UNAUDITED = '{"reason":"oops"}' const HALT = '{"reason":"over-deep rollback"}' // Hold the connection and gate modules the before hook loads. -let conn, mariadbExec, parseCountTokens, parseNewestClear +let conn, mariadbExec, parseCountTokens, parseNewestClear, parseRollbackGuardRows // Create the scratch database the gate's SQL reads. async function createScratchDb() { if (!HOST) this.skip() const mariadb = require('mariadb') mariadbExec = require('../../src/services/database_service/mariadb_exec') - ;({ parseCountTokens, parseNewestClear } = require('../../src/services/bootstrap_health_gate/halt_markers')) + ;({ parseCountTokens, parseNewestClear, parseRollbackGuardRows } = + require('../../src/services/bootstrap_health_gate/halt_markers')) conn = await mariadb.createConnection({ host: CFG.host, port: CFG.port, user: CFG.root_user, password: CFG.root_password }) @@ -203,6 +204,21 @@ async function countsPendingSyncHalts() { expect(await count(haltMarkers.pendingSyncHaltCountSql(DB))).to.equal(1) } +async function readsRollbackGuardStates() { + await conn.query(`TRUNCATE TABLE \`${DB}\`.sync_state`) + const rows = [ + ['rollback_guard:indexer', '{"peak":5,"low":3}'], + ['rollback_guard:decoder', '{"peak":1,"low":2}'], + ['rollback_guard:empty', ''], + ['rollback_guard:null', null], + ['rollbackXguard:ignored', 'not json'] + ] + for (const [key, value] of rows) + await conn.query(`INSERT INTO \`${DB}\`.sync_state (state_key, state_value) VALUES (?, ?)`, [key, value]) + const raw = await mariadbExec.executeNativeMariaDbCommand(CFG, haltMarkers.rollbackGuardStateSql(DB), '-BN') + expect(parseRollbackGuardRows(raw, 'test rollback guard probe')).to.equal(2) +} + // The reference above is a copy, so pin it to the decoder's source when that sibling is checked out. function pinsReferenceRule() { const src = path.join(__dirname, '../../../xchain-decoder/src/db/reorg_halt.js') @@ -236,5 +252,6 @@ describe('halt marker SQL on MariaDB', function () { it('reads the newest marker row, as the decoder does', readsNewestMarkerRow) it('counts every clear row raised inside the dump window, audited or not', countsClearsInWindow) it('counts every halt_pending copy that xchain-sync would restore', countsPendingSyncHalts) + it('reads valid, invalid, empty and NULL rollback guard rows', readsRollbackGuardStates) it('keeps the reference rule in step with the decoder', pinsReferenceRule) }) diff --git a/test/integration/hub_config_update.test/01_hub_update_retry_logic.test.js b/test/integration/hub_config_update.test/01_hub_update_retry_logic.test.js index 7ae6d016..288e92ba 100644 --- a/test/integration/hub_config_update.test/01_hub_update_retry_logic.test.js +++ b/test/integration/hub_config_update.test/01_hub_update_retry_logic.test.js @@ -69,6 +69,6 @@ hubConfigSuite('hub update retry logic', function (fixture) { expect(err).to.include('problem trying to update') } - expect(httpCapture.callCount('127.0.0.1:10000')).to.equal(10) + expect(httpCapture.callCount('127.0.0.1:10000')).to.equal(60) }) }) diff --git a/test/integration/hub_config_update.test/05_hub_update_retry_window.test.js b/test/integration/hub_config_update.test/05_hub_update_retry_window.test.js new file mode 100644 index 00000000..7b3f746f --- /dev/null +++ b/test/integration/hub_config_update.test/05_hub_update_retry_window.test.js @@ -0,0 +1,40 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') + +const { hubConfigSuite } = require('./support/fixture') + +hubConfigSuite('hub update retry window', function (fixture) { + + it('succeeds when the hub answers on the last attempt of its boot window', async function () { + const { env, httpCapture, makeHubService, TestEnv } = fixture() + const state = require('../../../src/state') + + const hubId = TestEnv.fakeContainerId('h') + await env.insertModule('xchain-hub', '', '', hubId) + + state.setStatusUpdated(true) + state.setLastStatus({}) + + httpCapture.when('127.0.0.1:10000').failsThenSucceeds( + 59, + new Error('Connection refused'), + { data: { result: true } } + ) + + const { HubService } = makeHubService() + await HubService.updateHubOrExplorer('xchain-hub') + + expect(httpCapture.callCount('127.0.0.1:10000')).to.equal(60) + }) +}) diff --git a/test/integration/module_lifecycle.test.js b/test/integration/module_lifecycle.test.js index c0735b0a..5bd1e34c 100644 --- a/test/integration/module_lifecycle.test.js +++ b/test/integration/module_lifecycle.test.js @@ -297,11 +297,11 @@ describe('Integration: Module Lifecycle (LevelDB state)', function () { beforeEach(setupModuleEnv) afterEach(cleanupModuleEnv) describe('moduleOperations uses LevelDB for container lookups', function () { - it('startModules gracefully handles missing LevelDB entry', async function () { - // No modules inserted; the registry is empty + it('startModules fails with a named no-installed-service error on a missing LevelDB entry', async function () { + const startContainer = sinon.stub().resolves(true) const moduleOps = proxyquire('../../src/operations/module_operations', { '../services/docker_service': { - startContainer: sinon.stub().resolves(true), + startContainer, createDockerNetwork: async () => true, stopContainer: async () => true, restartContainer: async () => true, @@ -315,10 +315,10 @@ describe('Integration: Module Lifecycle (LevelDB state)', function () { }) const serviceList = { 'bitcoin': { 'mainnet': ['xchain-encoder'] } } - // Should not throw; the operation catches errors and logs them. A missing - // container ID must not throw: the operation catches and logs it. - const result = await moduleOps.startModules(serviceList) - expect(result).to.be.true + // The registry is empty, so nothing selected is installed: fail with a reason. + const err = await moduleOps.startModules(serviceList).then(() => null, e => e) + expect(err && err.message).to.match(/no installed service: xchain-encoder \(bitcoin mainnet\)/) + expect(startContainer.called).to.be.false }) }) }) diff --git a/test/regression/regression.test/07_config_parsing_boundaries.test.js b/test/regression/regression.test/07_config_parsing_boundaries.test.js index ebd1ed75..49454d35 100644 --- a/test/regression/regression.test/07_config_parsing_boundaries.test.js +++ b/test/regression/regression.test/07_config_parsing_boundaries.test.js @@ -57,6 +57,7 @@ describe('Regression Suite', function () { // getDefaultConfig now persists generated RPC creds to the // .local sidecar (6648722): persistSidecarCreds needs these. writeFileSync: sinon.stub(), + renameSync: sinon.stub(), appendFileSync: sinon.stub(), chmodSync: sinon.stub() } diff --git a/test/regression/regression.test/08_precheck_error_handling.test.js b/test/regression/regression.test/08_precheck_error_handling.test.js index d87dac43..e6beb568 100644 --- a/test/regression/regression.test/08_precheck_error_handling.test.js +++ b/test/regression/regression.test/08_precheck_error_handling.test.js @@ -41,14 +41,15 @@ describe('Regression Suite', function () { expect(result).to.be.true }) - it('R-PRE-002: startModules gracefully handles missing LevelDB entry', async function () { + it('R-PRE-002: startModules fails with a named no-installed-service error on a missing LevelDB entry', async function () { const env = new TestEnv() + const startContainer = sinon.stub().resolves(true) await env.setup() try { const moduleOps = proxyquire('../../../src/operations/module_operations', { '../services/docker_service': { - startContainer: sinon.stub().resolves(true), + startContainer, createDockerNetwork: async () => true, stopContainer: async () => true, restartContainer: async () => true, @@ -62,8 +63,10 @@ describe('Regression Suite', function () { }) const serviceList = { 'bitcoin': { 'mainnet': ['xchain-encoder'] } } - const result = await moduleOps.startModules(serviceList) - expect(result).to.be.true + // The registry is empty, so nothing selected is installed: fail with a reason. + const err = await moduleOps.startModules(serviceList).then(() => null, e => e) + expect(err && err.message).to.match(/no installed service: xchain-encoder \(bitcoin mainnet\)/) + expect(startContainer.called).to.be.false } finally { await env.teardown() } diff --git a/test/regression/regression.test/support/helpers.js b/test/regression/regression.test/support/helpers.js index 722965d6..0e9514f4 100644 --- a/test/regression/regression.test/support/helpers.js +++ b/test/regression/regression.test/support/helpers.js @@ -37,6 +37,7 @@ function makeServiceWithConfig(configContent) { readFileSync: sinon.stub().returns(''), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub(), chmodSync: sinon.stub() diff --git a/test/unit/autoheal_service.test/12_restart_and_inspection_errors.test.js b/test/unit/autoheal_service.test/12_restart_and_inspection_errors.test.js index c808f9c6..1f8a6516 100644 --- a/test/unit/autoheal_service.test/12_restart_and_inspection_errors.test.js +++ b/test/unit/autoheal_service.test/12_restart_and_inspection_errors.test.js @@ -71,39 +71,42 @@ function loadService(stubs) { }) } -describe('AutohealService', () => { - let stubs, service, stateDir, logStub - +function restartErrorHarness() { + const h = {} beforeEach(() => { - stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoheal-test-')) - process.env.XCHAIN_NODE_AUTOHEAL_STATE_DIR = stateDir - stubs = makeStubs() - service = loadService(stubs) - logStub = sinon.stub(console, 'log') + h.stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoheal-test-')) + process.env.XCHAIN_NODE_AUTOHEAL_STATE_DIR = h.stateDir + h.stubs = makeStubs() + h.service = loadService(h.stubs) + h.logStub = sinon.stub(console, 'log') }) - afterEach(() => { - logStub.restore() + h.logStub.restore() delete process.env.XCHAIN_NODE_AUTOHEAL_STATE_DIR delete process.env.XCHAIN_NODE_AUTOHEAL_GRACE_MS delete process.env.XCHAIN_NODE_AUTOHEAL_COOLDOWN_MS - fs.rmSync(stateDir, { recursive: true, force: true }) + fs.rmSync(h.stateDir, { recursive: true, force: true }) }) + return h +} - function registryRow(module, containerId) { - return { module, coin: 'bitcoin', network: 'regtest', container_id: containerId } - } +function registryRow(module, containerId) { + return { module, coin: 'bitcoin', network: 'regtest', container_id: containerId } +} + +describe('AutohealService', () => { + const h = restartErrorHarness() it('records a failed restart without aborting the pass and reports it', async () => { - stubs.db.getAllModuleContainers.resolves([ + h.stubs.db.getAllModuleContainers.resolves([ registryRow('xchain-indexer', 'iii'), { module: 'xchain-decoder', coin: 'litecoin', network: 'regtest', container_id: 'jjj' } ]) - stubs.getStatusFromContainer.resolves(unhealthyPastGrace()) - stubs.stopContainer.withArgs('iii').rejects(new Error('docker daemon gone')) - stubs.stopContainer.withArgs('jjj').resolves({ stopped: true, seconds: 1, killed: false, exitCode: 0 }) + h.stubs.getStatusFromContainer.resolves(unhealthyPastGrace()) + h.stubs.stopContainer.withArgs('iii').rejects(new Error('docker daemon gone')) + h.stubs.stopContainer.withArgs('jjj').resolves({ stopped: true, seconds: 1, killed: false, exitCode: 0 }) - const result = await service.runAutoheal({ now: NOW }) + const result = await h.service.runAutoheal({ now: NOW }) expect(result.failed).to.have.length(1) expect(result.failed[0].containerId).to.equal('iii') @@ -112,12 +115,53 @@ describe('AutohealService', () => { }) it('skips a container that cannot be inspected instead of failing the pass', async () => { - stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'kkk')]) - stubs.getStatusFromContainer.rejects(new Error('No such container')) + h.stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'kkk')]) + h.stubs.getStatusFromContainer.rejects(new Error('No such container')) - const result = await service.runAutoheal({ now: NOW }) + const result = await h.service.runAutoheal({ now: NOW }) expect(result.skipped[0].reason).to.equal('inspect failed') expect(result.failed).to.have.length(0) }) }) + +describe('AutohealService', () => { + const h = restartErrorHarness() + + it('retries only the start after a failed restart left the container stopped', async () => { + h.stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'left-down')]) + h.stubs.getStatusFromContainer.onFirstCall().resolves(unhealthyPastGrace()) + h.stubs.getStatusFromContainer.onSecondCall().resolves(inspectStatus('unhealthy', [], 'exited')) + h.stubs.startContainer.onFirstCall().rejects(new Error('port is already allocated')) + h.stubs.startContainer.onSecondCall().resolves(true) + + const failed = await h.service.runAutoheal({ now: NOW }) + const stateAfterFailure = JSON.parse(fs.readFileSync(path.join(h.stateDir, 'autoheal-state.json'), 'utf8')) + const recovered = await h.service.runAutoheal({ now: NOW + 1000, resumeStopped: true }) + + expect(failed.failed).to.have.length(1) + expect(stateAfterFailure.stoppedByAutoheal['left-down']).to.equal(true) + expect(h.stubs.stopContainer.calledOnce).to.equal(true) + expect(h.stubs.startContainer.callCount).to.equal(2) + expect(recovered.restarted[0].containerId).to.equal('left-down') + const finalState = JSON.parse(fs.readFileSync(path.join(h.stateDir, 'autoheal-state.json'), 'utf8')) + expect(finalState.stoppedByAutoheal).to.not.have.property('left-down') + }) + + it('escalates a left-down container without overriding operator intent', async () => { + h.stubs.db.getAllModuleContainers.resolves([registryRow('xchain-indexer', 'left-down')]) + h.stubs.getStatusFromContainer.onFirstCall().resolves(unhealthyPastGrace()) + h.stubs.getStatusFromContainer.onSecondCall().resolves(inspectStatus('unhealthy', [], 'exited')) + h.stubs.startContainer.rejects(new Error('port is already allocated')) + + await h.service.runAutoheal({ now: NOW }) + const unattended = await h.service.runAutoheal({ now: NOW + 1000 }) + const state = JSON.parse(fs.readFileSync(path.join(h.stateDir, 'autoheal-state.json'), 'utf8')) + + expect(h.stubs.stopContainer.calledOnce).to.equal(true) + expect(h.stubs.startContainer.calledOnce).to.equal(true) + expect(unattended.escalated[0].containerId).to.equal('left-down') + expect(h.service.autohealExitCode(unattended)).to.equal(2) + expect(state.stoppedByAutoheal['left-down']).to.equal(true) + }) +}) diff --git a/test/unit/autoheal_service.test/backoff/20_escalation_signal.test.js b/test/unit/autoheal_service.test/backoff/20_escalation_signal.test.js index a5e29695..7288ff47 100644 --- a/test/unit/autoheal_service.test/backoff/20_escalation_signal.test.js +++ b/test/unit/autoheal_service.test/backoff/20_escalation_signal.test.js @@ -178,4 +178,24 @@ describe('AutohealService escalation', () => { exit.restore() } }) + + it('passes explicit resume-stopped intent to the autoheal service', async () => { + const { registerLifecycleCommands } = require('../../../../src/cli/options') + const program = new Command().exitOverride() + const result = { candidates: [], restarted: [], failed: [], skipped: [], escalated: [] } + let received + const loadModule = () => ({ + runAutoheal: async options => { received = options; return result }, + autohealExitCode: h.service.autohealExitCode + }) + registerLifecycleCommands(program, { loadModule }) + const exit = sinon.stub(process, 'exit') + try { + await program.parseAsync(['node', 'xchain-node', 'autoheal', '--resume-stopped']) + expect(received).to.deep.equal({ dryRun: false, resumeStopped: true }) + expect(exit.calledOnceWith(0)).to.be.true + } finally { + exit.restore() + } + }) }) diff --git a/test/unit/autoheal_service.test/probation/10_restart_stamp_after_start.test.js b/test/unit/autoheal_service.test/probation/10_restart_stamp_after_start.test.js new file mode 100644 index 00000000..41a261db --- /dev/null +++ b/test/unit/autoheal_service.test/probation/10_restart_stamp_after_start.test.js @@ -0,0 +1,95 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const fs = require('fs') +const os = require('os') +const path = require('path') +const sinon = require('sinon') +const { expect } = require('chai') +const proxyquire = require('proxyquire').noCallThru() + +const NOW = Date.parse('2026-07-21T12:00:00Z') +const RESTART_MS = 120000 + +function unhealthyPastGrace() { + const entry = (agoMs, exitCode) => { + const start = new Date(NOW - agoMs) + return { Start: start.toISOString(), End: new Date(start.getTime() + 1000).toISOString(), ExitCode: exitCode, Output: 'x' } + } + return { + State: { + Status: 'running', + Health: { Status: 'unhealthy', FailingStreak: 5, Log: [entry(11 * 60000, 0), entry(10 * 60000, 1), entry(5 * 60000, 1), entry(60000, 1)] } + } + } +} + +describe('AutohealService restart stamp', () => { + let clock, stateDir, logStub, service, stubs + + beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoheal-test-')) + process.env.XCHAIN_NODE_AUTOHEAL_STATE_DIR = stateDir + clock = sinon.useFakeTimers({ now: NOW, toFake: ['Date'] }) + logStub = sinon.stub(console, 'log') + stubs = { + db: { getAllModuleContainers: sinon.stub().resolves([ + { module: 'xchain-indexer', coin: 'bitcoin', network: 'regtest', container_id: 'a1' }, + { module: 'xchain-indexer', coin: 'dogecoin', network: 'regtest', container_id: 'b1' } + ]), assertReady: sinon.stub() }, + getStatusFromContainer: sinon.stub().callsFake(async () => unhealthyPastGrace()), + // A slow drain: the stop consumes RESTART_MS of wall time. + stopContainer: sinon.stub().callsFake(async () => { + clock.tick(RESTART_MS) + return { stopped: true, seconds: 120, killed: true, exitCode: 137 } + }), + startContainer: sinon.stub().resolves(true) + } + service = proxyquire('../../../../src/services/autoheal_service', { + '../state': { db: stubs.db }, + './docker_service': { + getStatusFromContainer: stubs.getStatusFromContainer, + stopContainer: stubs.stopContainer, + startContainer: stubs.startContainer, + getContainerStopSettings: sinon.stub().resolves(null) + }, + './module_service': { SERVICE_HEALTHCHECK: require('../../../../src/services/module_service').SERVICE_HEALTHCHECK } + }) + }) + + afterEach(() => { + clock.restore() + logStub.restore() + delete process.env.XCHAIN_NODE_AUTOHEAL_STATE_DIR + fs.rmSync(stateDir, { recursive: true, force: true }) + }) + + const readState = () => JSON.parse(fs.readFileSync(path.join(stateDir, 'autoheal-state.json'), 'utf8')) + + it('stamps each restart after its own stop and start, not at pass start', async () => { + await service.runAutoheal({ now: NOW }) + expect(stubs.startContainer.callCount).to.equal(2) + + const state = readState() + expect(state.restarts.a1).to.be.at.least(NOW + RESTART_MS) + expect(state.restarts.b1).to.be.at.least(NOW + 2 * RESTART_MS) + expect(state.restarts.b1).to.be.greaterThan(state.restarts.a1) + }) + + it('runs the cooldown from the finished restart', async () => { + await service.runAutoheal({ now: NOW }) + // 600s default cooldown: pass-start stamp would expire at NOW+600000. + const throttled = await service.runAutoheal({ now: NOW + 650000 }) + expect(stubs.startContainer.callCount).to.equal(2) + expect(throttled.skipped.map(s => s.reason)).to.include('inside restart cooldown') + }) +}) diff --git a/test/unit/bootstrap_health_gate.test/configured_database_names.test.js b/test/unit/bootstrap_health_gate.test/configured_database_names.test.js index 2e7bc799..05b42d07 100644 --- a/test/unit/bootstrap_health_gate.test/configured_database_names.test.js +++ b/test/unit/bootstrap_health_gate.test/configured_database_names.test.js @@ -30,7 +30,8 @@ function loadMarkers(cfg) { function nativeDeps() { // The newest-clear probe answers "no clear" as a count and an empty payload column. const executeNativeMariaDbCommand = sinon.stub().callsFake(async (c, sql) => - (/information_schema\.TABLES/.test(sql) ? '1\t1\t1' : (/HEX\(data\)/.test(sql) ? '0\t' : '0'))) + (/information_schema\.TABLES/.test(sql) ? '1\t1\t1' : + (/HEX\(data\)|rollback_guard/.test(sql) ? '0\t' : '0'))) return { executeNativeMariaDbCommand, getExternalDbConfig: sinon.stub().resolves({ host: 'h' }) } } diff --git a/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js b/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js index b166943a..0e0d4e5c 100644 --- a/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js +++ b/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js @@ -54,6 +54,52 @@ describe('BootstrapHealthGate', function () { }) }) +describe('BootstrapHealthGate rollback guard markers', function () { + + installEnvironmentHooks() + + const encoded = value => 'V' + Buffer.from(value, 'utf8').toString('hex').toUpperCase() + + it('REFUSES an invalid rollback guard row in the gated database', async function () { + const gate = loadGate() + const runner = makeRunner({ rollbackGuardRows: '1\t' + encoded('{"peak":1,"low":2}') }) + const err = await refusal(callGate(gate, { runner })) + expect(err.message).to.match(/database carries an invalid xchain-sync rollback_guard row/) + }) + + it('REFUSES an invalid rollback guard row in the paired decoder database', async function () { + const gate = loadGate() + const runner = makeRunner({ + decoder: { rollbackGuardRows: '1\t' + encoded('not json') } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(/paired decoder database \S+ carries an invalid xchain-sync rollback_guard row/) + }) + + it('accepts valid and NULL rollback guard rows', async function () { + const gate = loadGate() + await callGate(gate, { runner: makeRunner({ + rollbackGuardRows: '2\t' + encoded('{"peak":5,"low":3}') + ',N' + }) }) + }) + + it('REFUSES unreadable rollback guard probe output', async function () { + const gate = loadGate() + const err = await refusal(callGate(gate, { + runner: makeRunner({ rollbackGuardRows: '2\t' + encoded('{"peak":5,"low":3}') }) + })) + expect(err.message).to.match(/rollback_guard probe[\s\S]*returned unreadable output/) + }) + + it('does not probe rollback guards when sync_state is absent', async function () { + const gate = loadGate() + const runner = makeRunner({ tables: '1\t1\t0', rollbackGuardRows: 'broken' }) + await callGate(gate, { runner }) + const sqls = runner.getCalls().map(c => (c.args[1] || []).join(' ')) + expect(sqls.some(sql => /rollback_guard/.test(sql))).to.equal(false) + }) +}) + // xchain-sync parks a halt in sync_state when its sync_halt write fails, and // restorePendingHalt re-installs it on any replica that restores the archive. describe('BootstrapHealthGate', function () { diff --git a/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js b/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js index f37e4e53..0e701427 100644 --- a/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js +++ b/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js @@ -73,6 +73,11 @@ describe('BootstrapHealthGate', function () { expect(reasons).to.have.lengthOf(1) expect(reasons[0]).to.match(/WEDGED/) expect(reasons[0]).to.match(/vm_executor_host_fault/) + const held = gate.evaluateStatusPayload( + { status: 'healthy', lag: 3, stallClass: 'operator_action_hold', stallReason: 'retroactive_replay_diverged' }) + expect(held).to.have.lengthOf(1) + expect(held[0]).to.include('operator acts') + expect(held[0]).to.include('retroactive_replay_diverged') }) // A proof wait publishes its _stuck class past the hold ceiling, well before it wedges. diff --git a/test/unit/bootstrap_health_gate.test/published_stall_class_set.test.js b/test/unit/bootstrap_health_gate.test/published_stall_class_set.test.js index acf76820..4ca009d0 100644 --- a/test/unit/bootstrap_health_gate.test/published_stall_class_set.test.js +++ b/test/unit/bootstrap_health_gate.test/published_stall_class_set.test.js @@ -20,7 +20,7 @@ const GATE_SOURCE = path.join(__dirname, '../../../src/services/bootstrap_health // Every class the indexer publishes today; the grid below must reach each one. const EXPECTED = [ 'none', 'future_block_wait', 'bridge_proof_wait', 'train_activation_halt', 'price_landing_wait', - 'barrier_defer', 'wedged', 'upstream_halted', 'rollcall_proof_wait', 'rollcall_proof_stuck', + 'operator_action_hold', 'barrier_defer', 'wedged', 'upstream_halted', 'rollcall_proof_wait', 'rollcall_proof_stuck', 'capability_proof_wait', 'capability_proof_stuck' ] @@ -30,7 +30,8 @@ function publishedClasses({ stallClassOf, publishedStallClass }) { const now = 1760000000000 const graceMs = 120000 const reasons = [null, 'bridge_proof_barrier', 'train_activation_halt: rule set 3 required at 900', - 'price_landing_barrier', 'price_sync_barrier', 'rollcall_proof_unavailable', 'capability_proof_unavailable'] + 'retroactive_replay_diverged', 'price_landing_barrier', 'price_sync_barrier', + 'rollcall_proof_unavailable', 'capability_proof_unavailable'] const seen = new Set() for (const reason of reasons) for (const lastCommit of [null, now - 1000, now - 10 * 60000]) diff --git a/test/unit/bootstrap_health_gate.test/rollback_guard_parser.test.js b/test/unit/bootstrap_health_gate.test/rollback_guard_parser.test.js new file mode 100644 index 00000000..a9e6c47f --- /dev/null +++ b/test/unit/bootstrap_health_gate.test/rollback_guard_parser.test.js @@ -0,0 +1,41 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { expect } = require('chai') +const { + isValidRollbackGuardState, + parseRollbackGuardRows +} = require('../../../src/services/bootstrap_health_gate/halt_markers') + +const encoded = value => 'V' + Buffer.from(value, 'utf8').toString('hex').toUpperCase() + +describe('rollback guard marker parser', function () { + it('matches xchain-sync valid rollback streaks', function () { + expect(isValidRollbackGuardState('{"peak":5,"low":3}')).to.equal(true) + expect(isValidRollbackGuardState('{"peak":0,"low":0}')).to.equal(true) + }) + + it('rejects every incoherent or unreadable rollback streak shape', function () { + const invalid = [ + '{"peak":3,"low":5}', '{"peak":-1,"low":0}', '{"peak":1.5,"low":0}', + '{"peak":"5","low":3}', '{"low":0}', 'null', '5', '[]', '', ' ', 'not json' + ] + for (const value of invalid) + expect(isValidRollbackGuardState(value), JSON.stringify(value)).to.equal(false) + }) + + it('counts invalid values while accepting valid and NULL rows', function () { + const raw = '3\t' + encoded('{"peak":5,"low":3}') + ',N,' + encoded('not json') + expect(parseRollbackGuardRows(raw, 'rollback guard probe')).to.equal(1) + }) + + it('refuses count mismatches, malformed tokens and wrong field counts', function () { + for (const raw of ['2\t' + encoded('{"peak":1,"low":0}'), '1\tV0G', '0', '0\t\textra']) + expect(() => parseRollbackGuardRows(raw, 'rollback guard probe'), raw) + .to.throw(/returned unreadable output/) + }) +}) diff --git a/test/unit/bootstrap_health_gate.test/service_health_probe.test.js b/test/unit/bootstrap_health_gate.test/service_health_probe.test.js index 4c773139..dd77b937 100644 --- a/test/unit/bootstrap_health_gate.test/service_health_probe.test.js +++ b/test/unit/bootstrap_health_gate.test/service_health_probe.test.js @@ -94,6 +94,7 @@ describe('BootstrapHealthGate', function () { if (/information_schema\.TABLES/.test(sql)) return { stdout: '1\t1\t1' } // The newest-clear probe: no clear row, and the docker client prints NULL. if (/HEX\(data\)/.test(sql)) return { stdout: '0\tNULL\n' } + if (/rollback_guard/.test(sql)) return { stdout: '0\t' } return { stdout: '0' } }) const res = await callGate(gate, { runner }) diff --git a/test/unit/bootstrap_health_gate.test/support/bootstrap_health_gate.js b/test/unit/bootstrap_health_gate.test/support/bootstrap_health_gate.js index 07dd4d43..926d7bcd 100644 --- a/test/unit/bootstrap_health_gate.test/support/bootstrap_health_gate.js +++ b/test/unit/bootstrap_health_gate.test/support/bootstrap_health_gate.js @@ -39,7 +39,8 @@ function nativeHelperStub(nativeResolves) { if (nativeResolves !== null) return sinon.stub().resolves(nativeResolves) // The native client joins the clear probe's NULL payload column as ''. return sinon.stub().callsFake(async (cfg, sql) => - (/information_schema\.TABLES/.test(sql) ? '1\t1\t1' : (/HEX\(data\)/.test(sql) ? '0\t' : '0'))) + (/information_schema\.TABLES/.test(sql) ? '1\t1\t1' : + (/HEX\(data\)|rollback_guard/.test(sql) ? '0\t' : '0'))) } function loadGate({ external = false, nativeResolves = null } = {}) { @@ -97,6 +98,7 @@ function makeRunner({ syncHaltRows = '0', // Live halt_pending copies in sync_state (xchain-sync's fallback when sync_halt fails). pendingSyncHaltRows = '0', + rollbackGuardRows = '0\t', inspectThrows = null, statusThrows = null, sqlThrows = null, @@ -104,6 +106,7 @@ function makeRunner({ // null, that database answers exactly as the gated one does; set it to express // the scenario the gate exists for and a single-database fake cannot reach: // decoder dirty, indexer clean. Accepts { tables, reorgHaltRows, newestClear, syncHaltRows, pendingSyncHaltRows, + // rollbackGuardRows, // eventsWatermark, syncHaltWatermark, reorgHaltWindowRows, syncHaltWindowRows, // throws }. decoder = null, @@ -135,6 +138,7 @@ function makeRunner({ if (up && up.throws) throw up.throws const pick = (key, fallback) => (up && up[key] !== undefined) ? up[key] : fallback if (/information_schema\.TABLES/.test(sql)) return { stdout: pick('tables', tables) } + if (/rollback_guard/.test(sql)) return { stdout: pick('rollbackGuardRows', rollbackGuardRows) } if (/`[^`]+`\.sync_state/.test(sql)) return { stdout: pick('pendingSyncHaltRows', pendingSyncHaltRows) } // Watermark and dump-window queries first: both mention the table names the // live-marker branches below match on. diff --git a/test/unit/bootstrap_moved_port.test.js b/test/unit/bootstrap_moved_port.test.js new file mode 100644 index 00000000..7ba068cb --- /dev/null +++ b/test/unit/bootstrap_moved_port.test.js @@ -0,0 +1,121 @@ +'use strict' + +const { expect } = require('chai') +const proxyquire = require('proxyquire').noCallThru() +const { + configDir, + coinMain, + coinSidecar, + makeMemoryConfigService, + XChainService +} = require('./config_service.test/helpers.test') +const constants = require('../../src/config') +const { buildHubModuleConfig } = require('../../src/services/hub_service/hub_module_config') + +const FIRST_TRACKER = 'xchain-node-bitcoin-testnet-xchain-utxo-tracker' +const SECOND_TRACKER = 'xchain-node-bitcoin-mainnet-xchain-utxo-tracker' +const hostPortClaimPath = require.resolve('../../src/services/module_service/host_port_claim') +const bootstrapGuardPath = require.resolve('../../src/services/bootstrap_node_tip_guard') +const dockerArgsPath = require.resolve('../../src/services/module_service/docker_args') + +let baseClaimFreeHostPorts + +function loadDockerArgs(published) { + return proxyquire('../../src/services/module_service/docker_args', { + 'fs': {}, + 'path': {}, + '../../config': constants, + '../config_service': { getUtxoTrackerVolumeName: () => 'tracker-volume' }, + '../docker_service': { getPublishedHostPorts: async () => published }, + './healthcheck_command': { healthcheckCommand: () => '' }, + '../validator_service': {}, + '../observability/logger': { getLogger: () => ({ info: () => {} }) } + }) +} + +function installProductionWrapper(memory, published) { + const hostPortClaim = require(hostPortClaimPath) + baseClaimFreeHostPorts = hostPortClaim.claimFreeHostPorts.baseClaimFreeHostPorts || + hostPortClaim.claimFreeHostPorts + hostPortClaim.claimFreeHostPorts = baseClaimFreeHostPorts + delete require.cache[bootstrapGuardPath] + delete require.cache[dockerArgsPath] + + proxyquire('../../src/services/bootstrap_node_tip_guard', { + 'fs': { readFileSync: file => memory.files[file] || '' }, + './config_service': memory.cs + }) + + expect(hostPortClaim.claimFreeHostPorts).not.to.equal(baseClaimFreeHostPorts) + expect(hostPortClaim.claimFreeHostPorts.baseClaimFreeHostPorts).to.equal(baseClaimFreeHostPorts) + return loadDockerArgs(published) +} + +async function assertExplicitPortRefused(initialFiles, explicitPort) { + const memory = makeMemoryConfigService(initialFiles) + const published = new Map([[explicitPort, new Set([FIRST_TRACKER])]]) + const dockerArgs = installProductionWrapper(memory, published) + const installConfig = await memory.cs.getDefaultConfig( + XChainService.XCHAIN_UTXO_TRACKER, 'bitcoin', 'mainnet') + const portArgs = ['-p', `${installConfig.UTXO_TRACKER_PORT}:${installConfig.UTXO_TRACKER_API_PORT}`] + + let thrown = null + try { + await dockerArgs.assertNoHostPortConflicts(portArgs, SECOND_TRACKER, { + module: XChainService.XCHAIN_UTXO_TRACKER, + environmentVariables: installConfig + }) + } catch (err) { + thrown = err + } + + expect(thrown).to.be.an.instanceOf(Error) + expect(thrown.message).to.include(`host port ${explicitPort} is already published by`) + expect(portArgs).to.deep.equal(['-p', `${explicitPort}:3001`]) + return memory +} + +describe('moved install host ports', function () { + afterEach(function () { + const hostPortClaim = require(hostPortClaimPath) + if (baseClaimFreeHostPorts) hostPortClaim.claimFreeHostPorts = baseClaimFreeHostPorts + delete require.cache[bootstrapGuardPath] + delete require.cache[dockerArgsPath] + }) + + it('installs the production wrapper, persists a moved port, and reports it to the hub', async function () { + const memory = makeMemoryConfigService({ [coinMain]: '' }) + const published = new Map([['3001', new Set([FIRST_TRACKER])]]) + const dockerArgs = installProductionWrapper(memory, published) + const installConfig = await memory.cs.getDefaultConfig( + XChainService.XCHAIN_UTXO_TRACKER, 'bitcoin', 'mainnet') + const portArgs = ['-p', `${installConfig.UTXO_TRACKER_PORT}:${installConfig.UTXO_TRACKER_API_PORT}`] + + await dockerArgs.assertNoHostPortConflicts(portArgs, SECOND_TRACKER, { + module: XChainService.XCHAIN_UTXO_TRACKER, + environmentVariables: installConfig + }) + + expect(portArgs).to.deep.equal(['-p', '3101:3001']) + expect(memory.files[coinSidecar]).to.match(/(?:^|\n)UTXO_TRACKER_PORT=3101(?:\n|$)/) + + const reloaded = await memory.cs.getDefaultConfig('', 'bitcoin', 'mainnet') + const reported = buildHubModuleConfig( + XChainService.XCHAIN_UTXO_TRACKER, reloaded, {}) + expect(reported).to.include({ port: 3001, server_port: '3101' }) + }) + + it('refuses a clashing port explicitly configured in the main file', async function () { + await assertExplicitPortRefused({ + [coinMain]: 'UTXO_TRACKER_PORT=4101\n' + }, '4101') + }) + + it('refuses a clashing port explicitly configured in the local sidecar', async function () { + const memory = await assertExplicitPortRefused({ + [coinMain]: '', + [coinSidecar]: 'UTXO_TRACKER_PORT=4101\n' + }, '4101') + expect(memory.files[coinSidecar]).to.match(/(?:^|\n)UTXO_TRACKER_PORT=4101(?:\n|$)/) + }) +}) diff --git a/test/unit/cli/clear_reorg_halt_unknown_chain.test.js b/test/unit/cli/clear_reorg_halt_unknown_chain.test.js new file mode 100644 index 00000000..29c4d01f --- /dev/null +++ b/test/unit/cli/clear_reorg_halt_unknown_chain.test.js @@ -0,0 +1,80 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// clear-reorg-halt must refuse a chain or network it does not know with a +// reason that names the bad value as unknown, not with the misleading +// "no container is installed" text a typo would otherwise fall through to. + +const sinon = require('sinon') +const { expect } = require('chai') +const { Command } = require('commander') + +const { registerToolCommands } = require('../../../src/cli/commands') + +class ExitCalled extends Error { + constructor(code) { super('exit ' + code); this.code = code } +} + +async function run(argv, clearResult = true) { + const deps = { + filterCommandParameters: sinon.stub().returns([]), + clearDecoderReorgHalt: sinon.stub().resolves(clearResult), + execModules: sinon.stub(), + shellModule: sinon.stub() + } + const program = new Command() + program.exitOverride() + registerToolCommands(program, deps) + const logs = [] + sinon.stub(console, 'log').callsFake((...a) => { logs.push(a.join(' ')) }) + sinon.stub(process, 'exit').callsFake((code) => { throw new ExitCalled(code) }) + let exitCode = null + try { + await program.parseAsync(argv, { from: 'user' }) + } catch (e) { + if (!(e instanceof ExitCalled)) throw e + exitCode = e.code + } finally { + sinon.restore() + } + return { deps, logs: logs.join('\n'), exitCode } +} + +describe('CLI clear-reorg-halt: unknown chain or network', function () { + afterEach(() => sinon.restore()) + + it('names an unknown chain as unknown and touches nothing', async function () { + const { deps, logs, exitCode } = await run(['clear-reorg-halt', 'bitcion', 'regtest', '--dry-run']) + expect(logs).to.match(/unknown (coin|chain)/i) + expect(logs).to.include('bitcion') + expect(logs).to.not.include('no xchain-decoder container is installed') + expect(exitCode).to.equal(1) + expect(deps.filterCommandParameters.called).to.be.false + expect(deps.clearDecoderReorgHalt.called).to.be.false + }) + + it('names an unknown network as unknown and touches nothing', async function () { + const { deps, logs, exitCode } = await run(['clear-reorg-halt', 'bitcoin', 'regtset', '--dry-run']) + expect(logs).to.match(/unknown network/i) + expect(logs).to.include('regtset') + expect(exitCode).to.equal(1) + expect(deps.filterCommandParameters.called).to.be.false + expect(deps.clearDecoderReorgHalt.called).to.be.false + }) + + it('still reaches the clear for a valid chain and network', async function () { + const { deps, exitCode } = await run(['clear-reorg-halt', 'bitcoin', 'regtest', '--dry-run']) + expect(deps.filterCommandParameters.calledOnce).to.be.true + expect(deps.clearDecoderReorgHalt.calledOnce).to.be.true + expect(exitCode).to.equal(0) + }) +}) diff --git a/test/unit/cli/exec_exit_code.test.js b/test/unit/cli/exec_exit_code.test.js new file mode 100644 index 00000000..e26fcb93 --- /dev/null +++ b/test/unit/cli/exec_exit_code.test.js @@ -0,0 +1,98 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// `exec` must exit non-zero with a reason when the in-container command fails, +// or when the selection matched no installed container. It used to print the +// raw Error object and exit 0, so `exec ... && next-step` carried on after a +// command that failed or never ran. + +const { Command } = require('commander') +const { sinon, expect, makeStubs, loadOperations } = require('../module_operations.test/helpers/harness') +const { registerToolCommands } = require('../../../src/cli/commands') + +class ExitCalled extends Error { + constructor(code) { super('exit ' + code); this.code = code } +} + +// Builds `exec` over the real module controls, with only the registry and +// docker stubbed, so the exit status is the CLI's own. +function buildProgram(configure) { + const stubs = makeStubs() + configure(stubs) + const ops = loadOperations(stubs) + const program = new Command() + program.exitOverride() + registerToolCommands(program, { + filterCommandParameters: (_m, service, chain, network) => ({ [chain]: { [network]: [service] } }), + execModules: ops.execModules + }) + return { stubs, program } +} + +async function runExec(args, configure = () => {}) { + const { stubs, program } = buildProgram(configure) + const errors = [] + sinon.stub(console, 'error').callsFake((...a) => { errors.push(a.join(' ')) }) + sinon.stub(console, 'log') + sinon.stub(process, 'exit').callsFake((code) => { throw new ExitCalled(code) }) + let exitCode = null + try { + await program.parseAsync(['exec', ...args], { from: 'user' }) + } catch (e) { + if (!(e instanceof ExitCalled)) throw e + exitCode = e.code + } finally { + sinon.restore() + } + return { stubs, stderr: errors.join('\n'), exitCode } +} + +function commandFailure(code, stderr) { + const err = new Error('Command failed: docker exec -i container-id-123 ' + code) + err.code = code + err.stderr = stderr + err.stdout = '' + return err +} + +describe('CLI `exec` exit code', function () { + + it('exits 1 with the exit code as the reason when the in-container command fails (`false`)', async function () { + const { stderr, exitCode } = await runExec(['xchain-decoder', 'bitcoin', 'regtest', 'false'], + stubs => stubs.execContainer.rejects(commandFailure(1, ''))) + expect(exitCode).to.equal(1) + expect(stderr).to.match(/exec failed in xchain-decoder \(bitcoin regtest\): exit code 1/) + }) + + it('exits 1 and passes the command stderr through when it fails (`ls /no-such-dir`)', async function () { + const { stderr, exitCode } = await runExec(['xchain-decoder', 'bitcoin', 'regtest', 'ls /no-such-dir'], + stubs => stubs.execContainer.rejects(commandFailure(2, "ls: cannot access '/no-such-dir': No such file or directory"))) + expect(exitCode).to.equal(1) + expect(stderr).to.match(/exit code 2/) + expect(stderr).to.contain("ls: cannot access '/no-such-dir'") + }) + + it('exits 1 naming the selection when no installed container matched (typo service)', async function () { + const { stubs, stderr, exitCode } = await runExec(['xchain-decodr', 'bitcoin', 'regtest', 'ls'], + stubs => stubs.db.getModuleContainer.resolves(null)) + expect(exitCode).to.equal(1) + expect(stubs.execContainer.called).to.be.false + expect(stderr).to.match(/exec matched no installed service: xchain-decodr \(bitcoin regtest\)/) + }) + + it('exits 0 when the command ran and succeeded', async function () { + const { stubs, exitCode } = await runExec(['xchain-decoder', 'bitcoin', 'regtest', 'ls /app'], + stubs => stubs.execContainer.resolves('bin\nsrc')) + expect(exitCode).to.equal(0) + expect(stubs.execContainer.calledWith('container-id-123', ['ls', '/app'])).to.be.true + }) +}) diff --git a/test/unit/cli/lifecycle_zero_match_exit.test.js b/test/unit/cli/lifecycle_zero_match_exit.test.js new file mode 100644 index 00000000..8d88d8f3 --- /dev/null +++ b/test/unit/cli/lifecycle_zero_match_exit.test.js @@ -0,0 +1,140 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// start, stop and restart must exit non-zero with a reason when the selection +// matches no installed service. A typo'd service name, or a real service that +// is not installed on the named chain and network, used to act on nothing, +// print nothing and exit 0, so a script believed a service was stopped while +// it kept running. An unknown chain or network must give its reason the same way. + +const { Command } = require('commander') +const { sinon, expect, makeStubs, loadOperations } = require('../module_operations.test/helpers/harness') +const { filterCommandParameters } = require('../../../src/services/config_service') +const { registerLifecycleCommands } = require('../../../src/cli/options') + +const VERBS = [ + ['start', 'startModules', 'startContainer'], + ['stop', 'stopModules', 'stopContainer'], + ['restart', 'restartModules', 'stopContainer'] +] + +class ExitCalled extends Error { + constructor(code) { super('exit ' + code); this.code = code } +} + +// The registry of a regtest rail: real service names, installed on regtest only. +const RAIL_MODULES = ['node', 'xchain-encoder', 'xchain-decoder', 'xchain-indexer', 'xchain-utxo-tracker'] +const installedOnRegtestOnly = (module, coin, network) => + network === 'regtest' && RAIL_MODULES.includes(module) + +// Builds the commands over the real filter and the real module controls, with +// only the registry and docker stubbed, so the exit status is the CLI's own. +function buildProgram(installed) { + const stubs = makeStubs() + stubs.db.getModuleContainerStrict.callsFake(async (module, coin, network) => + installed(module, coin, network) ? `id-${module}-${coin}-${network}` : null) + const ops = loadOperations(stubs) + const program = new Command() + program.exitOverride() + registerLifecycleCommands(program, { + filterCommandParameters, + startModules: ops.startModules, + stopModules: ops.stopModules, + restartModules: ops.restartModules, + redactSecrets: (s) => String(s), + scanAndRegisterModules: sinon.stub().resolves(0), + loadModule: sinon.stub() + }) + return { stubs, program } +} + +async function run(argv, installed = () => true) { + const { stubs, program } = buildProgram(installed) + const errors = [] + sinon.stub(console, 'error').callsFake((...a) => { errors.push(a.join(' ')) }) + sinon.stub(console, 'log') + sinon.stub(process, 'exit').callsFake((code) => { throw new ExitCalled(code) }) + let exitCode = null + try { + await program.parseAsync(argv, { from: 'user' }) + } catch (e) { + if (!(e instanceof ExitCalled)) throw e + exitCode = e.code + } finally { + sinon.restore() + } + return { stubs, stderr: errors.join('\n'), exitCode } +} + +function moduleControlCases() { + for (const [verb, op, dockerCall] of VERBS) { + it(`${op} rejects with the unmatched module, chain and network and touches no container`, async function () { + const stubs = makeStubs() + stubs.db.getModuleContainerStrict.resolves(null) + const ops = loadOperations(stubs) + const err = await ops[op]({ bitcoin: { regtest: ['xchain-decodr'] } }).then(() => null, e => e) + expect(err, `${verb} of a service that is not installed must not resolve`).to.be.an('error') + expect(err.message).to.include('xchain-decodr') + expect(err.message).to.include('bitcoin') + expect(err.message).to.include('regtest') + expect(err.message).to.match(/no installed service/) + expect(stubs[dockerCall].called).to.be.false + expect(stubs.startContainer.called).to.be.false + }) + + it(`${op} still succeeds for an "all" selection with one module not installed`, async function () { + const stubs = makeStubs() + stubs.db.getModuleContainerStrict.callsFake(async (module) => + module === 'xchain-regtest-miner' ? null : `id-${module}`) + const ops = loadOperations(stubs) + const result = await ops[op]({ bitcoin: { regtest: ['node', 'xchain-decoder', 'xchain-regtest-miner'] } }) + expect(result).to.equal(true) + expect(stubs[dockerCall].called).to.be.true + }) + } +} + +const TYPO_CASES = [ + [['stop', 'xchain-decodr', 'bitcoin', 'regtest'], 'xchain-decodr'], + [['stop', 'xchain-decoder', 'bitcion', 'regtest'], 'bitcion'], + [['restart', 'xchain-decoder', 'bitcoin', 'regtset'], 'regtset'], + [['start', 'xchain-encoder', 'litecoin', 'mainnet'], 'xchain-encoder'], + [['stop', 'xchain-decoder;id', 'bitcoin', 'regtest'], 'xchain-decoder;id'], + [['stop', 'xchain-decoder', 'bitcoin$(id)', 'regtest'], 'bitcoin$(id)'] +] + +function cliExitCases() { + for (const [argv, named] of TYPO_CASES) { + it(`\`${argv.join(' ')}\` exits 1 and names ${named} on stderr`, async function () { + const { stubs, stderr, exitCode } = await run(argv, installedOnRegtestOnly) + expect(exitCode).to.equal(1) + expect(stderr).to.include(`${argv[0]} failed`) + expect(stderr).to.include(named) + expect(stubs.stopContainer.called).to.be.false + expect(stubs.startContainer.called).to.be.false + }) + } + + for (const verb of ['start', 'stop', 'restart']) { + it(`\`${verb}\` of an installed service still exits 0`, async function () { + const { stderr, exitCode } = await run([verb, 'xchain-decoder', 'bitcoin', 'regtest'], installedOnRegtestOnly) + expect(exitCode).to.equal(0) + expect(stderr).to.equal('') + }) + } +} + +describe('lifecycle commands: a selection that matches no installed service', function () { + afterEach(() => sinon.restore()) + describe('module controls', moduleControlCases) + describe('CLI exit status', cliExitCases) +}) diff --git a/test/unit/cli/reset_db_container.test.js b/test/unit/cli/reset_db_container.test.js new file mode 100644 index 00000000..98a5bc5c --- /dev/null +++ b/test/unit/cli/reset_db_container.test.js @@ -0,0 +1,105 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// The shipped `reset` command and dispatch hook, driven together: `--db-container` must reach resetModules and +// turn preCheck local, and a reset without it must call both exactly as before. + +const { expect } = require('chai') +const sinon = require('sinon') +const { Command } = require('commander') +const { installDispatch } = require('../../../src/cli/dispatch') +const { registerPreRecoveryCommands } = require('../../../src/cli/errors') + +const SIGNALS = ['exit', 'SIGINT', 'SIGTERM'] + +function makeDeps() { + return { + config: {}, + setVerbose: sinon.stub(), + maybeSelfUpdateBeforeUpdate: sinon.stub().resolves(), + redactSecrets: (s) => s, + acquireCommandLock: sinon.stub().returns(sinon.stub()), + preCheck: sinon.stub().resolves(), + refForPreCheck: sinon.stub().returns(null), + commandRepairsHub: sinon.stub().returns(false), + maybeReportTelemetry: sinon.stub().resolves(), + noticeNewerRelease: sinon.stub().resolves(), + resetModules: sinon.stub().resolves(true), + hubCheckpointDependencies: {} + } +} + +async function drive(argv) { + const deps = makeDeps() + const program = new Command() + program.exitOverride() + program.option('--verbose').option('--no-telemetry') + installDispatch(program, deps) + registerPreRecoveryCommands(program, deps) + const exit = sinon.stub(process, 'exit') + await program.parseAsync(['node', 'xchain-node', ...argv]) + return { deps, exit } +} + +describe('CLI reset --db-container', function () { + let listenersBefore + beforeEach(function () { + listenersBefore = {} + for (const ev of SIGNALS) listenersBefore[ev] = process.listeners(ev) + }) + afterEach(function () { + for (const ev of SIGNALS) { + for (const fn of process.listeners(ev)) if (!listenersBefore[ev].includes(fn)) process.removeListener(ev, fn) + } + sinon.restore() + }) + + it('hands the container to resetModules and runs a local-only preCheck', async function () { + const { deps, exit } = await drive(['reset', 'xchain-decoder', 'bitcoin', 'regtest', '--with-indexer', '--yes', + '--db-container', 'xchain-rail-database']) + sinon.assert.calledOnceWithExactly(deps.resetModules, 'xchain-decoder', 'bitcoin', 'regtest', true, true, + { dbContainer: 'xchain-rail-database', check: false }) + expect(deps.preCheck.firstCall.args).to.deep.equal([false, true, null, false, { localOnly: true }]) + sinon.assert.calledOnceWithExactly(exit, 0) + }) + + it('calls preCheck and resetModules as before when the flag is absent', async function () { + const { deps } = await drive(['reset', 'xchain-decoder', 'bitcoin', 'regtest', '--with-indexer', '--yes']) + sinon.assert.calledOnceWithExactly(deps.resetModules, 'xchain-decoder', 'bitcoin', 'regtest', true, true) + expect(deps.preCheck.firstCall.args).to.deep.equal([false, true, null, false]) + }) + + it('refuses the flag on hub-checkpoints without running that reset', async function () { + const write = sinon.stub(process.stderr, 'write') + const { deps, exit } = await drive(['reset', 'hub-checkpoints', '--yes', '--db-container', 'xchain-rail-database']) + write.restore() + sinon.assert.calledWith(exit, 1) + expect(write.args.map(a => String(a[0])).join('')).to.contain('--db-container does not apply to hub-checkpoints') + expect(deps.resetModules.called).to.be.false + }) + + it('passes --check through with the container', async function () { + const { deps } = await drive(['reset', 'xchain-decoder', 'bitcoin', 'regtest', '--with-indexer', + '--db-container', 'xchain-rail-database', '--check']) + sinon.assert.calledOnceWithExactly(deps.resetModules, 'xchain-decoder', 'bitcoin', 'regtest', false, true, + { dbContainer: 'xchain-rail-database', check: true }) + }) + + it('refuses --check without --db-container', async function () { + const write = sinon.stub(process.stderr, 'write') + const { deps, exit } = await drive(['reset', 'xchain-decoder', 'bitcoin', 'regtest', '--check']) + write.restore() + sinon.assert.calledWith(exit, 1) + expect(write.args.map(a => String(a[0])).join('')).to.contain('--check needs --db-container') + expect(deps.resetModules.called).to.be.false + }) +}) diff --git a/test/unit/cli/reset_hub_checkpoints.test.js b/test/unit/cli/reset_hub_checkpoints.test.js new file mode 100644 index 00000000..a269f289 --- /dev/null +++ b/test/unit/cli/reset_hub_checkpoints.test.js @@ -0,0 +1,148 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') +const sinon = require('sinon') +const { Command } = require('commander') +const { registerPreRecoveryCommands } = require('../../../src/cli/errors') + +const HUB = 'xchain-hub' +const INDEXER = 'xchain-indexer' + +function row(name, state = 'running') { + return { ID: name + '-id', Names: name, State: state } +} + +function railContainers() { + return [ + row('rail-a-xchain-hub'), row('rail-a-database'), + row('rail-a-bitcoin-regtest-xchain-indexer'), row('rail-a-xchain-explorer'), + row('rail-b-xchain-hub'), row('rail-b-database'), + row('rail-b-dogecoin-regtest-xchain-indexer'), row('rail-b-xchain-explorer') + ] +} + +function railEnvironments() { + return { + 'rail-a-xchain-hub-id': ['HUB_DB_NAME=RailA_Hub'], + 'rail-b-xchain-hub-id': ['HUB_DB_NAME=RailB_Hub'], + 'rail-a-bitcoin-regtest-xchain-indexer-id': ['INDEXER_DB_NAME=RailA_Bitcoin_Regtest_Indexer'], + 'rail-b-dogecoin-regtest-xchain-indexer-id': ['INDEXER_DB_NAME=RailB_Dogecoin_Regtest_Indexer'] + } +} + +function makeHarness({ failHub = null } = {}) { + const containers = railContainers() + const events = [] + const databaseCalls = [] + const environments = railEnvironments() + const execFileAsync = sinon.stub().callsFake(async (command, args) => { + expect(command).to.equal('docker') + if (args[0] === 'ps') return { stdout: containers.map(value => JSON.stringify(value)).join('\n') } + if (args[0] === 'inspect') return { stdout: JSON.stringify(environments[args[5]]) } + if (args[0] === 'exec') { + const [script, database, table] = [args[4], args[6], args[7]] + databaseCalls.push({ script, database, table }) + if (script.includes('mariadb-show')) return { stdout: `| ${table} |\n` } + if (database === 'RailA_Hub' || database === 'RailB_Hub') events.push('recreate:' + args[1]) + else events.push('purge:' + args[1]) + return { stdout: '' } + } + throw new Error('unexpected docker call: ' + args.join(' ')) + }) + const stopContainer = sinon.stub().callsFake(async (id) => { + events.push('stop:' + id) + if (id === failHub) throw new Error('hub refused SIGTERM') + return true + }) + const startContainer = sinon.stub().callsFake(async (id) => { + events.push('start:' + id) + return true + }) + const statusChanged = sinon.stub().resolves() + const dependencies = { + HUB_MODULE_NAME: HUB, + INDEXER_MODULE_NAME: INDEXER, + EXTERNAL_DB: false, + execFileAsync, + stopContainer, + startContainer, + stopModuleContainer: (...args) => args[0](args[4]), + getContainerStopSettings: sinon.stub().resolves(null), + statusChanged + } + return { events, databaseCalls, dependencies, statusChanged } +} + +async function driveReset(harness) { + const program = new Command() + program.exitOverride() + registerPreRecoveryCommands(program, { + resetModules: sinon.stub().rejects(new Error('ordinary reset path called')), + hubCheckpointDependencies: harness.dependencies + }) + const exit = sinon.stub(process, 'exit') + try { + await program.parseAsync(['node', 'xchain-node', 'reset', 'hub-checkpoints', '--yes']) + return exit + } catch (err) { + exit.restore() + throw err + } +} + +describe('CLI reset hub-checkpoints', function () { + afterEach(function () { + if (process.exit.restore) process.exit.restore() + }) + + it('stops every rail, recreates only checkpoint tables, purges every mirror, and restarts', async function () { + const harness = makeHarness() + const exit = await driveReset(harness) + + sinon.assert.calledOnceWithExactly(exit, 0) + const lastStop = Math.max(...harness.events.map((event, index) => event.startsWith('stop:') ? index : -1)) + const firstRecreate = harness.events.findIndex(event => event.startsWith('recreate:')) + const lastRecreate = Math.max(...harness.events.map((event, index) => event.startsWith('recreate:') ? index : -1)) + const firstPurge = harness.events.findIndex(event => event.startsWith('purge:')) + const lastPurge = Math.max(...harness.events.map((event, index) => event.startsWith('purge:') ? index : -1)) + const firstStart = harness.events.findIndex(event => event.startsWith('start:')) + expect(lastStop).to.be.lessThan(firstRecreate) + expect(lastRecreate).to.be.lessThan(firstPurge) + expect(lastPurge).to.be.lessThan(firstStart) + + expect(harness.events.filter(event => event.startsWith('recreate:'))).to.have.length(6) + expect(harness.events.filter(event => event.startsWith('purge:'))).to.have.length(4) + expect(harness.events.filter(event => event.startsWith('start:'))).to.have.length(6) + expect(harness.statusChanged.calledOnce).to.equal(true) + + const mutations = harness.databaseCalls.filter(call => call.script.includes('mariadb-dump')) + for (const table of ['state_checkpoints', 'snapshot_checkpoints', 'anchor_published_checkpoints']) { + expect(mutations.map(call => call.table)).to.contain(table) + } + expect(harness.databaseCalls.map(call => call.table).join('\n')).to.not.match(/price|validator|config|consensus_state/i) + }) + + it('refuses before any database mutation or mirror stop when one hub will not stop', async function () { + const harness = makeHarness({ failHub: 'rail-b-xchain-hub-id' }) + const exit = await driveReset(harness) + + sinon.assert.calledOnceWithExactly(exit, 1) + expect(harness.databaseCalls.some(call => call.script.includes('mariadb-dump'))).to.equal(false) + expect(harness.events).to.deep.equal([ + 'stop:rail-a-xchain-hub-id', + 'stop:rail-b-xchain-hub-id', + 'start:rail-a-xchain-hub-id' + ]) + expect(harness.statusChanged.called).to.equal(false) + }) +}) diff --git a/test/unit/cli/scoped_explorer_policy.test.js b/test/unit/cli/scoped_explorer_policy.test.js new file mode 100644 index 00000000..24cdcde3 --- /dev/null +++ b/test/unit/cli/scoped_explorer_policy.test.js @@ -0,0 +1,105 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { Command } = require('commander') +const sinon = require('sinon') +const { expect } = require('chai') +const { filterCommandParameters } = require('../../../src/services/config_service') +const { registerPrimaryCommands } = require('../../../src/cli/commands') +const { registerLifecycleCommands } = require('../../../src/cli/options') + +function hasExplorer(services) { + return services[''] && services[''][''] && services[''][''].includes('xchain-explorer') +} + +async function runPrimaryCommand(verb, chain = 'bitcoin') { + const captured = {} + const program = new Command() + registerPrimaryCommands(program, { + filterCommandParameters, + resolveArgs: () => ({ service: 'all', chain, network: 'regtest', branch: null }), + installModules: async services => { captured.install = services; return [] }, + syncSharedServicesAfterInstall: async () => true, + uninstallModules: async services => { captured.uninstall = services; return {} }, + updateModules: async services => { + captured.update = services + return { updated: [{ module: 'node' }], skipped: [], failed: [] } + }, + recreateModules: async services => { + captured.recreate = services + return { recreated: [{ module: 'node' }], skipped: [] } + }, + getStatus: async () => true, + redactSecrets: String, + config: {} + }) + sinon.stub(process, 'exit') + try { + await program.parseAsync([verb, 'all', chain, 'regtest'], { from: 'user' }) + } finally { + sinon.restore() + } + return captured[verb] +} + +async function runLifecycleCommand(verb, args = ['all', 'bitcoin', 'regtest']) { + let captured + const capture = async services => { captured = services } + const program = new Command() + registerLifecycleCommands(program, { + filterCommandParameters, + startModules: capture, + stopModules: capture, + restartModules: capture, + redactSecrets: String, + scanAndRegisterModules: async () => 0, + loadModule: () => ({}) + }) + sinon.stub(process, 'exit') + try { + await program.parseAsync([verb, ...args], { from: 'user' }) + } finally { + sinon.restore() + } + return captured +} + +describe('CLI chain-scoped all explorer policy', function () { + afterEach(() => sinon.restore()) + + for (const verb of ['install', 'update', 'recreate']) { + it(`${verb} all bitcoin regtest still lists xchain-explorer`, async function () { + expect(hasExplorer(await runPrimaryCommand(verb))).to.equal(true) + }) + } + + for (const verb of ['start', 'stop', 'restart']) { + it(`${verb} all bitcoin regtest leaves xchain-explorer out`, async function () { + expect(hasExplorer(await runLifecycleCommand(verb))).to.not.equal(true) + }) + } + + it('uninstall all bitcoin regtest leaves xchain-explorer out', async function () { + expect(hasExplorer(await runPrimaryCommand('uninstall'))).to.not.equal(true) + }) + + for (const verb of ['start', 'stop', 'restart']) { + it(`${verb} all with no chain, or all chains on one network, still lists xchain-explorer`, async function () { + expect(hasExplorer(await runLifecycleCommand(verb, ['all']))).to.equal(true) + expect(hasExplorer(await runLifecycleCommand(verb, ['all', 'all', 'regtest']))).to.equal(true) + }) + } + + it('uninstall all all regtest still lists xchain-explorer', async function () { + expect(hasExplorer(await runPrimaryCommand('uninstall', 'all'))).to.equal(true) + }) +}) diff --git a/test/unit/cli_error_handling.test.js b/test/unit/cli_error_handling.test.js index 832ed91b..ecbc4346 100644 --- a/test/unit/cli_error_handling.test.js +++ b/test/unit/cli_error_handling.test.js @@ -44,7 +44,7 @@ function loadCli(opsOverrides = {}) { restartModules: sinon.stub().resolves(), stopModules: sinon.stub().resolves(), startModules: sinon.stub().resolves(), - execModules: sinon.stub().resolves(), + execModules: sinon.stub().resolves(true), shellModule: sinon.stub().resolves(), runE2ETest: sinon.stub().resolves({ logFile: '', exitCode: 0 }), resetModules: sinon.stub().resolves(), diff --git a/test/unit/coins_conformance.test.js b/test/unit/coins_conformance.test.js index 05ffa3ec..cb2dfd4e 100644 --- a/test/unit/coins_conformance.test.js +++ b/test/unit/coins_conformance.test.js @@ -51,8 +51,9 @@ describe('coin-registry conformance (vendored copy) @regression', function(){ const pin = CONSENSUS_CONFIG_PIN[net]; if(pin === null || pin === undefined) continue; // pre-arm (mainnet) for(const tick of coins.ALLOWED_COINS){ - if(!pin[tick]) continue; // freshly added chain, not yet pinned it(`${tick}/${net} vendored pin matches the vendored consensusHash`, function(){ + // Require every coin pinned on an armed network, as the canonical hub pin test does. + assert.ok(pin[tick], `${tick}/${net} has no pin on an armed network; every ALLOWED_COINS coin must be pinned`); assert.strictEqual(coins.consensusHash(tick, net), String(pin[tick]).toLowerCase()); }); } diff --git a/test/unit/command_lock.test.js b/test/unit/command_lock.test.js index 831a99f2..bc4ecd28 100644 --- a/test/unit/command_lock.test.js +++ b/test/unit/command_lock.test.js @@ -134,6 +134,57 @@ describe('commandLock unreadable files', () => { }) }) +describe('commandLock publishes atomically', () => { + beforeEach(prepareCommandLock) + afterEach(cleanCommandLock) + + // A creator between open and write leaves an empty lock that a contender + // reclaims as stale. The lock path must therefore only ever appear complete. + it('never creates the lock path with a separate open-then-write', () => { + const lockFile = getLockFilePath() + const realWrite = fs.writeFileSync + const realLink = fs.linkSync + const directWrites = [] + let contentAtAppearance = null + fs.writeFileSync = function (file, ...args) { + if (file === lockFile) directWrites.push(file) + return realWrite.call(fs, file, ...args) + } + fs.linkSync = function (from, to) { + const r = realLink.call(fs, from, to) + if (to === lockFile) contentAtAppearance = fs.readFileSync(lockFile, 'utf8') + return r + } + let release + try { + release = acquireCommandLock({ command: 'install' }) + } finally { + fs.writeFileSync = realWrite + fs.linkSync = realLink + } + assert.deepStrictEqual(directWrites, [], 'lock path must not be created by a two-step open+write') + assert.ok(contentAtAppearance && JSON.parse(contentAtAppearance).pid === process.pid, + 'lock must be complete the moment it appears') + assert.deepStrictEqual(fs.readdirSync(tmpDir), ['command.lock'], 'no temp file left behind') + release() + }) + + it('a contender never sees an empty lock from a live creator mid-acquire', () => { + const lockFile = getLockFilePath() + const realLink = fs.linkSync + const seen = [] + fs.linkSync = function (from, to) { + // Just before the lock appears, nothing is at the path to be reclaimed. + seen.push(fs.existsSync(lockFile)) + return realLink.call(fs, from, to) + } + let release + try { release = acquireCommandLock({ command: 'install' }) } finally { fs.linkSync = realLink } + assert.deepStrictEqual(seen, [false]) + release() + }) +}) + describe('commandLock handoff', () => { beforeEach(prepareCommandLock) afterEach(cleanCommandLock) diff --git a/test/unit/command_lock_reclaim_marker.test.js b/test/unit/command_lock_reclaim_marker.test.js new file mode 100644 index 00000000..ec1670fb --- /dev/null +++ b/test/unit/command_lock_reclaim_marker.test.js @@ -0,0 +1,106 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const assert = require('assert') +const fs = require('fs') +const os = require('os') +const path = require('path') + +const { acquireCommandLock, getLockFilePath, isPidAlive } = require('../../src/utils/command_lock') + +let tmpDir + +function findDeadPid() { + let deadPid = 999999 + while (isPidAlive(deadPid)) deadPid-- + return deadPid +} + +function registerLockDirectoryHooks() { + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-lock-')) + process.env.XCHAIN_NODE_LOCK_DIR = tmpDir + }) + + afterEach(() => { + delete process.env.XCHAIN_NODE_LOCK_DIR + fs.rmSync(tmpDir, { recursive: true, force: true }) + }) +} + +describe('commandLock publishes the reclaim marker atomically', () => { + registerLockDirectoryHooks() + + // An empty marker reads as abandoned, so a contender could break a live reclaimer's marker. + it('never creates the reclaim marker with a separate open-then-write', () => { + const lockFile = getLockFilePath() + const reclaimFile = lockFile + '.reclaim' + fs.writeFileSync(lockFile, JSON.stringify({ pid: findDeadPid(), command: 'install' }) + '\n') + const realWrite = fs.writeFileSync + const realLink = fs.linkSync + const directWrites = [] + let markerAtAppearance = null + fs.writeFileSync = function (file, ...args) { + if (file === reclaimFile) directWrites.push(file) + return realWrite.call(fs, file, ...args) + } + fs.linkSync = function (from, to) { + const r = realLink.call(fs, from, to) + if (to === reclaimFile) markerAtAppearance = fs.readFileSync(reclaimFile, 'utf8') + return r + } + let release + try { + release = acquireCommandLock({ command: 'update' }) + } finally { + fs.writeFileSync = realWrite + fs.linkSync = realLink + } + assert.deepStrictEqual(directWrites, [], 'reclaim marker must not be created by a two-step open+write') + assert.ok(markerAtAppearance && JSON.parse(markerAtAppearance).pid === process.pid, + 'reclaim marker must be complete the moment it appears') + assert.strictEqual(JSON.parse(fs.readFileSync(lockFile, 'utf8')).pid, process.pid) + assert.deepStrictEqual(fs.readdirSync(tmpDir), ['command.lock'], 'no marker or temp file left behind') + release() + }) +}) + +describe('commandLock publishes the reclaim marker atomically', () => { + registerLockDirectoryHooks() + + it('does not remove a reclaim marker it no longer owns', () => { + const lockFile = getLockFilePath() + const reclaimFile = lockFile + '.reclaim' + const deadPid = findDeadPid() + fs.writeFileSync(lockFile, JSON.stringify({ pid: deadPid, command: 'install' }) + '\n') + const realKill = process.kill + let replaced = false + process.kill = function (pid, signal) { + if (!replaced && pid === deadPid && signal === 0 && fs.existsSync(reclaimFile)) { + replaced = true + fs.writeFileSync(reclaimFile, + JSON.stringify({ pid: process.pid, nonce: 'the-successor' }) + '\n') + } + return realKill.call(process, pid, signal) + } + let release + try { + release = acquireCommandLock({ command: 'update' }) + } finally { + process.kill = realKill + } + assert.ok(replaced, 'the marker ownership change must occur inside reclamation') + assert.strictEqual(JSON.parse(fs.readFileSync(reclaimFile, 'utf8')).nonce, 'the-successor') + release() + fs.unlinkSync(reclaimFile) + }) +}) diff --git a/test/unit/config_service.test/helpers.test.js b/test/unit/config_service.test/helpers.test.js index b0f2046e..51f1d7ca 100644 --- a/test/unit/config_service.test/helpers.test.js +++ b/test/unit/config_service.test/helpers.test.js @@ -69,6 +69,7 @@ function makeServiceWithConfig(configContent) { readFileSync: sinon.stub().returns(''), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } @@ -91,6 +92,7 @@ function makeMemoryConfigService(initialFiles = {}, { dbContainerId = null, exte readFileSync: (p) => files[p] != null ? String(files[p]) : '', writeFileSync: (p, body) => { files[p] = String(body) }, appendFileSync: (p, body) => { files[p] = (files[p] || '') + String(body) }, + renameSync: (from, to) => { files[to] = files[from]; delete files[from] }, chmodSync: () => {}, mkdirSync: () => {}, rmSync: (p) => { delete files[p] } diff --git a/test/unit/config_service.test/sidecars_module.test.js b/test/unit/config_service.test/sidecars_module.test.js index aec3d257..261f0b1f 100644 --- a/test/unit/config_service.test/sidecars_module.test.js +++ b/test/unit/config_service.test/sidecars_module.test.js @@ -108,21 +108,173 @@ function parsingAndGenerationTests() { }) } +function migrateFixture(mainBody, sidecarBody) { + const configFile = path.join(configDir, 'bitcoin-mainnet') + const sidecarFile = configFile + '.local' + fs.writeFileSync(configFile, mainBody) + if (sidecarBody !== undefined) fs.writeFileSync(sidecarFile, sidecarBody, { mode: 0o600 }) + const defaults = {} + for (const line of mainBody.split(/\r?\n/)) sidecars.readMainConfigLine(defaults, line) + sidecars.normalizeMainConfig(defaults, configFile) + if (sidecarBody !== undefined) { + const sidecar = {} + for (const line of sidecarBody.split(/\r?\n/)) sidecars.readSidecarConfigLine(sidecar, line) + sidecars.mergeSidecarConfig(defaults, sidecar, sidecarFile) + } + sidecars.migrateMainCredentials(defaults, configFile, sidecarFile) + return { configFile, sidecarFile } +} + +function migrationSafetyTests() { + it('preserves sidecar-only values and sidecar credential precedence during migration', function () { + const files = migrateFixture( + 'DUST_AMOUNT=546\nNODE_USER=alice\nNODE_PASSWORD=old\n', + 'NODE_SECRET=live\nDECODER_DB_PASS=d\nINDEXER_DB_PASS=i\nXCHAIN_NODE_BLOCKS_DIR=/data\n') + expect(fs.readFileSync(files.configFile, 'utf8')).to.equal('DUST_AMOUNT=546\n') + expect(fs.readFileSync(files.sidecarFile, 'utf8')).to.equal( + 'NODE_SECRET=live\nDECODER_DB_PASS=d\nINDEXER_DB_PASS=i\nXCHAIN_NODE_BLOCKS_DIR=/data\nNODE_USER=alice\n') + expect(fs.statSync(files.sidecarFile).mode & 0o777).to.equal(0o600) + }) + + it('peels glued credentials while preserving comments and main-file values', function () { + const files = migrateFixture( + 'DUST_AMOUNT=546NODE_USER=aliceNODE_PASSWORD=secret\n# keep me\nCUSTOM=main\n', + 'CUSTOM=sidecar\n') + expect(fs.readFileSync(files.configFile, 'utf8')).to.equal('DUST_AMOUNT=546\n# keep me\nCUSTOM=main\n') + expect(fs.readFileSync(files.sidecarFile, 'utf8')).to.equal( + 'CUSTOM=sidecar\nNODE_USER=alice\nNODE_SECRET=secret\n') + }) + + it('removes a safe password alias from the main file', function () { + const files = migrateFixture('KEEP=value\nNODE_SECRET=secret\n\n') + expect(fs.readFileSync(files.configFile, 'utf8')).to.equal('KEEP=value\n\n') + expect(fs.readFileSync(files.sidecarFile, 'utf8')).to.equal('NODE_SECRET=secret\n') + }) +} + +function configureSidecars(fsImpl) { + sidecars.configure({ + crypto, + fs: fsImpl, + path, + readline, + configDir, + preferredSecretEnvName, + foldSecretEnvAliases, + deprecatedSecretEnvNames: () => [], + logger: { warn() {} } + }) +} + +// Real fs with one method replaced, so a failure lands at an exact step of the write. +function fsWith(overrides) { + return Object.assign(Object.create(fs), overrides) +} + +function failingRename(match) { + return fsWith({ + renameSync(from, to) { + if (match(to)) throw Object.assign(new Error('rename failed'), { code: 'EIO' }) + return fs.renameSync(from, to) + } + }) +} + +function tempFilesIn(dir) { + return fs.readdirSync(dir).filter((name) => name.endsWith('.tmp')) +} + +function tornWriteTests() { + it('keeps the old sidecar byte for byte when the replacing rename fails', function () { + const file = path.join(configDir, 'bitcoin-mainnet.local') + const original = 'NODE_USER=alice\nNODE_SECRET=old-secret\n' + fs.writeFileSync(file, original, { mode: 0o600 }) + configureSidecars(failingRename((to) => to === fs.realpathSync(file))) + expect(() => sidecars.upsertSidecarValues(file, { DECODER_DB_PASS: 'new' })).to.throw('rename failed') + expect(fs.readFileSync(file, 'utf8')).to.equal(original) + expect(tempFilesIn(configDir)).to.deep.equal([]) + }) + + it('keeps the old sidecar when the disk fills while writing the replacement', function () { + const file = path.join(configDir, 'hub.local') + const original = 'HUB_DB_PASS=old\n' + fs.writeFileSync(file, original, { mode: 0o600 }) + configureSidecars(fsWith({ + writeFileSync(target, data, options) { + if (String(target).endsWith('.tmp')) { + fs.writeFileSync(target, String(data).slice(0, 3), options) + throw Object.assign(new Error('no space left on device'), { code: 'ENOSPC' }) + } + return fs.writeFileSync(target, data, options) + } + })) + expect(() => sidecars.upsertSidecarValues(file, { HUB_API_KEY: 'k' })).to.throw('no space left') + expect(fs.readFileSync(file, 'utf8')).to.equal(original) + expect(tempFilesIn(configDir)).to.deep.equal([]) + }) + + it('keeps the old sidecar when adding a missing credential fails mid-write', function () { + const file = path.join(configDir, 'bitcoin-regtest.local') + const original = 'NODE_USER=existing-user\n' + fs.writeFileSync(file, original, { mode: 0o600 }) + configureSidecars(failingRename(() => true)) + expect(() => sidecars.generateRpcCredentials({ NODE_USER: 'existing-user' }, file)).to.throw('rename failed') + expect(fs.readFileSync(file, 'utf8')).to.equal(original) + expect(tempFilesIn(configDir)).to.deep.equal([]) + }) +} + +function replacedFileShapeTests() { + it('leaves a replaced sidecar at 0600 with no temp file beside it', function () { + const file = path.join(configDir, 'bitcoin-mainnet.local') + fs.writeFileSync(file, 'NODE_USER=alice\n', { mode: 0o644 }) + sidecars.upsertSidecarValues(file, { INDEXER_DB_PASS: 'p' }) + expect(fs.readFileSync(file, 'utf8')).to.equal('NODE_USER=alice\nINDEXER_DB_PASS=p\n') + expect(fs.statSync(file).mode & 0o777).to.equal(0o600) + expect(fs.readdirSync(configDir)).to.deep.equal(['bitcoin-mainnet.local']) + }) + + it('rewrites the file a symlinked sidecar points at and keeps the link', function () { + const real = path.join(configDir, 'real.local') + const link = path.join(configDir, 'bitcoin-mainnet.local') + fs.writeFileSync(real, 'NODE_USER=alice\n', { mode: 0o600 }) + fs.symlinkSync(real, link) + sidecars.upsertSidecarValues(link, { DECODER_DB_PASS: 'd' }) + expect(fs.lstatSync(link).isSymbolicLink()).to.equal(true) + expect(fs.readFileSync(real, 'utf8')).to.equal('NODE_USER=alice\nDECODER_DB_PASS=d\n') + }) +} + +function migrationTornWriteTests() { + it('keeps the old main config when its rewrite fails during credential migration', function () { + const configFile = path.join(configDir, 'bitcoin-mainnet') + const sidecarFile = configFile + '.local' + const original = 'DUST_AMOUNT=546\nNODE_USER=alice\nNODE_PASSWORD=secret\n' + fs.writeFileSync(configFile, original, { mode: 0o640 }) + configureSidecars(failingRename((to) => to === fs.realpathSync(configFile))) + const defaults = { DUST_AMOUNT: '546', NODE_USER: 'alice', NODE_PASSWORD: 'secret' } + expect(() => sidecars.migrateMainCredentials(defaults, configFile, sidecarFile)).to.throw('rename failed') + expect(fs.readFileSync(configFile, 'utf8')).to.equal(original) + expect(tempFilesIn(configDir)).to.deep.equal([]) + }) + + it('keeps the main config permissions across a credential migration', function () { + const configFile = path.join(configDir, 'bitcoin-mainnet') + fs.writeFileSync(configFile, 'DUST_AMOUNT=546\nNODE_USER=alice\nNODE_PASSWORD=secret\n') + fs.chmodSync(configFile, 0o640) + const defaults = { DUST_AMOUNT: '546', NODE_USER: 'alice', NODE_PASSWORD: 'secret' } + sidecars.migrateMainCredentials(defaults, configFile, configFile + '.local') + expect(fs.readFileSync(configFile, 'utf8')).to.equal('DUST_AMOUNT=546\n') + expect(fs.statSync(configFile).mode & 0o777).to.equal(0o640) + expect(fs.readFileSync(configFile + '.local', 'utf8')).to.equal('NODE_USER=alice\nNODE_SECRET=secret\n') + }) +} + describe('config_service/sidecars', function () { beforeEach(function () { fs.mkdirSync(scratchRoot, { recursive: true }) configDir = fs.mkdtempSync(path.join(scratchRoot, 'case-')) - sidecars.configure({ - crypto, - fs, - path, - readline, - configDir, - preferredSecretEnvName, - foldSecretEnvAliases, - deprecatedSecretEnvNames: () => [], - logger: { warn() {} } - }) + configureSidecars(fs) }) afterEach(function () { @@ -136,4 +288,26 @@ describe('config_service/sidecars', function () { persistenceTests() hubKeyTests() parsingAndGenerationTests() + migrationSafetyTests() +}) + +describe('config_service/sidecars crash-safe writes', function () { + beforeEach(function () { + fs.mkdirSync(scratchRoot, { recursive: true }) + configDir = fs.mkdtempSync(path.join(scratchRoot, 'case-')) + configureSidecars(fs) + }) + + afterEach(function () { + configureSidecars(fs) + fs.rmSync(configDir, { recursive: true, force: true }) + }) + + after(function () { + fs.rmSync(scratchRoot, { recursive: true, force: true }) + }) + + tornWriteTests() + replacedFileShapeTests() + migrationTornWriteTests() }) diff --git a/test/unit/config_service.test/with_coin_and_network.test.js b/test/unit/config_service.test/with_coin_and_network.test.js index 8268a5a2..697c42da 100644 --- a/test/unit/config_service.test/with_coin_and_network.test.js +++ b/test/unit/config_service.test/with_coin_and_network.test.js @@ -31,6 +31,7 @@ function makeServiceWithValidator(validatorSettings) { readFileSync: sinon.stub().returns(''), appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), + renameSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() } diff --git a/test/unit/database_service.test/check_if_database_is_ready.test.js b/test/unit/database_service.test/check_if_database_is_ready.test.js index 817878e5..6a106204 100644 --- a/test/unit/database_service.test/check_if_database_is_ready.test.js +++ b/test/unit/database_service.test/check_if_database_is_ready.test.js @@ -10,77 +10,111 @@ // license (without AGPL source-disclosure terms) is available - // contact legal@dankest.llc. -const { sinon, expect, VALID_CONTAINER_ID, fakeSpawn, mariadbAttempts, makeStubs, loadDatabaseService } = require('./helpers/harness') +const { sinon, expect, VALID_CONTAINER_ID, mariadbAttempts, makeStubs, loadDatabaseService } = require('./helpers/harness') -describe('DatabaseService', function () { +// Answer `docker inspect` with the container id (or `inspect(n)`'s answer on the nth +// inspect) and every mariadb ping with `ping(n)` on the nth ping. +function answerDocker(stubs, { ping = () => true, inspect = () => VALID_CONTAINER_ID } = {}) { + let inspects = 0 + let pings = 0 + stubs.execFileAsync.callsFake(async (cmd, args) => { + if (args[0] === 'inspect') { + const id = inspect(++inspects) + if (!id) throw new Error('Error: No such container: xchain-node-database') + return { stdout: id + '\n' } + } + if (Array.isArray(args) && args.includes('mariadb')) { + if (ping(++pings)) return { stdout: '1\n' } + throw new Error('ERROR 2002 (HY000): Can\'t connect to local server through socket') + } + return { stdout: '' } + }) +} - describe('checkIfDatabaseIsReady()', function () { +function registerAnswerTests() { + it('returns true when database responds', async function () { + const stubs = makeStubs() + answerDocker(stubs) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass')).to.be.true + }) - it('returns true when database responds', async function () { - const stubs = makeStubs() - stubs.execFileAsync.resolves({ stdout: 'OK' }) - const ds = loadDatabaseService(stubs) - const result = await ds.checkIfDatabaseIsReady('root', 'rootpass') - expect(result).to.be.true - }) + it('passes -D database arg when database is specified', async function () { + const stubs = makeStubs() + answerDocker(stubs) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass', 'xchain_node')).to.be.true + const dockerCall = mariadbAttempts(stubs)[0] + expect(dockerCall.args[1]).to.include('-D') + expect(dockerCall.args[1]).to.include('xchain_node') + expect(dockerCall.args[1]).to.include(VALID_CONTAINER_ID) + }) - it('retries up to 10 times then returns false', async function () { - const stubs = makeStubs() - stubs.execFileAsync.rejects(new Error('connection refused')) - const ds = loadDatabaseService(stubs) - const result = await ds.checkIfDatabaseIsReady('root', 'badpass') - expect(result).to.be.false - }) + it('passes the password via MYSQL_PWD env, never in argv', async function () { + const stubs = makeStubs() + answerDocker(stubs) + const ds = loadDatabaseService(stubs) + await ds.checkIfDatabaseIsReady('root', 's3cret-pw') + const dockerCall = mariadbAttempts(stubs)[0] + expect(dockerCall.args[1]).to.include('MYSQL_PWD') + expect(dockerCall.args[1].some(a => String(a).includes('s3cret-pw'))).to.be.false + expect(dockerCall.args[2].env.MYSQL_PWD).to.equal('s3cret-pw') + }) +} - it('passes -D database arg when database is specified', async function () { - const stubs = makeStubs() - stubs.execFileAsync.resolves({ stdout: 'OK' }) - const ds = loadDatabaseService(stubs) - const result = await ds.checkIfDatabaseIsReady('root', 'rootpass', 'xchain_node') - expect(result).to.be.true - const dockerCall = stubs.execFileAsync.getCalls().find(c => - c.args[0] === 'docker' && Array.isArray(c.args[1]) && c.args[1].includes('mariadb')) - expect(dockerCall.args[1]).to.include('-D') - expect(dockerCall.args[1]).to.include('xchain_node') - }) +function registerBudgetTests() { + it('retries a server that is still starting and succeeds once it answers', async function () { + const stubs = makeStubs() + answerDocker(stubs, { ping: (n) => n >= 3 }) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass')).to.be.true + expect(mariadbAttempts(stubs)).to.have.length(3) + }) - it('passes the password via MYSQL_PWD env, never in argv', async function () { - const stubs = makeStubs() - stubs.execFileAsync.resolves({ stdout: 'OK' }) - const ds = loadDatabaseService(stubs) - await ds.checkIfDatabaseIsReady('root', 's3cret-pw') - const dockerCall = stubs.execFileAsync.getCalls().find(c => - c.args[0] === 'docker' && Array.isArray(c.args[1]) && c.args[1].includes('mariadb')) - expect(dockerCall.args[1]).to.include('MYSQL_PWD') - expect(dockerCall.args[1].some(a => String(a).includes('s3cret-pw'))).to.be.false - expect(dockerCall.args[2].env.MYSQL_PWD).to.equal('s3cret-pw') - }) + // The 10x10s budget belongs to the post-`docker run` readiness wait. A + // caller asking "do these credentials still work" gets a PERMANENT answer, + // so it must be able to buy a shorter budget than ~100s of sleeps. + it('honours a caller-supplied retry budget', async function () { + const stubs = makeStubs() + answerDocker(stubs, { ping: () => false }) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('u', 'p', 'xchain_node', { tries: 2, retryDelay: 1 })).to.be.false + expect(mariadbAttempts(stubs)).to.have.length(2) + }) + it('still spends the full readiness budget when the caller passes none', async function () { + const stubs = makeStubs() + answerDocker(stubs, { ping: () => false }) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass')).to.be.false + expect(mariadbAttempts(stubs)).to.have.length(10) + }) +} - // The 10x10s budget belongs to the post-`docker run` readiness wait. A - // caller asking "do these credentials still work" gets a PERMANENT answer, - // so it must be able to buy a shorter budget than ~100s of sleeps. - it('honours a caller-supplied retry budget', async function () { - const stubs = makeStubs() - stubs.execFileAsync.rejects(new Error('ERROR 1045 (28000): Access denied for user')) - const ds = loadDatabaseService(stubs) - const result = await ds.checkIfDatabaseIsReady('u', 'p', 'xchain_node', { tries: 2, retryDelay: 1 }) - expect(result).to.be.false - expect(mariadbAttempts(stubs)).to.have.length(2) - }) - }) -}) +function registerContainerLookupTests() { + it('looks the container up again on every attempt, so one missed inspect does not spend the budget', async function () { + const stubs = makeStubs() + answerDocker(stubs, { inspect: (n) => (n <= 2 ? null : VALID_CONTAINER_ID) }) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass')).to.be.true + expect(mariadbAttempts(stubs)).to.have.length(1) + expect(mariadbAttempts(stubs)[0].args[1]).to.include(VALID_CONTAINER_ID) + }) -describe('DatabaseService', function () { + it('never execs against a missing container, and gives up after the budget', async function () { + const stubs = makeStubs() + answerDocker(stubs, { inspect: () => null }) + const ds = loadDatabaseService(stubs) + expect(await ds.checkIfDatabaseIsReady('root', 'rootpass', null, { tries: 3, retryDelay: 1 })).to.be.false + expect(mariadbAttempts(stubs)).to.have.length(0) + }) +} - describe('checkIfDatabaseIsReady()', function () { - - it('still spends the full readiness budget when the caller passes none', async function () { - const stubs = makeStubs() - stubs.execFileAsync.rejects(new Error('connection refused')) - const ds = loadDatabaseService(stubs) - expect(await ds.checkIfDatabaseIsReady('root', 'rootpass')).to.be.false - expect(mariadbAttempts(stubs)).to.have.length(10) - }) - }) +describe('DatabaseService', function () { + afterEach(() => sinon.restore()) + describe('checkIfDatabaseIsReady()', function () { + registerAnswerTests() + registerBudgetTests() + registerContainerLookupTests() + }) }) diff --git a/test/unit/database_service.test/mariadb_exec/root_password_persisted_datadir.test.js b/test/unit/database_service.test/mariadb_exec/root_password_persisted_datadir.test.js index 7d7072a4..b8bfd72c 100644 --- a/test/unit/database_service.test/mariadb_exec/root_password_persisted_datadir.test.js +++ b/test/unit/database_service.test/mariadb_exec/root_password_persisted_datadir.test.js @@ -34,13 +34,20 @@ function makeDatadir({ initialised = true } = {}) { // Answer docker the way a host with no DB container, a working run, and a server whose root // password is `realRoot` would; `down` makes every ping fail as a server that never came up. +// The container is absent until `docker run`, then inspect finds it; a root login +// through the mariadb client (the first-boot wait) succeeds only with `realRoot`. function fakeDocker(stubs, { realRoot, down = false }) { + let started = false stubs.execFileAsync.callsFake(async (cmd, args, opts) => { - if (args[0] === 'inspect') throw new Error('Error: No such container: xchain-node-database') - if (args[0] === 'run') return { stdout: VALID_CONTAINER_ID + '\n', stderr: '' } - if (args[0] === 'exec' && args.includes('mariadb-admin')) { + if (args[0] === 'inspect') { + if (!started) throw new Error('Error: No such container: xchain-node-database') + return { stdout: VALID_CONTAINER_ID + '\n', stderr: '' } + } + if (args[0] === 'run') { started = true; return { stdout: VALID_CONTAINER_ID + '\n', stderr: '' } } + if (args[0] === 'exec' && (args.includes('mariadb-admin') || args.includes('mariadb'))) { if (down) throw Object.assign(new Error('ping failed'), { stderr: "Can't connect to local server through socket" }) if (opts && opts.env && opts.env.MYSQL_PWD === realRoot) return { stdout: 'mysqld is alive\n', stderr: '' } + if (args.includes('mariadb')) throw Object.assign(new Error('denied'), { stderr: DENIED }) return { stdout: '', stderr: DENIED } } return { stdout: '', stderr: '' } @@ -207,12 +214,22 @@ describe('DatabaseService', function () { expect(stubs.saveDbRootPassword.called).to.equal(false) }) - it('checks nothing after the start on a fresh datadir, which takes the password it was given', async function () { + it('waits on a fresh datadir until the password it was given logs in, with no persisted-password check', async function () { const { stubs, err } = await install({ datadir: makeDatadir({ initialised: false }), envPassword: 'env-root', realRoot: 'env-root' }) expect(err).to.equal(null) expect(stubs.saveDbRootPassword.calledOnceWithExactly('env-root')).to.equal(true) - const pings = stubs.execFileAsync.getCalls().filter(c => c.args[1].includes('mariadb-admin')) - expect(pings).to.have.lengthOf(0) + const confirms = stubs.execFileAsync.getCalls().filter(c => c.args[1].includes('mariadb-admin')) + expect(confirms).to.have.lengthOf(0) + const logins = stubs.execFileAsync.getCalls().filter(c => c.args[1].includes('mariadb') && c.args[1].includes('root')) + expect(logins.length).to.be.greaterThan(0) + expect(logins[0].args[2].env.MYSQL_PWD).to.equal('env-root') + }) + + it('stops the install with an actionable error when a fresh container never accepts a login', async function () { + const { err } = await install({ datadir: makeDatadir({ initialised: false }), envPassword: 'env-root', realRoot: 'env-root', down: true }) + expect(err).to.be.an('error') + expect(err.message).to.match(/did not accept connections within 5 min of starting/) + expect(err.message).to.include('docker logs xchain-node-database') }) }) }) diff --git a/test/unit/database_service.test/named_db_target.test.js b/test/unit/database_service.test/named_db_target.test.js new file mode 100644 index 00000000..2a877525 --- /dev/null +++ b/test/unit/database_service.test/named_db_target.test.js @@ -0,0 +1,345 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// resolveNamedDatabaseTarget answers for the MariaDB container a `reset --db-container` names: its id, a root +// password that logs in to it, and proof the target databases are already there. The root is another server's, +// so it must never reach the CLI's own cached or stored root. + +const { sinon, expect, fakeSpawn, makeStubs, loadDatabaseService } = require('./helpers/harness') + +const NAMED = 'xchain-rail-database' +const NAMED_ID = 'c'.repeat(64) + +// Docker as the named container answers it: inspect by name, one root env name set, a ping that logs in. +function namedServer(stubs, { rootVar = 'MYSQL_ROOT_PASSWORD', root = 'named-root', pingAlive = true } = {}) { + stubs.execFileAsync.callsFake(async (cmd, args, opts) => { + if (args[0] === 'inspect' && args[args.length - 1] === NAMED) return { stdout: NAMED_ID + '\n' } + if (args[0] === 'exec' && args[2] === 'printenv') { + if (args[3] === rootVar) return { stdout: root + '\n' } + throw new Error('exit status 1') + } + if (args.includes('mariadb-admin')) { + if (pingAlive && opts && opts.env && opts.env.MYSQL_PWD === root) return { stdout: 'mysqld is alive\n' } + throw new Error('Access denied for user root') + } + throw new Error('unexpected docker call: ' + args.join(' ')) + }) +} + +// The catalogue answer for every target database: 1 when present, 0 when not. +function schemaAnswer(stubs, present) { + stubs.spawn.callsFake(fakeSpawn((sql) => ({ stdout: /SCHEMATA/.test(sql) ? (present ? '1' : '0') : '' }))) +} + +async function rejection(promise) { + try { await promise } catch (err) { return err } + return null +} + +describe('DatabaseService', function () { + describe('resolveNamedDatabaseTarget()', function () { + + it('resolves the named container, its own root, and the databases it already holds', async function () { + const stubs = makeStubs() + namedServer(stubs) + schemaAnswer(stubs, true) + const ds = loadDatabaseService(stubs) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder', 'xchain-indexer']) + + expect(target).to.deep.equal({ containerName: NAMED, containerId: NAMED_ID, rootPassword: 'named-root', + databases: ['XChain_BTC_Mainnet_Decoder', 'XChain_BTC_Mainnet_Indexer'] }) + for (const call of stubs.spawn.getCalls()) expect(call.args[1]).to.include(NAMED_ID) + // Another server's root never lands in the CLI's own root cache or credentials store. + expect(stubs.setDbRootPassword.called).to.be.false + expect(stubs.saveDbRootPassword.called).to.be.false + }) + + it('reads the root from MARIADB_ROOT_PASSWORD when that is the name the container uses', async function () { + const stubs = makeStubs() + namedServer(stubs, { rootVar: 'MARIADB_ROOT_PASSWORD', root: 'other-root' }) + schemaAnswer(stubs, true) + const ds = loadDatabaseService(stubs) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder']) + expect(target.rootPassword).to.equal('other-root') + }) + + it('refuses a container name docker would not accept, before any docker call', async function () { + const stubs = makeStubs() + const ds = loadDatabaseService(stubs) + + const err = await rejection(ds.resolveNamedDatabaseTarget('db;reboot', 'bitcoin', 'mainnet', ['xchain-decoder'])) + expect(err && err.message).to.contain('not a valid docker container name') + expect(stubs.execFileAsync.called).to.be.false + }) + + it('refuses when no root password in the container logs in', async function () { + const stubs = makeStubs() + namedServer(stubs, { pingAlive: false }) + const ds = loadDatabaseService(stubs) + + const err = await rejection(ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder'])) + expect(err && err.message).to.contain('no root password') + expect(stubs.spawn.called).to.be.false + }) + + it('refuses a server that does not hold the target database', async function () { + const stubs = makeStubs() + namedServer(stubs) + schemaAnswer(stubs, false) + const ds = loadDatabaseService(stubs) + + const err = await rejection(ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder'])) + expect(err && err.message).to.contain('holds no database XChain_BTC_Mainnet_Decoder') + }) + }) +}) + +describe('DatabaseService', function () { + describe('resolveNamedDatabaseTarget() root override', function () { + const VAR = 'XCHAIN_NODE_DB_CONTAINER_ROOT_PASSWORD' + afterEach(function () { delete process.env[VAR] }) + + it('logs in with it when the container was started without a root env', async function () { + const stubs = makeStubs() + namedServer(stubs, { rootVar: 'NO_SUCH_VAR', root: 'moved-in-root' }) + schemaAnswer(stubs, true) + process.env[VAR] = 'moved-in-root' + const ds = loadDatabaseService(stubs) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'regtest', ['xchain-decoder', 'xchain-indexer']) + expect(target.rootPassword).to.equal('moved-in-root') + expect(stubs.setDbRootPassword.called).to.be.false + expect(stubs.saveDbRootPassword.called).to.be.false + }) + + it('refuses by name when it does not log in, before any database read', async function () { + const stubs = makeStubs() + namedServer(stubs, { rootVar: 'NO_SUCH_VAR', root: 'real-root' }) + process.env[VAR] = 'wrong-root' + const ds = loadDatabaseService(stubs) + + const err = await rejection(ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'regtest', ['xchain-decoder'])) + expect(err && err.message).to.contain(`${VAR} does not log in to ${NAMED}`) + expect(err.message).to.not.contain('wrong-root') + expect(stubs.spawn.called).to.be.false + }) + + it('is not read when the container env root logs in', async function () { + const stubs = makeStubs() + namedServer(stubs) + schemaAnswer(stubs, true) + process.env[VAR] = 'unused-root' + const ds = loadDatabaseService(stubs) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder']) + expect(target.rootPassword).to.equal('named-root') + }) + }) +}) + +// A named server with no usable root, whose stack's service containers carry their own database logins. Each user +// logs in only with its own password and answers SHOW GRANTS with the lines given, as `mariadb -B -N` prints them +// (backslashes doubled). `env` is the indexer container's environment; the decoder container's is `decoderEnv`. +const INDEXER_ID = 'b'.repeat(64) +const DECODER_ID = 'a'.repeat(64) +const DB = { decoder: 'XChain_BTC_Mainnet_Decoder', indexer: 'XChain_BTC_Mainnet_Indexer', hub: 'XChain_Hub' } +const SECRETS = { dec_user: 'dec-secret-1', idx_user: 'idx-secret-2', hub_user: 'hub-secret-3' } +const SERVICE_ENV = { + DECODER_DB_NAME: DB.decoder, DECODER_DB_USER: 'dec_user', DECODER_DB_PASS: SECRETS.dec_user, + INDEXER_DB_NAME: DB.indexer, INDEXER_DB_USER: 'idx_user', INDEXER_DB_PASS: SECRETS.idx_user, + HUB_DB_NAME: DB.hub, HUB_DB_USER: 'hub_user', HUB_DB_PASS: SECRETS.hub_user +} +const FULL_GRANTS = { + dec_user: ['GRANT USAGE ON *.* TO `dec_user`@`%` IDENTIFIED BY PASSWORD \'*AB12\'', + 'GRANT ALL PRIVILEGES ON `XChain\\\\_BTC\\\\_Mainnet\\\\_Decoder`.* TO `dec_user`@`%`'], + idx_user: ['GRANT USAGE ON *.* TO `idx_user`@`%`', 'GRANT SELECT, INSERT, DROP, CREATE ON `XChain_BTC_Mainnet_Indexer`.* TO `idx_user`@`%`'], + hub_user: ['GRANT USAGE ON *.* TO `hub_user`@`%`', 'GRANT SELECT, DELETE ON `XChain_Hub`.`price_ingest_watermarks` TO `hub_user`@`%`'] +} + +function serviceServer(stubs, { env = SERVICE_ENV, decoderEnv = {}, grants = FULL_GRANTS, passwords = SECRETS } = {}) { + const envs = { [INDEXER_ID]: env, [DECODER_ID]: decoderEnv } + stubs.execFileAsync.callsFake(async (cmd, args, opts) => { + if (args[0] === 'inspect' && args[args.length - 1] === NAMED) return { stdout: NAMED_ID + '\n' } + // Service env is read from the container config, which a stopped container still answers. + if (args[0] === 'inspect' && args.includes('{{json .Config.Env}}')) { + const env = envs[args[args.length - 1]] + if (!env) throw new Error('Error: No such object') + return { stdout: JSON.stringify(Object.entries(env).map(([k, v]) => k + '=' + v)) + '\n' } + } + if (args[0] === 'exec' && args[2] === 'printenv') throw new Error('exit status 1') + if (args.includes('mariadb-admin')) throw new Error('Access denied for user root') + throw new Error('unexpected docker call: ' + args.join(' ')) + }) + stubs.spawn.callsFake(fakeSpawn((sql, args, opts) => { + const user = args[args.indexOf('-u') + 1] + if (!passwords[user] || opts.env.MYSQL_PWD !== passwords[user]) { + return { code: 1, stderr: `ERROR 1045 (28000): Access denied for user '${user}'@'localhost' (using password: YES)` } + } + if (/SHOW GRANTS/.test(sql)) return { stdout: (grants[user] || []).join('\n') } + if (/SCHEMATA/.test(sql)) return { stdout: '1' } + return { stdout: '' } + })) +} + +// The stack's service containers as the reset's name lookup finds them. +function finder() { + return sinon.stub().callsFake(async (module) => ({ 'xchain-indexer': INDEXER_ID, 'xchain-decoder': DECODER_ID })[module] || null) +} + +function expectNoSecret(text) { + for (const secret of Object.values(SECRETS)) expect(String(text)).to.not.contain(secret) +} + +const BOTH = ['xchain-decoder', 'xchain-indexer'] + +describe('DatabaseService', function () { + describe('resolveNamedDatabaseTarget() service logins', function () { + + it('falls back to the stack\'s own users only when no root logs in, and reports their grants', async function () { + const stubs = makeStubs() + serviceServer(stubs) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH, finder()) + + expect(target.authMode).to.equal('service') + expect(target).to.not.have.property('rootPassword') + expect(target.logins[DB.decoder].user).to.equal('dec_user') + expect(target.logins[DB.indexer].user).to.equal('idx_user') + expect(target.hubLogin.user).to.equal('hub_user') + const report = target.authReport.join('\n') + expect(report).to.contain('no root password').and.to.contain('dec_user').and.to.contain('ALL PRIVILEGES ON') + .and.to.contain('SELECT, DELETE ON `XChain_Hub`.`price_ingest_watermarks`') + expectNoSecret(report) + expectNoSecret(JSON.stringify(target)) + for (const call of stubs.spawn.getCalls()) expect(call.args[1]).to.not.include('root') + }) + + it('reads the service logins from container config, so a stack stopped for the reset still answers', async function () { + const stubs = makeStubs() + serviceServer(stubs) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + + await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH, finder()) + + const serviceExecs = stubs.execFileAsync.getCalls() + .filter((c) => c.args[1][0] === 'exec' && [INDEXER_ID, DECODER_ID].includes(c.args[1][1])) + expect(serviceExecs).to.have.length(0) + }) + + it('never reads a service login when the container\'s root logs in', async function () { + const stubs = makeStubs() + namedServer(stubs) + schemaAnswer(stubs, true) + const find = finder() + const ds = loadDatabaseService(stubs) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH, find) + expect(target.rootPassword).to.equal('named-root') + expect(find.called).to.be.false + }) + + it('reads a login from the decoder container when the indexer\'s env lacks it', async function () { + const stubs = makeStubs() + const { DECODER_DB_USER, DECODER_DB_PASS, DECODER_DB_NAME, ...indexerOnly } = SERVICE_ENV + serviceServer(stubs, { env: indexerOnly, decoderEnv: { DECODER_DB_USER, DECODER_DB_PASS, DECODER_DB_NAME } }) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH, finder()) + expect(target.logins[DB.decoder].source).to.equal('xchain-decoder') + }) + }) +}) + +describe('DatabaseService', function () { + describe('resolveNamedDatabaseTarget() service hub login', function () { + + it('does not need a hub login for a decoder-only reset', async function () { + const stubs = makeStubs() + const { HUB_DB_USER, HUB_DB_PASS, ...noHub } = SERVICE_ENV + serviceServer(stubs, { env: noHub }) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', ['xchain-decoder'], finder()) + expect(target.hubLogin).to.equal(null) + }) + + it('skips the fence clear with a note when the stack\'s hub uses another database', async function () { + const stubs = makeStubs() + serviceServer(stubs, { env: { ...SERVICE_ENV, HUB_DB_NAME: 'Rail_Hub' } }) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + + const target = await ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH, finder()) + expect(target.hubLogin).to.equal(null) + expect(target.authReport.join('\n')).to.contain('names Rail_Hub, not XChain_Hub') + }) + }) +}) + +describe('DatabaseService', function () { + describe('resolveNamedDatabaseTarget() service login refusals', function () { + + async function refusal(options, modules = BOTH) { + const stubs = makeStubs() + serviceServer(stubs, options) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: DB.hub }) + const err = await rejection(ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', modules, finder())) + expect(err, 'expected a refusal').to.not.equal(null) + expectNoSecret(err.message) + // The root failure stays in the message beside the service one. + expect(err.message).to.contain('no root password') + return err.message + } + + it('refuses when a target database has no login in either container\'s env', async function () { + const { INDEXER_DB_PASS, ...env } = SERVICE_ENV + expect(await refusal({ env })).to.contain('carries INDEXER_DB_USER and INDEXER_DB_PASS') + }) + + it('refuses a login that does not log in, naming the user', async function () { + expect(await refusal({ passwords: { ...SECRETS, dec_user: 'not-the-env-one' } })) + .to.contain('DECODER_DB_USER dec_user does not log in to ' + NAMED) + }) + + it('refuses when the stack\'s services cannot be found', async function () { + const stubs = makeStubs() + serviceServer(stubs) + const ds = loadDatabaseService(stubs) + const err = await rejection(ds.resolveNamedDatabaseTarget(NAMED, 'bitcoin', 'mainnet', BOTH)) + expect(err && err.message).to.contain('no indexer or decoder container was found') + }) + + for (const [missing, privileges] of [['DROP', 'SELECT, INSERT, CREATE, CREATE VIEW'], ['CREATE', 'SELECT, DROP, CREATE VIEW, CREATE ROUTINE']]) { + it(`refuses a database user whose grants lack ${missing}`, async function () { + const grants = { ...FULL_GRANTS, idx_user: [`GRANT ${privileges} ON \`XChain_BTC_Mainnet_Indexer\`.* TO \`idx_user\`@\`%\``] } + expect(await refusal({ grants })).to.contain(`INDEXER_DB_USER idx_user has no ${missing} grant on ${DB.indexer}`) + }) + } + + it('refuses a grant on another database, even with ALL PRIVILEGES', async function () { + const grants = { ...FULL_GRANTS, dec_user: ['GRANT ALL PRIVILEGES ON `XChain_BTC_Regtest_Decoder`.* TO `dec_user`@`%`'] } + expect(await refusal({ grants })).to.contain(`has no DROP or CREATE grant on ${DB.decoder}`) + }) + + it('refuses a hub user without DELETE on the fence table', async function () { + const grants = { ...FULL_GRANTS, hub_user: ['GRANT SELECT ON `XChain_Hub`.* TO `hub_user`@`%`'] } + expect(await refusal({ grants })).to.contain('HUB_DB_USER hub_user has no DELETE grant on XChain_Hub.price_ingest_watermarks') + }) + + it('refuses when the stack names another database than the reset targets', async function () { + expect(await refusal({ env: { ...SERVICE_ENV, DECODER_DB_NAME: 'Other_Decoder' } })) + .to.contain(`DECODER_DB_NAME in the xchain-indexer container names Other_Decoder, not ${DB.decoder}`) + }) + }) +}) diff --git a/test/unit/database_service.test/reset_databases.test.js b/test/unit/database_service.test/reset_databases.test.js index 3be1b336..fcbae45c 100644 --- a/test/unit/database_service.test/reset_databases.test.js +++ b/test/unit/database_service.test/reset_databases.test.js @@ -191,3 +191,110 @@ describe('DatabaseService', function () { }) }) }) + +describe('DatabaseService', function () { + + // A `reset --db-container` hands both DB steps the server it resolved. They must run there with that server's + // root, and must not ask for the bundled container or the CLI's cached root at all. + describe('resetDatabases() and clearHubPriceIngestWatermark() on a named server', function () { + const NAMED = { containerName: 'xchain-rail-database', containerId: 'c'.repeat(64), rootPassword: 'named-root' } + + // Every mariadb child the source spawns, as the SQL it piped, its argv and its env. + function recordSpawns(stubs, respond) { + const calls = [] + stubs.spawn.callsFake(fakeSpawn((sql, args, opts) => { + calls.push({ sql, args, env: (opts && opts.env) || {} }) + return respond(sql) + })) + return calls + } + + it('drops and recreates each database on the named container with its root', async function () { + const stubs = makeStubs() + const calls = recordSpawns(stubs, () => ({ stdout: '' })) + const ds = loadDatabaseService(stubs) + await ds.resetDatabases('bitcoin', 'mainnet', ['xchain-decoder', 'xchain-indexer'], NAMED) + + expect(calls.map(c => c.sql).join(' | ')).to.contain('DROP DATABASE IF EXISTS XChain_BTC_Mainnet_Decoder') + .and.to.contain('DROP DATABASE IF EXISTS XChain_BTC_Mainnet_Indexer') + for (const call of calls) { + expect(call.args).to.include(NAMED.containerId) + expect(call.env.MYSQL_PWD).to.equal('named-root') + } + expect(stubs.getDbRootPassword.called).to.be.false + expect(stubs.execFileAsync.called).to.be.false + }) + + it('clears the price fence row on the named container', async function () { + const stubs = makeStubs() + const calls = recordSpawns(stubs, (sql) => ({ stdout: /information_schema/i.test(sql) ? '1' : '' })) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: 'XChain_Hub' }) + expect(await ds.clearHubPriceIngestWatermark('bitcoin', 'regtest', NAMED)).to.be.true + + expect(calls.map(c => c.sql).join(' | ')).to.match(/DELETE\s+FROM\s+`?XChain_Hub`?\.`?price_ingest_watermarks/i) + for (const call of calls) { + expect(call.args).to.include(NAMED.containerId) + expect(call.env.MYSQL_PWD).to.equal('named-root') + } + expect(stubs.getDbRootPassword.called).to.be.false + expect(stubs.execFileAsync.called).to.be.false + }) + }) +}) + +describe('DatabaseService', function () { + + // With no root for the named server, each database is dropped and recreated by the user that owns it, and the + // fence clear runs as the hub's user: the users resolveNamedDatabaseTarget checked before anything was stopped. + describe('resetDatabases() and clearHubPriceIngestWatermark() as the stack\'s own users', function () { + const login = (user, password) => ({ user, password, prefix: 'X', grants: [] }) + const SERVICE = { + containerName: 'xchain-rail-database', containerId: 'c'.repeat(64), authMode: 'service', + logins: { XChain_BTC_Mainnet_Decoder: login('dec_user', 'dec-pw'), XChain_BTC_Mainnet_Indexer: login('idx_user', 'idx-pw') }, + hubLogin: login('hub_user', 'hub-pw') + } + + function recordSpawns(stubs, respond) { + const calls = [] + stubs.spawn.callsFake(fakeSpawn((sql, args, opts) => { + calls.push({ sql, user: args[args.indexOf('-u') + 1], password: opts.env.MYSQL_PWD }) + return respond(sql) + })) + return calls + } + + it('drops and recreates each database as its own user', async function () { + const stubs = makeStubs() + const calls = recordSpawns(stubs, () => ({ stdout: '' })) + const ds = loadDatabaseService(stubs) + await ds.resetDatabases('bitcoin', 'mainnet', ['xchain-decoder', 'xchain-indexer'], SERVICE) + + expect(calls.map(({ sql, user, password }) => [sql.match(/DROP DATABASE IF EXISTS (\w+)/)[1], user, password])).to.deep.equal([ + ['XChain_BTC_Mainnet_Decoder', 'dec_user', 'dec-pw'], + ['XChain_BTC_Mainnet_Indexer', 'idx_user', 'idx-pw'] + ]) + expect(stubs.getDbRootPassword.called).to.be.false + }) + + it('clears the price fence as the hub user', async function () { + const stubs = makeStubs() + const calls = recordSpawns(stubs, (sql) => ({ stdout: /information_schema/i.test(sql) ? '1' : '' })) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: 'XChain_Hub' }) + expect(await ds.clearHubPriceIngestWatermark('bitcoin', 'regtest', SERVICE)).to.be.true + + expect(calls.some((c) => /DELETE\s+FROM\s+`XChain_Hub`\.price_ingest_watermarks/.test(c.sql))).to.be.true + for (const call of calls) expect([call.user, call.password]).to.deep.equal(['hub_user', 'hub-pw']) + }) + + it('runs nothing and throws when no hub login was resolved', async function () { + const stubs = makeStubs() + const calls = recordSpawns(stubs, () => ({ stdout: '1' })) + const ds = loadDatabaseService(stubs, {}, { HUB_DB_NAME: 'XChain_Hub' }) + let err = null + try { await ds.clearHubPriceIngestWatermark('bitcoin', 'regtest', { ...SERVICE, hubLogin: null }) } catch (e) { err = e } + + expect(err && err.message).to.contain('no hub login was resolved on xchain-rail-database') + expect(calls).to.have.length(0) + }) + }) +}) diff --git a/test/unit/db/halt_markers.test.js b/test/unit/db/halt_markers.test.js index decd2147..037b518f 100644 --- a/test/unit/db/halt_markers.test.js +++ b/test/unit/db/halt_markers.test.js @@ -15,6 +15,7 @@ const expectedExports = [ 'newestReorgClearSql', 'liveSyncHaltCountSql', 'pendingSyncHaltCountSql', + 'rollbackGuardStateSql', 'eventsWatermarkSql', 'syncHaltWatermarkSql', 'reorgHaltsSinceSql', @@ -67,6 +68,30 @@ describe('halt marker exports and live SQL builders', () => { expect(sql).to.include('CHAR_LENGTH(COALESCE(state_value') expect(sql).to.match(/;$/) }) +}) + +describe('halt marker exports and live SQL builders', () => { + it('reads exact rollback guard keys as encoded values', () => { + const sql = haltMarkers.rollbackGuardStateSql('xchain_btc') + + expect(sql).to.include('`xchain_btc`.sync_state') + expect(sql).to.include("LEFT(state_key, 15) = 'rollback_guard:'") + expect(sql).to.include("CONCAT('V', HEX(state_value))") + expect(sql).to.include("state_value IS NULL, 'N'") + expect(sql).to.not.include('LIKE') + expect(sql).to.match(/;$/) + }) + + it('reads exact rollback guard keys as encoded values', () => { + const sql = haltMarkers.rollbackGuardStateSql('xchain_btc') + + expect(sql).to.include('`xchain_btc`.sync_state') + expect(sql).to.include("LEFT(state_key, 15) = 'rollback_guard:'") + expect(sql).to.include("CONCAT('V', HEX(state_value))") + expect(sql).to.include("state_value IS NULL, 'N'") + expect(sql).to.not.include('LIKE') + expect(sql).to.match(/;$/) + }) it('builds the live sync halt query', () => { const sql = haltMarkers.liveSyncHaltCountSql('xchain_btc') diff --git a/test/unit/explorer_service.test/04_force_rebuild.test.js b/test/unit/explorer_service.test/04_force_rebuild.test.js index 6f0e7505..1580a2a3 100644 --- a/test/unit/explorer_service.test/04_force_rebuild.test.js +++ b/test/unit/explorer_service.test/04_force_rebuild.test.js @@ -100,56 +100,100 @@ function loadExplorerService(stubs) { }) } +function pingSecondCall() { + let pingCallCount = 0 + return sinon.stub().callsFake(() => { + pingCallCount++ + // Return true on the second call (post-install) to exit the while loop + return Promise.resolve(pingCallCount >= 2) + }) +} + +function expectOldExplorerUntouched(stubs) { + expect(stubs.killContainer.called).to.be.false + expect(stubs.removeContainer.called).to.be.false + expect(stubs.db.deleteModuleContainer.called).to.be.false +} + describe('ExplorerService: installExplorerModule() force=true', function () { + // buildAndUp stops and removes the container it is handed, after the image builds. it('kills and removes existing container when force=true and container exists', async function () { const existingId = 'existing-id' - let pingCallCount = 0 const stubs = makeExplorerServiceStubs({ dbGetModuleContainer: sinon.stub().resolves(existingId), - explorerPing: sinon.stub().callsFake(() => { - pingCallCount++ - // Return true on the second call (post-install) to exit the while loop - return Promise.resolve(pingCallCount >= 2) - }) + explorerPing: pingSecondCall() }) const es = loadExplorerService(stubs) const result = await es.installExplorerModule(true) expect(result).to.be.true - expect(stubs.killContainer.calledWith(existingId)).to.be.true - expect(stubs.removeContainer.calledWith(existingId)).to.be.true - expect(stubs.db.deleteModuleContainer.called).to.be.true + expect(stubs.buildAndUp.calledOnce).to.be.true + expect(stubs.buildAndUp.firstCall.args).to.deep.equal([EXPLORER_MODULE_NAME, null, null, existingId]) + expectOldExplorerUntouched(stubs) }) it('swallows errors from killContainer/removeContainer during force rebuild', async function () { const existingId = 'old-id' - let pingCallCount = 0 const stubs = makeExplorerServiceStubs({ dbGetModuleContainer: sinon.stub().resolves(existingId), killContainer: sinon.stub().rejects(new Error('already stopped')), removeContainer: sinon.stub().rejects(new Error('already gone')), - explorerPing: sinon.stub().callsFake(() => { - pingCallCount++ - return Promise.resolve(pingCallCount >= 2) - }) + explorerPing: pingSecondCall() }) const es = loadExplorerService(stubs) const result = await es.installExplorerModule(true) expect(result).to.be.true + expectOldExplorerUntouched(stubs) }) it('skips kill/remove when no existing container in DB (force=true)', async function () { - let pingCallCount = 0 const stubs = makeExplorerServiceStubs({ dbGetModuleContainer: sinon.stub().resolves(null), - explorerPing: sinon.stub().callsFake(() => { - pingCallCount++ - return Promise.resolve(pingCallCount >= 2) - }) + explorerPing: pingSecondCall() }) const es = loadExplorerService(stubs) await es.installExplorerModule(true) expect(stubs.killContainer.called).to.be.false expect(stubs.removeContainer.called).to.be.false + expect(stubs.buildAndUp.firstCall.args[3]).to.equal(null) + }) +}) + +describe('ExplorerService: installExplorerModule() force=true keeps the old explorer on failure', function () { + + it('leaves the running explorer and its registry row alone when the clone fails', async function () { + const stubs = makeExplorerServiceStubs({ + dbGetModuleContainer: sinon.stub().resolves('live-id'), + cloneGit: sinon.stub().rejects(new Error('clone failed')) + }) + const es = loadExplorerService(stubs) + let threw = null + try { await es.installExplorerModule(true) } catch (err) { threw = err } + expect(threw).to.be.an('error').with.property('message', 'clone failed') + expect(stubs.buildAndUp.called).to.be.false + expectOldExplorerUntouched(stubs) + }) + + it('leaves the registry row alone when the image build fails', async function () { + const stubs = makeExplorerServiceStubs({ + dbGetModuleContainer: sinon.stub().resolves('live-id'), + buildAndUp: sinon.stub().rejects('Error creating Docker image: build failed') + }) + const es = loadExplorerService(stubs) + let threw = null + try { await es.installExplorerModule(true) } catch (err) { threw = err } + expect(threw).to.not.equal(null) + expect(stubs.buildAndUp.firstCall.args[3]).to.equal('live-id') + expectOldExplorerUntouched(stubs) + }) + + it('does not look up an overwrite id when force=false', async function () { + const stubs = makeExplorerServiceStubs({ + dbGetModuleContainer: sinon.stub().resolves('live-id'), + explorerPing: sinon.stub().onFirstCall().resolves(false).resolves(true) + }) + const es = loadExplorerService(stubs) + await es.installExplorerModule(false) + expect(stubs.buildAndUp.firstCall.args[3]).to.equal(null) }) }) diff --git a/test/unit/hub_connector_failures.test.js b/test/unit/hub_connector_failures.test.js index 4db06448..f46731d3 100644 --- a/test/unit/hub_connector_failures.test.js +++ b/test/unit/hub_connector_failures.test.js @@ -73,6 +73,48 @@ describe('HubConnector: a non-throwing failure still records why', () => { }) }) +describe('HubConnector: a hub answer carrying an error is not a delivered push', () => { + let srv + + afterEach(async () => { + if (srv) await new Promise((r) => srv.close(r)) + srv = null + }) + + function answer(result) { + return server((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ jsonrpc: '2.0', id: 1, result })) + }) + } + + it('a {error} result (PBFT not-the-leader) makes updateConfig false and records the text', async () => { + srv = await answer({ error: 'Not the leader for seq 256 (leader: abc)' }) + const c = new HubConnector('127.0.0.1', srv.address().port) + + const ok = await c.updateConfig({}) + + expect(ok).to.equal(false) + expect(c.lastFailures.join('; ')).to.include('Not the leader for seq 256') + }) + + it('a non-string error value still fails the push', async () => { + srv = await answer({ error: { message: 'boom' } }) + const c = new HubConnector('127.0.0.1', srv.address().port) + + expect(await c.updateConfig({})).to.equal(false) + expect(c.lastFailures.join('; ')).to.include('boom') + }) + + it('control: a {status:success} result still reads as delivered', async () => { + srv = await answer({ status: 'success' }) + const c = new HubConnector('127.0.0.1', srv.address().port) + + expect(await c.updateConfig({})).to.equal(true) + expect(c.lastFailures).to.be.empty + }) +}) + describe('HubConnector: a non-throwing failure still records why', () => { let srv diff --git a/test/unit/hub_push_wall_clock_cap.test.js b/test/unit/hub_push_wall_clock_cap.test.js new file mode 100644 index 00000000..e221b2f4 --- /dev/null +++ b/test/unit/hub_push_wall_clock_cap.test.js @@ -0,0 +1,77 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * Hub config push: bounded by wall time when the hub hangs. + ********************************************************************/ + +const sinon = require('sinon') +const assert = require('node:assert/strict') +const { expect } = require('chai') + +const { updateHubOrExplorer } = require('../../src/services/hub_service/update_hub_or_explorer') + +function deps({ updateConfig, clock }) { + return { + HUB_MODULE_NAME: 'xchain-hub', + EXPLORER_MODULE_NAME: 'xchain-explorer', + EXTERNAL_DB: false, + XChainService: { XCHAIN_INDEXER: 'xchain-indexer' }, + db: { getModuleContainer: sinon.stub().resolves('c') }, + getLastStatus: () => ({}), + isStatusUpdated: () => true, + sleep: sinon.stub().resolves(), + redactSecrets: v => String(v && v.message ? v.message : v), + getDefaultConfig: sinon.stub().resolves({ HUB_PORT: 10000, EXPLORER_PORT: 10001 }), + getStatus: sinon.stub().resolves(), + getExternalDbConfig: sinon.stub().resolves(null), + readContainerEnv: sinon.stub().resolves(null), + HubConnector: class { constructor() { this.updateConfig = updateConfig } }, + ExplorerConnector: class {}, + dockerService: {}, + buildHubModuleConfig: sinon.stub().returns(null), + buildCheckpointConfig: sinon.stub().returns({}), + isCheckpointSelfSyncEnabled: sinon.stub().resolves(false), + logger: { info: sinon.stub() }, + now: clock ? () => clock.t : undefined + } +} + +describe('hub config push wall-clock cap', function () { + it('gives up after about three minutes when every attempt hangs for 60 s', async function () { + const clock = { t: 0 } + const updateConfig = sinon.spy(async () => { clock.t += 60000; return null }) + await assert.rejects( + updateHubOrExplorer('xchain-hub', deps({ updateConfig, clock })), + /There was a problem trying to update a config/ + ) + expect(updateConfig.callCount).to.be.at.most(4) + expect(clock.t).to.be.at.most(240000) + }) + + it('still retries fast failures up to the attempt limit', async function () { + const clock = { t: 0 } + const updateConfig = sinon.spy(async () => null) + await assert.rejects( + updateHubOrExplorer('xchain-hub', deps({ updateConfig, clock })), + /There was a problem trying to update a config/ + ) + expect(updateConfig.callCount).to.equal(60) + }) + + it('succeeds when the hub answers on a later attempt within the budget', async function () { + const clock = { t: 0 } + const updateConfig = sinon.spy(async () => { clock.t += 60000; return updateConfig.callCount >= 3 }) + expect(await updateHubOrExplorer('xchain-hub', deps({ updateConfig, clock }))).to.be.true + expect(updateConfig.callCount).to.equal(3) + }) +}) diff --git a/test/unit/hub_service.test.js b/test/unit/hub_service.test.js index 78eeedf5..935c7b11 100644 --- a/test/unit/hub_service.test.js +++ b/test/unit/hub_service.test.js @@ -139,9 +139,9 @@ describe('HubService.updateHubOrExplorer config delivery', function () { sinon.assert.calledOnceWithExactly(deps.sleep, 3000) }) - it('exhausts ten attempts and reports the last hub error', async function () { + it('exhausts the boot-window attempts and reports the last hub error', async function () { const updateConfig = sinon.stub().rejects(new Error('earlier refusal')) - updateConfig.onCall(9).rejects(new Error('final refusal')) + updateConfig.onCall(59).rejects(new Error('final refusal')) const deps = configDeliveryDependencies({ updateConfig }) let thrown @@ -151,8 +151,8 @@ describe('HubService.updateHubOrExplorer config delivery', function () { thrown = err } - expect(updateConfig.callCount).to.equal(10) - expect(deps.sleep.callCount).to.equal(9) + expect(updateConfig.callCount).to.equal(60) + expect(deps.sleep.callCount).to.equal(59) expect(thrown).to.include('last error: final refusal') expect(thrown).not.to.include('earlier refusal') }) diff --git a/test/unit/maria_db_store.test.js b/test/unit/maria_db_store.test.js index 4d3d8fe4..8db7cd79 100644 --- a/test/unit/maria_db_store.test.js +++ b/test/unit/maria_db_store.test.js @@ -45,9 +45,12 @@ function dispatchFakeSql(rows, sql, params = []) { return undefined } if (/^DELETE FROM modules WHERE/i.test(trimmed)) { - const [module, coin, network] = params - rows.delete(`${module}|${coin}|${network}`) - return undefined + const [module, coin, network, containerId] = params + const key = `${module}|${coin}|${network}` + const current = rows.get(key) + const matches = containerId === undefined || (current && current.container_id === containerId) + if (matches) rows.delete(key) + return { affectedRows: matches && current ? 1 : 0 } } throw new Error(`buildFakeMariadbModule: unhandled SQL: ${trimmed}`) } diff --git a/test/unit/maria_db_store.test/03_delete_module_container_if_matches.test.js b/test/unit/maria_db_store.test/03_delete_module_container_if_matches.test.js new file mode 100644 index 00000000..9d13d193 --- /dev/null +++ b/test/unit/maria_db_store.test/03_delete_module_container_if_matches.test.js @@ -0,0 +1,84 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { expect } = require('chai') +const proxyquire = require('proxyquire').noCallThru() + +function buildFakeMariadbModule() { + const rows = new Map() + const dispatch = (sql, params = []) => { + const trimmed = sql.replace(/\s+/g, ' ').trim() + if (/^CREATE TABLE IF NOT EXISTS modules/i.test(trimmed)) return undefined + if (/^INSERT INTO modules/i.test(trimmed)) { + const [module, coin, network, containerId] = params + rows.set(`${module}|${coin}|${network}`, containerId) + return undefined + } + if (/^SELECT container_id FROM modules WHERE/i.test(trimmed)) { + const [module, coin, network] = params + const containerId = rows.get(`${module}|${coin}|${network}`) + return containerId === undefined ? [] : [{ container_id: containerId }] + } + if (/^DELETE FROM modules WHERE/i.test(trimmed)) { + const [module, coin, network, containerId] = params + const key = `${module}|${coin}|${network}` + const matches = rows.get(key) === containerId + if (matches) rows.delete(key) + return { affectedRows: matches ? 1 : 0 } + } + throw new Error(`buildFakeMariadbModule: unhandled SQL: ${trimmed}`) + } + const connection = { query: async (sql, params) => dispatch(sql, params), release: () => {} } + const pool = { + getConnection: async () => connection, + query: async (sql, params) => dispatch(sql, params), + end: async () => {} + } + return { createPool: () => pool } +} + +function loadStore() { + return proxyquire('../../../src/db', { 'mariadb': buildFakeMariadbModule() }) +} + +describe('MariaDbStore', function () { + let store + + beforeEach(async function () { + const MariaDbStore = loadStore() + store = new MariaDbStore() + await store.createDatabase({ + host: '127.0.0.1', port: 3306, + user: 'u', password: 'p', database: 'xchain_node' + }) + }) + + afterEach(async function () { + await store.close() + }) + + describe('deleteModuleContainerIfMatches()', function () { + it('deletes only the registry row carrying the inspected container id', async function () { + await store.setModuleContainer('xchain-encoder', 'bitcoin', 'mainnet', 'new-id') + const staleResult = await store.deleteModuleContainerIfMatches( + 'xchain-encoder', 'bitcoin', 'mainnet', 'old-id') + expect(staleResult).to.equal(false) + expect(await store.getModuleContainer('xchain-encoder', 'bitcoin', 'mainnet')).to.equal('new-id') + + const currentResult = await store.deleteModuleContainerIfMatches( + 'xchain-encoder', 'bitcoin', 'mainnet', 'new-id') + expect(currentResult).to.equal(true) + expect(await store.getModuleContainer('xchain-encoder', 'bitcoin', 'mainnet')).to.equal(null) + }) + + it('returns false when the pool is not ready', async function () { + const MariaDbStore = loadStore() + const bare = new MariaDbStore() + expect(await bare.deleteModuleContainerIfMatches('xchain-hub', '', '', 'id')).to.equal(false) + }) + }) +}) diff --git a/test/unit/module_operations.test/03_module_controls.test.js b/test/unit/module_operations.test/03_module_controls.test.js index 8fca6772..8e6fc71d 100644 --- a/test/unit/module_operations.test/03_module_controls.test.js +++ b/test/unit/module_operations.test/03_module_controls.test.js @@ -107,12 +107,13 @@ describe('moduleOperations', function () { expect(stubs.startContainer.calledWith('container-id-123')).to.be.true }) - it('skips module when container ID is not found', async function () { + it('rejects with a no-installed-service error when the container ID is not found', async function () { const stubs = makeStubs() stubs.db.getModuleContainerStrict.resolves(null) const ops = loadOperations(stubs) - const result = await ops.startModules({ bitcoin: { mainnet: ['xchain-encoder'] } }) - expect(result).to.be.true + const err = await ops.startModules({ bitcoin: { mainnet: ['xchain-encoder'] } }).then(() => null, e => e) + expect(err).to.be.an('error') + expect(err.message).to.match(/start matched no installed service: xchain-encoder \(bitcoin mainnet\)/) expect(stubs.startContainer.called).to.be.false }) }) diff --git a/test/unit/module_operations.test/08_uninstall_and_e2e.test.js b/test/unit/module_operations.test/08_uninstall_and_e2e.test.js index b068d305..bb89bdce 100644 --- a/test/unit/module_operations.test/08_uninstall_and_e2e.test.js +++ b/test/unit/module_operations.test/08_uninstall_and_e2e.test.js @@ -328,8 +328,10 @@ function registerFullTeardownTests() { const { filterCommandParameters } = requireFromUnit('../../src/services/config_service/filters.js') const stubs = makeStubs() const ops = loadOperations(stubs) - await ops.uninstallModules(filterCommandParameters(null, 'all', 'bitcoin', 'mainnet'), true, { all: true }) - expect(called(stubs).slice(-3)).to.deep.equal(['xchain-explorer', 'xchain-sync', 'xchain-hub']) + await ops.uninstallModules(filterCommandParameters(null, 'all', 'bitcoin', 'mainnet', + { includeSharedExplorer: false }), true, { all: true }) + expect(called(stubs).slice(-2)).to.deep.equal(['xchain-sync', 'xchain-hub']) + expect(called(stubs)).to.not.include('xchain-explorer') expect(called(stubs)).to.not.include('database') }) } diff --git a/test/unit/module_operations.test/17_module_error_paths.test.js b/test/unit/module_operations.test/17_module_error_paths.test.js index aa7ead5d..3c2cfb20 100644 --- a/test/unit/module_operations.test/17_module_error_paths.test.js +++ b/test/unit/module_operations.test/17_module_error_paths.test.js @@ -124,17 +124,18 @@ describe('moduleOperations', function () { // ------------------------------------------------------------------- - // stopModules/startModules: skip when no container + // stopModules/startModules: reject when no container // ------------------------------------------------------------------- - describe('stopModules(): skip when no container', function () { + describe('stopModules(): reject when no container is registered', function () { - it('skips when container ID is not found', async function () { + it('rejects with a no-installed-service error and stops nothing', async function () { const stubs = makeStubs() stubs.db.getModuleContainerStrict.resolves(null) const ops = loadOperations(stubs) - const result = await ops.stopModules({ bitcoin: { mainnet: ['xchain-encoder'] } }) - expect(result).to.be.true + const err = await ops.stopModules({ bitcoin: { mainnet: ['xchain-encoder'] } }).then(() => null, e => e) + expect(err).to.be.an('error') + expect(err.message).to.match(/stop matched no installed service: xchain-encoder \(bitcoin mainnet\)/) expect(stubs.stopContainer.called).to.be.false }) @@ -171,33 +172,7 @@ describe('moduleOperations', function () { }) }) -describe('moduleOperations', function () { - let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub - registerLifecycleHooks(stubs => { - ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) - }) - - - describe('execModules(): error path', function () { - - it('continues after execContainer error', async function () { - const stubs = makeStubs() - stubs.execContainer.rejects(new Error('exec failed')) - const ops = loadOperations(stubs) - const result = await ops.execModules({ bitcoin: { mainnet: ['xchain-encoder'] } }, 'ls') - expect(result).to.be.true - }) - - it('skips when container ID is null', async function () { - const stubs = makeStubs() - stubs.db.getModuleContainer.resolves(null) - const ops = loadOperations(stubs) - const result = await ops.execModules({ bitcoin: { mainnet: ['xchain-encoder'] } }, 'ls') - expect(result).to.be.true - expect(stubs.execContainer.called).to.be.false - }) - }) -}) +// execModules failure and no-match cases live in 20_exec_failure.test.js. // The post-reset bounce of decoder + indexer is the last thing reset does to them, so a // restart failure swallowed there leaves a freshly wiped service stopped while reset exits 0. diff --git a/test/unit/module_operations.test/20_exec_failure.test.js b/test/unit/module_operations.test/20_exec_failure.test.js new file mode 100644 index 00000000..386e6bbd --- /dev/null +++ b/test/unit/module_operations.test/20_exec_failure.test.js @@ -0,0 +1,91 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// A failed in-container command, or a selection that matched no installed +// container, used to print the raw Error object and still report success, so +// `exec ... && next` carried on after a command that failed or never ran. + +const { sinon, expect, makeStubs, loadOperations } = require('./helpers/harness') + +// Runs execModules over stubbed docker and registry, capturing console output +// as argument arrays so a raw Error object passed to console is visible. +async function runExec(servicesList, command, configure) { + const stubs = makeStubs() + configure(stubs) + const ops = loadOperations(stubs) + const lines = { log: [], error: [] } + sinon.stub(console, 'log').callsFake((...a) => { lines.log.push(a) }) + sinon.stub(console, 'error').callsFake((...a) => { lines.error.push(a) }) + let result + try { + result = await ops.execModules(servicesList, command) + } finally { + sinon.restore() + } + const stderr = lines.error.map(a => a.join(' ')).join('\n') + return { stubs, lines, stderr, result } +} + +// The shape execContainer rejects with: execFile's error plus the child's output. +function commandFailure(code, stderr, stdout = '') { + const err = new Error('Command failed: docker exec -i container-id-123 ls /no-such-dir') + err.code = code + err.stderr = stderr + err.stdout = stdout + return err +} + +const DECODER = { bitcoin: { regtest: ['xchain-decoder'] } } + +describe('execModules(): failure is reported, never swallowed', function () { + + it('returns false when the in-container command exits non-zero, with its exit code and stderr as the reason', async function () { + const { lines, stderr, result } = await runExec(DECODER, 'ls /no-such-dir', stubs => + stubs.execContainer.rejects(commandFailure(2, "ls: cannot access '/no-such-dir': No such file or directory\n"))) + expect(result).to.equal(false) + const printed = [...lines.log, ...lines.error] + expect(printed.some(args => args.some(a => a instanceof Error)), 'no raw Error object is printed').to.be.false + expect(stderr).to.match(/exec failed in xchain-decoder \(bitcoin regtest\): exit code 2/) + expect(stderr).to.contain("ls: cannot access '/no-such-dir'") + }) + + it('returns false when the command fails before it runs (no exit code)', async function () { + const spawnError = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }) + const { stderr, result } = await runExec(DECODER, 'ls', stubs => stubs.execContainer.rejects(spawnError)) + expect(result).to.equal(false) + expect(stderr).to.match(/exec failed in xchain-decoder \(bitcoin regtest\): spawn docker ENOENT/) + }) + + it('returns false and names the selection when no installed container matched, running nothing', async function () { + const { stubs, stderr, result } = await runExec({ bitcoin: { regtest: ['xchain-decodr'] } }, 'ls', stubs => + stubs.db.getModuleContainer.resolves(null)) + expect(result).to.equal(false) + expect(stubs.execContainer.called).to.be.false + expect(stderr).to.match(/exec matched no installed service: xchain-decodr \(bitcoin regtest\)/) + }) + + it('returns false when one of several matched containers fails, after running every one', async function () { + const { stubs, result } = await runExec({ bitcoin: { regtest: ['xchain-decoder', 'xchain-encoder'] } }, 'false', stubs => { + stubs.execContainer.onFirstCall().rejects(commandFailure(1, '')) + stubs.execContainer.onSecondCall().resolves('ok') + }) + expect(result).to.equal(false) + expect(stubs.execContainer.callCount, 'the second container still runs').to.equal(2) + }) + + it('returns true and prints the output when the command succeeds', async function () { + const { lines, result } = await runExec(DECODER, 'ls /app', stubs => stubs.execContainer.resolves('bin\nsrc')) + expect(result).to.equal(true) + expect(lines.log.some(args => args[0] === 'bin\nsrc')).to.be.true + expect(lines.error).to.have.lengthOf(0) + }) +}) diff --git a/test/unit/module_operations.test/helpers/harness.js b/test/unit/module_operations.test/helpers/harness.js index 1e2e9e73..192df8ff 100644 --- a/test/unit/module_operations.test/helpers/harness.js +++ b/test/unit/module_operations.test/helpers/harness.js @@ -74,6 +74,11 @@ function makeStubs() { // Docker-mode reset resolves root before any stop; resolves by default so only the tests about it fail it. askMariadbRootPassword: sinon.stub().resolves('stub-root'), clearHubPriceIngestWatermark: sinon.stub().resolves(true), + // The server a --db-container reset runs its DB steps on, as resolveNamedDatabaseTarget answers it. + resolveNamedDatabaseTarget: sinon.stub().callsFake(async (containerName) => ({ + containerName, containerId: 'd'.repeat(64), rootPassword: 'named-root', + databases: ['XChain_BTC_Regtest_Decoder', 'XChain_BTC_Regtest_Indexer'] + })), // The regtest re-genesis purge of the hub's cross-chain relic rows. The // statements helper feeds the never-fatal catch's operator message. purgeHubCrossChainRows: sinon.stub().resolves(true), @@ -162,6 +167,7 @@ function loadOperations(stubs, constantsOverrides = null) { resetDatabases: stubs.resetDatabases, askMariadbRootPassword: stubs.askMariadbRootPassword, clearHubPriceIngestWatermark: stubs.clearHubPriceIngestWatermark, + resolveNamedDatabaseTarget: stubs.resolveNamedDatabaseTarget, purgeHubCrossChainRows: stubs.purgeHubCrossChainRows, manualHubCrossChainPurgeStatements: stubs.manualHubCrossChainPurgeStatements, getDatabaseContainerId: stubs.getDatabaseContainerId, diff --git a/test/unit/module_operations.test/reset/17_reset_db_container_override.test.js b/test/unit/module_operations.test/reset/17_reset_db_container_override.test.js new file mode 100644 index 00000000..4c6962eb --- /dev/null +++ b/test/unit/module_operations.test/reset/17_reset_db_container_override.test.js @@ -0,0 +1,274 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// `reset --db-container` serves a stack this CLI did not install, one that runs its own MariaDB container. +// Every DB step must land on that server, its services are found by container name rather than through this +// CLI's registry, nothing is pushed to the hub or explorer, and no node datadir is ever wiped on that path. + +const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks } = require('../helpers/harness') + +const NAMED_DB = 'xchain-rail-database' +const DECODER = 'xchain-decoder' +const INDEXER = 'xchain-indexer' +// The harness names containers `${coin}-${network}-${module}`. +const SERVICE_IDS = { + 'bitcoin-regtest-xchain-decoder': 'a'.repeat(64), + 'bitcoin-regtest-xchain-indexer': 'b'.repeat(64) +} + +// Answer `docker inspect` by container name: an id, docker's own "no such object", or a given failure. +// The callback hands back an object so the harness promisify resolves the { stdout } shape node's does. +function dockerAnswersByName(stubs, answers = SERVICE_IDS) { + stubs.execFile.callsFake((cmd, args, cb) => { + if (cmd === 'docker' && args[0] === 'inspect' && args[1] === '--type') { + const name = args[args.length - 1] + const answer = answers[name] + if (answer instanceof Error) return cb(answer) + if (answer === undefined) { + const gone = new Error('Error: No such object: ' + name) + gone.stderr = gone.message + return cb(gone) + } + return cb(null, { stdout: answer + '\n' }) + } + return cb(null, '', '') + }) +} + +// Run a reset past the post-reset bounce's settle sleep, with its console output captured. +async function runReset(ops, args, logged = []) { + const log = sinon.stub(console, 'log').callsFake((...a) => logged.push(a.join(' '))) + const clock = sinon.useFakeTimers() + try { + const promise = ops.resetModules(...args) + await clock.tickAsync(6000) + return await promise + } finally { + clock.restore() + log.restore() + } +} + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container', function () { + + it('runs the database drop and the hub fence clear on the named server', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + + const result = await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }]) + + expect(result).to.be.true + sinon.assert.calledOnce(stubs.resolveNamedDatabaseTarget) + const [name, coin, network, modules, findServiceContainer] = stubs.resolveNamedDatabaseTarget.firstCall.args + expect([name, coin, network, modules]).to.deep.equal([NAMED_DB, 'bitcoin', 'regtest', [DECODER, INDEXER]]) + // The resolver finds the stack's services by container name, the source of their own database logins. + expect(await findServiceContainer(INDEXER)).to.equal(SERVICE_IDS['bitcoin-regtest-xchain-indexer']) + const target = await stubs.resolveNamedDatabaseTarget.firstCall.returnValue + expect(stubs.resetDatabases.firstCall.args).to.deep.equal(['bitcoin', 'regtest', [DECODER, INDEXER], target]) + expect(stubs.clearHubPriceIngestWatermark.firstCall.args).to.deep.equal(['bitcoin', 'regtest', target]) + // The bundled server is never asked: not its container, not its cached root. + expect(stubs.getDatabaseContainerId.called).to.be.false + expect(stubs.askMariadbRootPassword.called).to.be.false + }) + + it('stops and restarts the containers found by name, never through the registry', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + + await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }]) + + const stopped = stubs.stopContainer.args.map((a) => a[0]) + const started = stubs.startContainer.args.map((a) => a[0]) + expect(stopped).to.include.members(Object.values(SERVICE_IDS)) + expect(started).to.include.members(Object.values(SERVICE_IDS)) + expect(stopped).to.not.include('container-id-123') + expect(stubs.db.getModuleContainerStrict.called).to.be.false + }) + + it('pushes no config to the hub or explorer after the restart', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }], logged)).to.be.true + + expect(stubs.statusChanged.called).to.be.false + expect(logged.join('\n')).to.contain('Skipped the hub and explorer config push') + }) + }) +}) + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container', function () { + + for (const service of ['all', 'node', 'xchain-utxo-tracker']) { + it(`refuses '${service}' before anything is resolved, stopped or wiped`, async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, [service, 'bitcoin', 'regtest', true, false, { dbContainer: NAMED_DB }], logged)).to.be.false + + expect(stubs.resolveNamedDatabaseTarget.called).to.be.false + expect(stubs.stopContainer.called).to.be.false + expect(stubs.resetDatabases.called).to.be.false + expect(stubs.execFile.args.some((a) => a[1] && a[1][0] === 'run')).to.be.false + expect(logged.join('\n')).to.contain('No data was touched') + }) + } + + it('refuses the override in external-DB mode', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs, { EXTERNAL_DB: true }) + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }])).to.be.false + expect(stubs.resolveNamedDatabaseTarget.called).to.be.false + expect(stubs.resetDatabases.called).to.be.false + }) + }) +}) + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container', function () { + + it('refuses before stopping anything when the named server cannot be used', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + stubs.resolveNamedDatabaseTarget.rejects(new Error('xchain-rail-database holds no database XChain_BTC_Regtest_Decoder')) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }], logged)).to.be.false + + expect(stubs.stopContainer.called).to.be.false + expect(stubs.resetDatabases.called).to.be.false + expect(logged.join('\n')).to.contain('holds no database XChain_BTC_Regtest_Decoder') + }) + + // Only docker saying "no such container" is absence: a failed inspect must not let the drop run under a live decoder. + it('refuses and drops nothing when a service container cannot be inspected', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs, { + ...SERVICE_IDS, + 'bitcoin-regtest-xchain-decoder': new Error('permission denied while trying to connect to the Docker daemon') + }) + const ops = loadOperations(stubs) + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }])).to.be.false + expect(stubs.resetDatabases.called).to.be.false + }) + }) +}) + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container', function () { + + it('leaves a reset without the flag on the registry, the bundled server and the config push', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true])).to.be.true + + expect(stubs.resolveNamedDatabaseTarget.called).to.be.false + expect(stubs.resetDatabases.firstCall.args).to.deep.equal(['bitcoin', 'regtest', [DECODER, INDEXER]]) + expect(stubs.clearHubPriceIngestWatermark.firstCall.args).to.deep.equal(['bitcoin', 'regtest']) + expect(stubs.askMariadbRootPassword.called).to.be.true + expect(stubs.db.getModuleContainerStrict.called).to.be.true + expect(stubs.stopContainer.args.map((a) => a[0])).to.include('container-id-123') + expect(stubs.statusChanged.calledOnce).to.be.true + }) + }) +}) + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container --check', function () { + + it('with check, resolves the named server and stops before any stop, wipe or prompt', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + const logged = [] + + const result = await runReset(ops, [DECODER, 'bitcoin', 'regtest', false, true, { dbContainer: NAMED_DB, check: true }], logged) + + expect(result).to.be.true + sinon.assert.calledOnce(stubs.resolveNamedDatabaseTarget) + expect(stubs.stopContainer.called).to.be.false + expect(stubs.resetDatabases.called).to.be.false + expect(stubs.clearHubPriceIngestWatermark.called).to.be.false + expect(logged.join('\n')).to.contain('Nothing was stopped or changed') + }) + }) +}) + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules() with --db-container --check, auth report', function () { + const checkArgs = [DECODER, 'bitcoin', 'regtest', false, true, { dbContainer: NAMED_DB, check: true }] + + it('prints the service users and grants it would run as, and stops', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const report = ['No root login works on ' + NAMED_DB + ' (no root password), so each step runs as the stack\'s own database user:', + ' XChain_BTC_Regtest_Decoder: dec_user (DECODER_DB_USER from the xchain-indexer container), granted ALL PRIVILEGES ON `XChain_BTC_Regtest_Decoder`.*'] + stubs.resolveNamedDatabaseTarget.callsFake(async (containerName) => ({ containerName, containerId: 'd'.repeat(64), + authMode: 'service', databases: ['XChain_BTC_Regtest_Decoder'], authReport: report })) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, checkArgs, logged)).to.be.true + const out = logged.join('\n') + for (const line of report) expect(out).to.contain(line) + expect(out).to.not.contain('as root') + expect(stubs.stopContainer.called).to.be.false + expect(stubs.resetDatabases.called).to.be.false + }) + + it('says it logs in as root when a root logged in', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, checkArgs, logged)).to.be.true + expect(logged.join('\n')).to.contain('Logs in to ' + NAMED_DB + ' as root.') + }) + + it('prints no auth line on a root reset without --check', async function () { + const stubs = makeStubs() + dockerAnswersByName(stubs) + const ops = loadOperations(stubs) + const logged = [] + + expect(await runReset(ops, [DECODER, 'bitcoin', 'regtest', true, true, { dbContainer: NAMED_DB }], logged)).to.be.true + expect(logged.join('\n')).to.not.contain('as root') + }) + }) +}) diff --git a/test/unit/module_operations.test/reset_wipes/wipe_failures.test.js b/test/unit/module_operations.test/reset_wipes/wipe_failures.test.js index d6686e8f..12f4ae12 100644 --- a/test/unit/module_operations.test/reset_wipes/wipe_failures.test.js +++ b/test/unit/module_operations.test/reset_wipes/wipe_failures.test.js @@ -111,4 +111,21 @@ describe('moduleOperations', function () { expect(output).to.not.include('No data was touched.') }) }) + + describe('resetModules() a database step that fails after the stop', function () { + + it('resolves false, names the stopped services as left down and restarts nothing', async function () { + const stubs = makeStubs() + stubs.resetDatabases.rejects(new Error('Container x is paused, unpause the container before exec')) + const { result, output } = await resetCapturing(stubs, 'xchain-indexer') + expect(stubs.stopContainer.called || stubs.execFile.called).to.be.true + expect(result).to.be.false + expect(stubs.startContainer.called).to.be.false + expect(output).to.include('Aborted: clearing the decoder/indexer databases failed') + expect(output).to.include('is paused') + expect(output).to.match(/The stopped services are left down: .*xchain-indexer/) + expect(output).to.include('re-run the same reset') + expect(output).to.not.include('NOT touched: the decoder/indexer databases') + }) + }) }) diff --git a/test/unit/module_service.test/03_buildandup_healthcheck_args.test.js b/test/unit/module_service.test/03_buildandup_healthcheck_args.test.js index d741af09..83beeb8d 100644 --- a/test/unit/module_service.test/03_buildandup_healthcheck_args.test.js +++ b/test/unit/module_service.test/03_buildandup_healthcheck_args.test.js @@ -200,6 +200,7 @@ moduleSuite('installModule() singleton guard', function () { // `docker run` with a duplicate name. The guard must // detect the existing named container and return without rebuilding. const stubs = makeStubs() + stubs.probeContainerPresenceByName.resolves('exists') const VALID_ID = 'b'.repeat(64) stubs.execFile.callsFake((cmd, args, ...rest) => { const cb = typeof rest[0] === 'function' ? rest[0] : rest[1] diff --git a/test/unit/module_service.test/04_installmodule_db_module.test.js b/test/unit/module_service.test/04_installmodule_db_module.test.js index 7396d6c8..c98aa6a7 100644 --- a/test/unit/module_service.test/04_installmodule_db_module.test.js +++ b/test/unit/module_service.test/04_installmodule_db_module.test.js @@ -244,7 +244,8 @@ moduleSuite('installModule(): generic module, container version already known', killContainer: stubs.killContainer, stopContainerByName: stubs.stopContainerByName, removeContainer: stubs.removeContainer, forceRemoveContainerByName: stubs.forceRemoveContainerByName, - getStatusFromContainer: stubs.getStatusFromContainer + getStatusFromContainer: stubs.getStatusFromContainer, + probeContainerPresenceByName: sinon.stub().resolves('exists') }, './database_service': { setDatabaseParameters: sinon.stub().resolves(), setHubDatabaseParameters: sinon.stub().resolves() diff --git a/test/unit/module_service.test/05_buildandup_module_specific_port_volume_branches.test.js b/test/unit/module_service.test/05_buildandup_module_specific_port_volume_branches.test.js index 17529695..7dc295a8 100644 --- a/test/unit/module_service.test/05_buildandup_module_specific_port_volume_branches.test.js +++ b/test/unit/module_service.test/05_buildandup_module_specific_port_volume_branches.test.js @@ -172,27 +172,29 @@ moduleSuite('buildAndUp(): module-specific port/volume branches', function () {i }) -moduleSuite('buildAndUp(): module-specific port/volume branches', function () {it('runs the host-port preflight BEFORE docker build so a conflict fails fast', async function () { +moduleSuite('buildAndUp(): module-specific port/volume branches', function () {it('runs the host-port preflight BEFORE docker build and moves a busy on-request port', async function () { const stubs = makeStubs() // ENCODER_PORT/ENCODER_API_PORT both resolve to 3003 (see makeStubs' // getDefaultConfig); report that host port as already published by a - // differently-named container so assertNoHostPortConflicts rejects. - stubs.getPublishedHostPorts = sinon.stub().resolves(new Map([['3003', new Set(['other-stack-container'])]])) - let buildInvoked = false + // differently-named container. The encoder port is published on request, + // so the preflight moves it (host_port_claim.js) before anything is built. + const order = [] + stubs.getPublishedHostPorts = sinon.stub().callsFake(async () => { + order.push('preflight') + return new Map([['3003', new Set(['other-stack-container'])]]) + }) + let runArgs = null stubs.execFile.callsFake((cmd, args, ...rest) => { let opts = {}, cb if (typeof rest[0] === 'function') { cb = rest[0] } else { opts = rest[0] || {}; cb = rest[1] } - if (args[0] === 'build') { buildInvoked = true; cb(null) } - else if (args[0] === 'run') { cb(null, 'a'.repeat(64) + '\n') } + if (args[0] === 'build') { order.push('build'); cb(null) } + else if (args[0] === 'run') { runArgs = args; cb(null, 'a'.repeat(64) + '\n') } }) const ms = loadModuleService(stubs) - try { - await ms.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet') - expect.fail('Should have rejected on host port conflict') - } catch (err) { - expect(String(err)).to.include('Host port conflict') - } - expect(buildInvoked).to.be.false + await ms.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet') + expect(order.indexOf('preflight')).to.be.within(0, order.indexOf('build')) + expect(runArgs).to.include('3103:3003') + expect(runArgs).to.not.include('3003:3003') }) it('includes --restart unless-stopped for non-execution containers', async function () { @@ -243,6 +245,22 @@ moduleSuite('buildAndUp(): module-specific port/volume branches', function () {i } }) + it('leaves the container it was asked to replace running when docker build fails', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, ...rest) => { + const cb = typeof rest[0] === 'function' ? rest[0] : rest[1] + if (args[0] === 'build') { cb(new Error('build failed')) } + else { cb(null, '') } + }) + const ms = loadModuleService(stubs) + let threw = null + try { await ms.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet', 'old-id-123') } catch (err) { threw = err } + expect(String(threw)).to.include('Error creating Docker image') + expect(stubs.stopContainerByName.called).to.be.false + expect(stubs.removeContainer.called).to.be.false + expect(stubs.forceRemoveContainerByName.called).to.be.false + }) + // xchain-hub issue 23: on a buildx-less host the legacy builder died on // the module Dockerfiles' optional COPY glob after the clone and the DB // provisioning had already run. The probe has to refuse BEFORE the diff --git a/test/unit/module_service.test/07_installmodule_node_module.test.js b/test/unit/module_service.test/07_installmodule_node_module.test.js index 15450611..9b6dd5e6 100644 --- a/test/unit/module_service.test/07_installmodule_node_module.test.js +++ b/test/unit/module_service.test/07_installmodule_node_module.test.js @@ -248,7 +248,7 @@ moduleSuite('installModule(): branch switch path', function () { it('reclones wh getDefaultConfig: sinon3.stub().resolves({ ENCODER_PORT: 3003, ENCODER_API_PORT: 3003 }) }, './status_service': { statusChanged: sinon3.stub().resolves(), getStatus: sinon3.stub().resolves({}) }, - './docker_service': { killContainer: sinon3.stub().resolves(true), removeContainer: sinon3.stub().resolves(true), forceRemoveContainerByName: sinon3.stub().resolves(true), getPublishedHostPorts: sinon3.stub().resolves(new Map()), checkBuildKitAvailable: sinon3.stub().resolves(true) }, + './docker_service': { killContainer: sinon3.stub().resolves(true), removeContainer: sinon3.stub().resolves(true), forceRemoveContainerByName: sinon3.stub().resolves(true), getPublishedHostPorts: sinon3.stub().resolves(new Map()), checkBuildKitAvailable: sinon3.stub().resolves(true), probeContainerPresenceByName: sinon3.stub().resolves('gone') }, './database_service': { setDatabaseParameters: sinon3.stub().resolves(), setHubDatabaseParameters: sinon3.stub().resolves() }, './version_service': { getLocalNodeVersion: sinon3.stub().resolves(null), diff --git a/test/unit/module_service.test/08_installmodule_node_checkremotenodeversion_called_when_coin_not_in_remoteversions.test.js b/test/unit/module_service.test/08_installmodule_node_checkremotenodeversion_called_when_coin_not_in_remoteversions.test.js index 0d9eb514..b1433ed4 100644 --- a/test/unit/module_service.test/08_installmodule_node_checkremotenodeversion_called_when_coin_not_in_remoteversions.test.js +++ b/test/unit/module_service.test/08_installmodule_node_checkremotenodeversion_called_when_coin_not_in_remoteversions.test.js @@ -110,7 +110,7 @@ moduleSuite('containerExistsByName(): via singleton installModule with inspect r // getPublishedHostPorts must be stubbed: this load calls through, and // HUB_PORT below is published as a host port, so the real probe would // shell out to the host's docker and fail wherever 10000 is taken. - './docker_service': { killContainer: stubs.killContainer, stopContainerByName: stubs.stopContainerByName, removeContainer: stubs.removeContainer, forceRemoveContainerByName: stubs.forceRemoveContainerByName, getPublishedHostPorts: stubs.getPublishedHostPorts, checkBuildKitAvailable: stubs.checkBuildKitAvailable }, + './docker_service': { killContainer: stubs.killContainer, stopContainerByName: stubs.stopContainerByName, removeContainer: stubs.removeContainer, forceRemoveContainerByName: stubs.forceRemoveContainerByName, getPublishedHostPorts: stubs.getPublishedHostPorts, checkBuildKitAvailable: stubs.checkBuildKitAvailable, probeContainerPresenceByName: sinon3.stub().resolves('gone') }, './database_service': { setDatabaseParameters: sinon3.stub().resolves(), setHubDatabaseParameters: sinon3.stub().resolves() }, './version_service': { getLocalNodeVersion: sinon3.stub().resolves(null), getLocalModuleVersion: sinon3.stub().resolves(null), checkRemoteNodeVersion: sinon3.stub().resolves() }, './node_service': { buildCryptoNode: sinon3.stub().resolves(true), getCryptoNode: sinon3.stub().resolves() }, diff --git a/test/unit/module_service.test/install/force_bootstrap_installed.test.js b/test/unit/module_service.test/install/force_bootstrap_installed.test.js new file mode 100644 index 00000000..2e52192f --- /dev/null +++ b/test/unit/module_service.test/install/force_bootstrap_installed.test.js @@ -0,0 +1,100 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Every failed bootstrap restore tells the operator to "re-run install with +// XCHAIN_NODE_FORCE_BOOTSTRAP=1". On an installed module install returned before it +// ever read that flag, so the remedy did nothing. These tests drive installModule +// with an installed module and the bootstrap service stubbed, and check whether the +// restore runs again. + +const { sinon, expect, makeStubs, loadModuleService, moduleSuite } = require('../support/helpers') +const { XChainService } = require('../../../../src/config') + +const COIN = 'bitcoin' +const NETWORK = 'mainnet' + +function installedStatus(module) { + return { [COIN]: { [NETWORK]: { [module]: { container_version: '1.2.3' } } } } +} + +function loadWith({ module, force }) { + const stubs = makeStubs() + stubs.probeContainerPresenceByName.resolves('exists') + const buildAndUp = sinon.stub().resolves('new-container-id') + const bootstrap = { + FRESHNESS_EMPTY: 'empty', + forceBootstrapRequested: () => force, + utxoTrackerVolumeFreshness: sinon.stub().resolves('populated'), + mariaDbModuleFreshness: sinon.stub().resolves('populated'), + ensureBootstrapUtxoTracker: sinon.stub().resolves(true), + ensureBootstrapMariaDb: sinon.stub().resolves(true) + } + const ms = loadModuleService(stubs, null, { + '../state': { + db: stubs.db, + getRemoteModuleVersions: () => ({}), + getLastStatus: () => installedStatus(module) + }, + './bootstrap_service': bootstrap, + './version_service': { + getLocalNodeVersion: sinon.stub().resolves(null), + getLocalModuleVersion: sinon.stub().resolves('1.2.3'), + checkRemoteNodeVersion: sinon.stub().resolves() + }, + './module_service/build_and_up.js': { configureDependencies: () => {}, buildAndUp } + }) + return { ms, stubs, buildAndUp, bootstrap } +} + +moduleSuite('installModule(): XCHAIN_NODE_FORCE_BOOTSTRAP on an installed module', function () { + + it('reinstalls an installed decoder and takes the restore again when the flag is set', async function () { + const { ms, stubs, buildAndUp, bootstrap } = loadWith({ module: XChainService.XCHAIN_DECODER, force: true }) + + const result = await ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK) + + expect(result).to.equal('new-container-id') + expect(buildAndUp.calledOnce).to.equal(true) + // The running container is handed over by id, so buildAndUp stops it gracefully. + expect(buildAndUp.firstCall.args.slice(0, 4)) + .to.deep.equal([XChainService.XCHAIN_DECODER, COIN, NETWORK, 'old-container-id']) + expect(stubs.db.getModuleContainerStrict.calledWith(XChainService.XCHAIN_DECODER, COIN, NETWORK)).to.equal(true) + expect(bootstrap.ensureBootstrapMariaDb.calledOnceWith(COIN, NETWORK, XChainService.XCHAIN_DECODER)).to.equal(true) + }) + + it('takes the restore again for an installed utxo-tracker', async function () { + const { ms, bootstrap } = loadWith({ module: XChainService.XCHAIN_UTXO_TRACKER, force: true }) + + await ms.installModule(XChainService.XCHAIN_UTXO_TRACKER, COIN, NETWORK) + + expect(bootstrap.ensureBootstrapUtxoTracker.calledOnceWith(COIN, NETWORK)).to.equal(true) + }) + + it('leaves an installed module alone without the flag', async function () { + const { ms, buildAndUp, bootstrap } = loadWith({ module: XChainService.XCHAIN_INDEXER, force: false }) + + const result = await ms.installModule(XChainService.XCHAIN_INDEXER, COIN, NETWORK) + + expect(result).to.equal(false) + expect(buildAndUp.called).to.equal(false) + expect(bootstrap.ensureBootstrapMariaDb.called).to.equal(false) + }) + + it('never rebuilds a module that has no bootstrap restore, flag or not', async function () { + const { ms, buildAndUp } = loadWith({ module: XChainService.XCHAIN_ENCODER, force: true }) + + const result = await ms.installModule(XChainService.XCHAIN_ENCODER, COIN, NETWORK) + + expect(result).to.equal(false) + expect(buildAndUp.called).to.equal(false) + }) +}) diff --git a/test/unit/module_service.test/install/host_port_claim_second_install.test.js b/test/unit/module_service.test/install/host_port_claim_second_install.test.js new file mode 100644 index 00000000..7f282a6a --- /dev/null +++ b/test/unit/module_service.test/install/host_port_claim_second_install.test.js @@ -0,0 +1,106 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Every install defaults the utxo-tracker host port to 3001, so a second install on +// the same host (testnet beside mainnet) asked for a port the first already +// published and stopped on a host port conflict. A port the registry publishes only +// on request now moves to a free one; the hub and explorer ports still refuse. + +const { expect, makeStubs, loadModuleService, moduleSuite } = require('../support/helpers') +const { XChainService, HUB_MODULE_NAME } = require('../../../../src/config') +const { claimFreeHostPorts, nextFreeHostPort } = require('../../../../src/services/module_service/host_port_claim') + +const FIRST_TRACKER = 'xchain-node-bitcoin-mainnet-xchain-utxo-tracker' +const SECOND_TRACKER = 'xchain-node-bitcoin-testnet-xchain-utxo-tracker' + +function trackerEnv() { + return { UTXO_TRACKER_PORT: 3001, UTXO_TRACKER_API_PORT: 3001 } +} + +function loadWithPublished(entries) { + const stubs = makeStubs() + stubs.getPublishedHostPorts.resolves(new Map(entries.map(([port, name]) => [port, new Set([name])]))) + return loadModuleService(stubs) +} + +function registerSecondInstallTests() { + it('publishes a second install\'s utxo-tracker on a free host port', async function () { + const ms = loadWithPublished([['3001', FIRST_TRACKER]]) + const env = trackerEnv() + const portArgs = ['-p', '3001:3001'] + + await ms.assertNoHostPortConflicts(portArgs, SECOND_TRACKER, { module: XChainService.XCHAIN_UTXO_TRACKER, environmentVariables: env }) + + expect(portArgs).to.deep.equal(['-p', '3101:3001']) + expect(env.UTXO_TRACKER_PORT).to.equal(3101) + expect(env.UTXO_TRACKER_API_PORT).to.equal(3001, 'the container side is untouched') + }) + + it('skips host ports already taken by a third install', async function () { + const ms = loadWithPublished([['3001', FIRST_TRACKER], ['3101', 'xchain-node-litecoin-mainnet-xchain-utxo-tracker']]) + const env = trackerEnv() + const portArgs = ['-p', '3001:3001'] + + await ms.assertNoHostPortConflicts(portArgs, SECOND_TRACKER, { module: XChainService.XCHAIN_UTXO_TRACKER, environmentVariables: env }) + + expect(portArgs).to.deep.equal(['-p', '3201:3001']) + }) + + it('leaves a free port alone, and does not count its own container as a holder', async function () { + const ms = loadWithPublished([['3001', SECOND_TRACKER]]) + const env = trackerEnv() + const portArgs = ['-p', '3001:3001'] + + await ms.assertNoHostPortConflicts(portArgs, SECOND_TRACKER, { module: XChainService.XCHAIN_UTXO_TRACKER, environmentVariables: env }) + + expect(portArgs).to.deep.equal(['-p', '3001:3001']) + expect(env.UTXO_TRACKER_PORT).to.equal(3001) + }) +} + +function registerRefusalTests() { + it('still refuses a conflict when the caller does not opt in', async function () { + const ms = loadWithPublished([['3001', FIRST_TRACKER]]) + await ms.assertNoHostPortConflicts(['-p', '3001:3001'], SECOND_TRACKER) + .then(() => { throw new Error('expected a host port conflict') }, + (err) => expect(err.message).to.match(/host port 3001 is already published by/)) + }) + + it('still refuses a conflict on the hub port, which the CLI dials on 127.0.0.1', async function () { + const ms = loadWithPublished([['10000', 'some-other-hub']]) + const env = { HUB_PORT: 10000 } + const portArgs = ['-p', '10000:10000'] + await ms.assertNoHostPortConflicts(portArgs, 'xchain-node-xchain-hub', { module: HUB_MODULE_NAME, environmentVariables: env }) + .then(() => { throw new Error('expected a host port conflict') }, + (err) => expect(err.message).to.match(/host port 10000 is already published by: some-other-hub/)) + expect(portArgs).to.deep.equal(['-p', '10000:10000']) + }) +} + +function registerPickTests() { + it('steps by 100 and gives up past the last port', function () { + expect(nextFreeHostPort(3001, new Set(['3101']))).to.equal(3201) + expect(nextFreeHostPort(65500, new Set())).to.equal(null) + }) + + it('changes nothing without a claim', function () { + const conflicts = [{ hostPort: '3001', holders: [FIRST_TRACKER] }] + expect(claimFreeHostPorts(conflicts, ['-p', '3001:3001'], ['3001'], new Map(), null)).to.equal(conflicts) + expect(conflicts).to.have.length(1) + }) +} + +moduleSuite('assertNoHostPortConflicts(): a second install on the same host', function () { + registerSecondInstallTests() + registerRefusalTests() + registerPickTests() +}) diff --git a/test/unit/module_service.test/install/install_presence_gate.test.js b/test/unit/module_service.test/install/install_presence_gate.test.js new file mode 100644 index 00000000..a80dab99 --- /dev/null +++ b/test/unit/module_service.test/install/install_presence_gate.test.js @@ -0,0 +1,95 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { sinon, expect, makeStubs, loadModuleService, moduleSuite } = require('../support/helpers') +const { XChainService } = require('../../../../src/config') + +const COIN = 'bitcoin' +const NETWORK = 'mainnet' + +function loadForPresence({ presence, force = false, registryId = 'registered-id' }) { + const stubs = makeStubs() + stubs.probeContainerPresenceByName.resolves(presence) + stubs.db.getModuleContainerStrict.resolves(registryId) + const buildAndUp = sinon.stub().resolves('new-id') + const bootstrap = { + FRESHNESS_EMPTY: 'empty', + forceBootstrapRequested: () => force, + utxoTrackerVolumeFreshness: sinon.stub().resolves('populated'), + mariaDbModuleFreshness: sinon.stub().resolves('populated'), + ensureBootstrapUtxoTracker: sinon.stub().resolves(), + ensureBootstrapMariaDb: sinon.stub().resolves() + } + const ms = loadModuleService(stubs, null, { + '../state': { db: stubs.db, getRemoteModuleVersions: () => ({}), getLastStatus: () => ({}) }, + './bootstrap_service': bootstrap, + './version_service': { + getLocalNodeVersion: sinon.stub().resolves(null), + getLocalModuleVersion: sinon.stub().resolves('1.0.0'), + checkRemoteNodeVersion: sinon.stub().resolves() + }, + './module_service/build_and_up.js': { configureDependencies: () => {}, buildAndUp } + }) + return { ms, stubs, buildAndUp } +} + +async function expectRejection(promise, message) { + try { + await promise + expect.fail('expected rejection') + } catch (err) { + expect(err.message).to.include(message) + } +} + +moduleSuite('installModule(): Docker presence gate', function () { + it('leaves a live decoder unchanged when cached status has no version', async function () { + const { ms, buildAndUp } = loadForPresence({ presence: 'exists' }) + expect(await ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK)).to.equal(false) + expect(buildAndUp.called).to.equal(false) + }) + + it('refuses an ambiguous decoder probe before clone or build', async function () { + const { ms, stubs, buildAndUp } = loadForPresence({ presence: 'unknown' }) + await expectRejection(ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK), 'nothing was changed') + expect(buildAndUp.called).to.equal(false) + expect(stubs.execFile.called).to.equal(false) + }) + + it('builds a decoder only after Docker confirms it is gone', async function () { + const { ms, buildAndUp } = loadForPresence({ presence: 'gone' }) + expect(await ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK)).to.equal('new-id') + expect(buildAndUp.calledOnce).to.equal(true) + }) + + it('refuses an ambiguous singleton probe', async function () { + const { ms, buildAndUp } = loadForPresence({ presence: 'unknown' }) + await expectRejection(ms.installModule('xchain-hub', null, null), 'Docker could not confirm') + expect(buildAndUp.called).to.equal(false) + }) + + it('hands forced bootstrap the strictly resolved container id', async function () { + const { ms, stubs, buildAndUp } = loadForPresence({ presence: 'exists', force: true }) + await ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK) + expect(stubs.db.getModuleContainerStrict.calledOnceWith(XChainService.XCHAIN_DECODER, COIN, NETWORK)).to.equal(true) + expect(buildAndUp.firstCall.args[3]).to.equal('registered-id') + }) + + it('refuses forced bootstrap when the strict registry lookup fails', async function () { + const { ms, stubs, buildAndUp } = loadForPresence({ presence: 'exists', force: true }) + stubs.db.getModuleContainerStrict.rejects(new Error('registry unavailable')) + await expectRejection(ms.installModule(XChainService.XCHAIN_DECODER, COIN, NETWORK), 'registry unavailable') + expect(buildAndUp.called).to.equal(false) + }) + + it('preserves an explicit rebuild handover without probing presence', async function () { + const { ms, stubs, buildAndUp } = loadForPresence({ presence: 'exists' }) + await ms.installModule(XChainService.XCHAIN_ENCODER, COIN, NETWORK, false, 'menu-selected-id') + expect(stubs.probeContainerPresenceByName.called).to.equal(false) + expect(buildAndUp.firstCall.args[3]).to.equal('menu-selected-id') + }) +}) diff --git a/test/unit/module_service.test/support/helpers.js b/test/unit/module_service.test/support/helpers.js index 27fa9dbd..9952b260 100644 --- a/test/unit/module_service.test/support/helpers.js +++ b/test/unit/module_service.test/support/helpers.js @@ -44,6 +44,7 @@ function makeStubs() { db: { setModuleContainer: sinon.stub().resolves(true), getModuleContainer: sinon.stub().resolves('old-container-id'), + getModuleContainerStrict: sinon.stub().resolves('old-container-id'), deleteModuleContainer: sinon.stub().resolves('removed-id') }, statusChanged: sinon.stub().resolves(), @@ -56,6 +57,7 @@ function makeStubs() { // No published host ports by default → buildAndUp's pre-flight conflict // check is a no-op. Conflict tests override this with a populated Map. getPublishedHostPorts: sinon.stub().resolves(new Map()), + probeContainerPresenceByName: sinon.stub().resolves('gone'), // buildx present by default → buildAndUp's BuildKit probe passes. The // legacy-builder tests override this with a rejection. checkBuildKitAvailable: sinon.stub().resolves(true) @@ -138,7 +140,8 @@ function makeBaseProxies(stubs, configServiceStub) { forceRemoveContainerByName: stubs.forceRemoveContainerByName, getStatusFromContainer: stubs.getStatusFromContainer, getPublishedHostPorts: stubs.getPublishedHostPorts, - checkBuildKitAvailable: stubs.checkBuildKitAvailable + checkBuildKitAvailable: stubs.checkBuildKitAvailable, + probeContainerPresenceByName: stubs.probeContainerPresenceByName }, './database_service': { setDatabaseParameters: sinon.stub().resolves(), setHubDatabaseParameters: sinon.stub().resolves() diff --git a/test/unit/nightly_e2e_workflow.test/two_stack_legs.test.js b/test/unit/nightly_e2e_workflow.test/two_stack_legs.test.js index 0eb502e1..2c0a7a5f 100644 --- a/test/unit/nightly_e2e_workflow.test/two_stack_legs.test.js +++ b/test/unit/nightly_e2e_workflow.test/two_stack_legs.test.js @@ -47,7 +47,7 @@ function loadSteps() { } return { ports: find('Publish distinct host ports'), boot: find('Boot the regtest stack'), db: find('Start headless MariaDB'), - validator: find('Initialize the validator identity'), + validator: find('Initialize the validator identity'), seed: find('Enable the regtest validator seed'), // Step order matters for the validator step: it must precede the install. order: steps.map(s => s.name || ''), } @@ -78,14 +78,32 @@ function runStep(step, env) { const bin = path.join(dir, 'stub-bin') fs.mkdirSync(bin) const log = path.join(dir, 'node-calls.log') - fs.writeFileSync(path.join(bin, 'node'), '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >> "' + log + '"\n', { mode: 0o755 }) + const envLog = path.join(dir, 'node-env.log') + fs.writeFileSync(path.join(bin, 'node'), '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >> "' + log + '"\nprintf "%s\\n" "${XDEX_SEED_LOCAL_VALIDATOR-unset}" >> "' + envLog + '"\n', { mode: 0o755 }) const stdout = execFileSync('bash', ['-e', '-c', step.run], { cwd: dir, env: Object.assign({ PATH: bin + ':' + process.env.PATH, HOME: dir }, env), encoding: 'utf8', }) const calls = fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean) : [] - return { dir, calls, stdout } + const callEnvs = fs.existsSync(envLog) ? fs.readFileSync(envLog, 'utf8').trim().split('\n').filter(Boolean) : [] + return { dir, calls, callEnvs, stdout } +} + +function runStepWithGithubEnv(step, coin, label) { + const githubEnv = path.join(os.tmpdir(), 'nightly-e2e-' + label + '-env-' + process.pid + '-' + coin) + fs.writeFileSync(githubEnv, '') + const out = runStep(step, { COIN: coin, GITHUB_ENV: githubEnv }) + out.exported = parseConfigFile(githubEnv) + return out +} + +function runValidatorStep(coin) { + return runStepWithGithubEnv(steps.validator, coin, 'github') +} + +function runSeedStep(coin) { + return runStepWithGithubEnv(steps.seed, coin, 'seed') } let steps @@ -160,64 +178,65 @@ function registerCoinLegChecks(coin) { }) } -function registerValidatorModeChecks() { - // The bridged credit needs a hub that FINALIZES transfers, which a standalone - // hub never does: startCrossChain returns before constructing - // CrossChainBridgeEngine without a peerManager, and even with an identity the - // engine signs against the cross_chain capability set, empty on a fresh - // regtest until XDEX_SEED_LOCAL_VALIDATOR=1 seeds the hub's own key into it - // (xchain-hub src/cross_chain/bridge/plumbing.js resolveCapabilityValidators). - // Runs 35115449692 and 35115452598 got the BTC lock valid and debited with no - // credit on the destination for exactly this reason. - describe('validator mode on the two-stack legs', function () { - function runValidatorStep(coin) { - const githubEnv = path.join(os.tmpdir(), 'nightly-e2e-github-env-' + process.pid + '-' + coin) - fs.writeFileSync(githubEnv, '') - const out = runStep(steps.validator, { COIN: coin, GITHUB_ENV: githubEnv }) - out.exported = parseConfigFile(githubEnv) - return out - } - - it('runs the validator init for every non-bitcoin leg, and for bitcoin only on the opt-in input', function () { - expect(steps.validator.if).to.equal("github.event.inputs.validator == 'true' || env.COIN != 'bitcoin'") - }) +function registerValidatorContractChecks() { + it('runs the validator init for every non-bitcoin leg, and for bitcoin only on the opt-in input', function () { + expect(steps.validator.if).to.equal("github.event.inputs.validator == 'true' || env.COIN != 'bitcoin'") + }) - it('initializes the identity BEFORE the ports and boot steps, which render the hub env from it', function () { - const at = (prefix) => steps.order.findIndex(n => n.startsWith(prefix)) - expect(at('Initialize the validator identity')).to.be.below(at('Publish distinct host ports')) - expect(at('Publish distinct host ports')).to.be.below(at('Boot the regtest stack')) - }) + it('initializes the identity BEFORE the ports and boot steps, which render the hub env from it', function () { + const at = (prefix) => steps.order.findIndex(n => n.startsWith(prefix)) + expect(at('Initialize the validator identity')).to.be.below(at('Publish distinct host ports')) + expect(at('Publish distinct host ports')).to.be.below(at('Boot the regtest stack')) + expect(at('Boot the regtest stack')).to.be.below(at('Enable the regtest validator seed')) + expect(at('Enable the regtest validator seed')).to.be.below(at('Select this shard')) + }) - for (const coin of ['litecoin', 'dogecoin']) { - it(coin + ': inits a cross_chain-capable identity and exports the seed so the hub finalizes the gas lock', function () { - const { calls, exported } = runValidatorStep(coin) - expect(calls[0]).to.match(/^src\/index\.js validator init --network regtest --oracle-epoch-start \d+ --capabilities [a-z_,]+$/) - expect(calls[0].split('--capabilities ')[1].split(',')).to.include('cross_chain') - expect(calls[calls.length - 1]).to.equal('src/index.js validator status') - // The indexer URLs ride the same export: the hub's cross-chain - // engines resolve them once at start, before either stack exists - // on this runner, so the configs table cannot supply them in time. - const code = { litecoin: 'LTC', dogecoin: 'DOGE' }[coin] - // The confirmation depths ride it too: at the hub's default six - // BTC blocks nothing on the runner mines behind the lock, so the - // engine held the leg "below depth 6" for the whole credit wait - // (run 35124072478). The hub clamps these up off regtest. - expect(exported).to.deep.equal({ - HUB_NETWORK: 'regtest', ORACLE_MIN_SUBMISSIONS: '1', XDEX_SEED_LOCAL_VALIDATOR: '1', - BTC_INDEXER_API_URL: 'http://xchain-node-bitcoin-regtest-xchain-indexer:3004', - BTC_INDEXER_URL: 'http://xchain-node-bitcoin-regtest-xchain-indexer:3004', - [code + '_INDEXER_URL']: 'http://xchain-node-' + coin + '-regtest-xchain-indexer:3004', - XCHAIN_CONFIRMATIONS_BTC: '1', XCHAIN_CONFIRMATIONS_LTC: '1', XCHAIN_CONFIRMATIONS_DOGE: '1', - }) - }) - } + it('gates validator seeding off the bitcoin leg', function () { + expect(steps.seed.if).to.equal("env.COIN != 'bitcoin'") + }) +} - it('bitcoin (opt-in): keeps the identity but never seeds, so the input changes nothing beyond the price regime it documents', function () { - const { calls, exported } = runValidatorStep('bitcoin') +function registerNonBitcoinValidatorChecks() { + for (const coin of ['litecoin', 'dogecoin']) { + it(coin + ': inits a cross_chain-capable identity without seeding it during stack boot', function () { + const { calls, callEnvs, exported } = runValidatorStep(coin) expect(calls[0]).to.match(/^src\/index\.js validator init --network regtest --oracle-epoch-start \d+ --capabilities [a-z_,]+$/) + expect(calls[0].split('--capabilities ')[1].split(',')).to.include('cross_chain') expect(calls[calls.length - 1]).to.equal('src/index.js validator status') - expect(exported).to.deep.equal({ HUB_NETWORK: 'regtest', ORACLE_MIN_SUBMISSIONS: '1' }) + expect(callEnvs).to.deep.equal(['unset', 'unset']) + const code = { litecoin: 'LTC', dogecoin: 'DOGE' }[coin] + expect(exported).to.deep.equal({ + HUB_NETWORK: 'regtest', ORACLE_MIN_SUBMISSIONS: '1', + BTC_INDEXER_API_URL: 'http://xchain-node-bitcoin-regtest-xchain-indexer:3004', + BTC_INDEXER_URL: 'http://xchain-node-bitcoin-regtest-xchain-indexer:3004', + [code + '_INDEXER_URL']: 'http://xchain-node-' + coin + '-regtest-xchain-indexer:3004', + XCHAIN_CONFIRMATIONS_BTC: '1', XCHAIN_CONFIRMATIONS_LTC: '1', XCHAIN_CONFIRMATIONS_DOGE: '1', + }) }) + + it(coin + ': seeds and recreates the hub only after both stacks are ready', function () { + const { calls, callEnvs, exported } = runSeedStep(coin) + expect(exported).to.deep.equal({ XDEX_SEED_LOCAL_VALIDATOR: '1' }) + expect(calls).to.deep.equal(['src/index.js recreate xchain-hub']) + expect(callEnvs).to.deep.equal(['1']) + }) + } +} + +function registerBitcoinValidatorCheck() { + it('bitcoin (opt-in): keeps the identity but never seeds, so the input changes nothing beyond the price regime it documents', function () { + const { calls, exported } = runValidatorStep('bitcoin') + expect(calls[0]).to.match(/^src\/index\.js validator init --network regtest --oracle-epoch-start \d+ --capabilities [a-z_,]+$/) + expect(calls[calls.length - 1]).to.equal('src/index.js validator status') + expect(exported).to.deep.equal({ HUB_NETWORK: 'regtest', ORACLE_MIN_SUBMISSIONS: '1' }) + }) +} + +function registerValidatorModeChecks() { + describe('validator mode on the two-stack legs', function () { + registerValidatorContractChecks() + registerNonBitcoinValidatorChecks() + registerBitcoinValidatorCheck() }) } diff --git a/test/unit/precheck.test.js b/test/unit/precheck.test.js index 0f6aef5d..5319b931 100644 --- a/test/unit/precheck.test.js +++ b/test/unit/precheck.test.js @@ -276,3 +276,32 @@ describe('preCheck(): hub/explorer config push @regression', function () { expect(updateExplorer.calledOnce).to.be.true }) }) + +// A read-only command (ps, tail, logs, monitor) must never change a container's run +// state. installHubModule ends in useInstalledHub, which restarts a hub whose State is +// 'exited', so running it for `ps` started a hub the operator had stopped and then +// printed the stale 'exited' status. Read-only commands pass syncHubConfig=false, and +// that must also skip the hub install; mutating commands still install. +describe('preCheck(): read-only commands never install or restart the hub @regression', function () { + + it('does not call installHubModule or updateHub when syncHubConfig is false', async function () { + const installHubModule = sinon.stub().resolves() + const updateHub = sinon.stub().resolves() + const { precheck } = loadPrecheck({ installHubModule, updateHub }) + + await precheck.preCheck(false, false, null, false) + + expect(installHubModule.called).to.be.false + expect(updateHub.called).to.be.false + }) + + it('still installs the hub once for a mutating command (syncHubConfig true)', async function () { + const installHubModule = sinon.stub().resolves() + const { precheck } = loadPrecheck({ installHubModule }) + + await precheck.preCheck(false, true, 'v1.2.3', false) + + expect(installHubModule.calledOnce).to.be.true + expect(installHubModule.firstCall.args[0]).to.equal('v1.2.3') + }) +}) diff --git a/test/unit/precheck.test/04_a_reset_on_another_stacks_db_server_stays_local.test.js b/test/unit/precheck.test/04_a_reset_on_another_stacks_db_server_stays_local.test.js new file mode 100644 index 00000000..6bf54668 --- /dev/null +++ b/test/unit/precheck.test/04_a_reset_on_another_stacks_db_server_stays_local.test.js @@ -0,0 +1,81 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// `reset --db-container` resets a stack this CLI did not install. Its preCheck stops after the docker and +// directory checks: provisioning the bundled DB, scanning containers into the registry, installing the hub and +// pushing config would all act on this CLI's own stack, and the push would overwrite the hub's and explorer's +// view of the other stack with this CLI's DB hosts. + +const sinon = require('sinon') +const { configStub } = require('../../helpers/config_stub') +const { expect } = require('chai') +const proxyquire = require('proxyquire').noCallThru() + +function loadPrecheck() { + const stubs = { + checkDockerInstalledAndReachable: sinon.stub().resolves(), + createDatabase: sinon.stub().resolves(), + buildDatabaseModule: sinon.stub().resolves(), + ensureXchainNodeAccess: sinon.stub().resolves({ user: 'u', password: 'p', database: 'xchain_node' }), + scanAndRegisterModules: sinon.stub().resolves(), + getStatus: sinon.stub().resolves(), + installHubModule: sinon.stub().resolves(), + updateHub: sinon.stub().resolves(), + updateExplorer: sinon.stub().resolves() + } + const precheck = proxyquire('../../../src/precheck.js', { + 'fs': { existsSync: () => true, mkdirSync: () => {} }, + './config': configStub({ + dataDir: '/tmp/x', moduleDir: '/tmp/x', tmpDir: '/tmp/x', containersFilesDir: '/tmp/x', EXTERNAL_DB: false + }), + './state': { db: { createDatabase: stubs.createDatabase }, isVerbose: () => false }, + './utils/helpers': { redactSecrets: (e) => e }, + './services/docker_service': { + checkDockerInstalledAndReachable: stubs.checkDockerInstalledAndReachable, + createDockerNetwork: sinon.stub().resolves(), + checkContainerdDataRootRelocation: sinon.stub().resolves(null), + checkMemoryLimitSupport: sinon.stub().resolves(null) + }, + './services/config_service': { getDockerNetwork: () => 'xchain', applyHubApiKeyFromSidecar: sinon.stub().resolves() }, + './services/version_service': { checkAllRemoteVersions: sinon.stub().resolves() }, + './services/status_service': { getStatus: stubs.getStatus }, + './services/hub_service': { installHubModule: stubs.installHubModule, updateHub: stubs.updateHub, isHubAnswering: sinon.stub().resolves(true) }, + './services/explorer_service': { updateExplorer: stubs.updateExplorer }, + './services/database_service': { + buildDatabaseModule: stubs.buildDatabaseModule, + ensureXchainNodeAccess: stubs.ensureXchainNodeAccess, + getDatabaseHostPort: sinon.stub().resolves(13306), + getExternalDbConfig: sinon.stub().resolves({}) + }, + './services/discovery_service': { scanAndRegisterModules: stubs.scanAndRegisterModules } + }) + return { precheck, stubs } +} + +const SHARED_STEPS = ['buildDatabaseModule', 'ensureXchainNodeAccess', 'createDatabase', 'scanAndRegisterModules', + 'getStatus', 'installHubModule', 'updateHub', 'updateExplorer'] + +describe('preCheck: a reset on another stack\'s DB server stays local @regression', function () { + + it('runs only the docker check, and none of the bundled DB, registry, hub or push steps', async function () { + const { precheck, stubs } = loadPrecheck() + expect(await precheck.preCheck(false, true, null, false, { localOnly: true })).to.be.true + expect(stubs.checkDockerInstalledAndReachable.calledOnce).to.be.true + for (const step of SHARED_STEPS) expect(stubs[step].called, step).to.be.false + }) + + it('still runs every step when the option is absent', async function () { + const { precheck, stubs } = loadPrecheck() + await precheck.preCheck(false, true, null, false) + for (const step of SHARED_STEPS) expect(stubs[step].calledOnce, step).to.be.true + }) +}) diff --git a/test/unit/publish_bootstraps_refusal_reason.test.js b/test/unit/publish_bootstraps_refusal_reason.test.js index a887a0bb..ea980d54 100644 --- a/test/unit/publish_bootstraps_refusal_reason.test.js +++ b/test/unit/publish_bootstraps_refusal_reason.test.js @@ -38,6 +38,7 @@ function runWithCreateOutput(binDir, workDir, lines) { env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, + TMPDIR: path.join(workDir, 'unavailable-host-tmp'), STAGE_DIR: path.join(workDir, 'stage'), TMP_DIR: path.join(workDir, 'tmp'), LOCK_FILE: path.join(workDir, 'publish.lock'), diff --git a/test/unit/secret_env.test.js b/test/unit/secret_env.test.js index 4a45d068..2f1bc478 100644 --- a/test/unit/secret_env.test.js +++ b/test/unit/secret_env.test.js @@ -66,6 +66,7 @@ function makeMemoryConfigService(initialFiles = {}, { dbContainerId = null, exte readFileSync: (p) => files[p] != null ? String(files[p]) : '', writeFileSync: (p, body) => { files[p] = String(body) }, appendFileSync: (p, body) => { files[p] = (files[p] || '') + String(body) }, + renameSync: (from, to) => { files[to] = files[from]; delete files[from] }, statSync: (p) => ({ size: String(files[p] || '').length }), openSync: () => 1, readSync: () => 0, @@ -344,7 +345,7 @@ describe('secret-env', function () { const config = await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') expect(config['NODE_PASSWORD']).to.equal('inmainfile') expect(files[coinMain]).to.not.include('inmainfile') - expect(files[coinSidecar]).to.include('NODE_PASSWORD=inmainfile') + expect(files[coinSidecar]).to.include('NODE_SECRET=inmainfile') }) }) }) diff --git a/test/unit/status_reorg_halt.test.js b/test/unit/status_reorg_halt.test.js index 17dc8c92..62a26f5c 100644 --- a/test/unit/status_reorg_halt.test.js +++ b/test/unit/status_reorg_halt.test.js @@ -145,6 +145,12 @@ describe('ps: indexer STALL surface', function () { const parse = note('parse_exception: TypeError: boom', 'wedged') expect(parse).to.not.match(/hub-mirror barrier/) expect(parse).to.match(/same exception hit the same block twice/).and.to.match(/xchain-node logs xchain-indexer bitcoin mainnet/) + const needsGo = note('retroactive_replay_needs_go', 'operator_action_hold') + expect(needsGo).to.not.match(/no specific remedy/) + expect(needsGo).to.match(/operator go row/).and.to.match(/exact go command/) + const diverged = note('retroactive_replay_diverged', 'operator_action_hold') + expect(diverged).to.not.match(/no specific remedy/) + expect(diverged).to.match(/data fault/).and.to.match(/hub resync clears/).and.to.match(/rewitness command/) }) it('an unknown reason gets the hub-mirror line only when its word ends in _barrier', function () { diff --git a/test/unit/status_service.test/get_status_with_installed_modules.test.js b/test/unit/status_service.test/get_status_with_installed_modules.test.js index adedab9c..845a3f21 100644 --- a/test/unit/status_service.test/get_status_with_installed_modules.test.js +++ b/test/unit/status_service.test/get_status_with_installed_modules.test.js @@ -121,7 +121,7 @@ describe('StatusService: getStatus() with installed modules', function () { it('removes module from installedModules AND reconciles the registry when docker confirms the container is gone', async function () { const installedModulesObj = {} - const deleteModuleContainer = sinon.stub().resolves(true) + const deleteModuleContainerIfMatches = sinon.stub().resolves(true) const state = makeStateStub({ isStatusUpdated: sinon.stub().returns(false), getInstalledModules: sinon.stub().callsFake(() => installedModulesObj), @@ -133,7 +133,7 @@ describe('StatusService: getStatus() with installed modules', function () { getAllModuleContainers: sinon.stub().resolves([ { module: 'xchain-encoder', coin: 'bitcoin', network: 'mainnet', container_id: 'bbb' } ]), - deleteModuleContainer + deleteModuleContainerIfMatches } }) @@ -147,7 +147,8 @@ describe('StatusService: getStatus() with installed modules', function () { const coinNetModules = (result.bitcoin || {})[`mainnet`] || {} expect(coinNetModules['xchain-encoder']).to.be.undefined // ...and the persistent registry row reconciled, not only in-memory status. - expect(deleteModuleContainer.calledOnceWith('xchain-encoder', 'bitcoin', 'mainnet')).to.be.true + expect(deleteModuleContainerIfMatches.calledOnceWith( + 'xchain-encoder', 'bitcoin', 'mainnet', 'bbb')).to.be.true }) }) @@ -166,7 +167,7 @@ describe('StatusService: getStatus() with installed modules', function () { getAllModuleContainers: sinon.stub().resolves([ { module: 'xchain-encoder', coin: 'bitcoin', network: 'mainnet', container_id: 'ccc' } ]), - deleteModuleContainer: sinon.stub().resolves(true) + deleteModuleContainerIfMatches: sinon.stub().resolves(true) } }) @@ -183,7 +184,7 @@ describe('StatusService: getStatus() with installed modules', function () { it('does NOT prune or touch the registry on a transient inspect failure (daemon unreachable)', async function () { const installedModulesObj = {} - const deleteModuleContainer = sinon.stub().resolves(true) + const deleteModuleContainerIfMatches = sinon.stub().resolves(true) const state = makeStateStub({ isStatusUpdated: sinon.stub().returns(false), getInstalledModules: sinon.stub().callsFake(() => installedModulesObj), @@ -195,7 +196,7 @@ describe('StatusService: getStatus() with installed modules', function () { getAllModuleContainers: sinon.stub().resolves([ { module: 'xchain-encoder', coin: 'bitcoin', network: 'mainnet', container_id: 'ddd' } ]), - deleteModuleContainer + deleteModuleContainerIfMatches } }) @@ -211,7 +212,7 @@ describe('StatusService: getStatus() with installed modules', function () { expect(mod).to.exist expect(mod.status.State.Status).to.equal('unknown') // ...and no registry row is deleted on an ambiguous/transient failure. - expect(deleteModuleContainer.called).to.be.false + expect(deleteModuleContainerIfMatches.called).to.be.false }) }) diff --git a/test/unit/stop_budget_service_restart.test.js b/test/unit/stop_budget_service_restart.test.js index 4eb6cf87..ecbc6541 100644 --- a/test/unit/stop_budget_service_restart.test.js +++ b/test/unit/stop_budget_service_restart.test.js @@ -98,6 +98,7 @@ describe('StopBudgetService restartModuleContainer() failures', function () { .then(() => null, e => e) expect(err).to.be.an('error') expect(err.message).to.equal('xchain-indexer was stopped but failed to start again (port is already allocated)') + expect(err.containerStopped).to.equal(true) }) it('never starts a container whose stop failed, and passes docker\'s error through', async function () {