feat(keepers): v2 expiry keeper, engine keeper instance, pager checks (#258)

.github/workflows/services-perp-feeds.yml

@@ -1,6 +1,6 @@
 # .github/workflows/services-perp-feeds.yml
 # The USDCcNGN-PERP price publishers, and the cNGN future's forward publisher (forward-main.ts). The
-# anvil suites run too: they deploy the real feed contracts (and FutureAssetV1, the live series the keepers run) from risk-core and check
+# anvil suites run too: they deploy the real feed contracts (and FutureAssetV1, the live v1 series, and the v2 FutureAsset) from risk-core and check
 # they accept what the service signs, so they need forge and anvil.
 name: services-perp-feeds
 on:
@@ -48,7 +48,7 @@ jobs:
       - run: pnpm --filter @numo/kms-signer build
       - run: pnpm --filter perp-feeds run check
       - run: pnpm --filter perp-feeds run build
-      - run: forge build src/feeds/LyraSpotFeed.sol src/feeds/LyraSpotDiffFeed.sol src/feeds/LyraForwardFeed.sol src/periphery/OracleDataSubmitter.sol src/legacy/v1/FutureAssetV1.sol src/SubAccounts.sol
+      - run: forge build src/feeds/LyraSpotFeed.sol src/feeds/LyraSpotDiffFeed.sol src/feeds/LyraForwardFeed.sol src/periphery/OracleDataSubmitter.sol src/legacy/v1/FutureAssetV1.sol src/assets/FutureAsset.sol src/SubAccounts.sol
         working-directory: contracts/risk-core
       - run: |
           anvil --port 8599 --silent &

contracts/risk-core/scripts/ops/engine-keeper-v2.env

@@ -0,0 +1,39 @@
+# Template for /etc/numo/engine-keeper-v2.env (numo-engine-keeper-v2.service): the liquidation keeper for MarginEngine v2.
+# NOT INSTALLED (#258). Non-secret: KEEPER_KEY, RPC_URL and ALERT_WEBHOOK_URL come from SSM in the unit's ExecStart.
+# Addresses from risk-core deployments/8453/MARGIN_ENGINE_V2.json. v1's keeper (engine-keeper.env) is unchanged and runs
+# beside it until USDC-cNGN-30NOV26 settles.
+#
+# Per engine, not per series: a v2 series (EURC-30NOV26 once listed) is one more address in FUTURE.
+CHAIN_ID=8453
+# TODO before install: the keeper's funding account under v2, created by the v2 keeper EOA
+# (SubAccounts.createAccount(eoa, 0x0705…0950)) and funded with USDC cash. The keeper refuses to start without it.
+KEEPER_ACCOUNT=
+SUB_ACCOUNTS=0x7019244E25FA416e6Ca2ed2F3cA25277aef72843
+MANAGER=0x0705219a1594d5Fc5dC2573da4082cAf3e1B0950
+AUCTION=0xd07176B006cE8246e930d37b8bc44B1154f8c4cE
+CASH=0xA74E49b4Ed7cb176bc02ef4D8a1A3240C9aD4272
+# No PERP: v2 lists none. Comma-separated, one per series.
+FUTURE=0xcACEe66ad41fD1472a47D7D5Fe372FFbFac9AA75
+POOL_ACCOUNT=37
+SECURITY_MODULE_ACCOUNT=21
+# v2's deploy (2026-10-27, blocks 53164124–53164160): no account was under it before.
+START_BLOCK=53164124
+DRY_RUN=true
+MAX_BID_USD=1500
+# 9464 is the perp keeper's, 9465 v1's engine keeper's.
+HEALTH_PORT=9466
+HEALTH_HOST=127.0.0.1
+MIN_KEEPER_ETH=0.002
+# A full futures side's IM at the enable cap: 15 contracts x 1,000,000 cNGN x ~0.000743 x 25% = ~2,790 USDC. It moves
+# with the mark; propose_margin_engine_v2_batch.py's keeper gate re-reads it (engine_keeper_min_cash).
+MIN_KEEPER_CASH_USD=2800
+CNGN_ESCROW=0x37c976bb5d4887a714ef19AF6B83e34fe2f37c98
+# The engine's market-1 spot feed (getMarketFeeds(1)[0]), USDC per cNGN: v2 has no perp index to price cNGN by.
+CNGN_SPOT_FEED=0xFaC420d160C7c219A72DC676971670980bAC20a5
+MAX_CNGN_INVENTORY=8000000
+# EURC (v2 market 2): the escrow is whitelisted for v2 at cap 0 until `eurc-cap` (margin-engine-v2.md, step 6). Set now so
+# a keeper paid in EURC prices and limits it; revisit MAX_EURC_INVENTORY with the eurc-cap step.
+EURC_ESCROW=0xfD2AFbF4eF6E7a7f34A04D16Ff6da249Ca086cD9
+EURC_SPOT_FEED=0xaaE62Ae5df3747B8f43F895561043FB77D97C435
+EURC_HAIRCUT_BPS=200
+MAX_EURC_INVENTORY=40000

contracts/risk-core/scripts/ops/future-feeds-v2-cngn-30nov26.env

@@ -0,0 +1,38 @@
+# /etc/numo/future-feeds-v2-cngn-30nov26.env ([email protected]), for cNGN-30NOV26 on
+# MarginEngine v2: from risk-core/deployments/8453/MARGIN_ENGINE_V2.json. NOT INSTALLED: a template until the v2
+# forward keeper goes on the box (#258). A change here is copied to the box and recorded in OPS_BOX_CHANGES.md
+# the same day. Holds no secret.
+#
+# A v2 series is config only: an EURC-30NOV26 instance is a copy of this file with its own FORWARD_FEED,
+# FUTURE_ASSET, INDEX_FEED (the EURC spot feed), CARRY, BASE_SYMBOL and relayer, installed as
+# numo-future-feeds-v2@eurc-30nov26.
+CHAIN_ID=8453
+FUTURE_VERSION=v2
+BASE_SYMBOL=cNGN
+DATA_SUBMITTER=0xe0C06DD245f1e8C8bC516c66C66e64648987F912
+# The cNGN spot feed (USDC per cNGN), v1's index too: read as is.
+INDEX_FEED=0xFaC420d160C7c219A72DC676971670980bAC20a5
+FORWARD_FEED=0x24474D8A95EB8Be9070ede58E09405B410c277Df
+FUTURE_ASSET=0xcACEe66ad41fD1472a47D7D5Fe372FFbFac9AA75
+# The expiry, 2026-11-30 14:00 UTC: the same sub id as v1's USDC-cNGN-30NOV26, on a different FUTURE_ASSET.
+FUTURE_SUB_ID=1796047200
+# MarginEngine v2. Its VM pool is account #37, cash only.
+FUTURE_MANAGER=0x0705219a1594d5Fc5dC2573da4082cAf3e1B0950
+# Annual NGN-over-USD carry, 18dp: with no book the mark is index / (1 + CARRY * tau). 11.8%, the figure the
+# deploy's INITIAL_MARK was checked against (MARGIN_ENGINE_V2.json carry).
+CARRY=118000000000000000
+# v1's forward KMS key (0xd65F…3650), the v2 feed's only signer: each feed's EIP-712 domain binds signatures to it.
+FORWARD_SIGNER_KMS_KEY_ID=arn:aws:kms:us-east-1:957261948743:key/4312f074-ddcb-4db3-89d5-02d042beaacd
+# A relayer of this instance's own (run-with-ssm-forward.sh): sharing v1's would race it for nonces.
+FORWARD_RELAYER_SSM_NAME=forward_relayer_key_v2_cngn
+MARKETS_SERVICE_URL=https://api.numofx.com
+FORWARD_TICK_MS=60000
+# Each side of the futures book must fill this much (USDC) before it sets the mark; else carry. A v2 contract is
+# 1,000,000 cNGN, ~$720 at 0.00072: about 3 contracts a side.
+FORWARD_MIN_BOOK_DEPTH_USDC=2000
+FORWARD_MAX_BASIS_BPS=300
+FORWARD_SAFETY_BPS=50
+FORWARD_MARK_LATE_MS=7200000
+FIXING_SAMPLE_INTERVAL_MS=300000
+CLOSE_OUT_BATCH=20
+SETTLE_PER_TICK=10

contracts/risk-core/scripts/ops/numo-engine-keeper-v2.service

@@ -0,0 +1,38 @@
+# MarginEngine v2's liquidation keeper: services/perp-keeper with MANAGER = v2 (engine-keeper-v2.env). Watches every account
+# under v2, starts auctions on those under maintenance and bids from its own funding account under v2. v1's keeper
+# (numo-engine-keeper.service) is unchanged.
+#
+# NOT INSTALLED (#258). propose_margin_engine_v2_batch.py's keeper gate reads its /health (ENGINE_V2_KEEPER_HEALTH_URL,
+# http://127.0.0.1:9466/health) and refuses `enable` until it is live with a full side's IM in cash.
+#
+# Its own key (/numo/keeper/engine_v2_keeper_key) and funding account: sharing v1's engine keeper EOA would race it for
+# nonces. Runs from /home/ec2-user/exchange-future-v2 (the v2 future keeper's checkout), never exchange-future (v1's).
+#
+# Before starting:
+#   - SSM /numo/keeper/engine_v2_keeper_key: a fresh key, its EOA funded with gas;
+#   - its funding account: SubAccounts.createAccount(v2 keeper EOA, v2) from that EOA, then a cash deposit;
+#   - /etc/numo/engine-keeper-v2.env from engine-keeper-v2.env with KEEPER_ACCOUNT filled in, DRY_RUN=true first.
+#
+# Install: (cd /home/ec2-user/exchange-future-v2 && pnpm install --frozen-lockfile --filter perp-keeper... && pnpm --filter perp-keeper build)
+#          cp contracts/risk-core/scripts/ops/numo-engine-keeper-v2.service /etc/systemd/system/
+#          systemctl daemon-reload && systemctl enable --now numo-engine-keeper-v2
+# and record it in OPS_BOX_CHANGES.md the same day.
+[Unit]
+Description=Numo MarginEngine v2 liquidation keeper (Base mainnet, SSM secrets)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=ec2-user
+WorkingDirectory=/home/ec2-user/exchange-future-v2/services/perp-keeper
+EnvironmentFile=/etc/numo/engine-keeper-v2.env
+ExecStart=/bin/bash -c 'export KEEPER_KEY="$(aws ssm get-parameter --name /numo/keeper/engine_v2_keeper_key --with-decryption --query Parameter.Value --output text --region "$AWS_REGION")" RPC_URL="$(aws ssm get-parameter --name /numo/feeds/rpc_url --with-decryption --query Parameter.Value --output text --region "$AWS_REGION")" ALERT_WEBHOOK_URL="$(aws ssm get-parameter --name /numo/feeds/alert_webhook_url --with-decryption --query Parameter.Value --output text --region "$AWS_REGION" 2>/dev/null || true)" ALERT_PREFIX="[engine v2] " && exec /usr/bin/node dist/main.js'
+Environment=AWS_REGION=us-east-1
+Restart=always
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

contracts/risk-core/scripts/ops/numo-future-feeds-v2@.service

@@ -0,0 +1,39 @@
+# The keeper for a MarginEngine v2 dated future (services/perp-feeds, dist/forward-main.js with FUTURE_VERSION=v2),
+# one instance per series: numo-future-feeds-v2@cngn-30nov26 reads /etc/numo/future-feeds-v2-cngn-30nov26.env
+# (future-feeds-v2-cngn-30nov26.env), and an EURC series is another instance and env file, no code.
+#
+# NOT INSTALLED (#258). v1's USDC-cNGN-30NOV26 keeps numo-future-feeds.service, unchanged, until it settles.
+#
+# Its own checkout, /home/ec2-user/exchange-future-v2: exchange-future is v1's keeper and the engine keeper's,
+# and building there would move their code on their next restart.
+#
+# Before starting an instance:
+#   - its env file in /etc/numo, with FORWARD_RELAYER_SSM_NAME naming a funded relayer of its own under
+#     /numo/feeds (watched by check_signer_balance.py);
+#   - the forward feed's signer is the KMS key the env names (v2 cNGN: v1's forward key);
+#   - a dry run first: DRY_RUN=true in the env file, then off.
+#
+# Install: git clone https://github.com/numofx/exchange /home/ec2-user/exchange-future-v2 && git -C ... checkout <sha>
+#          (cd /home/ec2-user/exchange-future-v2 && pnpm install --frozen-lockfile --filter perp-feeds... && pnpm --filter @numo/kms-signer build && pnpm --filter perp-feeds build)
+#          cp contracts/risk-core/scripts/ops/[email protected] /etc/systemd/system/
+#          systemctl daemon-reload && systemctl enable --now numo-future-feeds-v2@cngn-30nov26
+# and record it in OPS_BOX_CHANGES.md the same day.
+[Unit]
+Description=Numo v2 future keeper %i: forward marks, fixing, close-out (Base mainnet, KMS signer, SSM relayer)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=ec2-user
+WorkingDirectory=/home/ec2-user/exchange-future-v2/services/perp-feeds
+EnvironmentFile=/etc/numo/future-feeds-v2-%i.env
+ExecStart=/home/ec2-user/exchange-future-v2/contracts/risk-core/scripts/ops/run-with-ssm-forward.sh /usr/bin/node dist/forward-main.js
+Environment=AWS_REGION=us-east-1
+Restart=always
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

contracts/risk-core/scripts/ops/perp-pager.env

@@ -6,3 +6,7 @@ PERP_INDEX_STATUS_FILE=/var/lib/numo/perp-index-status.json
 PAGER_STATE_FILE=/var/lib/numo/perp-pager-state.json
 KEEPER_EOA=0xdE57b9E91358BB2f58DB7A7D588533f996373f18
 KEEPER_ACCOUNT=23
+# MarginEngine v2's liquidation keeper (numo-engine-keeper-v2, #258). NOT ON THE BOX YET: uncomment once it is installed,
+# and the pager pages keeper-unhealthy-v2 / insolvent-account-v2 for it. v2's futures are read from
+# deployments/8453/MARGIN_ENGINE_V2.json by default (FUTURE_V2_ARTIFACTS, comma-separated, adds a series).
+# ENGINE_V2_KEEPER_HEALTH_URL=http://127.0.0.1:9466/health

contracts/risk-core/scripts/ops/run-with-ssm-forward.sh

@@ -8,7 +8,9 @@
 #
 #   rpc_url              the same RPC the perp publishers use
 #   forward_relayer_key  a relayer of its OWN: sharing perp-feeds' relayer across two processes races
-#                        for nonces (the 2026-10-03 mark outage was one process doing that with itself)
+#                        for nonces (the 2026-10-03 mark outage was one process doing that with itself).
+#                        A second keeper (a v2 series, [email protected]) names its own in
+#                        FORWARD_RELAYER_SSM_NAME, from its env file; unset, it is forward_relayer_key.
 #   alert_webhook_url    optional
 #
 # The forward SIGNER is not here: it is a KMS key, FORWARD_SIGNER_KMS_KEY_ID in the env file, signed
@@ -28,7 +30,7 @@ get_optional() {
 }
 
 export RPC_URL="$(get rpc_url)"
-export RELAYER_KEY="$(get forward_relayer_key)"
+export RELAYER_KEY="$(get "${FORWARD_RELAYER_SSM_NAME:-forward_relayer_key}")"
 export ALERT_WEBHOOK_URL="$(get_optional alert_webhook_url)"
 
 exec "$@"

scripts/verify.sh

@@ -101,12 +101,12 @@ step node . "cngn-rebalance check" pnpm --filter cngn-rebalance run check
 step node . "cngn-rebalance test" pnpm --filter cngn-rebalance test
 
 # ---- services-perp-feeds.yml -----------------------------------------------------------
-# The anvil suites deploy the real feed contracts and FutureAssetV1 (the live 30NOV26 series the keepers run), so they need forge and anvil, and
+# The anvil suites deploy the real feed contracts, FutureAssetV1 (the live 30NOV26 series) and the v2 FutureAsset, so they need forge and anvil, and
 # a fresh anvil on the port the tests read: started here, stopped however the tests end.
 perp_feeds_anvil_tests() {
   ( cd "$ROOT/contracts/risk-core" \
     && forge build src/feeds/LyraSpotFeed.sol src/feeds/LyraSpotDiffFeed.sol src/feeds/LyraForwardFeed.sol \
-      src/periphery/OracleDataSubmitter.sol src/legacy/v1/FutureAssetV1.sol src/SubAccounts.sol >/dev/null ) || return 1
+      src/periphery/OracleDataSubmitter.sol src/legacy/v1/FutureAssetV1.sol src/assets/FutureAsset.sol src/SubAccounts.sol >/dev/null ) || return 1
   anvil --port 8599 --silent &
   local anvil_pid=$!
   sleep 2

services/perp-feeds/README.md

@@ -168,3 +168,33 @@ the ops box since 2026-10-13 for USDC-cNGN-30NOV26, from its own checkout `/home
 on `FutureAsset`. The anvil suites share one chain clock, so `pnpm test` runs test files one at a time
 (`--test-concurrency=1`): concurrently, one suite's `evm_increaseTime` expires another's signatures or
 skips its fixing window.
+
+### v2 series (`FUTURE_VERSION=v2`)
+
+The same process marks a MarginEngine v2 future (`src/assets/FutureAsset.sol`: cNGN-30NOV26, and
+EURC-30NOV26 once deployed), one instance per series (`numo-future-feeds-v2@<series>.service`,
+`/etc/numo/future-feeds-v2-<series>.env`; `future-feeds-v2-cngn-30nov26.env` is the first). v1's
+USDC-cNGN-30NOV26 keeps its unit and code path, unchanged, until it settles. What v2 changes:
+
+- **Units.** Every price is USDC per unit of the base, the feeds' units and the contract's, so
+  nothing is inverted: the diff signed is `forward − index`, and `pullMark` reads `index + diff` as is.
+- **Carry.** The forward without a book is `index / (1 + CARRY × τ)`, as the v2 deploy script listed
+  it. `CARRY` is signed: positive for cNGN (a forward discount, a negative diff), negative for EURC
+  (a premium). v1 refuses a negative one.
+- **Depth.** A v2 contract is `contractSize` of the base (1,000,000 cNGN), so its notional is
+  `contracts × contractSize × price`; `FORWARD_MIN_BOOK_DEPTH_USDC` stays in USDC.
+- **Startup.** It refuses to start when the contract's mark sits nearer the index read the other
+  version's way (`checkUnits`), and its alerts are headed `[future-feeds <BASE_SYMBOL> v2]`: both
+  30NOV26 series have the same sub id.
+- **Relayer.** Its own, named by `FORWARD_RELAYER_SSM_NAME` under `/numo/feeds`. The signer is the
+  same KMS key as v1's: each feed's EIP-712 domain binds a signature to that feed.
+
+- **The VM pool** (`coverPoolDeficit`): v2's VM is USDC and moves through the engine's pool (#37) as
+  cash, nothing converted, so the pool's net is its cash and a loser's settlement pays exactly what it
+  owes. Losers are settled largest first while the cash is below zero (the same $1 tolerance and
+  `SETTLE_PER_TICK`); no escrow leg, no spot feed, no `assetMap` read. Every series on an engine shares
+  its pool, so each series' keeper sees the same cash and settles its own series' losers.
+- **Expiry** is the same calls as v1's: fixing samples, `fixSettlementFromIndex`, `closeOutExpired`.
+
+`src/forward-v2.anvil.e2e.test.ts` and `src/expiry-v2.anvil.e2e.test.ts` run it against the real v2
+`FutureAsset` for a cNGN and an EURC series.

services/perp-feeds/src/alert.ts

@@ -1,7 +1,8 @@
 /**
  * Posts to ALERT_WEBHOOK_URL in the shape the ops scripts already use ({text, content}, which Slack
  * and Discord both read). Each alert key is held back for `cooldownMs` after it fires, so a refusal
- * that lasts an hour pages a few times rather than sixty.
+ * that lasts an hour pages a few times rather than sixty. `tag` heads every message: a second
+ * instance of a keeper (a v2 series) names itself, since its alert keys are the first one's.
  */
 export type Alerter = (key: string, message: string) => Promise<void>;
 
@@ -10,6 +11,7 @@ export function createAlerter(
   cooldownMs = 15 * 60_000,
   now: () => number = Date.now,
   fetchImpl: typeof fetch = fetch,
+  tag = 'perp-feeds',
 ): Alerter {
   const lastSent = new Map<string, number>();
 
@@ -23,7 +25,7 @@ export function createAlerter(
       const response = await fetchImpl(webhookUrl, {
         method: 'POST',
         headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({ text: `[perp-feeds] ${message}`, content: `[perp-feeds] ${message}` }),
+        body: JSON.stringify({ text: `[${tag}] ${message}`, content: `[${tag}] ${message}` }),
         signal: AbortSignal.timeout(10_000),
       });
       // A 2xx means the webhook took it, not that anyone read it; a non-2xx means nobody will.