feat: store slices refactor, credit system, DR automation, API docs (#915 #918 #911 #909)

.github/workflows/disaster-recovery.yml

@@ -0,0 +1,73 @@
+name: Disaster Recovery Automation
+
+# Automated disaster-recovery routine:
+#  1. Runs a full DR drill (backup → verify → restore + monitor health check)
+#  2. Creates a DR backup (with pre-check)
+#  3. Captures and uploads DR status + backup artefacts
+#
+# Wired to a schedule so backups/status are taken on a routine cadence
+# independent of human action, plus a manual dispatch for on-demand runs.
+
+on:
+  schedule:
+    # Daily at 03:17 UTC
+    - cron: '17 3 * * *'
+  workflow_dispatch:
+
+env:
+  NODE_VERSION: '20'
+
+jobs:
+  dr-routine:
+    name: DR Backup + Status
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci --legacy-peer-deps
+
+      - name: Run DR drill (backup → verify → restore + health)
+        run: node scripts/dr-test.js
+        continue-on-error: true
+
+      - name: Create DR backup (with pre-check)
+        run: ./scripts/dr-backup.sh --pre-check --region "${DR_REGION:-us-east-1}" --env "${DR_ENVIRONMENT:-production}"
+        env:
+          DR_REGION: us-east-1
+          DR_ENVIRONMENT: production
+
+      - name: Capture DR status (JSON)
+        run: |
+          STATUS_FILE="dr-status-${{ github.run_id }}.json"
+          ./scripts/dr-status.sh --json > "$STATUS_FILE" 2>&1 || true
+          echo "STATUS_FILE=$STATUS_FILE" >> "$GITHUB_ENV"
+        id: status
+
+      - name: Upload DR backup artefact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: dr-backups-${{ github.run_id }}
+          path: |
+            .dr-backups/*.tar.gz
+            .dr-recovery-log.jsonl
+            ${{ env.STATUS_FILE }}
+
+      - name: Notify on degraded/critical DR health
+        if: always()
+        run: |
+          if ! ./scripts/dr-status.sh --short; then
+            echo "::warning::DR system is in a degraded/critical state — review the DR status artefact."
+          else
+            echo "DR system is healthy."
+          fi

app/stores/__tests__/creditStore.test.ts

@@ -2,7 +2,7 @@ import { useCreditStore } from '../creditStore';
 
 let clock = 1000;
 const reset = () =>
-  useCreditStore.setState({ accounts: {}, nextId: 0, now: () => clock });
+  useCreditStore.setState({ accounts: {}, wallets: {}, nextId: 0, now: () => clock });
 
 beforeEach(() => {
   clock = 1000;
@@ -66,4 +66,54 @@ describe('useCreditStore', () => {
     clock = 1200;
     expect(s().getBalance('alice')).toBe(0);
   });
+
+  it('deposits credit into an account balance', () => {
+    s().depositCredit('alice', 200, 'prepaid top-up');
+    expect(s().getBalance('alice')).toBe(200);
+    expect(
+      s()
+        .getAccount('alice')
+        .transactions.some((t) => t.kind === 'deposit' && t.amount === 200)
+    ).toBe(true);
+  });
+
+  it('withdraws available credit and rejects overdrafts', () => {
+    s().issueCredit('alice', 300, 'promo');
+    expect(s().withdrawCredit('alice', 100, 'cash-out')).toBe(true);
+    expect(s().getBalance('alice')).toBe(200);
+    expect(s().withdrawCredit('alice', 500, 'cash-out')).toBe(false);
+    expect(s().getBalance('alice')).toBe(200);
+  });
+
+  it('computes a consolidated account balance summary', () => {
+    useCreditStore.getState().wallets = {
+      'w-1': {
+        id: 'w-1',
+        subscriber: 'alice',
+        currency: 'USD',
+        balance: 75,
+        totalDeposited: 100,
+        totalWithdrawn: 25,
+      },
+    };
+
+    s().issueCredit('alice', 250, 'refund');
+    s().applyCredit('alice', 'sub_1', 50);
+
+    const balance = s().getAccountBalance('alice');
+    expect(balance.subscriber).toBe('alice');
+    expect(balance.availableCredit).toBe(200);
+    expect(balance.totalIssued).toBe(250);
+    expect(balance.totalApplied).toBe(50);
+    expect(balance.prepaymentBalance).toBe(75);
+    expect(balance.netBalance).toBe(275);
+  });
+
+  it('returns account balances for all known subscribers', () => {
+    s().issueCredit('alice', 100, 'promo');
+    s().issueCredit('bob', 200, 'promo');
+    const balances = s().getAccountBalances();
+    expect(balances).toHaveLength(2);
+    expect(balances.map((b) => b.subscriber).sort()).toEqual(['alice', 'bob']);
+  });
 });

app/stores/creditStore.ts

@@ -10,7 +10,14 @@ import { create } from 'zustand';
 import { persist, createJSONStorage } from 'zustand/middleware';
 import { asyncStorageAdapter } from '../../src/utils/storage';
 
-export type CreditTxKind = 'issue' | 'apply' | 'transfer_in' | 'transfer_out' | 'expire';
+export type CreditTxKind =
+  | 'issue'
+  | 'apply'
+  | 'transfer_in'
+  | 'transfer_out'
+  | 'expire'
+  | 'deposit'
+  | 'withdraw';
 
 export type ExpirationPolicy = { kind: 'never' } | { kind: 'after_secs'; seconds: number };
 
@@ -46,6 +53,26 @@ export interface CreditApplied {
   balanceAfter: number;
 }
 
+/**
+ * Consolidated account-balance summary for a subscriber, combining on-book
+ * credit (available/issued/expired/applied/transferred) with any prepayment
+ * wallet funds. Used for high-level balance display and reconciliation.
+ */
+export interface AccountBalance {
+  subscriber: string;
+  availableCredit: number;
+  totalIssued: number;
+  totalApplied: number;
+  totalExpired: number;
+  totalTransferredIn: number;
+  totalTransferredOut: number;
+  totalDeposited: number;
+  totalWithdrawn: number;
+  prepaymentBalance: number;
+  netBalance: number;
+  nextExpirationAt?: number;
+}
+
 const isExpired = (lot: CreditLot, now: number): boolean =>
   lot.expiresAt !== undefined && lot.expiresAt <= now;
 
@@ -57,16 +84,31 @@ const availableOf = (account: AccountCredit, now: number): number =>
 
 interface CreditStoreState {
   accounts: Record<string, AccountCredit>;
+  wallets: Record<string, CreditWallet>;
   nextId: number;
   now: () => number;
 
   issueCredit: (subscriber: string, amount: number, reason: string, expiresAt?: number) => void;
   setExpirationPolicy: (subscriber: string, policy: ExpirationPolicy) => void;
   applyCredit: (subscriber: string, subscriptionId: string, amountDue: number) => CreditApplied;
   transferCredit: (from: string, to: string, amount: number, reason: string) => boolean;
+  depositCredit: (subscriber: string, amount: number, reason: string) => void;
+  withdrawCredit: (subscriber: string, amount: number, reason: string) => boolean;
   expireCredits: (subscriber: string) => number;
   getBalance: (subscriber: string) => number;
   getAccount: (subscriber: string) => AccountCredit;
+  getAccountBalance: (subscriber: string) => AccountBalance;
+  getAccountBalances: () => AccountBalance[];
+}
+
+/** Prepayment wallet tracked alongside credit accounts. */
+export interface CreditWallet {
+  id: string;
+  subscriber: string;
+  currency: string;
+  balance: number;
+  totalDeposited: number;
+  totalWithdrawn: number;
 }
 
 const blankAccount = (subscriber: string): AccountCredit => ({
@@ -141,6 +183,7 @@ export const useCreditStore = create<CreditStoreState>()(
 
       return {
         accounts: {},
+        wallets: {},
         nextId: 0,
         now: () => Math.floor(Date.now() / 1000),
 
@@ -217,13 +260,89 @@ export const useCreditStore = create<CreditStoreState>()(
 
         getBalance: (subscriber) => availableOf(account(subscriber), get().now()),
         getAccount: (subscriber) => account(subscriber),
+
+        depositCredit: (subscriber, amount, reason) => {
+          if (amount <= 0) return;
+          const now = get().now();
+          const acc = cloneAccount(account(subscriber));
+          realizeExpiry(acc, now);
+          acc.balance += amount;
+          acc.lots.push({ id: nextId(), remaining: amount, issuedAt: now });
+          record(acc, 'deposit', amount, reason);
+          commit(acc);
+        },
+
+        withdrawCredit: (subscriber, amount, reason) => {
+          if (amount <= 0) return false;
+          const now = get().now();
+          const acc = cloneAccount(account(subscriber));
+          realizeExpiry(acc, now);
+          if (availableOf(acc, now) < amount) return false;
+          const moved = consume(acc, now, amount);
+          acc.balance -= moved;
+          record(acc, 'withdraw', -moved, reason);
+          commit(acc);
+          return true;
+        },
+
+        getAccountBalance: (subscriber): AccountBalance => {
+          const now = get().now();
+          const acc = account(subscriber);
+          const availableCredit = availableOf(acc, now);
+          const totalApplied = acc.transactions
+            .filter((t) => t.kind === 'apply')
+            .reduce((sum, t) => sum + Math.abs(t.amount), 0);
+          const totalExpired = acc.transactions
+            .filter((t) => t.kind === 'expire')
+            .reduce((sum, t) => sum + Math.abs(t.amount), 0);
+          const totalTransferredIn = acc.transactions
+            .filter((t) => t.kind === 'transfer_in')
+            .reduce((sum, t) => sum + Math.abs(t.amount), 0);
+          const totalTransferredOut = acc.transactions
+            .filter((t) => t.kind === 'transfer_out')
+            .reduce((sum, t) => sum + Math.abs(t.amount), 0);
+          const totalIssued = acc.transactions
+            .filter((t) => t.kind === 'issue' || t.kind === 'deposit')
+            .reduce((sum, t) => sum + Math.abs(t.amount), 0);
+
+          const wallets = Object.values(get().wallets).filter(
+            (w) => w.subscriber === subscriber
+          );
+          const totalDeposited = wallets.reduce((s, w) => s + w.totalDeposited, 0);
+          const totalWithdrawn = wallets.reduce((s, w) => s + w.totalWithdrawn, 0);
+          const prepaymentBalance = wallets.reduce((s, w) => s + w.balance, 0);
+
+          const expiringLots = acc.lots
+            .filter((lot) => lot.remaining > 0 && lot.expiresAt !== undefined && lot.expiresAt > now)
+            .sort((a, b) => (a.expiresAt ?? 0) - (b.expiresAt ?? 0));
+
+          return {
+            subscriber,
+            availableCredit,
+            totalIssued,
+            totalApplied,
+            totalExpired,
+            totalTransferredIn,
+            totalTransferredOut,
+            totalDeposited,
+            totalWithdrawn,
+            prepaymentBalance,
+            netBalance: availableCredit + prepaymentBalance,
+            nextExpirationAt: expiringLots[0]?.expiresAt,
+          };
+        },
+
+        getAccountBalances: () =>
+          [...new Set([...Object.keys(get().accounts), ...Object.keys(get().wallets)])]
+            .map((sub) => get().getAccountBalance(sub)),
       };
     },
     {
       name: 'subtrackr-credit-store',
       storage: createJSONStorage(() => asyncStorageAdapter),
       partialize: (state) => ({
         accounts: state.accounts,
+        wallets: state.wallets,
         nextId: state.nextId,
       }),
     }

chaos/__tests__/failure-injection.test.ts

@@ -0,0 +1,28 @@
+import {
+  injectFailure,
+  runFailureInjectionExperiment,
+} from '../experiments/failure-injection';
+
+describe('Failure Injection Experiment', () => {
+  it('injects failure into marked steps', async () => {
+    const result = await injectFailure([
+      { name: 'charge', inject: true },
+      { name: 'notify', inject: false },
+    ]);
+    expect(result.ok).toBe(false);
+    expect(result.failedSteps).toEqual(['charge']);
+  });
+
+  it('succeeds when no steps are marked', async () => {
+    const result = await injectFailure([{ name: 'charge', inject: false }]);
+    expect(result.ok).toBe(true);
+    expect(result.failedSteps).toEqual([]);
+  });
+
+  it('runFailureInjectionExperiment passes', async () => {
+    const result = await runFailureInjectionExperiment();
+    expect(result.experiment).toBe('failure-injection');
+    expect(result.passed).toBe(true);
+    expect(result.recovery).toBe('failure-contained-and-recovered');
+  });
+});

chaos/__tests__/network-partition.test.ts

@@ -0,0 +1,32 @@
+import {
+  simulateNetworkPartition,
+  runNetworkPartitionExperiment,
+  PartitionNode,
+} from '../experiments/network-partition';
+
+describe('Network Partition Experiment', () => {
+  it('reports unreachable nodes during partition', async () => {
+    const nodes: PartitionNode[] = [
+      { name: 'a', reachable: true, value: 'ok' },
+      { name: 'b', reachable: false, value: 'ok' },
+    ];
+    const result = await simulateNetworkPartition(nodes);
+    expect(result.find((r) => r.name === 'b')?.ok).toBe(false);
+    expect(result.find((r) => r.name === 'a')?.ok).toBe(true);
+  });
+
+  it('recovers once partition heals', async () => {
+    const nodes: PartitionNode[] = [
+      { name: 'a', reachable: false, value: 'ok' },
+    ];
+    const recovered = await simulateNetworkPartition(nodes, true);
+    expect(recovered[0].ok).toBe(true);
+  });
+
+  it('runNetworkPartitionExperiment passes', async () => {
+    const result = await runNetworkPartitionExperiment();
+    expect(result.experiment).toBe('network-partition');
+    expect(result.passed).toBe(true);
+    expect(result.recovery).toBe('partition-healed');
+  });
+});