diff --git a/docs.json b/docs.json
index ab3d6de..aa86399 100644
--- a/docs.json
+++ b/docs.json
@@ -171,7 +171,8 @@
"guides/stellar/stellar-quickstart",
"guides/stellar/wraith-names-lifecycle",
"guides/wraith-names-stellar",
- "guides/ops/self-hosted-deployment"
+ "guides/ops/self-hosted-deployment",
+ "guides/ops/monitoring-and-on-call"
]
}
]
diff --git a/guides/ops/alerts/wraith-alerts.yml b/guides/ops/alerts/wraith-alerts.yml
new file mode 100644
index 0000000..73de305
--- /dev/null
+++ b/guides/ops/alerts/wraith-alerts.yml
@@ -0,0 +1,274 @@
+groups:
+ - name: wraith_rpc_alerts
+ interval: 30s
+ rules:
+ - alert: HighRPCLatency
+ expr: histogram_quantile(0.95, sum(rate(wraith_rpc_latency_seconds_bucket[5m])) by (le, method)) > 2
+ for: 5m
+ labels:
+ severity: high
+ component: rpc
+ team: platform
+ annotations:
+ summary: "RPC P95 latency > 2s for 5 minutes"
+ description: "{{ $labels.method }} RPC calls are experiencing high latency ({{ $value }}s). Check Horizon/Soroban RPC health and network connectivity."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage"
+ dashboard: "https://grafana.example.com/d/wraith-rpc-health"
+ runbook_steps: |
+ 1. Check RPC endpoint status
+ 2. Verify network connectivity
+ 3. Switch to backup RPC if available
+ 4. Enable request queueing
+
+ - alert: HighRPCErrorRate
+ expr: sum(rate(wraith_rpc_error_rate[5m])) / sum(rate(wraith_rpc_latency_seconds_count[5m])) > 0.05
+ for: 2m
+ labels:
+ severity: critical
+ component: rpc
+ team: platform
+ annotations:
+ summary: "RPC error rate > 5%"
+ description: "{{ $value | humanizePercentage }} of RPC calls are failing. Immediate investigation required."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage"
+
+ - alert: HorizonBackpressure
+ expr: rate(wraith_horizon_backpressure[1m]) > 10
+ for: 2m
+ labels:
+ severity: critical
+ component: rpc
+ team: platform
+ page: "true"
+ annotations:
+ summary: "Horizon rate-limiting detected (> 10 429s/min)"
+ description: "Application is being rate-limited by Horizon at {{ $value }} 429 responses/min. Immediate intervention required to prevent service disruption."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-2-horizon-backpressure"
+ impact: "Users cannot submit transactions. Service degraded."
+
+ - alert: SorobanInvocationFailures
+ expr: rate(wraith_soroban_invocation_failures[1h]) > 5
+ for: 5m
+ labels:
+ severity: high
+ component: rpc
+ team: platform
+ annotations:
+ summary: "Soroban invocation failure rate high"
+ description: "{{ $value }} Soroban invocations failed in the last hour due to RPC issues."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage"
+
+ - name: wraith_indexer_alerts
+ interval: 1m
+ rules:
+ - alert: IndexerBacklogBuildup
+ expr: wraith_indexer_backlog_count > 1000
+ for: 5m
+ labels:
+ severity: high
+ component: indexer
+ team: data
+ annotations:
+ summary: "Indexer backlog > 1000 announcements"
+ description: "Indexer is falling behind with {{ $value }} unprocessed announcements. Check database performance and announcement ingestion rate."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+ dashboard: "https://grafana.example.com/d/wraith-indexer-performance"
+ impact: "Users may experience delayed stealth payment notifications."
+
+ - alert: AnnouncementLagHigh
+ expr: wraith_announcement_lag_seconds > 300
+ for: 5m
+ labels:
+ severity: medium
+ component: indexer
+ team: data
+ annotations:
+ summary: "Announcement lag > 5 minutes"
+ description: "Time between announcement emission and indexing is {{ $value }}s. Users may experience delays seeing incoming stealth payments."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+
+ - alert: IndexerSyncHeightLagging
+ expr: (stellar_network_ledger_height - wraith_indexer_sync_height) > 50
+ for: 10m
+ labels:
+ severity: high
+ component: indexer
+ team: data
+ annotations:
+ summary: "Indexer sync height lagging by > 50 ledgers"
+ description: "Indexer is {{ $value }} ledgers behind the network tip. Risk of announcement processing delays."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+
+ - alert: ScanMissRateHigh
+ expr: rate(wraith_scan_miss_rate[10m]) > 0.05
+ for: 10m
+ labels:
+ severity: medium
+ component: indexer
+ team: data
+ annotations:
+ summary: "View-tag scan miss rate > 5%"
+ description: "{{ $value | humanizePercentage }} of scanned announcements are not matching. Check view-tag scanner configuration."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+
+ - name: wraith_contract_alerts
+ interval: 1m
+ rules:
+ - alert: ContractErrorRateHigh
+ expr: sum(rate(wraith_contract_error_rate[5m])) by (contract) > 0.01
+ for: 5m
+ labels:
+ severity: critical
+ component: contracts
+ team: blockchain
+ page: "true"
+ annotations:
+ summary: "Contract {{ $labels.contract }} error rate > 1%"
+ description: "Contract {{ $labels.contract }} invocations are failing at {{ $value | humanizePercentage }}. Check contract state, RPC health, and recent deployments."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish"
+ dashboard: "https://grafana.example.com/d/wraith-contract-monitoring"
+ impact: "Core Wraith functionality may be unavailable to users."
+
+ - alert: SenderInvocationLatencyHigh
+ expr: histogram_quantile(0.95, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le)) > 20
+ for: 5m
+ labels:
+ severity: high
+ component: contracts
+ team: blockchain
+ annotations:
+ summary: "Sender invocation P95 latency > 20s"
+ description: "Stealth payment submissions are taking {{ $value }}s at P95. Check RPC performance and network congestion."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish"
+
+ - alert: NamesRegistrationFailures
+ expr: increase(wraith_names_registration_failures[1h]) > 3
+ for: 5m
+ labels:
+ severity: high
+ component: contracts
+ team: blockchain
+ annotations:
+ summary: "Wraith Names registrations failing"
+ description: "{{ $value }} wraith-names registration attempts failed in the last hour. Check contract admin authorization and state."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish"
+
+ - alert: UnauthorizedAccessAttempt
+ expr: increase(wraith_unauthorized_access_attempts[5m]) > 0
+ for: 1m
+ labels:
+ severity: critical
+ component: contracts
+ team: security
+ page: "true"
+ annotations:
+ summary: "SECURITY: Unauthorized contract access detected"
+ description: "{{ $value }} unauthorized access attempt(s) detected on Wraith contracts. Potential security incident - investigate immediately."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-6-key-rotation-incident"
+ impact: "Potential security breach. Admin keys may be compromised."
+ action_required: "1. Verify contract admin keys. 2. Check transaction history. 3. Initiate key rotation if needed. 4. Notify security team."
+
+ - name: wraith_watcher_alerts
+ interval: 1m
+ rules:
+ - alert: WatcherEventDropSpike
+ expr: rate(wraith_watcher_event_drop_rate[5m]) > 5
+ for: 2m
+ labels:
+ severity: high
+ component: watcher
+ team: platform
+ annotations:
+ summary: "Watcher event drop rate > 5/hour"
+ description: "Events are being dropped at {{ $value }}/hour. Users may miss stealth payment notifications."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike"
+ impact: "Users may not see incoming stealth payments in wallets."
+
+ - alert: ViewTagScanLatencyHigh
+ expr: histogram_quantile(0.95, rate(wraith_view_tag_scan_duration_seconds_bucket[5m])) > 0.1
+ for: 10m
+ labels:
+ severity: medium
+ component: watcher
+ team: platform
+ annotations:
+ summary: "View-tag scan P95 latency > 100ms"
+ description: "View-tag scanning is taking {{ $value }}s at P95. Risk of announcement processing bottleneck."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike"
+
+ - alert: ViewTagFalsePositiveRateHigh
+ expr: rate(wraith_view_tag_false_positive_rate[10m]) > 0.1
+ for: 10m
+ labels:
+ severity: low
+ component: watcher
+ team: platform
+ annotations:
+ summary: "View-tag false positive rate > 10%"
+ description: "{{ $value | humanizePercentage }} of view-tag matches are failing full ECDH check. Consider view-tag configuration review."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike"
+
+ - alert: WatcherDown
+ expr: up{job="wraith-watcher"} == 0
+ for: 2m
+ labels:
+ severity: critical
+ component: watcher
+ team: platform
+ page: "true"
+ annotations:
+ summary: "Watcher service is DOWN"
+ description: "Watcher process is not responding. All event monitoring is offline."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike"
+ impact: "NO stealth payment events are being processed. Service is effectively down for users."
+ action_required: "Restart watcher service immediately."
+
+ - name: wraith_meta_alerts
+ interval: 5m
+ rules:
+ - alert: TooManyFiringAlerts
+ expr: count(ALERTS{alertstate="firing", severity=~"critical|high"}) > 5
+ for: 5m
+ labels:
+ severity: critical
+ component: meta
+ team: sre
+ page: "true"
+ annotations:
+ summary: "Multiple critical/high alerts firing simultaneously"
+ description: "{{ $value }} critical or high severity alerts are firing. Potential cascading failure or infrastructure issue."
+ action_required: "Escalate to incident commander. Begin coordinated multi-team response."
+
+ - alert: AlertmanagerDown
+ expr: up{job="alertmanager"} == 0
+ for: 5m
+ labels:
+ severity: critical
+ component: monitoring
+ team: sre
+ annotations:
+ summary: "Alertmanager is DOWN"
+ description: "Alertmanager process is not responding. Alert routing is offline."
+ impact: "No alerts are being delivered to on-call engineers."
+
+# Rationale per rule:
+#
+# RPC Alerts:
+# - HighRPCLatency: Slow RPC degrades UX and can cause transaction timeouts. 2s threshold based on typical Stellar finality.
+# - HorizonBackpressure: 429 rate limiting is critical - service is effectively down if sustained.
+# - SorobanInvocationFailures: Contract calls failing means core Wraith features unavailable.
+#
+# Indexer Alerts:
+# - IndexerBacklogBuildup: Large backlog means users won't see payments promptly. 1000 is ~10min at 100 announcements/min.
+# - AnnouncementLagHigh: 5min lag is poor UX for real-time payment notifications.
+# - ScanMissRateHigh: High miss rate suggests view-tag misconfiguration or scanning bugs.
+#
+# Contract Alerts:
+# - ContractErrorRateHigh: Any sustained contract error rate is critical - means core protocol broken.
+# - UnauthorizedAccessAttempt: Security incident - requires immediate response per auditor guide severity matrix.
+#
+# Watcher Alerts:
+# - WatcherEventDropSpike: Dropped events = missed payments = broken core functionality.
+# - ViewTagScanLatencyHigh: Slow scanning causes backlog and eventual drops.
+#
+# All thresholds tuned to balance false positives vs. catching real incidents early.
diff --git a/guides/ops/dashboards/contract-monitoring.json b/guides/ops/dashboards/contract-monitoring.json
new file mode 100644
index 0000000..f45d03d
--- /dev/null
+++ b/guides/ops/dashboards/contract-monitoring.json
@@ -0,0 +1,263 @@
+{
+ "dashboard": {
+ "id": null,
+ "uid": "wraith-contract-monitoring",
+ "title": "Wraith Contract Monitoring",
+ "tags": ["wraith", "contracts", "stellar", "soroban"],
+ "timezone": "browser",
+ "schemaVersion": 38,
+ "version": 1,
+ "refresh": "1m",
+ "panels": [
+ {
+ "id": 1,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "type": "heatmap",
+ "title": "Contract Error Rate by Contract",
+ "targets": [
+ {
+ "expr": "sum(rate(wraith_contract_error_rate[5m])) by (contract)",
+ "format": "time_series",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "hideFrom": {
+ "tooltip": false,
+ "viz": false,
+ "legend": false
+ }
+ }
+ }
+ },
+ "options": {
+ "calculate": true,
+ "cellGap": 2,
+ "color": {
+ "mode": "scheme",
+ "scheme": "Spectral",
+ "steps": 128
+ },
+ "yAxis": {
+ "unit": "short"
+ },
+ "tooltip": {
+ "show": true,
+ "yHistogram": false
+ }
+ }
+ },
+ {
+ "id": 2,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "type": "histogram",
+ "title": "Sender Invocation Latency Distribution",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.5, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))",
+ "legendFormat": "P50",
+ "refId": "A"
+ },
+ {
+ "expr": "histogram_quantile(0.95, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))",
+ "legendFormat": "P95",
+ "refId": "B"
+ },
+ {
+ "expr": "histogram_quantile(0.99, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))",
+ "legendFormat": "P99",
+ "refId": "C"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 5,
+ "color": "yellow"
+ },
+ {
+ "value": 20,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 3,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "type": "stat",
+ "title": "Names Registration Failures (Last Hour)",
+ "targets": [
+ {
+ "expr": "sum(increase(wraith_names_registration_failures[1h]))",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 1,
+ "color": "yellow"
+ },
+ {
+ "value": 3,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "area",
+ "textMode": "value_and_name",
+ "reduceOptions": {
+ "values": false,
+ "calcs": ["lastNotNull"]
+ }
+ }
+ },
+ {
+ "id": 4,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "type": "alertlist",
+ "title": "Unauthorized Access Attempts",
+ "targets": [
+ {
+ "expr": "wraith_unauthorized_access_attempts",
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "showOptions": "current",
+ "maxItems": 10,
+ "sortOrder": 1,
+ "dashboardAlerts": false,
+ "alertName": "",
+ "dashboardTitle": "",
+ "tags": ["wraith", "security"],
+ "stateFilter": {
+ "firing": true,
+ "pending": true,
+ "noData": false,
+ "normal": false,
+ "error": true
+ }
+ }
+ },
+ {
+ "id": 5,
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 17
+ },
+ "type": "timeseries",
+ "title": "Contract Invocations by Method",
+ "targets": [
+ {
+ "expr": "sum(rate(wraith_contract_invocations_total[5m])) by (contract, method)",
+ "legendFormat": "{{contract}}.{{method}}",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops",
+ "custom": {
+ "lineWidth": 1,
+ "fillOpacity": 10
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "multi"
+ },
+ "legend": {
+ "displayMode": "table",
+ "placement": "right",
+ "calcs": ["mean", "lastNotNull", "max"]
+ }
+ }
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "name": "contract",
+ "type": "query",
+ "query": "label_values(wraith_contract_error_rate, contract)",
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "multi": true,
+ "includeAll": true
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "name": "Contract Deployments",
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "iconColor": "purple"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ }
+ },
+ "overwrite": true
+}
diff --git a/guides/ops/dashboards/indexer-performance.json b/guides/ops/dashboards/indexer-performance.json
new file mode 100644
index 0000000..d8804a7
--- /dev/null
+++ b/guides/ops/dashboards/indexer-performance.json
@@ -0,0 +1,248 @@
+{
+ "dashboard": {
+ "id": null,
+ "uid": "wraith-indexer-performance",
+ "title": "Wraith Indexer Performance",
+ "tags": ["wraith", "indexer", "announcements"],
+ "timezone": "browser",
+ "schemaVersion": 38,
+ "version": 1,
+ "refresh": "30s",
+ "panels": [
+ {
+ "id": 1,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "type": "timeseries",
+ "title": "Announcement Lag",
+ "targets": [
+ {
+ "expr": "wraith_announcement_lag_seconds",
+ "legendFormat": "Lag (seconds)",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 30,
+ "color": "yellow"
+ },
+ {
+ "value": 300,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "multi"
+ },
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 2,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "type": "gauge",
+ "title": "Indexer Backlog Count",
+ "targets": [
+ {
+ "expr": "wraith_indexer_backlog_count",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "min": 0,
+ "max": 2000,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 100,
+ "color": "yellow"
+ },
+ {
+ "value": 1000,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "showThresholdLabels": true,
+ "showThresholdMarkers": true
+ }
+ },
+ {
+ "id": 3,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "type": "timeseries",
+ "title": "Scan Miss Rate",
+ "targets": [
+ {
+ "expr": "rate(wraith_scan_miss_rate[5m]) * 100",
+ "legendFormat": "Miss Rate %",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percent",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 0.1,
+ "color": "yellow"
+ },
+ {
+ "value": 5,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "single"
+ }
+ }
+ },
+ {
+ "id": 4,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "type": "timeseries",
+ "title": "Sync Height Lag (Ledgers)",
+ "targets": [
+ {
+ "expr": "stellar_network_ledger_height - wraith_indexer_sync_height",
+ "legendFormat": "Ledger lag",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 10,
+ "color": "yellow"
+ },
+ {
+ "value": 50,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "multi"
+ },
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ }
+ }
+ },
+ {
+ "id": 5,
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "type": "table",
+ "title": "Recent Announcements Processed",
+ "targets": [
+ {
+ "expr": "topk(10, wraith_announcement_processed_timestamp)",
+ "format": "table",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "options": {
+ "showHeader": true,
+ "sortBy": [
+ {
+ "displayName": "Time",
+ "desc": true
+ }
+ ]
+ }
+ }
+ ],
+ "templating": {
+ "list": []
+ },
+ "annotations": {
+ "list": [
+ {
+ "name": "Indexer Restarts",
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "iconColor": "orange"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ }
+ },
+ "overwrite": true
+}
diff --git a/guides/ops/dashboards/rpc-health.json b/guides/ops/dashboards/rpc-health.json
new file mode 100644
index 0000000..1bd77da
--- /dev/null
+++ b/guides/ops/dashboards/rpc-health.json
@@ -0,0 +1,228 @@
+{
+ "dashboard": {
+ "id": null,
+ "uid": "wraith-rpc-health",
+ "title": "Wraith RPC Health",
+ "tags": ["wraith", "rpc", "stellar"],
+ "timezone": "browser",
+ "schemaVersion": 38,
+ "version": 1,
+ "refresh": "30s",
+ "panels": [
+ {
+ "id": 1,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "type": "timeseries",
+ "title": "RPC P95 Latency",
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.95, sum(rate(wraith_rpc_latency_seconds_bucket[5m])) by (le, method))",
+ "legendFormat": "{{method}} P95",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 0.5,
+ "color": "yellow"
+ },
+ {
+ "value": 2,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "multi"
+ },
+ "legend": {
+ "displayMode": "table",
+ "placement": "right",
+ "calcs": ["lastNotNull", "max"]
+ }
+ }
+ },
+ {
+ "id": 2,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "type": "gauge",
+ "title": "RPC Error Rate",
+ "targets": [
+ {
+ "expr": "sum(rate(wraith_rpc_error_rate[5m])) / sum(rate(wraith_rpc_latency_seconds_count[5m])) * 100",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percent",
+ "min": 0,
+ "max": 100,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 0.5,
+ "color": "yellow"
+ },
+ {
+ "value": 5,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "showThresholdLabels": true,
+ "showThresholdMarkers": true
+ }
+ },
+ {
+ "id": 3,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 8
+ },
+ "type": "timeseries",
+ "title": "Horizon Backpressure (429 Responses)",
+ "targets": [
+ {
+ "expr": "rate(wraith_horizon_backpressure[1m])",
+ "legendFormat": "429 rate/min",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqpm",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 5,
+ "color": "yellow"
+ },
+ {
+ "value": 10,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "tooltip": {
+ "mode": "single"
+ }
+ }
+ },
+ {
+ "id": 4,
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 8
+ },
+ "type": "stat",
+ "title": "Soroban Invocation Failures",
+ "targets": [
+ {
+ "expr": "sum(increase(wraith_soroban_invocation_failures[1h]))",
+ "refId": "A"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "value": null,
+ "color": "green"
+ },
+ {
+ "value": 1,
+ "color": "yellow"
+ },
+ {
+ "value": 5,
+ "color": "red"
+ }
+ ]
+ }
+ }
+ },
+ "options": {
+ "colorMode": "background",
+ "graphMode": "area",
+ "textMode": "value_and_name"
+ }
+ }
+ ],
+ "templating": {
+ "list": [
+ {
+ "name": "network",
+ "type": "query",
+ "query": "label_values(wraith_rpc_latency_seconds, network)",
+ "current": {
+ "text": "All",
+ "value": "$__all"
+ },
+ "multi": true,
+ "includeAll": true
+ }
+ ]
+ },
+ "annotations": {
+ "list": [
+ {
+ "name": "Deployments",
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "iconColor": "blue"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ }
+ },
+ "overwrite": true
+}
diff --git a/guides/ops/monitoring-and-on-call.mdx b/guides/ops/monitoring-and-on-call.mdx
new file mode 100644
index 0000000..2641a36
--- /dev/null
+++ b/guides/ops/monitoring-and-on-call.mdx
@@ -0,0 +1,693 @@
+---
+title: 'Monitoring, Alerting, and On‑Call Runbook'
+sidebarTitle: 'Monitoring & On‑Call'
+description: 'Production runbook for Wraith Protocol: metrics, dashboards, alerts, incident playbooks, and tabletop exercises'
+---
+
+This guide provides a complete operational runbook for teams running self-hosted Wraith Protocol instances in production. It covers healthy metric baselines, Grafana dashboards, Prometheus alerting rules, incident response playbooks, and a quarterly tabletop exercise.
+
+
+ This guide assumes you have already deployed Wraith Protocol using the [Self-Hosted Deployment](/guides/ops/self-hosted-deployment) guide. If not, complete that first.
+
+
+## Overview
+
+Production Wraith deployments require monitoring across four domains:
+
+| Domain | Key Components | Primary Risk |
+|---|---|---|
+| **RPC Layer** | Stellar Horizon RPC, Soroban RPC | Transaction submission failures, ledger query timeouts |
+| **Indexer** | Announcement scanner, event log sync | Missed announcements, backlog buildup |
+| **Contracts** | `stealth-announcer`, `stealth-sender`, `wraith-names` | Contract invocation errors, unauthorized access |
+| **Watcher** | Event drop detection, view-tag scanning | Dropped events, privacy leakage via missed scans |
+
+This runbook is structured around these four domains. Each section defines healthy baselines, provides ready-to-import Grafana dashboards, and links to incident playbooks.
+
+---
+
+## Key Metrics
+
+### RPC Metrics
+
+Monitor these metrics to ensure reliable access to the Stellar network:
+
+| Metric | Description | Healthy Baseline | Alert Threshold |
+|---|---|---|---|
+| `wraith_rpc_latency_seconds` | P95 latency for Horizon/Soroban RPC calls | < 500ms | > 2s for 5min |
+| `wraith_rpc_error_rate` | Percentage of RPC calls returning 5xx or network errors | < 0.5% | > 5% for 2min |
+| `wraith_horizon_backpressure` | Rate-limit 429 responses from Horizon | 0/min | > 10/min |
+| `wraith_soroban_invocation_failures` | Failed contract invocations due to RPC issues | < 1/hour | > 5/hour |
+
+**Collection**: Instrument your SDK or middleware layer with Prometheus client libraries. Example Node.js middleware:
+
+```typescript
+import promClient from 'prom-client';
+
+const rpcLatency = new promClient.Histogram({
+ name: 'wraith_rpc_latency_seconds',
+ help: 'RPC call latency in seconds',
+ labelNames: ['method', 'network'],
+ buckets: [0.1, 0.5, 1, 2, 5],
+});
+
+const rpcErrors = new promClient.Counter({
+ name: 'wraith_rpc_error_rate',
+ help: 'RPC error count',
+ labelNames: ['method', 'status'],
+});
+```
+
+### Indexer Metrics
+
+Track announcement scanning and event synchronization health:
+
+| Metric | Description | Healthy Baseline | Alert Threshold |
+|---|---|---|---|
+| `wraith_announcement_lag_seconds` | Time between announcement emission and indexer ingestion | < 30s | > 300s for 5min |
+| `wraith_indexer_backlog_count` | Number of unprocessed announcements | < 100 | > 1000 |
+| `wraith_scan_miss_rate` | Announcements scanned but not matched (view-tag misses) | < 0.1% | > 5% for 10min |
+| `wraith_indexer_sync_height` | Current ledger height being indexed vs. network tip | lag < 10 ledgers | lag > 50 ledgers |
+
+**Collection**: Expose metrics from your indexer service. Example Rust Prometheus exporter:
+
+```rust
+use prometheus::{Histogram, IntGauge, register_histogram, register_int_gauge};
+
+lazy_static! {
+ static ref ANNOUNCEMENT_LAG: Histogram = register_histogram!(
+ "wraith_announcement_lag_seconds",
+ "Lag between announcement emission and ingestion"
+ ).unwrap();
+
+ static ref INDEXER_BACKLOG: IntGauge = register_int_gauge!(
+ "wraith_indexer_backlog_count",
+ "Count of unprocessed announcements"
+ ).unwrap();
+}
+```
+
+### Contract Metrics
+
+Monitor on-chain contract health and error rates:
+
+| Metric | Description | Healthy Baseline | Alert Threshold |
+|---|---|---|---|
+| `wraith_contract_error_rate` | Failed contract invocations (all contracts) | < 0.1% | > 1% for 5min |
+| `wraith_sender_invocation_latency` | Time from transaction submission to ledger inclusion | < 5s | > 20s for 5min |
+| `wraith_names_registration_failures` | Failed `wraith-names` registrations | 0/hour | > 3/hour |
+| `wraith_unauthorized_access_attempts` | Contract calls with invalid authorization | 0/day | > 1/day |
+
+**Collection**: Parse Soroban contract events and transaction results. Example:
+
+```javascript
+const contractErrors = new promClient.Counter({
+ name: 'wraith_contract_error_rate',
+ help: 'Contract invocation errors',
+ labelNames: ['contract', 'method'],
+});
+
+sorobanClient.on('transactionFailed', (tx) => {
+ contractErrors.inc({ contract: tx.contract, method: tx.method });
+});
+```
+
+### Watcher Metrics
+
+Track event drop detection and view-tag scanning performance:
+
+| Metric | Description | Healthy Baseline | Alert Threshold |
+|---|---|---|---|
+| `wraith_watcher_event_drop_rate` | Events detected as dropped by the watcher | 0/hour | > 5/hour |
+| `wraith_view_tag_scan_duration_seconds` | Time to scan view tags for a single announcement | < 10ms | > 100ms |
+| `wraith_view_tag_false_positive_rate` | View-tag matches that fail full ECDH check | < 1% | > 10% |
+
+**Collection**: Instrument your watcher/scanner service:
+
+```python
+from prometheus_client import Counter, Histogram
+
+event_drops = Counter('wraith_watcher_event_drop_rate', 'Dropped event count')
+scan_duration = Histogram('wraith_view_tag_scan_duration_seconds', 'View tag scan time')
+
+def scan_announcement(announcement):
+ with scan_duration.time():
+ # scanning logic
+ pass
+```
+
+---
+
+## Grafana Dashboards
+
+Import these dashboards into your Grafana instance to visualize Wraith metrics. All dashboards are in the `guides/ops/dashboards/` directory.
+
+### Dashboard 1: RPC Health
+
+**File**: `guides/ops/dashboards/rpc-health.json`
+
+**Panels**:
+- RPC P95 Latency (time series)
+- RPC Error Rate (gauge)
+- Horizon Backpressure (429 responses over time)
+- Soroban Invocation Failures (counter)
+
+**Import**:
+```bash
+curl -X POST http://localhost:3000/api/dashboards/db \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer YOUR_GRAFANA_API_KEY" \
+ -d @guides/ops/dashboards/rpc-health.json
+```
+
+### Dashboard 2: Indexer Performance
+
+**File**: `guides/ops/dashboards/indexer-performance.json`
+
+**Panels**:
+- Announcement Lag (time series)
+- Indexer Backlog Count (gauge)
+- Scan Miss Rate (percentage over time)
+- Sync Height Lag (ledger difference graph)
+
+### Dashboard 3: Contract Monitoring
+
+**File**: `guides/ops/dashboards/contract-monitoring.json`
+
+**Panels**:
+- Contract Error Rate by Contract (heatmap)
+- Sender Invocation Latency (histogram)
+- Names Registration Failures (counter)
+- Unauthorized Access Attempts (alert panel)
+
+**Cross-Reference**: See [Stellar Cryptography](/architecture/stellar-cryptography) for contract architecture details.
+
+---
+
+## Prometheus Alerting Rules
+
+Import these alerting rules into your Prometheus configuration. The rules file is at `guides/ops/alerts/wraith-alerts.yml`.
+
+### Alerting Philosophy
+
+Each alert includes:
+- **Severity**: Maps to the [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix)
+- **Rationale**: Why this threshold triggers an alert
+- **Playbook**: Link to the incident response playbook
+
+### Rules File: `wraith-alerts.yml`
+
+```yaml
+groups:
+ - name: wraith_rpc_alerts
+ interval: 30s
+ rules:
+ - alert: HighRPCLatency
+ expr: histogram_quantile(0.95, wraith_rpc_latency_seconds) > 2
+ for: 5m
+ labels:
+ severity: high
+ component: rpc
+ annotations:
+ summary: "RPC P95 latency > 2s for 5 minutes"
+ description: "{{ $labels.method }} RPC calls are experiencing high latency. Check Horizon/Soroban RPC health."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage"
+
+ - alert: HorizonBackpressure
+ expr: rate(wraith_horizon_backpressure[1m]) > 10
+ for: 2m
+ labels:
+ severity: critical
+ component: rpc
+ annotations:
+ summary: "Horizon rate-limiting detected (> 10 429s/min)"
+ description: "Application is being rate-limited by Horizon. Immediate intervention required."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-2-horizon-backpressure"
+
+ - name: wraith_indexer_alerts
+ interval: 1m
+ rules:
+ - alert: IndexerBacklogBuildup
+ expr: wraith_indexer_backlog_count > 1000
+ for: 5m
+ labels:
+ severity: high
+ component: indexer
+ annotations:
+ summary: "Indexer backlog > 1000 announcements"
+ description: "Indexer is falling behind. Check database performance and announcement rate."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+
+ - alert: AnnouncementLagHigh
+ expr: wraith_announcement_lag_seconds > 300
+ for: 5m
+ labels:
+ severity: medium
+ component: indexer
+ annotations:
+ summary: "Announcement lag > 5 minutes"
+ description: "Users may experience delays seeing incoming stealth payments."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall"
+
+ - name: wraith_contract_alerts
+ interval: 1m
+ rules:
+ - alert: ContractErrorRateHigh
+ expr: rate(wraith_contract_error_rate[5m]) > 0.01
+ for: 5m
+ labels:
+ severity: critical
+ component: contracts
+ annotations:
+ summary: "Contract error rate > 1%"
+ description: "Contract invocations are failing at an elevated rate. Check contract state and RPC health."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish"
+
+ - alert: UnauthorizedAccessAttempt
+ expr: wraith_unauthorized_access_attempts > 0
+ for: 1m
+ labels:
+ severity: critical
+ component: contracts
+ annotations:
+ summary: "Unauthorized contract access detected"
+ description: "A contract received a call with invalid authorization. Potential security incident."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-6-key-rotation-incident"
+
+ - name: wraith_watcher_alerts
+ interval: 1m
+ rules:
+ - alert: WatcherEventDropSpike
+ expr: rate(wraith_watcher_event_drop_rate[5m]) > 5
+ for: 2m
+ labels:
+ severity: high
+ component: watcher
+ annotations:
+ summary: "Watcher event drop rate > 5/hour"
+ description: "Events are being dropped. Users may miss stealth payments."
+ playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike"
+```
+
+**Validation**:
+```bash
+promtool check rules guides/ops/alerts/wraith-alerts.yml
+```
+
+Expected output:
+```
+Checking guides/ops/alerts/wraith-alerts.yml
+ SUCCESS: 8 rules found
+```
+
+---
+
+## Incident Playbooks
+
+Each playbook follows a standard structure: **Symptoms → Diagnosis → Immediate Actions → Long-term Fix → Post-Incident Review**.
+
+### Playbook 1: RPC Outage
+
+**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 15 minutes to acknowledge, 1 hour to mitigation
+
+#### Symptoms
+- `HighRPCLatency` alert firing
+- Users reporting "transaction not found" errors
+- Dashboard shows RPC error rate > 10%
+
+#### Diagnosis
+1. Check Horizon/Soroban RPC status:
+ ```bash
+ curl https://horizon-futurenet.stellar.org/
+ curl https://soroban-testnet.stellar.org/health
+ ```
+2. Verify network connectivity:
+ ```bash
+ traceroute horizon-futurenet.stellar.org
+ ```
+3. Check application logs for RPC timeout patterns:
+ ```bash
+ grep "RPC timeout" /var/log/wraith-app.log | tail -50
+ ```
+
+#### Immediate Actions
+1. **Switch to backup RPC endpoint** (if configured):
+ ```typescript
+ const wraith = new Wraith({
+ rpcUrl: process.env.BACKUP_RPC_URL,
+ });
+ ```
+2. **Enable RPC request queueing** to prevent overwhelming the endpoint:
+ ```typescript
+ const rpcQueue = new PQueue({ concurrency: 10, interval: 1000, intervalCap: 10 });
+ ```
+3. **Post status update** to users via status page or Twitter.
+
+#### Long-term Fix
+- Set up RPC load balancing with multiple Horizon instances
+- Implement exponential backoff for RPC retries
+- Add RPC health checks to pre-deployment tests
+
+#### Post-Incident Review
+- Document RPC failure patterns
+- Update runbook with new diagnostic steps
+- Review RPC provider SLA and consider alternatives
+
+---
+
+### Playbook 2: Horizon Backpressure
+
+**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 15 minutes to acknowledge, 30 minutes to mitigation
+
+#### Symptoms
+- `HorizonBackpressure` alert firing
+- HTTP 429 "Rate Limit Exceeded" responses from Horizon
+- Transaction submission queue building up
+
+#### Diagnosis
+1. Check current rate limit status:
+ ```bash
+ curl -I https://horizon-futurenet.stellar.org/ | grep X-RateLimit
+ ```
+2. Identify which service is generating excessive requests:
+ ```bash
+ grep "429" /var/log/wraith-app.log | cut -d' ' -f1 | sort | uniq -c
+ ```
+
+#### Immediate Actions
+1. **Implement request throttling immediately**:
+ ```typescript
+ import Bottleneck from 'bottleneck';
+ const limiter = new Bottleneck({
+ minTime: 200, // 5 requests/second max
+ maxConcurrent: 1,
+ });
+ ```
+2. **Disable non-critical polling loops** (e.g., balance checks, ledger monitoring).
+3. **Cache RPC responses** for frequently accessed data (account balances, contract state).
+
+#### Long-term Fix
+- Upgrade to a dedicated Horizon instance (not public)
+- Implement intelligent request batching
+- Add rate-limit monitoring to CI/CD pipeline
+
+#### Post-Incident Review
+- Audit all RPC call sites for unnecessary requests
+- Set up proactive rate-limit monitoring (alert at 80% of limit)
+
+---
+
+### Playbook 3: Indexer Stall
+
+**Severity**: High ([Auditor Guide Sev 2](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 1 hour to acknowledge, 4 hours to mitigation
+
+#### Symptoms
+- `IndexerBacklogBuildup` or `AnnouncementLagHigh` alert firing
+- Users report delayed stealth payment notifications
+- Indexer sync height lagging network tip by > 50 ledgers
+
+#### Diagnosis
+1. Check indexer process health:
+ ```bash
+ systemctl status wraith-indexer
+ journalctl -u wraith-indexer -n 100
+ ```
+2. Check database query performance:
+ ```sql
+ SELECT query, mean_exec_time, calls FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10;
+ ```
+3. Verify disk I/O isn't saturated:
+ ```bash
+ iostat -x 5 3
+ ```
+
+#### Immediate Actions
+1. **Restart the indexer** if it's stuck:
+ ```bash
+ systemctl restart wraith-indexer
+ ```
+2. **Increase indexer concurrency** temporarily:
+ ```bash
+ export INDEXER_WORKERS=8 # default is 4
+ systemctl restart wraith-indexer
+ ```
+3. **Prioritize recent announcements** by adjusting sync strategy:
+ ```typescript
+ await indexer.syncFrom({ ledger: currentLedger - 100 });
+ ```
+
+#### Long-term Fix
+- Add database indexes on announcement query columns
+- Implement announcement batching (process 100 at a time instead of 1)
+- Scale indexer horizontally with ledger range sharding
+
+#### Post-Incident Review
+- Benchmark indexer throughput under load
+- Add capacity planning metrics (CPU, memory, disk I/O)
+
+---
+
+### Playbook 4: Contract Mispublish
+
+**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 15 minutes to acknowledge, 2 hours to mitigation
+
+#### Symptoms
+- `ContractErrorRateHigh` alert firing
+- Users unable to register names or send stealth payments
+- Soroban RPC returns `contract not found` or `invalid wasm`
+
+#### Diagnosis
+1. Verify deployed contract IDs match expected values:
+ ```bash
+ cat stellar/deployments/futurenet.json | jq '.contracts'
+ ```
+2. Query contract state on-chain:
+ ```bash
+ stellar contract invoke --id --network futurenet -- --help
+ ```
+3. Check deployment logs for errors:
+ ```bash
+ grep "deploy failed" /var/log/deploy.log
+ ```
+
+#### Immediate Actions
+1. **Roll back to last known-good contract** (if available):
+ ```bash
+ stellar contract deploy --wasm stealth-sender-v1.2.wasm --network futurenet
+ ```
+2. **Update frontend to point to backup contract** (if multi-contract setup):
+ ```typescript
+ const SENDER_ID = process.env.BACKUP_SENDER_CONTRACT_ID;
+ ```
+3. **Disable contract-dependent features** in the UI to prevent user errors.
+
+#### Long-term Fix
+- Implement blue-green deployments for contracts
+- Add contract state verification to deployment scripts
+- Set up contract upgrade simulations in staging
+
+#### Post-Incident Review
+- Document contract deployment checklist
+- Add contract state smoke tests to CI
+- Review contract upgrade governance process
+
+---
+
+### Playbook 5: Watcher Drop Spike
+
+**Severity**: High ([Auditor Guide Sev 2](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 1 hour to acknowledge, 4 hours to mitigation
+
+#### Symptoms
+- `WatcherEventDropSpike` alert firing
+- Users report missing stealth payments
+- View-tag scan logs show high false-positive rates
+
+#### Diagnosis
+1. Check watcher process logs:
+ ```bash
+ journalctl -u wraith-watcher -n 200 | grep "drop"
+ ```
+2. Verify event subscription is active:
+ ```typescript
+ const isConnected = await eventStream.isConnected();
+ console.log('Event stream connected:', isConnected);
+ ```
+3. Check for network partitions or RPC instability.
+
+#### Immediate Actions
+1. **Restart the watcher service**:
+ ```bash
+ systemctl restart wraith-watcher
+ ```
+2. **Re-scan recent ledgers** to catch missed events:
+ ```bash
+ wraith-watcher rescan --from-ledger
+ ```
+3. **Notify users** to check for missing payments manually via explorer.
+
+#### Long-term Fix
+- Implement event stream redundancy (subscribe to multiple RPC endpoints)
+- Add automatic gap detection and backfill
+- Set up watcher health checks with auto-restart
+
+#### Post-Incident Review
+- Analyze root cause of event drops (network, RPC, bug)
+- Add integration tests for event stream resilience
+
+---
+
+### Playbook 6: Key Rotation Incident
+
+**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix))
+**Response SLA**: 15 minutes to acknowledge, immediate mitigation
+
+#### Symptoms
+- `UnauthorizedAccessAttempt` alert firing
+- Contract admin key compromised or needs rotation
+- Suspicious transactions targeting Wraith contracts
+
+#### Diagnosis
+1. Identify unauthorized transactions:
+ ```bash
+ stellar account transactions --network futurenet | grep "failed"
+ ```
+2. Check contract authorization settings:
+ ```bash
+ stellar contract invoke --id --network futurenet -- get_admin
+ ```
+3. Review audit logs for anomalous access patterns.
+
+#### Immediate Actions
+1. **Rotate admin keys immediately** (see [Multisig Authority Rotation](/guides/stellar/multisig-authority-rotation)):
+ ```bash
+ stellar contract invoke --id --source new-admin --network futurenet -- set_admin --new_admin
+ ```
+2. **Pause affected contracts** if available:
+ ```bash
+ stellar contract invoke --id --source admin --network futurenet -- pause
+ ```
+3. **Notify security team** and begin incident response process.
+
+#### Long-term Fix
+- Implement multisig admin controls for all contracts
+- Add hardware wallet support for admin operations
+- Set up automated key rotation schedule (quarterly)
+
+#### Post-Incident Review
+- Conduct full security audit of contract access patterns
+- Update [Security Disclosure Policy](/reference/security-disclosure) if needed
+- File internal postmortem with root cause analysis
+
+---
+
+## Severity Matrix and Response SLA
+
+This table maps incident types to response SLAs, aligned with the [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix).
+
+| Incident Type | Severity | User Impact | Response SLA | Fix SLA |
+|---|---|---|---|---|
+| **RPC Outage** | Critical | No transactions can be submitted | 15min | 1 hour |
+| **Horizon Backpressure** | Critical | Service degradation, failed submissions | 15min | 30min |
+| **Indexer Stall** | High | Delayed payment notifications | 1 hour | 4 hours |
+| **Contract Mispublish** | Critical | Core functionality unavailable | 15min | 2 hours |
+| **Watcher Drop Spike** | High | Missed stealth payments | 1 hour | 4 hours |
+| **Key Rotation Incident** | Critical | Potential unauthorized access | 15min | Immediate |
+
+**Response SLA**: Time from alert firing to on-call engineer acknowledging and beginning diagnosis.
+**Fix SLA**: Time from acknowledgment to incident resolved or mitigated.
+
+### Severity Definitions
+
+These align with the [Auditor Guide](/reference/auditor-guide#severity-matrix):
+
+- **Critical**: Complete loss of functionality, security breach, or privacy violation affecting all users.
+- **High**: Partial functionality loss or degradation affecting a subset of users.
+- **Medium**: Limited impact requiring unusual conditions, no direct fund loss.
+- **Low**: Minor issues with no realistic path to user harm.
+
+---
+
+## Tabletop Exercise Script
+
+Run this quarterly drill to validate your team's incident response readiness. The exercise takes 90 minutes and requires:
+- 3-5 participants (on-call engineers, product lead, security contact)
+- Access to a staging environment
+- This runbook and all playbooks printed or open
+
+### Exercise Structure
+
+**Duration**: 90 minutes
+**Facilitator**: Rotating role (different person each quarter)
+**Participants**: On-call engineers, product manager, security lead
+
+#### Phase 1: Scenario Briefing (15 minutes)
+
+**Facilitator reads**:
+> "It's 2 AM on a Saturday. The on-call pager fires: `HorizonBackpressure` alert is critical. Users are reporting failed stealth payments on Twitter. Your staging environment is configured identically to production. You have 60 minutes to resolve the incident."
+
+**Distribute roles**:
+- **Incident Commander**: Coordinates response, makes decisions
+- **Diagnostics Lead**: Runs diagnostic commands, interprets metrics
+- **Comms Lead**: Drafts user-facing status updates
+- **Scribe**: Documents all actions taken for post-incident review
+
+#### Phase 2: Incident Response (45 minutes)
+
+Participants work through [Playbook 2: Horizon Backpressure](#playbook-2-horizon-backpressure) on staging:
+
+1. **Diagnosis** (15 min): Run diagnostic commands, identify root cause
+2. **Mitigation** (20 min): Implement rate limiting, disable non-critical services
+3. **Validation** (10 min): Verify alerts clear, test transaction submission
+
+**Facilitator injects complications**:
+- At T+15min: "The backup RPC endpoint is also returning 429s. What now?"
+- At T+30min: "Marketing is asking for an ETA to post on Twitter. What do you tell them?"
+
+#### Phase 3: Post-Incident Review (30 minutes)
+
+**Discussion prompts**:
+1. What went well? What slowed you down?
+2. Were the diagnostic steps clear? What was missing?
+3. Did you have the access and tools you needed?
+4. What would you change in the playbook?
+5. What should we add to the runbook?
+
+**Action items**:
+- Update playbooks based on learnings
+- Add missing diagnostic tools to on-call toolkit
+- Schedule follow-up training for gaps identified
+
+### Quarterly Rotation
+
+Run different scenarios each quarter:
+
+- **Q1**: Horizon Backpressure (Playbook 2)
+- **Q2**: Indexer Stall (Playbook 3)
+- **Q3**: Key Rotation Incident (Playbook 6)
+- **Q4**: Multi-incident cascade (RPC + Watcher drop)
+
+---
+
+## Cross-References
+
+- [Self-Hosted Deployment](/guides/ops/self-hosted-deployment) — prerequisite deployment guide
+- [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix) — severity definitions and response SLAs
+- [Multisig Authority Rotation](/guides/stellar/multisig-authority-rotation) — key rotation procedures
+- [Stellar Cryptography](/architecture/stellar-cryptography) — contract architecture for debugging
+- [Security Disclosure Policy](/reference/security-disclosure) — incident reporting and safe harbor
+
+---
+
+## Next Steps
+
+1. **Import dashboards**: Load all three Grafana dashboards into your instance.
+2. **Validate alerts**: Run `promtool check rules` on `wraith-alerts.yml`.
+3. **Schedule tabletop**: Put the quarterly drill on the calendar.
+4. **Assign on-call rotation**: Define primary and secondary on-call engineers.
+5. **Test a playbook**: Walk through Playbook 1 (RPC Outage) on staging to validate all commands work.
+
+For operational questions, see the [Self-Hosted Deployment guide](/guides/ops/self-hosted-deployment) or reach out in the Wraith Protocol community Discord.
diff --git a/guides/ops/self-hosted-deployment.mdx b/guides/ops/self-hosted-deployment.mdx
index b98556f..ad9160d 100644
--- a/guides/ops/self-hosted-deployment.mdx
+++ b/guides/ops/self-hosted-deployment.mdx
@@ -141,3 +141,11 @@ You should see ledger entries confirming the deployment.
**Your self‑hosted Wraith instance is now live.** You can register names,
announce stealth meta‑addresses, and send private payments against these contracts.
+
+## Next Steps
+
+For production deployments, see the [Monitoring, Alerting, and On‑Call Runbook](/guides/ops/monitoring-and-on-call) to set up:
+- Grafana dashboards for RPC, indexer, and contract health
+- Prometheus alerting rules with severity thresholds
+- Incident response playbooks for common operational issues
+- Quarterly tabletop exercises to validate readiness