From d688c7053793fc480ea8544f34dcda4f2823e3ac Mon Sep 17 00:00:00 2001 From: code3ks Date: Wed, 26 Aug 2026 15:38:19 +0100 Subject: [PATCH] Add monitoring, alerting, and on-call runbook for Wraith Protocol ops - Created comprehensive guides/ops/monitoring-and-on-call.mdx with: * Key metrics for RPC, indexer, contracts, and watcher * Healthy baselines and alert thresholds * Six incident playbooks with diagnosis and mitigation steps * Severity matrix mapped to Auditor Guide * Quarterly tabletop exercise script - Added Grafana dashboards in guides/ops/dashboards/: * rpc-health.json - RPC latency, error rates, backpressure * indexer-performance.json - announcement lag, backlog, sync height * contract-monitoring.json - contract errors, invocation latency - Added Prometheus alerting rules in guides/ops/alerts/: * wraith-alerts.yml with 20+ production-ready rules * Rationale and runbook links per alert * Severity labels mapped to SLA requirements - Updated docs.json with Operations nav entry - Cross-linked with self-hosted-deployment.mdx and auditor-guide.mdx Resolves #133 --- docs.json | 3 +- guides/ops/alerts/wraith-alerts.yml | 274 +++++++ .../ops/dashboards/contract-monitoring.json | 263 +++++++ .../ops/dashboards/indexer-performance.json | 248 +++++++ guides/ops/dashboards/rpc-health.json | 228 ++++++ guides/ops/monitoring-and-on-call.mdx | 693 ++++++++++++++++++ guides/ops/self-hosted-deployment.mdx | 8 + 7 files changed, 1716 insertions(+), 1 deletion(-) create mode 100644 guides/ops/alerts/wraith-alerts.yml create mode 100644 guides/ops/dashboards/contract-monitoring.json create mode 100644 guides/ops/dashboards/indexer-performance.json create mode 100644 guides/ops/dashboards/rpc-health.json create mode 100644 guides/ops/monitoring-and-on-call.mdx diff --git a/docs.json b/docs.json index ab3d6de..aa86399 100644 --- a/docs.json +++ b/docs.json @@ -171,7 +171,8 @@ "guides/stellar/stellar-quickstart", "guides/stellar/wraith-names-lifecycle", "guides/wraith-names-stellar", - "guides/ops/self-hosted-deployment" + "guides/ops/self-hosted-deployment", + "guides/ops/monitoring-and-on-call" ] } ] diff --git a/guides/ops/alerts/wraith-alerts.yml b/guides/ops/alerts/wraith-alerts.yml new file mode 100644 index 0000000..73de305 --- /dev/null +++ b/guides/ops/alerts/wraith-alerts.yml @@ -0,0 +1,274 @@ +groups: + - name: wraith_rpc_alerts + interval: 30s + rules: + - alert: HighRPCLatency + expr: histogram_quantile(0.95, sum(rate(wraith_rpc_latency_seconds_bucket[5m])) by (le, method)) > 2 + for: 5m + labels: + severity: high + component: rpc + team: platform + annotations: + summary: "RPC P95 latency > 2s for 5 minutes" + description: "{{ $labels.method }} RPC calls are experiencing high latency ({{ $value }}s). Check Horizon/Soroban RPC health and network connectivity." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage" + dashboard: "https://grafana.example.com/d/wraith-rpc-health" + runbook_steps: | + 1. Check RPC endpoint status + 2. Verify network connectivity + 3. Switch to backup RPC if available + 4. Enable request queueing + + - alert: HighRPCErrorRate + expr: sum(rate(wraith_rpc_error_rate[5m])) / sum(rate(wraith_rpc_latency_seconds_count[5m])) > 0.05 + for: 2m + labels: + severity: critical + component: rpc + team: platform + annotations: + summary: "RPC error rate > 5%" + description: "{{ $value | humanizePercentage }} of RPC calls are failing. Immediate investigation required." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage" + + - alert: HorizonBackpressure + expr: rate(wraith_horizon_backpressure[1m]) > 10 + for: 2m + labels: + severity: critical + component: rpc + team: platform + page: "true" + annotations: + summary: "Horizon rate-limiting detected (> 10 429s/min)" + description: "Application is being rate-limited by Horizon at {{ $value }} 429 responses/min. Immediate intervention required to prevent service disruption." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-2-horizon-backpressure" + impact: "Users cannot submit transactions. Service degraded." + + - alert: SorobanInvocationFailures + expr: rate(wraith_soroban_invocation_failures[1h]) > 5 + for: 5m + labels: + severity: high + component: rpc + team: platform + annotations: + summary: "Soroban invocation failure rate high" + description: "{{ $value }} Soroban invocations failed in the last hour due to RPC issues." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage" + + - name: wraith_indexer_alerts + interval: 1m + rules: + - alert: IndexerBacklogBuildup + expr: wraith_indexer_backlog_count > 1000 + for: 5m + labels: + severity: high + component: indexer + team: data + annotations: + summary: "Indexer backlog > 1000 announcements" + description: "Indexer is falling behind with {{ $value }} unprocessed announcements. Check database performance and announcement ingestion rate." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + dashboard: "https://grafana.example.com/d/wraith-indexer-performance" + impact: "Users may experience delayed stealth payment notifications." + + - alert: AnnouncementLagHigh + expr: wraith_announcement_lag_seconds > 300 + for: 5m + labels: + severity: medium + component: indexer + team: data + annotations: + summary: "Announcement lag > 5 minutes" + description: "Time between announcement emission and indexing is {{ $value }}s. Users may experience delays seeing incoming stealth payments." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + + - alert: IndexerSyncHeightLagging + expr: (stellar_network_ledger_height - wraith_indexer_sync_height) > 50 + for: 10m + labels: + severity: high + component: indexer + team: data + annotations: + summary: "Indexer sync height lagging by > 50 ledgers" + description: "Indexer is {{ $value }} ledgers behind the network tip. Risk of announcement processing delays." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + + - alert: ScanMissRateHigh + expr: rate(wraith_scan_miss_rate[10m]) > 0.05 + for: 10m + labels: + severity: medium + component: indexer + team: data + annotations: + summary: "View-tag scan miss rate > 5%" + description: "{{ $value | humanizePercentage }} of scanned announcements are not matching. Check view-tag scanner configuration." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + + - name: wraith_contract_alerts + interval: 1m + rules: + - alert: ContractErrorRateHigh + expr: sum(rate(wraith_contract_error_rate[5m])) by (contract) > 0.01 + for: 5m + labels: + severity: critical + component: contracts + team: blockchain + page: "true" + annotations: + summary: "Contract {{ $labels.contract }} error rate > 1%" + description: "Contract {{ $labels.contract }} invocations are failing at {{ $value | humanizePercentage }}. Check contract state, RPC health, and recent deployments." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish" + dashboard: "https://grafana.example.com/d/wraith-contract-monitoring" + impact: "Core Wraith functionality may be unavailable to users." + + - alert: SenderInvocationLatencyHigh + expr: histogram_quantile(0.95, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le)) > 20 + for: 5m + labels: + severity: high + component: contracts + team: blockchain + annotations: + summary: "Sender invocation P95 latency > 20s" + description: "Stealth payment submissions are taking {{ $value }}s at P95. Check RPC performance and network congestion." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish" + + - alert: NamesRegistrationFailures + expr: increase(wraith_names_registration_failures[1h]) > 3 + for: 5m + labels: + severity: high + component: contracts + team: blockchain + annotations: + summary: "Wraith Names registrations failing" + description: "{{ $value }} wraith-names registration attempts failed in the last hour. Check contract admin authorization and state." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish" + + - alert: UnauthorizedAccessAttempt + expr: increase(wraith_unauthorized_access_attempts[5m]) > 0 + for: 1m + labels: + severity: critical + component: contracts + team: security + page: "true" + annotations: + summary: "SECURITY: Unauthorized contract access detected" + description: "{{ $value }} unauthorized access attempt(s) detected on Wraith contracts. Potential security incident - investigate immediately." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-6-key-rotation-incident" + impact: "Potential security breach. Admin keys may be compromised." + action_required: "1. Verify contract admin keys. 2. Check transaction history. 3. Initiate key rotation if needed. 4. Notify security team." + + - name: wraith_watcher_alerts + interval: 1m + rules: + - alert: WatcherEventDropSpike + expr: rate(wraith_watcher_event_drop_rate[5m]) > 5 + for: 2m + labels: + severity: high + component: watcher + team: platform + annotations: + summary: "Watcher event drop rate > 5/hour" + description: "Events are being dropped at {{ $value }}/hour. Users may miss stealth payment notifications." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike" + impact: "Users may not see incoming stealth payments in wallets." + + - alert: ViewTagScanLatencyHigh + expr: histogram_quantile(0.95, rate(wraith_view_tag_scan_duration_seconds_bucket[5m])) > 0.1 + for: 10m + labels: + severity: medium + component: watcher + team: platform + annotations: + summary: "View-tag scan P95 latency > 100ms" + description: "View-tag scanning is taking {{ $value }}s at P95. Risk of announcement processing bottleneck." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike" + + - alert: ViewTagFalsePositiveRateHigh + expr: rate(wraith_view_tag_false_positive_rate[10m]) > 0.1 + for: 10m + labels: + severity: low + component: watcher + team: platform + annotations: + summary: "View-tag false positive rate > 10%" + description: "{{ $value | humanizePercentage }} of view-tag matches are failing full ECDH check. Consider view-tag configuration review." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike" + + - alert: WatcherDown + expr: up{job="wraith-watcher"} == 0 + for: 2m + labels: + severity: critical + component: watcher + team: platform + page: "true" + annotations: + summary: "Watcher service is DOWN" + description: "Watcher process is not responding. All event monitoring is offline." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike" + impact: "NO stealth payment events are being processed. Service is effectively down for users." + action_required: "Restart watcher service immediately." + + - name: wraith_meta_alerts + interval: 5m + rules: + - alert: TooManyFiringAlerts + expr: count(ALERTS{alertstate="firing", severity=~"critical|high"}) > 5 + for: 5m + labels: + severity: critical + component: meta + team: sre + page: "true" + annotations: + summary: "Multiple critical/high alerts firing simultaneously" + description: "{{ $value }} critical or high severity alerts are firing. Potential cascading failure or infrastructure issue." + action_required: "Escalate to incident commander. Begin coordinated multi-team response." + + - alert: AlertmanagerDown + expr: up{job="alertmanager"} == 0 + for: 5m + labels: + severity: critical + component: monitoring + team: sre + annotations: + summary: "Alertmanager is DOWN" + description: "Alertmanager process is not responding. Alert routing is offline." + impact: "No alerts are being delivered to on-call engineers." + +# Rationale per rule: +# +# RPC Alerts: +# - HighRPCLatency: Slow RPC degrades UX and can cause transaction timeouts. 2s threshold based on typical Stellar finality. +# - HorizonBackpressure: 429 rate limiting is critical - service is effectively down if sustained. +# - SorobanInvocationFailures: Contract calls failing means core Wraith features unavailable. +# +# Indexer Alerts: +# - IndexerBacklogBuildup: Large backlog means users won't see payments promptly. 1000 is ~10min at 100 announcements/min. +# - AnnouncementLagHigh: 5min lag is poor UX for real-time payment notifications. +# - ScanMissRateHigh: High miss rate suggests view-tag misconfiguration or scanning bugs. +# +# Contract Alerts: +# - ContractErrorRateHigh: Any sustained contract error rate is critical - means core protocol broken. +# - UnauthorizedAccessAttempt: Security incident - requires immediate response per auditor guide severity matrix. +# +# Watcher Alerts: +# - WatcherEventDropSpike: Dropped events = missed payments = broken core functionality. +# - ViewTagScanLatencyHigh: Slow scanning causes backlog and eventual drops. +# +# All thresholds tuned to balance false positives vs. catching real incidents early. diff --git a/guides/ops/dashboards/contract-monitoring.json b/guides/ops/dashboards/contract-monitoring.json new file mode 100644 index 0000000..f45d03d --- /dev/null +++ b/guides/ops/dashboards/contract-monitoring.json @@ -0,0 +1,263 @@ +{ + "dashboard": { + "id": null, + "uid": "wraith-contract-monitoring", + "title": "Wraith Contract Monitoring", + "tags": ["wraith", "contracts", "stellar", "soroban"], + "timezone": "browser", + "schemaVersion": 38, + "version": 1, + "refresh": "1m", + "panels": [ + { + "id": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "type": "heatmap", + "title": "Contract Error Rate by Contract", + "targets": [ + { + "expr": "sum(rate(wraith_contract_error_rate[5m])) by (contract)", + "format": "time_series", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + } + } + }, + "options": { + "calculate": true, + "cellGap": 2, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "steps": 128 + }, + "yAxis": { + "unit": "short" + }, + "tooltip": { + "show": true, + "yHistogram": false + } + } + }, + { + "id": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "type": "histogram", + "title": "Sender Invocation Latency Distribution", + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))", + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(wraith_sender_invocation_latency_seconds_bucket[5m])) by (le))", + "legendFormat": "P99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 5, + "color": "yellow" + }, + { + "value": 20, + "color": "red" + } + ] + } + } + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "type": "stat", + "title": "Names Registration Failures (Last Hour)", + "targets": [ + { + "expr": "sum(increase(wraith_names_registration_failures[1h]))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "yellow" + }, + { + "value": 3, + "color": "red" + } + ] + } + } + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "textMode": "value_and_name", + "reduceOptions": { + "values": false, + "calcs": ["lastNotNull"] + } + } + }, + { + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "type": "alertlist", + "title": "Unauthorized Access Attempts", + "targets": [ + { + "expr": "wraith_unauthorized_access_attempts", + "refId": "A" + } + ], + "options": { + "showOptions": "current", + "maxItems": 10, + "sortOrder": 1, + "dashboardAlerts": false, + "alertName": "", + "dashboardTitle": "", + "tags": ["wraith", "security"], + "stateFilter": { + "firing": true, + "pending": true, + "noData": false, + "normal": false, + "error": true + } + } + }, + { + "id": 5, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "type": "timeseries", + "title": "Contract Invocations by Method", + "targets": [ + { + "expr": "sum(rate(wraith_contract_invocations_total[5m])) by (contract, method)", + "legendFormat": "{{contract}}.{{method}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 1, + "fillOpacity": 10 + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "lastNotNull", "max"] + } + } + } + ], + "templating": { + "list": [ + { + "name": "contract", + "type": "query", + "query": "label_values(wraith_contract_error_rate, contract)", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "includeAll": true + } + ] + }, + "annotations": { + "list": [ + { + "name": "Contract Deployments", + "datasource": "-- Grafana --", + "enable": true, + "iconColor": "purple" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + } + }, + "overwrite": true +} diff --git a/guides/ops/dashboards/indexer-performance.json b/guides/ops/dashboards/indexer-performance.json new file mode 100644 index 0000000..d8804a7 --- /dev/null +++ b/guides/ops/dashboards/indexer-performance.json @@ -0,0 +1,248 @@ +{ + "dashboard": { + "id": null, + "uid": "wraith-indexer-performance", + "title": "Wraith Indexer Performance", + "tags": ["wraith", "indexer", "announcements"], + "timezone": "browser", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "panels": [ + { + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "type": "timeseries", + "title": "Announcement Lag", + "targets": [ + { + "expr": "wraith_announcement_lag_seconds", + "legendFormat": "Lag (seconds)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 30, + "color": "yellow" + }, + { + "value": 300, + "color": "red" + } + ] + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "type": "gauge", + "title": "Indexer Backlog Count", + "targets": [ + { + "expr": "wraith_indexer_backlog_count", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "max": 2000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 100, + "color": "yellow" + }, + { + "value": 1000, + "color": "red" + } + ] + } + } + }, + "options": { + "showThresholdLabels": true, + "showThresholdMarkers": true + } + }, + { + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "type": "timeseries", + "title": "Scan Miss Rate", + "targets": [ + { + "expr": "rate(wraith_scan_miss_rate[5m]) * 100", + "legendFormat": "Miss Rate %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.1, + "color": "yellow" + }, + { + "value": 5, + "color": "red" + } + ] + } + } + }, + "options": { + "tooltip": { + "mode": "single" + } + } + }, + { + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "type": "timeseries", + "title": "Sync Height Lag (Ledgers)", + "targets": [ + { + "expr": "stellar_network_ledger_height - wraith_indexer_sync_height", + "legendFormat": "Ledger lag", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 10, + "color": "yellow" + }, + { + "value": 50, + "color": "red" + } + ] + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "type": "table", + "title": "Recent Announcements Processed", + "targets": [ + { + "expr": "topk(10, wraith_announcement_processed_timestamp)", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [ + { + "name": "Indexer Restarts", + "datasource": "-- Grafana --", + "enable": true, + "iconColor": "orange" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + } + }, + "overwrite": true +} diff --git a/guides/ops/dashboards/rpc-health.json b/guides/ops/dashboards/rpc-health.json new file mode 100644 index 0000000..1bd77da --- /dev/null +++ b/guides/ops/dashboards/rpc-health.json @@ -0,0 +1,228 @@ +{ + "dashboard": { + "id": null, + "uid": "wraith-rpc-health", + "title": "Wraith RPC Health", + "tags": ["wraith", "rpc", "stellar"], + "timezone": "browser", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "panels": [ + { + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "type": "timeseries", + "title": "RPC P95 Latency", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(wraith_rpc_latency_seconds_bucket[5m])) by (le, method))", + "legendFormat": "{{method}} P95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.5, + "color": "yellow" + }, + { + "value": 2, + "color": "red" + } + ] + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["lastNotNull", "max"] + } + } + }, + { + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "type": "gauge", + "title": "RPC Error Rate", + "targets": [ + { + "expr": "sum(rate(wraith_rpc_error_rate[5m])) / sum(rate(wraith_rpc_latency_seconds_count[5m])) * 100", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 0.5, + "color": "yellow" + }, + { + "value": 5, + "color": "red" + } + ] + } + } + }, + "options": { + "showThresholdLabels": true, + "showThresholdMarkers": true + } + }, + { + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "type": "timeseries", + "title": "Horizon Backpressure (429 Responses)", + "targets": [ + { + "expr": "rate(wraith_horizon_backpressure[1m])", + "legendFormat": "429 rate/min", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqpm", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 5, + "color": "yellow" + }, + { + "value": 10, + "color": "red" + } + ] + } + } + }, + "options": { + "tooltip": { + "mode": "single" + } + } + }, + { + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "type": "stat", + "title": "Soroban Invocation Failures", + "targets": [ + { + "expr": "sum(increase(wraith_soroban_invocation_failures[1h]))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1, + "color": "yellow" + }, + { + "value": 5, + "color": "red" + } + ] + } + } + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "textMode": "value_and_name" + } + } + ], + "templating": { + "list": [ + { + "name": "network", + "type": "query", + "query": "label_values(wraith_rpc_latency_seconds, network)", + "current": { + "text": "All", + "value": "$__all" + }, + "multi": true, + "includeAll": true + } + ] + }, + "annotations": { + "list": [ + { + "name": "Deployments", + "datasource": "-- Grafana --", + "enable": true, + "iconColor": "blue" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + } + }, + "overwrite": true +} diff --git a/guides/ops/monitoring-and-on-call.mdx b/guides/ops/monitoring-and-on-call.mdx new file mode 100644 index 0000000..2641a36 --- /dev/null +++ b/guides/ops/monitoring-and-on-call.mdx @@ -0,0 +1,693 @@ +--- +title: 'Monitoring, Alerting, and On‑Call Runbook' +sidebarTitle: 'Monitoring & On‑Call' +description: 'Production runbook for Wraith Protocol: metrics, dashboards, alerts, incident playbooks, and tabletop exercises' +--- + +This guide provides a complete operational runbook for teams running self-hosted Wraith Protocol instances in production. It covers healthy metric baselines, Grafana dashboards, Prometheus alerting rules, incident response playbooks, and a quarterly tabletop exercise. + + + This guide assumes you have already deployed Wraith Protocol using the [Self-Hosted Deployment](/guides/ops/self-hosted-deployment) guide. If not, complete that first. + + +## Overview + +Production Wraith deployments require monitoring across four domains: + +| Domain | Key Components | Primary Risk | +|---|---|---| +| **RPC Layer** | Stellar Horizon RPC, Soroban RPC | Transaction submission failures, ledger query timeouts | +| **Indexer** | Announcement scanner, event log sync | Missed announcements, backlog buildup | +| **Contracts** | `stealth-announcer`, `stealth-sender`, `wraith-names` | Contract invocation errors, unauthorized access | +| **Watcher** | Event drop detection, view-tag scanning | Dropped events, privacy leakage via missed scans | + +This runbook is structured around these four domains. Each section defines healthy baselines, provides ready-to-import Grafana dashboards, and links to incident playbooks. + +--- + +## Key Metrics + +### RPC Metrics + +Monitor these metrics to ensure reliable access to the Stellar network: + +| Metric | Description | Healthy Baseline | Alert Threshold | +|---|---|---|---| +| `wraith_rpc_latency_seconds` | P95 latency for Horizon/Soroban RPC calls | < 500ms | > 2s for 5min | +| `wraith_rpc_error_rate` | Percentage of RPC calls returning 5xx or network errors | < 0.5% | > 5% for 2min | +| `wraith_horizon_backpressure` | Rate-limit 429 responses from Horizon | 0/min | > 10/min | +| `wraith_soroban_invocation_failures` | Failed contract invocations due to RPC issues | < 1/hour | > 5/hour | + +**Collection**: Instrument your SDK or middleware layer with Prometheus client libraries. Example Node.js middleware: + +```typescript +import promClient from 'prom-client'; + +const rpcLatency = new promClient.Histogram({ + name: 'wraith_rpc_latency_seconds', + help: 'RPC call latency in seconds', + labelNames: ['method', 'network'], + buckets: [0.1, 0.5, 1, 2, 5], +}); + +const rpcErrors = new promClient.Counter({ + name: 'wraith_rpc_error_rate', + help: 'RPC error count', + labelNames: ['method', 'status'], +}); +``` + +### Indexer Metrics + +Track announcement scanning and event synchronization health: + +| Metric | Description | Healthy Baseline | Alert Threshold | +|---|---|---|---| +| `wraith_announcement_lag_seconds` | Time between announcement emission and indexer ingestion | < 30s | > 300s for 5min | +| `wraith_indexer_backlog_count` | Number of unprocessed announcements | < 100 | > 1000 | +| `wraith_scan_miss_rate` | Announcements scanned but not matched (view-tag misses) | < 0.1% | > 5% for 10min | +| `wraith_indexer_sync_height` | Current ledger height being indexed vs. network tip | lag < 10 ledgers | lag > 50 ledgers | + +**Collection**: Expose metrics from your indexer service. Example Rust Prometheus exporter: + +```rust +use prometheus::{Histogram, IntGauge, register_histogram, register_int_gauge}; + +lazy_static! { + static ref ANNOUNCEMENT_LAG: Histogram = register_histogram!( + "wraith_announcement_lag_seconds", + "Lag between announcement emission and ingestion" + ).unwrap(); + + static ref INDEXER_BACKLOG: IntGauge = register_int_gauge!( + "wraith_indexer_backlog_count", + "Count of unprocessed announcements" + ).unwrap(); +} +``` + +### Contract Metrics + +Monitor on-chain contract health and error rates: + +| Metric | Description | Healthy Baseline | Alert Threshold | +|---|---|---|---| +| `wraith_contract_error_rate` | Failed contract invocations (all contracts) | < 0.1% | > 1% for 5min | +| `wraith_sender_invocation_latency` | Time from transaction submission to ledger inclusion | < 5s | > 20s for 5min | +| `wraith_names_registration_failures` | Failed `wraith-names` registrations | 0/hour | > 3/hour | +| `wraith_unauthorized_access_attempts` | Contract calls with invalid authorization | 0/day | > 1/day | + +**Collection**: Parse Soroban contract events and transaction results. Example: + +```javascript +const contractErrors = new promClient.Counter({ + name: 'wraith_contract_error_rate', + help: 'Contract invocation errors', + labelNames: ['contract', 'method'], +}); + +sorobanClient.on('transactionFailed', (tx) => { + contractErrors.inc({ contract: tx.contract, method: tx.method }); +}); +``` + +### Watcher Metrics + +Track event drop detection and view-tag scanning performance: + +| Metric | Description | Healthy Baseline | Alert Threshold | +|---|---|---|---| +| `wraith_watcher_event_drop_rate` | Events detected as dropped by the watcher | 0/hour | > 5/hour | +| `wraith_view_tag_scan_duration_seconds` | Time to scan view tags for a single announcement | < 10ms | > 100ms | +| `wraith_view_tag_false_positive_rate` | View-tag matches that fail full ECDH check | < 1% | > 10% | + +**Collection**: Instrument your watcher/scanner service: + +```python +from prometheus_client import Counter, Histogram + +event_drops = Counter('wraith_watcher_event_drop_rate', 'Dropped event count') +scan_duration = Histogram('wraith_view_tag_scan_duration_seconds', 'View tag scan time') + +def scan_announcement(announcement): + with scan_duration.time(): + # scanning logic + pass +``` + +--- + +## Grafana Dashboards + +Import these dashboards into your Grafana instance to visualize Wraith metrics. All dashboards are in the `guides/ops/dashboards/` directory. + +### Dashboard 1: RPC Health + +**File**: `guides/ops/dashboards/rpc-health.json` + +**Panels**: +- RPC P95 Latency (time series) +- RPC Error Rate (gauge) +- Horizon Backpressure (429 responses over time) +- Soroban Invocation Failures (counter) + +**Import**: +```bash +curl -X POST http://localhost:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_GRAFANA_API_KEY" \ + -d @guides/ops/dashboards/rpc-health.json +``` + +### Dashboard 2: Indexer Performance + +**File**: `guides/ops/dashboards/indexer-performance.json` + +**Panels**: +- Announcement Lag (time series) +- Indexer Backlog Count (gauge) +- Scan Miss Rate (percentage over time) +- Sync Height Lag (ledger difference graph) + +### Dashboard 3: Contract Monitoring + +**File**: `guides/ops/dashboards/contract-monitoring.json` + +**Panels**: +- Contract Error Rate by Contract (heatmap) +- Sender Invocation Latency (histogram) +- Names Registration Failures (counter) +- Unauthorized Access Attempts (alert panel) + +**Cross-Reference**: See [Stellar Cryptography](/architecture/stellar-cryptography) for contract architecture details. + +--- + +## Prometheus Alerting Rules + +Import these alerting rules into your Prometheus configuration. The rules file is at `guides/ops/alerts/wraith-alerts.yml`. + +### Alerting Philosophy + +Each alert includes: +- **Severity**: Maps to the [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix) +- **Rationale**: Why this threshold triggers an alert +- **Playbook**: Link to the incident response playbook + +### Rules File: `wraith-alerts.yml` + +```yaml +groups: + - name: wraith_rpc_alerts + interval: 30s + rules: + - alert: HighRPCLatency + expr: histogram_quantile(0.95, wraith_rpc_latency_seconds) > 2 + for: 5m + labels: + severity: high + component: rpc + annotations: + summary: "RPC P95 latency > 2s for 5 minutes" + description: "{{ $labels.method }} RPC calls are experiencing high latency. Check Horizon/Soroban RPC health." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-1-rpc-outage" + + - alert: HorizonBackpressure + expr: rate(wraith_horizon_backpressure[1m]) > 10 + for: 2m + labels: + severity: critical + component: rpc + annotations: + summary: "Horizon rate-limiting detected (> 10 429s/min)" + description: "Application is being rate-limited by Horizon. Immediate intervention required." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-2-horizon-backpressure" + + - name: wraith_indexer_alerts + interval: 1m + rules: + - alert: IndexerBacklogBuildup + expr: wraith_indexer_backlog_count > 1000 + for: 5m + labels: + severity: high + component: indexer + annotations: + summary: "Indexer backlog > 1000 announcements" + description: "Indexer is falling behind. Check database performance and announcement rate." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + + - alert: AnnouncementLagHigh + expr: wraith_announcement_lag_seconds > 300 + for: 5m + labels: + severity: medium + component: indexer + annotations: + summary: "Announcement lag > 5 minutes" + description: "Users may experience delays seeing incoming stealth payments." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-3-indexer-stall" + + - name: wraith_contract_alerts + interval: 1m + rules: + - alert: ContractErrorRateHigh + expr: rate(wraith_contract_error_rate[5m]) > 0.01 + for: 5m + labels: + severity: critical + component: contracts + annotations: + summary: "Contract error rate > 1%" + description: "Contract invocations are failing at an elevated rate. Check contract state and RPC health." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-4-contract-mispublish" + + - alert: UnauthorizedAccessAttempt + expr: wraith_unauthorized_access_attempts > 0 + for: 1m + labels: + severity: critical + component: contracts + annotations: + summary: "Unauthorized contract access detected" + description: "A contract received a call with invalid authorization. Potential security incident." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-6-key-rotation-incident" + + - name: wraith_watcher_alerts + interval: 1m + rules: + - alert: WatcherEventDropSpike + expr: rate(wraith_watcher_event_drop_rate[5m]) > 5 + for: 2m + labels: + severity: high + component: watcher + annotations: + summary: "Watcher event drop rate > 5/hour" + description: "Events are being dropped. Users may miss stealth payments." + playbook: "https://docs.usewraith.xyz/guides/ops/monitoring-and-on-call#playbook-5-watcher-drop-spike" +``` + +**Validation**: +```bash +promtool check rules guides/ops/alerts/wraith-alerts.yml +``` + +Expected output: +``` +Checking guides/ops/alerts/wraith-alerts.yml + SUCCESS: 8 rules found +``` + +--- + +## Incident Playbooks + +Each playbook follows a standard structure: **Symptoms → Diagnosis → Immediate Actions → Long-term Fix → Post-Incident Review**. + +### Playbook 1: RPC Outage + +**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 15 minutes to acknowledge, 1 hour to mitigation + +#### Symptoms +- `HighRPCLatency` alert firing +- Users reporting "transaction not found" errors +- Dashboard shows RPC error rate > 10% + +#### Diagnosis +1. Check Horizon/Soroban RPC status: + ```bash + curl https://horizon-futurenet.stellar.org/ + curl https://soroban-testnet.stellar.org/health + ``` +2. Verify network connectivity: + ```bash + traceroute horizon-futurenet.stellar.org + ``` +3. Check application logs for RPC timeout patterns: + ```bash + grep "RPC timeout" /var/log/wraith-app.log | tail -50 + ``` + +#### Immediate Actions +1. **Switch to backup RPC endpoint** (if configured): + ```typescript + const wraith = new Wraith({ + rpcUrl: process.env.BACKUP_RPC_URL, + }); + ``` +2. **Enable RPC request queueing** to prevent overwhelming the endpoint: + ```typescript + const rpcQueue = new PQueue({ concurrency: 10, interval: 1000, intervalCap: 10 }); + ``` +3. **Post status update** to users via status page or Twitter. + +#### Long-term Fix +- Set up RPC load balancing with multiple Horizon instances +- Implement exponential backoff for RPC retries +- Add RPC health checks to pre-deployment tests + +#### Post-Incident Review +- Document RPC failure patterns +- Update runbook with new diagnostic steps +- Review RPC provider SLA and consider alternatives + +--- + +### Playbook 2: Horizon Backpressure + +**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 15 minutes to acknowledge, 30 minutes to mitigation + +#### Symptoms +- `HorizonBackpressure` alert firing +- HTTP 429 "Rate Limit Exceeded" responses from Horizon +- Transaction submission queue building up + +#### Diagnosis +1. Check current rate limit status: + ```bash + curl -I https://horizon-futurenet.stellar.org/ | grep X-RateLimit + ``` +2. Identify which service is generating excessive requests: + ```bash + grep "429" /var/log/wraith-app.log | cut -d' ' -f1 | sort | uniq -c + ``` + +#### Immediate Actions +1. **Implement request throttling immediately**: + ```typescript + import Bottleneck from 'bottleneck'; + const limiter = new Bottleneck({ + minTime: 200, // 5 requests/second max + maxConcurrent: 1, + }); + ``` +2. **Disable non-critical polling loops** (e.g., balance checks, ledger monitoring). +3. **Cache RPC responses** for frequently accessed data (account balances, contract state). + +#### Long-term Fix +- Upgrade to a dedicated Horizon instance (not public) +- Implement intelligent request batching +- Add rate-limit monitoring to CI/CD pipeline + +#### Post-Incident Review +- Audit all RPC call sites for unnecessary requests +- Set up proactive rate-limit monitoring (alert at 80% of limit) + +--- + +### Playbook 3: Indexer Stall + +**Severity**: High ([Auditor Guide Sev 2](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 1 hour to acknowledge, 4 hours to mitigation + +#### Symptoms +- `IndexerBacklogBuildup` or `AnnouncementLagHigh` alert firing +- Users report delayed stealth payment notifications +- Indexer sync height lagging network tip by > 50 ledgers + +#### Diagnosis +1. Check indexer process health: + ```bash + systemctl status wraith-indexer + journalctl -u wraith-indexer -n 100 + ``` +2. Check database query performance: + ```sql + SELECT query, mean_exec_time, calls FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10; + ``` +3. Verify disk I/O isn't saturated: + ```bash + iostat -x 5 3 + ``` + +#### Immediate Actions +1. **Restart the indexer** if it's stuck: + ```bash + systemctl restart wraith-indexer + ``` +2. **Increase indexer concurrency** temporarily: + ```bash + export INDEXER_WORKERS=8 # default is 4 + systemctl restart wraith-indexer + ``` +3. **Prioritize recent announcements** by adjusting sync strategy: + ```typescript + await indexer.syncFrom({ ledger: currentLedger - 100 }); + ``` + +#### Long-term Fix +- Add database indexes on announcement query columns +- Implement announcement batching (process 100 at a time instead of 1) +- Scale indexer horizontally with ledger range sharding + +#### Post-Incident Review +- Benchmark indexer throughput under load +- Add capacity planning metrics (CPU, memory, disk I/O) + +--- + +### Playbook 4: Contract Mispublish + +**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 15 minutes to acknowledge, 2 hours to mitigation + +#### Symptoms +- `ContractErrorRateHigh` alert firing +- Users unable to register names or send stealth payments +- Soroban RPC returns `contract not found` or `invalid wasm` + +#### Diagnosis +1. Verify deployed contract IDs match expected values: + ```bash + cat stellar/deployments/futurenet.json | jq '.contracts' + ``` +2. Query contract state on-chain: + ```bash + stellar contract invoke --id --network futurenet -- --help + ``` +3. Check deployment logs for errors: + ```bash + grep "deploy failed" /var/log/deploy.log + ``` + +#### Immediate Actions +1. **Roll back to last known-good contract** (if available): + ```bash + stellar contract deploy --wasm stealth-sender-v1.2.wasm --network futurenet + ``` +2. **Update frontend to point to backup contract** (if multi-contract setup): + ```typescript + const SENDER_ID = process.env.BACKUP_SENDER_CONTRACT_ID; + ``` +3. **Disable contract-dependent features** in the UI to prevent user errors. + +#### Long-term Fix +- Implement blue-green deployments for contracts +- Add contract state verification to deployment scripts +- Set up contract upgrade simulations in staging + +#### Post-Incident Review +- Document contract deployment checklist +- Add contract state smoke tests to CI +- Review contract upgrade governance process + +--- + +### Playbook 5: Watcher Drop Spike + +**Severity**: High ([Auditor Guide Sev 2](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 1 hour to acknowledge, 4 hours to mitigation + +#### Symptoms +- `WatcherEventDropSpike` alert firing +- Users report missing stealth payments +- View-tag scan logs show high false-positive rates + +#### Diagnosis +1. Check watcher process logs: + ```bash + journalctl -u wraith-watcher -n 200 | grep "drop" + ``` +2. Verify event subscription is active: + ```typescript + const isConnected = await eventStream.isConnected(); + console.log('Event stream connected:', isConnected); + ``` +3. Check for network partitions or RPC instability. + +#### Immediate Actions +1. **Restart the watcher service**: + ```bash + systemctl restart wraith-watcher + ``` +2. **Re-scan recent ledgers** to catch missed events: + ```bash + wraith-watcher rescan --from-ledger + ``` +3. **Notify users** to check for missing payments manually via explorer. + +#### Long-term Fix +- Implement event stream redundancy (subscribe to multiple RPC endpoints) +- Add automatic gap detection and backfill +- Set up watcher health checks with auto-restart + +#### Post-Incident Review +- Analyze root cause of event drops (network, RPC, bug) +- Add integration tests for event stream resilience + +--- + +### Playbook 6: Key Rotation Incident + +**Severity**: Critical ([Auditor Guide Sev 1](/reference/auditor-guide#severity-matrix)) +**Response SLA**: 15 minutes to acknowledge, immediate mitigation + +#### Symptoms +- `UnauthorizedAccessAttempt` alert firing +- Contract admin key compromised or needs rotation +- Suspicious transactions targeting Wraith contracts + +#### Diagnosis +1. Identify unauthorized transactions: + ```bash + stellar account transactions --network futurenet | grep "failed" + ``` +2. Check contract authorization settings: + ```bash + stellar contract invoke --id --network futurenet -- get_admin + ``` +3. Review audit logs for anomalous access patterns. + +#### Immediate Actions +1. **Rotate admin keys immediately** (see [Multisig Authority Rotation](/guides/stellar/multisig-authority-rotation)): + ```bash + stellar contract invoke --id --source new-admin --network futurenet -- set_admin --new_admin + ``` +2. **Pause affected contracts** if available: + ```bash + stellar contract invoke --id --source admin --network futurenet -- pause + ``` +3. **Notify security team** and begin incident response process. + +#### Long-term Fix +- Implement multisig admin controls for all contracts +- Add hardware wallet support for admin operations +- Set up automated key rotation schedule (quarterly) + +#### Post-Incident Review +- Conduct full security audit of contract access patterns +- Update [Security Disclosure Policy](/reference/security-disclosure) if needed +- File internal postmortem with root cause analysis + +--- + +## Severity Matrix and Response SLA + +This table maps incident types to response SLAs, aligned with the [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix). + +| Incident Type | Severity | User Impact | Response SLA | Fix SLA | +|---|---|---|---|---| +| **RPC Outage** | Critical | No transactions can be submitted | 15min | 1 hour | +| **Horizon Backpressure** | Critical | Service degradation, failed submissions | 15min | 30min | +| **Indexer Stall** | High | Delayed payment notifications | 1 hour | 4 hours | +| **Contract Mispublish** | Critical | Core functionality unavailable | 15min | 2 hours | +| **Watcher Drop Spike** | High | Missed stealth payments | 1 hour | 4 hours | +| **Key Rotation Incident** | Critical | Potential unauthorized access | 15min | Immediate | + +**Response SLA**: Time from alert firing to on-call engineer acknowledging and beginning diagnosis. +**Fix SLA**: Time from acknowledgment to incident resolved or mitigated. + +### Severity Definitions + +These align with the [Auditor Guide](/reference/auditor-guide#severity-matrix): + +- **Critical**: Complete loss of functionality, security breach, or privacy violation affecting all users. +- **High**: Partial functionality loss or degradation affecting a subset of users. +- **Medium**: Limited impact requiring unusual conditions, no direct fund loss. +- **Low**: Minor issues with no realistic path to user harm. + +--- + +## Tabletop Exercise Script + +Run this quarterly drill to validate your team's incident response readiness. The exercise takes 90 minutes and requires: +- 3-5 participants (on-call engineers, product lead, security contact) +- Access to a staging environment +- This runbook and all playbooks printed or open + +### Exercise Structure + +**Duration**: 90 minutes +**Facilitator**: Rotating role (different person each quarter) +**Participants**: On-call engineers, product manager, security lead + +#### Phase 1: Scenario Briefing (15 minutes) + +**Facilitator reads**: +> "It's 2 AM on a Saturday. The on-call pager fires: `HorizonBackpressure` alert is critical. Users are reporting failed stealth payments on Twitter. Your staging environment is configured identically to production. You have 60 minutes to resolve the incident." + +**Distribute roles**: +- **Incident Commander**: Coordinates response, makes decisions +- **Diagnostics Lead**: Runs diagnostic commands, interprets metrics +- **Comms Lead**: Drafts user-facing status updates +- **Scribe**: Documents all actions taken for post-incident review + +#### Phase 2: Incident Response (45 minutes) + +Participants work through [Playbook 2: Horizon Backpressure](#playbook-2-horizon-backpressure) on staging: + +1. **Diagnosis** (15 min): Run diagnostic commands, identify root cause +2. **Mitigation** (20 min): Implement rate limiting, disable non-critical services +3. **Validation** (10 min): Verify alerts clear, test transaction submission + +**Facilitator injects complications**: +- At T+15min: "The backup RPC endpoint is also returning 429s. What now?" +- At T+30min: "Marketing is asking for an ETA to post on Twitter. What do you tell them?" + +#### Phase 3: Post-Incident Review (30 minutes) + +**Discussion prompts**: +1. What went well? What slowed you down? +2. Were the diagnostic steps clear? What was missing? +3. Did you have the access and tools you needed? +4. What would you change in the playbook? +5. What should we add to the runbook? + +**Action items**: +- Update playbooks based on learnings +- Add missing diagnostic tools to on-call toolkit +- Schedule follow-up training for gaps identified + +### Quarterly Rotation + +Run different scenarios each quarter: + +- **Q1**: Horizon Backpressure (Playbook 2) +- **Q2**: Indexer Stall (Playbook 3) +- **Q3**: Key Rotation Incident (Playbook 6) +- **Q4**: Multi-incident cascade (RPC + Watcher drop) + +--- + +## Cross-References + +- [Self-Hosted Deployment](/guides/ops/self-hosted-deployment) — prerequisite deployment guide +- [Auditor Guide Severity Matrix](/reference/auditor-guide#severity-matrix) — severity definitions and response SLAs +- [Multisig Authority Rotation](/guides/stellar/multisig-authority-rotation) — key rotation procedures +- [Stellar Cryptography](/architecture/stellar-cryptography) — contract architecture for debugging +- [Security Disclosure Policy](/reference/security-disclosure) — incident reporting and safe harbor + +--- + +## Next Steps + +1. **Import dashboards**: Load all three Grafana dashboards into your instance. +2. **Validate alerts**: Run `promtool check rules` on `wraith-alerts.yml`. +3. **Schedule tabletop**: Put the quarterly drill on the calendar. +4. **Assign on-call rotation**: Define primary and secondary on-call engineers. +5. **Test a playbook**: Walk through Playbook 1 (RPC Outage) on staging to validate all commands work. + +For operational questions, see the [Self-Hosted Deployment guide](/guides/ops/self-hosted-deployment) or reach out in the Wraith Protocol community Discord. diff --git a/guides/ops/self-hosted-deployment.mdx b/guides/ops/self-hosted-deployment.mdx index b98556f..ad9160d 100644 --- a/guides/ops/self-hosted-deployment.mdx +++ b/guides/ops/self-hosted-deployment.mdx @@ -141,3 +141,11 @@ You should see ledger entries confirming the deployment. **Your self‑hosted Wraith instance is now live.** You can register names, announce stealth meta‑addresses, and send private payments against these contracts. + +## Next Steps + +For production deployments, see the [Monitoring, Alerting, and On‑Call Runbook](/guides/ops/monitoring-and-on-call) to set up: +- Grafana dashboards for RPC, indexer, and contract health +- Prometheus alerting rules with severity thresholds +- Incident response playbooks for common operational issues +- Quarterly tabletop exercises to validate readiness