diff --git a/bot/bot.go b/bot/bot.go index b089d46..c656661 100644 --- a/bot/bot.go +++ b/bot/bot.go @@ -23,6 +23,7 @@ type Bot struct { Plugins []Plugin Log *zap.Logger Queue *Queue + networkStats *networkStats client *irc.Client reloadHandler func(Message) mu sync.RWMutex @@ -45,6 +46,9 @@ func New(cfg Config, db *storage.DB, plugins []Plugin, log *zap.Logger) *Bot { } func NewWithStats(cfg Config, db *storage.DB, plugins []Plugin, log *zap.Logger, stats *Stats) *Bot { + if stats == nil { + stats = NewStats() + } enabledPlugins := make(map[string]bool, len(plugins)) startedPlugins := make(map[string]bool, len(plugins)) for _, plugin := range plugins { @@ -52,6 +56,7 @@ func NewWithStats(cfg Config, db *storage.DB, plugins []Plugin, log *zap.Logger, } b := &Bot{Config: cfg, DB: db, Stats: stats, Plugins: plugins, Log: log, enabledPlugins: enabledPlugins, startedPlugins: startedPlugins, lastCommands: make(map[string]time.Time), lastWarnings: make(map[string]time.Time), lastInvites: make(map[string]time.Time), warmupUntil: make(map[string]time.Time)} b.Queue = NewQueue(cfg.RateLimit.MessagesPerSecond, cfg.RateLimit.Burst, func(o Outgoing) { b.sendNow(o.Target, o.Text) }) + b.networkStats = stats.registerNetwork(cfg.NetworkName, len(cfg.Channels), b.Queue) return b } @@ -175,6 +180,9 @@ func clonePluginOverrides(overrides map[string]map[string]bool) map[string]map[s func (b *Bot) Send(target, text string) { if !b.Queue.Enqueue(Outgoing{target, text}) { b.Stats.dropped.Add(1) + if b.networkStats != nil { + b.networkStats.dropped.Add(1) + } b.Log.Warn("outgoing queue full", zap.String("target", target)) } } @@ -185,6 +193,9 @@ func (b *Bot) sendNow(target, text string) { if c != nil { c.WriteMessage(&irc.Message{Command: "PRIVMSG", Params: []string{target, text}}) b.Stats.sent.Add(1) + if b.networkStats != nil { + b.networkStats.sent.Add(1) + } } } func (b *Bot) connect(ctx context.Context) error { @@ -295,13 +306,16 @@ func (b *Bot) connect(ctx context.Context) error { } if m.Command == "PRIVMSG" { b.Stats.received.Add(1) + if b.networkStats != nil { + b.networkStats.received.Add(1) + } b.dispatch(ParseMessage(m)) } })}) b.mu.Lock() b.client = client b.mu.Unlock() - b.Stats.connected.Store(1) + b.Stats.setNetworkConnected(b.networkStats, true) if mechanism != "" { // irc.v3 has CAP parsing but no SASL mechanism implementation. Start // the capability exchange manually so authentication can complete before 001. @@ -321,10 +335,10 @@ func (b *Bot) connect(ctx context.Context) error { client.Write("QUIT :shutting down") conn.Close() <-errc - b.Stats.connected.Store(0) + b.Stats.setNetworkConnected(b.networkStats, false) return nil case err := <-errc: - b.Stats.connected.Store(0) + b.Stats.setNetworkConnected(b.networkStats, false) b.mu.Lock() b.client = nil b.mu.Unlock() @@ -734,7 +748,7 @@ func (b *Bot) dispatch(msg Message) { if _, _, ok := IsCommand(msg, b.Config.CommandPrefix); ok && !b.AllowCommand(msg) { return } - command := false + _, _, command := IsCommand(msg, b.Config.CommandPrefix) for _, p := range b.Plugins { if !b.pluginEnabled(p.Name()) { continue @@ -748,6 +762,7 @@ func (b *Bot) dispatch(msg Message) { defer func() { b.pluginMu.RUnlock() if r := recover(); r != nil { + b.Stats.recordPluginPanic(b.networkStats, p.Name(), "message") b.Log.Error("plugin panic", zap.String("plugin", p.Name()), zap.Any("panic", r)) } }() @@ -755,13 +770,10 @@ func (b *Bot) dispatch(msg Message) { }() if consumed { if command { - b.Stats.commands.Add(1) + b.Stats.recordCommand(b.networkStats, p.Name()) } return } - if _, _, ok := IsCommand(msg, b.Config.CommandPrefix); ok { - command = true - } } } @@ -782,6 +794,7 @@ func (b *Bot) dispatchEvent(msg Message) { defer func() { b.pluginMu.RUnlock() if r := recover(); r != nil { + b.Stats.recordPluginPanic(b.networkStats, p.Name(), "event") b.Log.Error("plugin event panic", zap.String("plugin", p.Name()), zap.Any("panic", r)) } }() @@ -814,6 +827,9 @@ func (b *Bot) Run(ctx context.Context) error { return nil } b.Stats.reconnects.Add(1) + if b.networkStats != nil { + b.networkStats.reconnects.Add(1) + } b.Log.Warn("IRC connection ended", zap.Error(err), zap.Duration("retry_in", backoff)) jitter := time.Duration(rand.Int63n(int64(backoff/5 + 1))) select { diff --git a/bot/queue.go b/bot/queue.go index 743de2b..45e0fed 100644 --- a/bot/queue.go +++ b/bot/queue.go @@ -43,6 +43,13 @@ func (q *Queue) Enqueue(o Outgoing) bool { return false } } + +// Depth reports the number of messages currently waiting to be sent. +func (q *Queue) Depth() int { return len(q.ch) } + +// Capacity reports the maximum number of queued outbound messages. +func (q *Queue) Capacity() int { return cap(q.ch) } + func (q *Queue) loop() { ticker := time.NewTicker(q.interval) defer ticker.Stop() diff --git a/bot/queue_test.go b/bot/queue_test.go index a220964..ca2b5d8 100644 --- a/bot/queue_test.go +++ b/bot/queue_test.go @@ -30,3 +30,17 @@ func TestQueueClampsExtremeRates(t *testing.T) { t.Fatalf("queue interval = %s, want positive", q.interval) } } + +func TestQueueReportsDepthAndCapacity(t *testing.T) { + q := NewQueue(0.01, 2, func(Outgoing) {}) + defer q.Drain(context.Background()) + if got := q.Capacity(); got != 40 { + t.Fatalf("Capacity() = %d, want 40", got) + } + if !q.Enqueue(Outgoing{Text: "one"}) || !q.Enqueue(Outgoing{Text: "two"}) { + t.Fatal("failed to enqueue messages") + } + if got := q.Depth(); got != 2 { + t.Fatalf("Depth() = %d, want 2", got) + } +} diff --git a/bot/stats.go b/bot/stats.go index 4dfc7bd..fbec85f 100644 --- a/bot/stats.go +++ b/bot/stats.go @@ -5,7 +5,10 @@ import ( "fmt" "net" "net/http" + "sort" "strconv" + "strings" + "sync" "sync/atomic" "time" @@ -15,9 +18,26 @@ import ( type Stats struct { started time.Time received, sent, commands, reconnects, dropped, connected atomic.Uint64 + connectedNetworks atomic.Int64 db *storage.DB persistDone chan struct{} persistOnce atomic.Bool + networkMu sync.RWMutex + networks map[string]*networkStats +} + +type networkStats struct { + name string + configuredChannels int + queue *Queue + connected atomic.Uint64 + received atomic.Uint64 + sent atomic.Uint64 + commands atomic.Uint64 + reconnects atomic.Uint64 + dropped atomic.Uint64 + pluginCommands sync.Map + pluginPanics sync.Map } type persistedStats struct { @@ -28,8 +48,18 @@ type persistedStats struct { Dropped uint64 `json:"messages_dropped"` } +type metricLabel struct { + name string + value string +} + +type prometheusWriter struct { + output strings.Builder + described map[string]struct{} +} + func NewStats(dbs ...*storage.DB) *Stats { - s := &Stats{started: time.Now()} + s := &Stats{started: time.Now(), networks: make(map[string]*networkStats)} if len(dbs) > 0 && dbs[0] != nil { s.db = dbs[0] if raw, err := s.db.Get("stats", "global"); err == nil { @@ -48,6 +78,74 @@ func NewStats(dbs ...*storage.DB) *Stats { return s } +func (s *Stats) registerNetwork(name string, configuredChannels int, queue *Queue) *networkStats { + name = strings.TrimSpace(name) + if name == "" { + name = "default" + } + if configuredChannels < 0 { + configuredChannels = 0 + } + s.networkMu.Lock() + defer s.networkMu.Unlock() + if existing := s.networks[name]; existing != nil { + return existing + } + network := &networkStats{name: name, configuredChannels: configuredChannels, queue: queue} + s.networks[name] = network + return network +} + +func (s *Stats) setNetworkConnected(network *networkStats, value bool) { + if network == nil { + if value { + s.connected.Store(1) + } else { + s.connected.Store(0) + } + return + } + desired := uint64(0) + delta := int64(-1) + if value { + desired = 1 + delta = 1 + } + if network.connected.Swap(desired) == desired { + return + } + active := s.connectedNetworks.Add(delta) + if active > 0 { + s.connected.Store(1) + return + } + if active < 0 { + s.connectedNetworks.Store(0) + } + s.connected.Store(0) +} + +func (s *Stats) recordCommand(network *networkStats, plugin string) { + s.commands.Add(1) + if network == nil { + return + } + network.commands.Add(1) + atomicCounter(&network.pluginCommands, plugin).Add(1) +} + +func (s *Stats) recordPluginPanic(network *networkStats, plugin, handler string) { + if network == nil { + return + } + atomicCounter(&network.pluginPanics, plugin+"\x00"+handler).Add(1) +} + +func atomicCounter(counters *sync.Map, key string) *atomic.Uint64 { + value, _ := counters.LoadOrStore(key, &atomic.Uint64{}) + return value.(*atomic.Uint64) +} + func (s *Stats) persistLoop() { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() @@ -79,33 +177,156 @@ func (s *Stats) Close() { } func (s *Stats) Snapshot() map[string]interface{} { - return map[string]interface{}{"uptime": time.Since(s.started).Round(time.Second).String(), "connected": s.connected.Load() == 1, "reconnects": s.reconnects.Load(), "messages_received": s.received.Load(), "messages_sent": s.sent.Load(), "messages_dropped": s.dropped.Load(), "commands_handled": s.commands.Load()} + return map[string]interface{}{ + "uptime": time.Since(s.started).Round(time.Second).String(), + "connected": s.connected.Load() == 1, + "reconnects": s.reconnects.Load(), + "messages_received": s.received.Load(), + "messages_sent": s.sent.Load(), + "messages_dropped": s.dropped.Load(), + "commands_handled": s.commands.Load(), + "networks": s.networkSnapshot(), + } } func (s *Stats) MetricsSnapshot() map[string]interface{} { snapshot := s.Snapshot() delete(snapshot, "uptime") + delete(snapshot, "networks") snapshot["uptime_seconds"] = time.Since(s.started).Seconds() return snapshot } + +func (s *Stats) networkSnapshot() map[string]interface{} { + networks := make(map[string]interface{}) + for _, network := range s.sortedNetworks() { + depth, capacity := 0, 0 + if network.queue != nil { + depth = network.queue.Depth() + capacity = network.queue.Capacity() + } + networks[network.name] = map[string]interface{}{ + "connected": network.connected.Load() == 1, + "reconnects": network.reconnects.Load(), + "messages_received": network.received.Load(), + "messages_sent": network.sent.Load(), + "messages_dropped": network.dropped.Load(), + "commands_handled": network.commands.Load(), + "configured_channels": network.configuredChannels, + "queue_depth": depth, + "queue_capacity": capacity, + } + } + return networks +} + +func (s *Stats) sortedNetworks() []*networkStats { + s.networkMu.RLock() + networks := make([]*networkStats, 0, len(s.networks)) + for _, network := range s.networks { + networks = append(networks, network) + } + s.networkMu.RUnlock() + sort.Slice(networks, func(i, j int) bool { return networks[i].name < networks[j].name }) + return networks +} + +func (s *Stats) PrometheusSnapshot() string { + writer := prometheusWriter{described: make(map[string]struct{})} + writer.metric("bot_connected", "Whether at least one IRC network is connected.", "gauge", nil, s.connected.Load()) + writer.metric("bot_reconnects", "Persistent cumulative IRC reconnect count.", "untyped", nil, s.reconnects.Load()) + writer.metric("bot_messages_received", "Persistent cumulative IRC messages received.", "untyped", nil, s.received.Load()) + writer.metric("bot_messages_sent", "Persistent cumulative IRC messages sent.", "untyped", nil, s.sent.Load()) + writer.metric("bot_commands_handled", "Persistent cumulative commands handled.", "untyped", nil, s.commands.Load()) + writer.metric("bot_uptime_seconds", "Current GoBot process uptime in seconds.", "gauge", nil, time.Since(s.started).Seconds()) + writer.metric("bot_messages_dropped", "Persistent cumulative messages dropped because an outbound queue was full.", "untyped", nil, s.dropped.Load()) + writer.metric("bot_process_start_time_seconds", "Unix timestamp when the current GoBot process started.", "gauge", nil, float64(s.started.Unix())) + networks := s.sortedNetworks() + writer.metric("bot_networks_configured", "Number of configured IRC networks.", "gauge", nil, len(networks)) + writer.metric("bot_networks_connected", "Number of currently connected IRC networks.", "gauge", nil, s.connectedNetworks.Load()) + + for _, network := range networks { + labels := []metricLabel{{name: "network", value: network.name}} + writer.metric("bot_network_connected", "Whether the IRC network is connected.", "gauge", labels, network.connected.Load()) + writer.metric("bot_network_reconnects_total", "IRC reconnects during the current process lifetime.", "counter", labels, network.reconnects.Load()) + writer.metric("bot_network_messages_received_total", "IRC messages received during the current process lifetime.", "counter", labels, network.received.Load()) + writer.metric("bot_network_messages_sent_total", "IRC messages sent during the current process lifetime.", "counter", labels, network.sent.Load()) + writer.metric("bot_network_commands_handled_total", "Commands handled during the current process lifetime.", "counter", labels, network.commands.Load()) + writer.metric("bot_network_messages_dropped_total", "Messages dropped because the network outbound queue was full.", "counter", labels, network.dropped.Load()) + writer.metric("bot_network_configured_channels", "Configured IRC channels for the network.", "gauge", labels, network.configuredChannels) + depth, capacity := 0, 0 + if network.queue != nil { + depth = network.queue.Depth() + capacity = network.queue.Capacity() + } + writer.metric("bot_outgoing_queue_depth", "Messages currently waiting in the network outbound queue.", "gauge", labels, depth) + writer.metric("bot_outgoing_queue_capacity", "Maximum messages supported by the network outbound queue.", "gauge", labels, capacity) + writer.syncMapCounters("bot_plugin_commands_handled_total", "Commands handled by each plugin during the current process lifetime.", network.name, &network.pluginCommands, false) + writer.syncMapCounters("bot_plugin_panics_total", "Recovered plugin panics during the current process lifetime.", network.name, &network.pluginPanics, true) + } + return writer.output.String() +} + +func (w *prometheusWriter) syncMapCounters(name, help, network string, counters *sync.Map, includeHandler bool) { + values := make(map[string]uint64) + keys := make([]string, 0) + counters.Range(func(key, value interface{}) bool { + text, ok := key.(string) + counter, counterOK := value.(*atomic.Uint64) + if ok && counterOK { + keys = append(keys, text) + values[text] = counter.Load() + } + return true + }) + sort.Strings(keys) + for _, key := range keys { + plugin, handler := key, "" + if includeHandler { + plugin, handler, _ = strings.Cut(key, "\x00") + } + labels := []metricLabel{{name: "network", value: network}, {name: "plugin", value: plugin}} + if includeHandler { + labels = append(labels, metricLabel{name: "handler", value: handler}) + } + w.metric(name, help, "counter", labels, values[key]) + } +} + +func (w *prometheusWriter) metric(name, help, metricType string, labels []metricLabel, value interface{}) { + if _, ok := w.described[name]; !ok { + fmt.Fprintf(&w.output, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, metricType) + w.described[name] = struct{}{} + } + w.output.WriteString(name) + if len(labels) > 0 { + w.output.WriteByte('{') + for i, label := range labels { + if i > 0 { + w.output.WriteByte(',') + } + fmt.Fprintf(&w.output, `%s="%s"`, label.name, escapePrometheusLabel(label.value)) + } + w.output.WriteByte('}') + } + fmt.Fprintf(&w.output, " %v\n", value) +} + +func escapePrometheusLabel(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, "\n", `\n`) + return strings.ReplaceAll(value, `"`, `\"`) +} + func (s *Stats) Serve(address string, port int) { mux := http.NewServeMux() mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(s.Snapshot()) + _ = json.NewEncoder(w).Encode(s.Snapshot()) }) mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; version=0.0.4") - for k, v := range s.MetricsSnapshot() { - if k == "connected" { - if v == true { - v = 1 - } else { - v = 0 - } - } - fmt.Fprintf(w, "bot_%s %v\n", k, v) - } + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + _, _ = w.Write([]byte(s.PrometheusSnapshot())) }) if address == "" { address = "127.0.0.1" @@ -118,7 +339,7 @@ func (s *Stats) Serve(address string, port int) { WriteTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, } - go server.ListenAndServe() + go func() { _ = server.ListenAndServe() }() } func statsListenAddress(address string, port int) string { diff --git a/bot/stats_test.go b/bot/stats_test.go index 81e37a5..322d583 100644 --- a/bot/stats_test.go +++ b/bot/stats_test.go @@ -1,6 +1,13 @@ package bot -import "testing" +import ( + "context" + "strings" + "testing" + + "github.com/variablenix/GoBot/storage" + "go.uber.org/zap" +) func TestStatsListenAddress(t *testing.T) { tests := []struct { @@ -33,3 +40,125 @@ func TestMetricsSnapshotIncludesUptimeAndDroppedMessages(t *testing.T) { t.Fatal("metrics snapshot should not include human-readable uptime") } } + +func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) { + stats := NewStats() + queue := NewQueue(0.01, 2, func(Outgoing) {}) + defer queue.Drain(context.Background()) + network := stats.registerNetwork("libera", 3, queue) + stats.setNetworkConnected(network, true) + stats.received.Store(12) + stats.sent.Store(8) + network.received.Store(7) + network.sent.Store(5) + network.reconnects.Store(2) + stats.recordCommand(network, "help") + stats.recordPluginPanic(network, "weather", "message") + if !queue.Enqueue(Outgoing{Target: "#test", Text: "queued"}) { + t.Fatal("failed to enqueue test message") + } + + metrics := stats.PrometheusSnapshot() + for _, want := range []string{ + "bot_connected 1\n", + "bot_messages_received 12\n", + "bot_messages_sent 8\n", + "bot_commands_handled 1\n", + "bot_networks_configured 1\n", + "bot_networks_connected 1\n", + `bot_network_connected{network="libera"} 1`, + `bot_network_reconnects_total{network="libera"} 2`, + `bot_network_messages_received_total{network="libera"} 7`, + `bot_network_messages_sent_total{network="libera"} 5`, + `bot_network_configured_channels{network="libera"} 3`, + `bot_outgoing_queue_depth{network="libera"} 1`, + `bot_outgoing_queue_capacity{network="libera"} 40`, + `bot_plugin_commands_handled_total{network="libera",plugin="help"} 1`, + `bot_plugin_panics_total{network="libera",plugin="weather",handler="message"} 1`, + } { + if !strings.Contains(metrics, want) { + t.Errorf("PrometheusSnapshot() missing %q\n%s", want, metrics) + } + } + networks, ok := stats.Snapshot()["networks"].(map[string]interface{}) + if !ok { + t.Fatal("Snapshot() networks has an unexpected type") + } + libera, ok := networks["libera"].(map[string]interface{}) + if !ok || libera["configured_channels"] != 3 || libera["queue_depth"] != 1 { + t.Fatalf("Snapshot() network details = %#v", networks["libera"]) + } +} + +type panicMetricsPlugin struct{} + +func (*panicMetricsPlugin) Name() string { return "panic-test" } +func (*panicMetricsPlugin) Commands() []string { return []string{"panic-test"} } +func (*panicMetricsPlugin) Help() string { return "panic-test" } +func (*panicMetricsPlugin) Init(PluginConfig, *storage.DB) error { return nil } +func (*panicMetricsPlugin) Handle(*Bot, Message) bool { panic("test panic") } + +type commandMetricsPlugin struct{} + +func (*commandMetricsPlugin) Name() string { return "choose" } +func (*commandMetricsPlugin) Commands() []string { return []string{"choose"} } +func (*commandMetricsPlugin) Help() string { return "choose" } +func (*commandMetricsPlugin) Init(PluginConfig, *storage.DB) error { return nil } +func (*commandMetricsPlugin) Handle(*Bot, Message) bool { return true } + +func TestPluginPanicsAreRecoveredAndCounted(t *testing.T) { + plugin := &panicMetricsPlugin{} + config := Config{NetworkName: "test", CommandPrefix: "!"} + stats := NewStats() + instance := NewWithStats(config, nil, []Plugin{plugin}, zap.NewNop(), stats) + defer instance.Queue.Drain(context.Background()) + + instance.dispatch(Message{Command: "PRIVMSG", Target: "#test", Text: "!panic-test", IsChannel: true}) + metrics := stats.PrometheusSnapshot() + if !strings.Contains(metrics, `bot_plugin_panics_total{network="test",plugin="panic-test",handler="message"} 1`) { + t.Fatalf("plugin panic metric missing:\n%s", metrics) + } +} + +func TestConnectionGaugeAggregatesMultipleNetworks(t *testing.T) { + stats := NewStats() + first := stats.registerNetwork("first", 1, nil) + second := stats.registerNetwork("second", 1, nil) + + stats.setNetworkConnected(first, true) + stats.setNetworkConnected(second, true) + stats.setNetworkConnected(first, false) + if stats.connected.Load() != 1 || stats.connectedNetworks.Load() != 1 { + t.Fatalf("one connected network reported global=%d active=%d", stats.connected.Load(), stats.connectedNetworks.Load()) + } + stats.setNetworkConnected(second, false) + if stats.connected.Load() != 0 || stats.connectedNetworks.Load() != 0 { + t.Fatalf("zero connected networks reported global=%d active=%d", stats.connected.Load(), stats.connectedNetworks.Load()) + } +} + +func TestPrometheusLabelsAreEscaped(t *testing.T) { + stats := NewStats() + stats.registerNetwork("bad\\\"\nname", 0, nil).reconnects.Store(1) + metrics := stats.PrometheusSnapshot() + if !strings.Contains(metrics, `network="bad\\\"\nname"`) { + t.Fatalf("network label was not escaped safely:\n%s", metrics) + } +} + +func TestFirstPluginHandledCommandIsCounted(t *testing.T) { + plugin := &commandMetricsPlugin{} + config := Config{NetworkName: "test", CommandPrefix: "!"} + stats := NewStats() + instance := NewWithStats(config, nil, []Plugin{plugin}, zap.NewNop(), stats) + defer instance.Queue.Drain(context.Background()) + + instance.dispatch(Message{Command: "PRIVMSG", Target: "#test", Text: "!choose a | b", IsChannel: true}) + if got := stats.commands.Load(); got != 1 { + t.Fatalf("commands handled = %d, want 1", got) + } + metrics := stats.PrometheusSnapshot() + if !strings.Contains(metrics, `bot_plugin_commands_handled_total{network="test",plugin="choose"} 1`) { + t.Fatalf("plugin command metric missing:\n%s", metrics) + } +} diff --git a/docs/monitoring.md b/docs/monitoring.md index 632c17a..9e2cc39 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -18,6 +18,8 @@ firewall. ## Metrics +The original global metrics remain available for backward compatibility: + - bot_connected: 1 while at least one IRC connection is active, otherwise 0 - bot_reconnects: cumulative reconnect count - bot_messages_received: cumulative IRC messages received @@ -30,6 +32,35 @@ Cumulative counters are persisted in BoltDB and survive restarts. Connection status and process uptime naturally reset or change when the process restarts. The !stats command reports persistent per-channel message and user statistics. +GoBot also exposes operational metrics for richer dashboards: + +| Metric | Type | Meaning | +| --- | --- | --- | +| `bot_process_start_time_seconds` | gauge | Current process start time as a Unix timestamp | +| `bot_networks_configured` | gauge | Number of configured IRC networks | +| `bot_networks_connected` | gauge | Number of currently connected IRC networks | +| `bot_network_connected{network}` | gauge | Per-network connection state | +| `bot_network_reconnects_total{network}` | counter | Per-network reconnects since process start | +| `bot_network_messages_received_total{network}` | counter | Per-network messages received since process start | +| `bot_network_messages_sent_total{network}` | counter | Per-network messages sent since process start | +| `bot_network_commands_handled_total{network}` | counter | Per-network handled commands since process start | +| `bot_network_messages_dropped_total{network}` | counter | Per-network outbound messages dropped since process start | +| `bot_network_configured_channels{network}` | gauge | Configured channels per network | +| `bot_outgoing_queue_depth{network}` | gauge | Messages currently waiting to be sent | +| `bot_outgoing_queue_capacity{network}` | gauge | Maximum outbound queue size | +| `bot_plugin_commands_handled_total{network,plugin}` | counter | Handled commands grouped by plugin | +| `bot_plugin_panics_total{network,plugin,handler}` | counter | Recovered message/event handler panics | + +Per-network and per-plugin counters reset when the GoBot process restarts; +Prometheus `rate()` and `increase()` account for counter resets. Labels are +limited to configured network names, built-in plugin names, and the bounded +handler type. Channel names, nicknames, accounts, and message contents are not +exported. + +The `/stats` JSON response includes a `networks` object with connection, +traffic, command, reconnect, queue, and configured-channel details for each +network. + ## Prometheus scrape configuration If Prometheus runs on another host, set GoBot's listener to a WireGuard or @@ -71,5 +102,6 @@ curl http://:8082/metrics Import [grafana/gobot-dashboard.json](../grafana/gobot-dashboard.json) into Grafana and select your Prometheus data source. The dashboard variables use -Prometheus queries for the job and instance labels. A sample export is -included at [grafana/gobot-dashboard-example.png](../grafana/gobot-dashboard-example.png). +Prometheus queries for job, environment, hostname, instance, and IRC network +labels. Follow the complete step-by-step instructions in +[grafana/README.md](../grafana/README.md). diff --git a/grafana/README.md b/grafana/README.md index 620e02d..22baf42 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -1,22 +1,83 @@ # GoBot Grafana dashboard -`gobot-dashboard.json` is an importable Grafana dashboard for the metrics exposed by GoBot at `/metrics`. +[`gobot-dashboard.json`](gobot-dashboard.json) is the importable Grafana +dashboard for GoBot's `/metrics` endpoint. It covers: -## Import +- Prometheus scrape and IRC connection health +- process uptime and reliability events +- per-network incoming and outgoing message rates +- handled-command rates grouped by plugin +- outbound queue depth and capacity by network +- filtering by Prometheus job, environment, hostname, instance, and IRC network -1. In Grafana, open **Dashboards → New → Import**. -2. Upload `gobot-dashboard.json` or paste its contents. -3. Select the Prometheus datasource that scrapes GoBot. -4. Click **Import**. +The dashboard defaults match the production example below, but every filter +is selectable after import. -The dashboard expects the `gobot` Prometheus job and uses these metrics: +## 1. Deploy the expanded metrics -- `bot_connected` -- `bot_reconnects` -- `bot_messages_received` -- `bot_messages_sent` -- `bot_commands_handled` -- `bot_uptime_seconds` -- `bot_messages_dropped` +Pull the current GoBot `main` branch, rebuild the binary, and restart the +service. Confirm the endpoint from the Prometheus host: -The Prometheus job and dashboard are intentionally kept separate: the Ansible monitoring role deploys the scrape configuration, while this repository carries the dashboard definition alongside the application. +~~~sh +curl http://10.69.0.22:8082/metrics +~~~ + +Along with the original `bot_*` metrics, the response should include +`bot_network_connected`, `bot_network_messages_received_total`, +`bot_plugin_commands_handled_total`, and `bot_outgoing_queue_depth`. + +## 2. Configure Prometheus + +Add this job under `scrape_configs` in `prometheus.yml`: + +~~~yaml + # GoBot IRC bot + - job_name: gobot + scrape_interval: 30s + scrape_timeout: 10s + metrics_path: /metrics + static_configs: + - targets: ["10.69.0.22:8082"] + labels: + role: irc-bot + use: gobot + hostname: nexus-node + environment: production +~~~ + +Check the Prometheus configuration and reload it. For a package installation, +the usual commands are: + +~~~sh +promtool check config /etc/prometheus/prometheus.yml +sudo systemctl reload prometheus +~~~ + +In Prometheus, run `up{job="gobot"}`. A value of `1` confirms that scraping is +working. If it is `0` or absent, check routing/firewall access from the +Prometheus host to `10.69.0.22:8082` before importing the dashboard. + +## 3. Import into Grafana + +1. Open **Dashboards → New → Import** in Grafana. +2. Upload `grafana/gobot-dashboard.json` from this repository, or paste the + file contents. +3. Choose the Prometheus data source that contains the `gobot` job when + Grafana asks for a data source. +4. Keep the dashboard name **GoBot Operations** and click **Import**. +5. At the top of the dashboard, verify these filters: + - **Job:** `gobot` + - **Environment:** `production` + - **Host:** `nexus-node` + - **Instance:** `10.69.0.22:8082` + - **IRC network:** `All`, or one configured GoBot network + +The dashboard refreshes every 30 seconds, matching the Prometheus scrape +interval. A newly started bot may need two scrapes (about one minute) before +rate panels have enough samples to draw a line. + +## Security note + +GoBot's `/stats` and `/metrics` endpoints do not provide authentication. Bind +the listener to a private address and allow access only from the Prometheus +host through the firewall. diff --git a/grafana/dashboard_test.go b/grafana/dashboard_test.go new file mode 100644 index 0000000..43618b9 --- /dev/null +++ b/grafana/dashboard_test.go @@ -0,0 +1,87 @@ +package grafana + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +type dashboardResource struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Spec struct { + Title string `json:"title"` + Elements map[string]dashboardElement `json:"elements"` + Variables []struct { + Spec struct { + Name string `json:"name"` + } `json:"spec"` + } `json:"variables"` + } `json:"spec"` +} + +type dashboardElement struct { + Spec struct { + Title string `json:"title"` + } `json:"spec"` +} + +func TestDashboardJSONUsesExpandedMetrics(t *testing.T) { + raw, err := os.ReadFile("gobot-dashboard.json") + if err != nil { + t.Fatal(err) + } + var dashboard dashboardResource + if err := json.Unmarshal(raw, &dashboard); err != nil { + t.Fatalf("dashboard JSON is invalid: %v", err) + } + if dashboard.APIVersion == "" || dashboard.Kind != "Dashboard" || dashboard.Spec.Title != "GoBot Operations" { + t.Fatalf("unexpected dashboard identity: %#v", dashboard) + } + + panelTitles := make(map[string]bool) + for _, element := range dashboard.Spec.Elements { + panelTitles[element.Spec.Title] = true + } + for _, title := range []string{ + "IRC connection", + "Prometheus scrape", + "Reliability events", + "Message throughput by network", + "Command throughput by plugin", + "Outbound queue pressure", + "Uptime", + } { + if !panelTitles[title] { + t.Errorf("dashboard is missing panel %q", title) + } + } + + variables := make(map[string]bool) + for _, variable := range dashboard.Spec.Variables { + variables[variable.Spec.Name] = true + } + for _, name := range []string{"job", "environment", "hostname", "instance", "network"} { + if !variables[name] { + t.Errorf("dashboard is missing variable %q", name) + } + } + + text := string(raw) + for _, metric := range []string{ + "bot_network_connected", + "bot_network_reconnects_total", + "bot_network_messages_received_total", + "bot_network_messages_sent_total", + "bot_network_messages_dropped_total", + "bot_plugin_commands_handled_total", + "bot_plugin_panics_total", + "bot_outgoing_queue_depth", + "bot_outgoing_queue_capacity", + } { + if !strings.Contains(text, metric) { + t.Errorf("dashboard does not query %s", metric) + } + } +} diff --git a/grafana/gobot-dashboard-example.png b/grafana/gobot-dashboard-example.png deleted file mode 100644 index 01f459b..0000000 Binary files a/grafana/gobot-dashboard-example.png and /dev/null differ diff --git a/grafana/gobot-dashboard.json b/grafana/gobot-dashboard.json index 98fce55..f1a566d 100644 --- a/grafana/gobot-dashboard.json +++ b/grafana/gobot-dashboard.json @@ -2,8 +2,8 @@ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { - "name": "gobot-overview", - "generation": 5, + "name": "gobot-operations", + "generation": 6, "creationTimestamp": "2026-08-01T07:11:36Z", "labels": {}, "annotations": {} @@ -38,7 +38,7 @@ "spec": { "id": 1, "title": "IRC connection", - "description": "", + "description": "Connected only when every selected IRC network reports an active connection.", "links": [], "data": { "kind": "QueryGroup", @@ -52,7 +52,7 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_connected{job=~\"$job\", instance=~\"$instance\"})" + "expr": "min(bot_network_connected{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"})" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -134,8 +134,8 @@ "kind": "Panel", "spec": { "id": 2, - "title": "Reconnects", - "description": "", + "title": "Prometheus scrape", + "description": "Whether Prometheus can currently scrape every selected GoBot target.", "links": [], "data": { "kind": "QueryGroup", @@ -149,7 +149,7 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_reconnects{job=~\"$job\", instance=~\"$instance\"})" + "expr": "min(up{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -170,9 +170,9 @@ "version": "13.0.2", "spec": { "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { @@ -189,22 +189,36 @@ "fieldConfig": { "defaults": { "unit": "short", - "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down", + "color": "red" + }, + "1": { + "text": "Up", + "color": "green" + } + } + } + ], "thresholds": { "mode": "absolute", "steps": [ { "value": 0, - "color": "green" + "color": "red" }, { - "value": 80, - "color": "red" + "value": 1, + "color": "green" } ] }, "color": { - "mode": "palette-classic" + "mode": "thresholds" } }, "overrides": [] @@ -217,8 +231,8 @@ "kind": "Panel", "spec": { "id": 3, - "title": "Commands handled", - "description": "", + "title": "Reliability events", + "description": "Reconnects, dropped outbound messages, and recovered plugin panics during the selected time range.", "links": [], "data": { "kind": "QueryGroup", @@ -232,7 +246,7 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_commands_handled{job=~\"$job\", instance=~\"$instance\"})" + "expr": "sum(increase(bot_network_reconnects_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__range])) + sum(increase(bot_network_messages_dropped_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__range])) + (sum(increase(bot_plugin_panics_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__range])) or vector(0))" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -281,13 +295,13 @@ "color": "green" }, { - "value": 80, + "value": 1, "color": "red" } ] }, "color": { - "mode": "palette-classic" + "mode": "thresholds" } }, "overrides": [] @@ -300,8 +314,8 @@ "kind": "Panel", "spec": { "id": 4, - "title": "Message throughput", - "description": "", + "title": "Message throughput by network", + "description": "Per-second IRC message rates by network, calculated over Grafana's safe rate interval.", "links": [], "data": { "kind": "QueryGroup", @@ -315,8 +329,8 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "sum(rate(bot_messages_received{job=~\"$job\", instance=~\"$instance\"}[5m]))", - "legendFormat": "received" + "expr": "sum by (network) (rate(bot_network_messages_received_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__rate_interval]))", + "legendFormat": "{{network}} received" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -334,8 +348,8 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "sum(rate(bot_messages_sent{job=~\"$job\", instance=~\"$instance\"}[5m]))", - "legendFormat": "sent" + "expr": "sum by (network) (rate(bot_network_messages_sent_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__rate_interval]))", + "legendFormat": "{{network}} sent" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -440,8 +454,8 @@ "kind": "Panel", "spec": { "id": 5, - "title": "Command throughput", - "description": "", + "title": "Command throughput by plugin", + "description": "Per-second handled-command rate, grouped by plugin.", "links": [], "data": { "kind": "QueryGroup", @@ -455,8 +469,8 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "sum(rate(bot_commands_handled{job=~\"$job\", instance=~\"$instance\"}[5m]))", - "legendFormat": "commands" + "expr": "sum by (plugin) (rate(bot_plugin_commands_handled_total{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"}[$__rate_interval]))", + "legendFormat": "{{plugin}}" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -561,8 +575,8 @@ "kind": "Panel", "spec": { "id": 6, - "title": "Cumulative counters", - "description": "", + "title": "Outbound queue pressure", + "description": "Current queued messages and queue capacity by IRC network. Sustained depth indicates rate limiting or an unhealthy connection.", "links": [], "data": { "kind": "QueryGroup", @@ -576,8 +590,8 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_messages_received{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "received" + "expr": "max by (network) (bot_outgoing_queue_depth{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"})", + "legendFormat": "{{network}} depth" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -595,8 +609,8 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_messages_sent{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "sent" + "expr": "max by (network) (bot_outgoing_queue_capacity{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\", network=~\"$network\"})", + "legendFormat": "{{network}} capacity" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -605,44 +619,6 @@ "refId": "B", "hidden": false } - }, - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", - "spec": { - "expr": "max(bot_commands_handled{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "commands" - }, - "labels": { - "grafana.app/export-label": "prometheus-1" - } - }, - "refId": "C", - "hidden": false - } - }, - { - "kind": "PanelQuery", - "spec": { - "query": { - "kind": "DataQuery", - "group": "prometheus", - "version": "v0", - "spec": { - "expr": "max(bot_messages_dropped{job=~\"$job\", instance=~\"$instance\"})", - "legendFormat": "dropped" - }, - "labels": { - "grafana.app/export-label": "prometheus-1" - } - }, - "refId": "D", - "hidden": false - } } ], "transformations": [], @@ -739,7 +715,7 @@ "spec": { "id": 7, "title": "Uptime", - "description": "", + "description": "Uptime of the selected GoBot process. This resets after a rebuild or service restart.", "links": [], "data": { "kind": "QueryGroup", @@ -753,7 +729,7 @@ "group": "prometheus", "version": "v0", "spec": { - "expr": "max(bot_uptime_seconds{job=~\"$job\", instance=~\"$instance\"})" + "expr": "max(bot_uptime_seconds{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})" }, "labels": { "grafana.app/export-label": "prometheus-1" @@ -800,10 +776,6 @@ { "value": 0, "color": "green" - }, - { - "value": 80, - "color": "red" } ] }, @@ -944,15 +916,16 @@ "hideTimepicker": false, "fiscalYearStartMonth": 0 }, - "title": "GoBot Overview", + "title": "GoBot Operations", + "description": "Operational health, traffic, plugin activity, reliability, and outbound queue pressure for GoBot.", "variables": [ { "kind": "QueryVariable", "spec": { "name": "job", "current": { - "text": "", - "value": "" + "text": "gobot", + "value": "gobot" }, "label": "Job", "hide": "dontHide", @@ -981,13 +954,83 @@ "allowCustomValue": true } }, + { + "kind": "QueryVariable", + "spec": { + "name": "environment", + "current": { + "text": "production", + "value": "production" + }, + "label": "Environment", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "qryType": 5, + "query": "label_values(bot_connected{job=~\"$job\"}, environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "alphabeticalAsc", + "definition": "label_values(bot_connected{job=~\"$job\"}, environment)", + "options": [], + "multi": false, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "hostname", + "current": { + "text": "nexus-node", + "value": "nexus-node" + }, + "label": "Host", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "qryType": 5, + "query": "label_values(bot_connected{job=~\"$job\", environment=~\"$environment\"}, hostname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "alphabeticalAsc", + "definition": "label_values(bot_connected{job=~\"$job\", environment=~\"$environment\"}, hostname)", + "options": [], + "multi": false, + "includeAll": true, + "allowCustomValue": true + } + }, { "kind": "QueryVariable", "spec": { "name": "instance", "current": { - "text": "", - "value": "" + "text": "10.69.0.22:8082", + "value": "10.69.0.22:8082" }, "label": "Instance", "hide": "dontHide", @@ -999,7 +1042,7 @@ "version": "v0", "spec": { "qryType": 5, - "query": "label_values(bot_connected{job=~\"$job\"}, instance)", + "query": "label_values(bot_connected{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\"}, instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "labels": { @@ -1009,13 +1052,48 @@ "regex": "", "regexApplyTo": "value", "sort": "disabled", - "definition": "label_values(bot_connected{job=~\"$job\"}, instance)", + "definition": "label_values(bot_connected{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\"}, instance)", "options": [], "multi": false, "includeAll": true, "allowCustomValue": true } + }, + { + "kind": "QueryVariable", + "spec": { + "name": "network", + "current": { + "text": "All", + "value": "$__all" + }, + "label": "IRC network", + "hide": "dontHide", + "refresh": "onDashboardLoad", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "qryType": 5, + "query": "label_values(bot_network_connected{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"}, network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "alphabeticalAsc", + "definition": "label_values(bot_network_connected{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"}, network)", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } } ] } -} \ No newline at end of file +}