Persist plugin usage metrics for Grafana

bot/stats.go

@@ -24,6 +24,8 @@ type Stats struct {
 	persistOnce                                              atomic.Bool
 	networkMu                                                sync.RWMutex
 	networks                                                 map[string]*networkStats
+	persistedPluginCommands                                  map[string]map[string]uint64
+	persistWG                                                sync.WaitGroup
 }
 
 type networkStats struct {
@@ -43,11 +45,12 @@ type networkStats struct {
 }
 
 type persistedStats struct {
-	Received   uint64 `json:"messages_received"`
-	Sent       uint64 `json:"messages_sent"`
-	Commands   uint64 `json:"commands_handled"`
-	Reconnects uint64 `json:"reconnects"`
-	Dropped    uint64 `json:"messages_dropped"`
+	Received       uint64                       `json:"messages_received"`
+	Sent           uint64                       `json:"messages_sent"`
+	Commands       uint64                       `json:"commands_handled"`
+	Reconnects     uint64                       `json:"reconnects"`
+	Dropped        uint64                       `json:"messages_dropped"`
+	PluginCommands map[string]map[string]uint64 `json:"plugin_commands,omitempty"`
 }
 
 type metricLabel struct {
@@ -61,7 +64,7 @@ type prometheusWriter struct {
 }
 
 func NewStats(dbs ...*storage.DB) *Stats {
-	s := &Stats{started: time.Now(), networks: make(map[string]*networkStats)}
+	s := &Stats{started: time.Now(), networks: make(map[string]*networkStats), persistedPluginCommands: make(map[string]map[string]uint64)}
 	if len(dbs) > 0 && dbs[0] != nil {
 		s.db = dbs[0]
 		if raw, err := s.db.Get("stats", "global"); err == nil {
@@ -72,10 +75,17 @@ func NewStats(dbs ...*storage.DB) *Stats {
 				s.commands.Store(saved.Commands)
 				s.reconnects.Store(saved.Reconnects)
 				s.dropped.Store(saved.Dropped)
+				if saved.PluginCommands != nil {
+					s.persistedPluginCommands = saved.PluginCommands
+				}
 			}
 		}
 		s.persistDone = make(chan struct{})
-		go s.persistLoop()
+		s.persistWG.Add(1)
+		go func() {
+			defer s.persistWG.Done()
+			s.persistLoop()
+		}()
 	}
 	return s
 }
@@ -94,6 +104,9 @@ func (s *Stats) registerNetwork(name string, configuredChannels int, queue *Queu
 		return existing
 	}
 	network := &networkStats{name: name, configuredChannels: configuredChannels, queue: queue, joinedChannels: make(map[string]string)}
+	for plugin, count := range s.persistedPluginCommands[name] {
+		atomicCounter(&network.pluginCommands, plugin).Store(count)
+	}
 	s.networks[name] = network
 	return network
 }
@@ -213,8 +226,15 @@ func (s *Stats) Persist() {
 	if s.db == nil {
 		return
 	}
+	pluginCommands := make(map[string]map[string]uint64)
+	for _, network := range s.sortedNetworks() {
+		counters := snapshotCounters(&network.pluginCommands)
+		if len(counters) > 0 {
+			pluginCommands[network.name] = counters
+		}
+	}
 	_ = s.db.Set("stats", "global", persistedStats{
-		Received: s.received.Load(), Sent: s.sent.Load(), Commands: s.commands.Load(), Reconnects: s.reconnects.Load(), Dropped: s.dropped.Load(),
+		Received: s.received.Load(), Sent: s.sent.Load(), Commands: s.commands.Load(), Reconnects: s.reconnects.Load(), Dropped: s.dropped.Load(), PluginCommands: pluginCommands,
 	})
 }
 
@@ -223,6 +243,7 @@ func (s *Stats) Close() {
 		return
 	}
 	close(s.persistDone)
+	s.persistWG.Wait()
 	s.Persist()
 }
 
@@ -327,17 +348,11 @@ func (s *Stats) PrometheusSnapshot() string {
 }
 
 func (w *prometheusWriter) syncMapCounters(name, help, network string, counters *sync.Map, includeHandler bool) {
-	values := make(map[string]uint64)
-	keys := make([]string, 0)
-	counters.Range(func(key, value interface{}) bool {
-		text, ok := key.(string)
-		counter, counterOK := value.(*atomic.Uint64)
-		if ok && counterOK {
-			keys = append(keys, text)
-			values[text] = counter.Load()
-		}
-		return true
-	})
+	values := snapshotCounters(counters)
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
 	sort.Strings(keys)
 	for _, key := range keys {
 		plugin, handler := key, ""
@@ -352,6 +367,19 @@ func (w *prometheusWriter) syncMapCounters(name, help, network string, counters
 	}
 }
 
+func snapshotCounters(counters *sync.Map) map[string]uint64 {
+	values := make(map[string]uint64)
+	counters.Range(func(key, value interface{}) bool {
+		text, ok := key.(string)
+		counter, counterOK := value.(*atomic.Uint64)
+		if ok && counterOK {
+			values[text] = counter.Load()
+		}
+		return true
+	})
+	return values
+}
+
 func (w *prometheusWriter) metric(name, help, metricType string, labels []metricLabel, value interface{}) {
 	if _, ok := w.described[name]; !ok {
 		fmt.Fprintf(&w.output, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, metricType)

bot/stats_test.go

@@ -2,6 +2,7 @@ package bot
 
 import (
 	"context"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -95,6 +96,36 @@ func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) {
 	}
 }
 
+func TestPluginCommandMetricsPersistAcrossRestart(t *testing.T) {
+	dbPath := filepath.Join(t.TempDir(), "stats.db")
+	db, err := storage.Open(dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	first := NewStats(db)
+	network := first.registerNetwork("ouch", 0, nil)
+	first.recordCommand(network, "help")
+	first.Close()
+	if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	db, err = storage.Open(dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	second := NewStats(db)
+	second.registerNetwork("ouch", 0, nil)
+	metrics := second.PrometheusSnapshot()
+	if !strings.Contains(metrics, `bot_plugin_commands_handled_total{network="ouch",plugin="help"} 1`) {
+		t.Fatalf("persisted plugin command metric missing:\n%s", metrics)
+	}
+	second.Close()
+	if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestOwnChannelMembershipTracksJoinPartKickAndDisconnect(t *testing.T) {
 	stats := NewStats()
 	instance := NewWithStats(Config{NetworkName: "libera", Identity: IdentityConfig{Nick: "GoBot"}}, nil, nil, zap.NewNop(), stats)

docs/monitoring.md

@@ -50,15 +50,17 @@ GoBot also exposes operational metrics for richer dashboards:
 | `bot_network_channel_joined{network,channel}` | gauge | Current channel membership; one series with value 1 per joined channel |
 | `bot_outgoing_queue_depth{network}` | gauge | Messages currently waiting to be sent |
 | `bot_outgoing_queue_capacity{network}` | gauge | Maximum outbound queue size |
-| `bot_plugin_commands_handled_total{network,plugin}` | counter | Handled commands grouped by plugin |
+| `bot_plugin_commands_handled_total{network,plugin}` | counter | Persistent handled-command totals grouped by plugin |
 | `bot_plugin_panics_total{network,plugin,handler}` | counter | Recovered message/event handler panics |
 
-Per-network and per-plugin counters reset when the GoBot process restarts;
-Prometheus `rate()` and `increase()` account for counter resets. Labels are
-limited to network names, the bot's current joined channels, built-in plugin
-names, and the bounded handler type. Nicknames, accounts, and message contents
-are not exported. Because joined channel names are exposed as metric labels,
-keep `/metrics` on a private monitoring network.
+Per-network traffic, reconnect, dropped-message, and panic counters reset when
+the GoBot process restarts; Prometheus `rate()` and `increase()` account for
+those counter resets. Plugin command totals are persisted in BoltDB and remain
+available after a restart, so the Grafana most-used-plugins panel does not lose
+its ranking. Labels are limited to network names, the bot's current joined
+channels, built-in plugin names, and the bounded handler type. Nicknames,
+accounts, and message contents are not exported. Because joined channel names
+are exposed as metric labels, keep `/metrics` on a private monitoring network.
 
 The `/stats` JSON response includes a `networks` object with connection,
 traffic, command, reconnect, queue, configured-channel, and current joined-

grafana/README.md

@@ -9,7 +9,7 @@ identifier. It covers:
 - process uptime and reliability events
 - per-network incoming and outgoing message rates
 - handled-command rates grouped by plugin
-- the ten most-used plugins during the current GoBot process lifetime
+- the ten most-used plugins across the persisted GoBot command history
 - current networks and channels GoBot has actually joined
 - outbound queue depth and capacity by network
 - filtering by Prometheus job, environment, hostname, instance, and IRC network
@@ -86,10 +86,11 @@ The dashboard refreshes every 30 seconds, matching the Prometheus scrape
 interval. A newly started bot may need two scrapes (about one minute) before
 rate panels have enough samples to draw a line.
 
-The **Most-used plugins** panel counts handled commands since the current
-GoBot process started, so its ranking resets after a service restart. The
-**Joined networks and channels** panel reflects live JOIN/PART/KICK state and
-clears a network's channels when its IRC connection ends.
+The **Most-used plugins** panel uses persistent per-network command totals from
+BoltDB, so its ranking survives service restarts. A new database has no plugin
+series until the first command is handled. The **Joined networks and channels**
+panel reflects live JOIN/PART/KICK state and clears a network's channels when
+its IRC connection ends.
 
 ## Security note
 

grafana/gobot-dashboard.json

@@ -215,7 +215,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "min(up{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})",
+          "expr": "min(up{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"} and on (job, environment, hostname, instance) bot_process_start_time_seconds{job=~\"$job\", environment=~\"$environment\", hostname=~\"$hostname\", instance=~\"$instance\"})",
           "legendFormat": "__auto",
           "range": true,
           "refId": "A"
@@ -702,7 +702,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "Top plugins by commands handled during the current GoBot process lifetime. Counts reset when GoBot restarts.",
+      "description": "Top plugins by persistent per-network command totals. Counts survive GoBot restarts while the BoltDB database is retained.",
       "fieldConfig": {
         "defaults": {
           "unit": "short",