Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions bot/bot.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ func (b *Bot) connect(ctx context.Context) error {
client := irc.NewClient(conn, irc.ClientConfig{Nick: b.Config.Identity.Nick, User: b.Config.Identity.User, Name: b.Config.Identity.Realname, Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) {
handleSASL(c, m, b.Config.Identity.SASLUser, b.Config.Identity.SASLPass, mechanism, capState, b.Log)
b.logIRCEvent(m)
b.trackOwnChannelMembership(c.CurrentNick(), m)
if m.Command == "INVITE" {
b.handleInvite(m)
}
Expand Down Expand Up @@ -336,6 +337,9 @@ func (b *Bot) connect(ctx context.Context) error {
conn.Close()
<-errc
b.Stats.setNetworkConnected(b.networkStats, false)
b.mu.Lock()
b.client = nil
b.mu.Unlock()
return nil
case err := <-errc:
b.Stats.setNetworkConnected(b.networkStats, false)
Expand All @@ -346,6 +350,30 @@ func (b *Bot) connect(ctx context.Context) error {
}
}

func (b *Bot) trackOwnChannelMembership(currentNick string, message *irc.Message) {
if b.networkStats == nil || message == nil || message.Prefix == nil {
return
}
currentNick = strings.TrimSpace(currentNick)
if currentNick == "" {
currentNick = b.Config.Identity.Nick
}
switch message.Command {
case "JOIN":
if strings.EqualFold(message.Prefix.Name, currentNick) && len(message.Params) > 0 {
b.networkStats.joinChannel(message.Params[0])
}
case "PART":
if strings.EqualFold(message.Prefix.Name, currentNick) && len(message.Params) > 0 {
b.networkStats.leaveChannel(message.Params[0])
}
case "KICK":
if len(message.Params) > 1 && strings.EqualFold(message.Params[1], currentNick) {
b.networkStats.leaveChannel(message.Params[0])
}
}
}

func (b *Bot) handleInvite(m *irc.Message) {
if !b.Config.Invites.Enabled || len(m.Params) < 2 {
return
Expand Down
79 changes: 69 additions & 10 deletions bot/stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ type networkStats struct {
name string
configuredChannels int
queue *Queue
channelMu sync.RWMutex
joinedChannels map[string]string
connected atomic.Uint64
received atomic.Uint64
sent atomic.Uint64
Expand Down Expand Up @@ -91,11 +93,56 @@ func (s *Stats) registerNetwork(name string, configuredChannels int, queue *Queu
if existing := s.networks[name]; existing != nil {
return existing
}
network := &networkStats{name: name, configuredChannels: configuredChannels, queue: queue}
network := &networkStats{name: name, configuredChannels: configuredChannels, queue: queue, joinedChannels: make(map[string]string)}
s.networks[name] = network
return network
}

func (network *networkStats) joinChannel(channel string) {
if network == nil {
return
}
channel = strings.TrimSpace(channel)
if channel == "" {
return
}
network.channelMu.Lock()
network.joinedChannels[strings.ToLower(channel)] = channel
network.channelMu.Unlock()
}

func (network *networkStats) leaveChannel(channel string) {
if network == nil {
return
}
network.channelMu.Lock()
delete(network.joinedChannels, strings.ToLower(strings.TrimSpace(channel)))
network.channelMu.Unlock()
}

func (network *networkStats) clearJoinedChannels() {
if network == nil {
return
}
network.channelMu.Lock()
clear(network.joinedChannels)
network.channelMu.Unlock()
}

func (network *networkStats) sortedJoinedChannels() []string {
if network == nil {
return nil
}
network.channelMu.RLock()
channels := make([]string, 0, len(network.joinedChannels))
for _, channel := range network.joinedChannels {
channels = append(channels, channel)
}
network.channelMu.RUnlock()
sort.Slice(channels, func(i, j int) bool { return strings.ToLower(channels[i]) < strings.ToLower(channels[j]) })
return channels
}

func (s *Stats) setNetworkConnected(network *networkStats, value bool) {
if network == nil {
if value {
Expand All @@ -105,6 +152,9 @@ func (s *Stats) setNetworkConnected(network *networkStats, value bool) {
}
return
}
if !value {
network.clearJoinedChannels()
}
desired := uint64(0)
delta := int64(-1)
if value {
Expand Down Expand Up @@ -201,20 +251,23 @@ func (s *Stats) networkSnapshot() map[string]interface{} {
networks := make(map[string]interface{})
for _, network := range s.sortedNetworks() {
depth, capacity := 0, 0
joinedChannels := network.sortedJoinedChannels()
if network.queue != nil {
depth = network.queue.Depth()
capacity = network.queue.Capacity()
}
networks[network.name] = map[string]interface{}{
"connected": network.connected.Load() == 1,
"reconnects": network.reconnects.Load(),
"messages_received": network.received.Load(),
"messages_sent": network.sent.Load(),
"messages_dropped": network.dropped.Load(),
"commands_handled": network.commands.Load(),
"configured_channels": network.configuredChannels,
"queue_depth": depth,
"queue_capacity": capacity,
"connected": network.connected.Load() == 1,
"reconnects": network.reconnects.Load(),
"messages_received": network.received.Load(),
"messages_sent": network.sent.Load(),
"messages_dropped": network.dropped.Load(),
"commands_handled": network.commands.Load(),
"configured_channels": network.configuredChannels,
"joined_channel_count": len(joinedChannels),
"joined_channels": joinedChannels,
"queue_depth": depth,
"queue_capacity": capacity,
}
}
return networks
Expand Down Expand Up @@ -254,6 +307,12 @@ func (s *Stats) PrometheusSnapshot() string {
writer.metric("bot_network_commands_handled_total", "Commands handled during the current process lifetime.", "counter", labels, network.commands.Load())
writer.metric("bot_network_messages_dropped_total", "Messages dropped because the network outbound queue was full.", "counter", labels, network.dropped.Load())
writer.metric("bot_network_configured_channels", "Configured IRC channels for the network.", "gauge", labels, network.configuredChannels)
joinedChannels := network.sortedJoinedChannels()
writer.metric("bot_network_joined_channels", "IRC channels the bot is currently joined to on the network.", "gauge", labels, len(joinedChannels))
for _, channel := range joinedChannels {
channelLabels := append(append([]metricLabel{}, labels...), metricLabel{name: "channel", value: channel})
writer.metric("bot_network_channel_joined", "Current IRC channel membership for the bot.", "gauge", channelLabels, 1)
}
depth, capacity := 0, 0
if network.queue != nil {
depth = network.queue.Depth()
Expand Down
42 changes: 41 additions & 1 deletion bot/stats_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/variablenix/GoBot/storage"
"go.uber.org/zap"
"gopkg.in/irc.v3"
)

func TestStatsListenAddress(t *testing.T) {
Expand Down Expand Up @@ -52,6 +53,7 @@ func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) {
network.received.Store(7)
network.sent.Store(5)
network.reconnects.Store(2)
network.joinChannel("#GoBot")
stats.recordCommand(network, "help")
stats.recordPluginPanic(network, "weather", "message")
if !queue.Enqueue(Outgoing{Target: "#test", Text: "queued"}) {
Expand All @@ -71,6 +73,8 @@ func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) {
`bot_network_messages_received_total{network="libera"} 7`,
`bot_network_messages_sent_total{network="libera"} 5`,
`bot_network_configured_channels{network="libera"} 3`,
`bot_network_joined_channels{network="libera"} 1`,
`bot_network_channel_joined{network="libera",channel="#GoBot"} 1`,
`bot_outgoing_queue_depth{network="libera"} 1`,
`bot_outgoing_queue_capacity{network="libera"} 40`,
`bot_plugin_commands_handled_total{network="libera",plugin="help"} 1`,
Expand All @@ -85,11 +89,47 @@ func TestExpandedPrometheusMetricsRemainBackwardCompatible(t *testing.T) {
t.Fatal("Snapshot() networks has an unexpected type")
}
libera, ok := networks["libera"].(map[string]interface{})
if !ok || libera["configured_channels"] != 3 || libera["queue_depth"] != 1 {
joinedChannels, channelsOK := libera["joined_channels"].([]string)
if !ok || libera["configured_channels"] != 3 || libera["joined_channel_count"] != 1 || !channelsOK || len(joinedChannels) != 1 || joinedChannels[0] != "#GoBot" || libera["queue_depth"] != 1 {
t.Fatalf("Snapshot() network details = %#v", networks["libera"])
}
}

func TestOwnChannelMembershipTracksJoinPartKickAndDisconnect(t *testing.T) {
stats := NewStats()
instance := NewWithStats(Config{NetworkName: "libera", Identity: IdentityConfig{Nick: "GoBot"}}, nil, nil, zap.NewNop(), stats)
defer instance.Queue.Drain(context.Background())

instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "GoBot"}, Command: "JOIN", Params: []string{"#One"}})
instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "someone"}, Command: "JOIN", Params: []string{"#Ignored"}})
instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "GoBot"}, Command: "JOIN", Params: []string{"#two"}})
instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "GoBot"}, Command: "PART", Params: []string{"#ONE"}})
instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "operator"}, Command: "KICK", Params: []string{"#two", "gobot", "reason"}})

if channels := instance.networkStats.sortedJoinedChannels(); len(channels) != 0 {
t.Fatalf("joined channels after PART and KICK = %v, want none", channels)
}
instance.trackOwnChannelMembership("GoBot", &irc.Message{Prefix: &irc.Prefix{Name: "GoBot"}, Command: "JOIN", Params: []string{"#rejoined"}})
stats.setNetworkConnected(instance.networkStats, false)
if channels := instance.networkStats.sortedJoinedChannels(); len(channels) != 0 {
t.Fatalf("joined channels after disconnect clear = %v, want none", channels)
}
}

func TestJoinedChannelMetricLabelsAreEscapedAndSorted(t *testing.T) {
stats := NewStats()
network := stats.registerNetwork("libera", 0, nil)
network.joinChannel("#z")
network.joinChannel("#a\\\"")

metrics := stats.PrometheusSnapshot()
first := strings.Index(metrics, `channel="#a\\\""`)
second := strings.Index(metrics, `channel="#z"`)
if first < 0 || second < 0 || first >= second {
t.Fatalf("joined channel labels were not escaped and sorted:\n%s", metrics)
}
}

type panicMetricsPlugin struct{}

func (*panicMetricsPlugin) Name() string { return "panic-test" }
Expand Down
14 changes: 9 additions & 5 deletions docs/monitoring.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,20 +46,24 @@ GoBot also exposes operational metrics for richer dashboards:
| `bot_network_commands_handled_total{network}` | counter | Per-network handled commands since process start |
| `bot_network_messages_dropped_total{network}` | counter | Per-network outbound messages dropped since process start |
| `bot_network_configured_channels{network}` | gauge | Configured channels per network |
| `bot_network_joined_channels{network}` | gauge | Channels the bot is currently joined to per network |
| `bot_network_channel_joined{network,channel}` | gauge | Current channel membership; one series with value 1 per joined channel |
| `bot_outgoing_queue_depth{network}` | gauge | Messages currently waiting to be sent |
| `bot_outgoing_queue_capacity{network}` | gauge | Maximum outbound queue size |
| `bot_plugin_commands_handled_total{network,plugin}` | counter | Handled commands grouped by plugin |
| `bot_plugin_panics_total{network,plugin,handler}` | counter | Recovered message/event handler panics |

Per-network and per-plugin counters reset when the GoBot process restarts;
Prometheus `rate()` and `increase()` account for counter resets. Labels are
limited to configured network names, built-in plugin names, and the bounded
handler type. Channel names, nicknames, accounts, and message contents are not
exported.
limited to network names, the bot's current joined channels, built-in plugin
names, and the bounded handler type. Nicknames, accounts, and message contents
are not exported. Because joined channel names are exposed as metric labels,
keep `/metrics` on a private monitoring network.

The `/stats` JSON response includes a `networks` object with connection,
traffic, command, reconnect, queue, and configured-channel details for each
network.
traffic, command, reconnect, queue, configured-channel, and current joined-
channel details for each network. Membership is updated from the bot's own
JOIN, PART, and KICK events and cleared whenever that IRC connection ends.

## Prometheus scrape configuration

Expand Down
13 changes: 11 additions & 2 deletions grafana/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ dashboard for GoBot's `/metrics` endpoint. It covers:
- process uptime and reliability events
- per-network incoming and outgoing message rates
- handled-command rates grouped by plugin
- the ten most-used plugins during the current GoBot process lifetime
- current networks and channels GoBot has actually joined
- outbound queue depth and capacity by network
- filtering by Prometheus job, environment, hostname, instance, and IRC network

Expand All @@ -24,7 +26,8 @@ curl http://10.69.0.22:8082/metrics

Along with the original `bot_*` metrics, the response should include
`bot_network_connected`, `bot_network_messages_received_total`,
`bot_plugin_commands_handled_total`, and `bot_outgoing_queue_depth`.
`bot_plugin_commands_handled_total`, `bot_network_channel_joined`, and
`bot_outgoing_queue_depth`.

## 2. Configure Prometheus

Expand Down Expand Up @@ -76,8 +79,14 @@ The dashboard refreshes every 30 seconds, matching the Prometheus scrape
interval. A newly started bot may need two scrapes (about one minute) before
rate panels have enough samples to draw a line.

The **Most-used plugins** panel counts handled commands since the current
GoBot process started, so its ranking resets after a service restart. The
**Joined networks and channels** panel reflects live JOIN/PART/KICK state and
clears a network's channels when its IRC connection ends.

## Security note

GoBot's `/stats` and `/metrics` endpoints do not provide authentication. Bind
the listener to a private address and allow access only from the Prometheus
host through the firewall.
host through the firewall. The membership metric includes joined channel
names as labels; it does not expose users, accounts, or message contents.
3 changes: 3 additions & 0 deletions grafana/dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func TestDashboardJSONUsesExpandedMetrics(t *testing.T) {
"Command throughput by plugin",
"Outbound queue pressure",
"Uptime",
"Most-used plugins",
"Joined networks and channels",
} {
if !panelTitles[title] {
t.Errorf("dashboard is missing panel %q", title)
Expand Down Expand Up @@ -79,6 +81,7 @@ func TestDashboardJSONUsesExpandedMetrics(t *testing.T) {
"bot_plugin_panics_total",
"bot_outgoing_queue_depth",
"bot_outgoing_queue_capacity",
"bot_network_channel_joined",
} {
if !strings.Contains(text, metric) {
t.Errorf("dashboard does not query %s", metric)
Expand Down
Loading