windows stats

internal/processmanager/README.md

@@ -91,10 +91,11 @@ forwards to the panel via gRPC.
 | `docker`                              |        yes         |                yes                |           yes            |                  yes                  |              yes (Linux)               |             yes              |
 | `podman`                              |        yes         |                yes                |           yes            |                  yes                  |                  yes                   |             yes              |
 | `systemd`                             |        yes         |                yes                |           yes            |                  yes                  |                  yes                   |             yes              |
-| `tmux` / `simple` / `winsw` / `shawl` |        yes         |                 —                 |            —             |                   —                   |                   —                    |              —               |
+| `shawl`                               |        yes         |                yes                |        usage only        |                   —                   |           yes (logical I/O)            |        yes (threads)         |
+| `tmux` / `simple` / `winsw`           |        yes         |                 —                 |            —             |                   —                   |                   —                    |              —               |
 
 Container-backed managers tag their metrics with `{server_id, server_uuid, container}`.
-The systemd manager tags its metrics with `{server_id, server_uuid, service}`.
+The systemd and shawl managers tag their metrics with `{server_id, server_uuid, service}`.
 
 The systemd manager reads metrics from `systemctl show` and relies on the
 `CPUAccounting=yes`, `MemoryAccounting=yes`, `IOAccounting=yes`,
@@ -105,7 +106,7 @@ CPU%) until the next start/restart regenerates the unit. Metrics are also
 suppressed for the first sample after each restart, since the cumulative
 CPU counter has no baseline yet.
 
-PID-based stats for `tmux` / `simple` / `winsw` / `shawl` are tracked as a follow-up.
+PID-based stats for `tmux` / `simple` / `winsw` are tracked as a follow-up.
 
 ## SystemD scopes
 
@@ -208,7 +209,33 @@ and the start command's program is in neither the working directory nor PATH, th
 so before the log tail, naming both — the difference between an archive that unpacked into a
 subdirectory and a game that crashed on startup, which are fixed in entirely different places.
 
-Metrics are liveness-only; see the table above.
+### Metrics
+
+The metrics describe the processes shawl started for the server: the game server and anything it
+runs through or starts, such as `cmd.exe` for a `.bat` or `.cmd` start command. shawl itself is not
+counted, just as systemd and container runtimes stay out of a unit's or a container's accounting.
+The shawl process is the one the service control manager reports for the service, and a single
+`NtQuerySystemInformation(SystemProcessInformation)` call per metrics tick reads every process on
+the host. No process has to be opened, so the account a server runs under does not matter.
+
+- `gameap_server_cpu_usage_percent` is a percentage of one core, as for docker and systemd. Task
+  Manager divides by the number of cores instead.
+- `gameap_server_memory_usage_bytes` is the private working set, the "Memory" column of Task
+  Manager. Shared DLL pages are left out, so the sum over several processes is not inflated. A
+  service has no memory limit, so `gameap_server_memory_limit_bytes` and
+  `gameap_server_memory_usage_percent` are not reported.
+- `gameap_server_block_io_*_bytes_total` count the bytes the processes read and wrote through files
+  and pipes, including reads served from the file cache: the I/O the game asked for, not what
+  reached the disk.
+- `gameap_server_process_pids` is the number of threads, the unit `pids.current` counts on Linux.
+- Windows keeps no per-process network counters, only ETW traces carry them, so no network metrics
+  are reported.
+
+A process counts as a child only when it is not older than its parent: Windows keeps the parent ID
+of a process whose parent has exited and hands that ID to the next process that starts. CPU time and
+I/O are tracked per process, so the counters keep growing when shawl restarts a crashed game, and
+CPU is reported from the second sample after a start. What a process used between the last sample
+and its exit is not counted.
 
 ### Changing the restart policy
 

internal/processmanager/errors.go

@@ -18,4 +18,7 @@ var (
 	ErrUserMismatch          = errors.New(
 		"server user does not match daemon user (required for systemctl --user mode)",
 	)
+
+	ErrProcessSnapshotMalformed = errors.New("malformed process list")
+	ErrProcessSnapshotTooLarge  = errors.New("process list does not fit the buffer")
 )

internal/processmanager/process_snapshot_windows.go

@@ -0,0 +1,158 @@
+//go:build windows
+
+package processmanager
+
+import (
+	"context"
+	"sync"
+	"time"
+	"unsafe"
+
+	"github.com/gameap/daemon/internal/app/config"
+	"github.com/gameap/daemon/internal/app/domain"
+	"github.com/gameap/daemon/pkg/logger"
+	"github.com/pkg/errors"
+	"golang.org/x/sys/windows"
+)
+
+const (
+	processSnapshotInitialSize = 512 * 1024
+	processSnapshotMaxSize     = 64 * 1024 * 1024
+	processSnapshotAttempts    = 5
+
+	// processSnapshotMaxAge lets every server of one metrics tick share a snapshot. It is half the
+	// shortest collection interval, so two ticks never do.
+	processSnapshotMaxAge = config.MetricsMinCollectionInterval / 2
+)
+
+var nativeProcessRecordLayout = processRecordLayout{
+	size:              int(unsafe.Sizeof(windows.SYSTEM_PROCESS_INFORMATION{})),
+	threads:           int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.NumberOfThreads)),
+	privateWorkingSet: int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.WorkingSetPrivateSize)),
+	createTime:        int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.CreateTime)),
+	userTime:          int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.UserTime)),
+	kernelTime:        int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.KernelTime)),
+	pid:               int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.UniqueProcessID)),
+	parentPID:         int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.InheritedFromUniqueProcessID)),
+	readBytes:         int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.ReadTransferCount)),
+	writeBytes:        int(unsafe.Offsetof(windows.SYSTEM_PROCESS_INFORMATION{}.WriteTransferCount)),
+}
+
+// processSnapshotter reads the process list of the host. The list covers every process and thread
+// on the host, hundreds of kilobytes, so one read serves every server of a metrics tick and the
+// buffer is kept for the next tick.
+type processSnapshotter struct {
+	mu      sync.Mutex
+	buf     []uint64
+	procs   []processInfo
+	takenAt time.Time
+}
+
+// snapshot returns the process list and the time it was read. The list is shared between callers
+// and must not be modified.
+func (s *processSnapshotter) snapshot() ([]processInfo, time.Time, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.procs != nil && time.Since(s.takenAt) < processSnapshotMaxAge {
+		return s.procs, s.takenAt, nil
+	}
+
+	procs, takenAt, err := s.read()
+	if err != nil {
+		return nil, time.Time{}, err
+	}
+
+	s.procs, s.takenAt = procs, takenAt
+
+	return procs, takenAt, nil
+}
+
+// read queries the process list. The buffer is a []uint64 because the records carry 64-bit fields,
+// which the kernel writes aligned.
+func (s *processSnapshotter) read() ([]processInfo, time.Time, error) {
+	if len(s.buf) == 0 {
+		s.buf = make([]uint64, processSnapshotInitialSize/8)
+	}
+
+	for range processSnapshotAttempts {
+		size := len(s.buf) * 8
+		takenAt := time.Now()
+
+		var written uint32
+
+		err := windows.NtQuerySystemInformation(
+			windows.SystemProcessInformation, unsafe.Pointer(&s.buf[0]), uint32(size), &written,
+		)
+		if errors.Is(err, windows.STATUS_INFO_LENGTH_MISMATCH) {
+			// Processes start between two calls, so the buffer gets more room than the kernel
+			// asked for a moment ago.
+			grown := max(2*size, int(written)+int(written)/4)
+			if grown > processSnapshotMaxSize {
+				return nil, time.Time{}, errors.WithMessagef(
+					ErrProcessSnapshotTooLarge, "%d bytes needed, at most %d allowed", written, processSnapshotMaxSize,
+				)
+			}
+
+			s.buf = make([]uint64, (grown+7)/8)
+
+			continue
+		}
+		if err != nil {
+			return nil, time.Time{}, errors.Wrap(err, "failed to query the process list")
+		}
+
+		buf := unsafe.Slice((*byte)(unsafe.Pointer(&s.buf[0])), min(int(written), size))
+
+		procs, err := parseProcessSnapshot(buf, nativeProcessRecordLayout)
+		if err != nil {
+			return nil, time.Time{}, err
+		}
+
+		return procs, takenAt, nil
+	}
+
+	return nil, time.Time{}, errors.WithMessagef(
+		ErrProcessSnapshotTooLarge, "the process list kept growing over %d attempts", processSnapshotAttempts,
+	)
+}
+
+// serviceProcessMetrics reports what the processes a Windows service started are using: the game
+// server and whatever it runs through, such as cmd.exe for a script. The service process itself is
+// a supervisor, and systemd and container runtimes keep theirs out of a unit's or container's
+// accounting as well.
+//
+// It returns nothing while the service has no process; the caller reports liveness on its own.
+func serviceProcessMetrics(
+	ctx context.Context, serviceName string, snapshots *processSnapshotter, usage *processTreeSampler,
+) []domain.Metric {
+	status, err := queryService(serviceName)
+	if err != nil || status.ProcessId == 0 {
+		usage.forget(serviceName)
+
+		if err != nil && !errors.Is(err, ErrServiceNotFound) {
+			logger.WithError(ctx, err).Debug("Failed to query service " + serviceName + " for metrics")
+		}
+
+		return nil
+	}
+
+	procs, takenAt, err := snapshots.snapshot()
+	if err != nil {
+		logger.WithError(ctx, err).Debug("Failed to read the process list for metrics")
+
+		return nil
+	}
+
+	// The snapshot can be up to processSnapshotMaxAge older than the query. A service that started
+	// in between is not in it yet and is measured on the next tick; in the rare case that its
+	// process ID was still held by another process then, that process is reported for one tick.
+	tree, found := processDescendants(procs, status.ProcessId)
+	if !found {
+		usage.forget(serviceName)
+
+		return nil
+	}
+
+	return processTreeMetrics(takenAt, serviceName, usage.observe(serviceName, tree, takenAt))
+}

internal/processmanager/process_snapshot_windows_test.go

@@ -0,0 +1,98 @@
+//go:build windows
+
+package processmanager
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+	"unsafe"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNativeProcessRecordLayout(t *testing.T) {
+	want := processRecordLayout64
+	if unsafe.Sizeof(uintptr(0)) == 4 {
+		want = processRecordLayout386
+	}
+
+	assert.Equal(t, want, nativeProcessRecordLayout)
+}
+
+func TestProcessSnapshotterRead_ContainsCurrentProcess(t *testing.T) {
+	procs, takenAt, err := (&processSnapshotter{}).read()
+	require.NoError(t, err)
+	assert.WithinDuration(t, time.Now(), takenAt, time.Minute)
+
+	var self *processInfo
+
+	for i := range procs {
+		if procs[i].PID == uint32(os.Getpid()) {
+			self = &procs[i]
+		}
+	}
+
+	require.NotNil(t, self, "the snapshot must contain the test process")
+	assert.Equal(t, uint32(os.Getppid()), self.ParentPID)
+	assert.Positive(t, self.Threads)
+	assert.Positive(t, self.PrivateWorkingSet)
+	assert.Positive(t, self.CPUTime)
+	assert.Less(t, self.CreateTime, filetimeTicks(time.Now()))
+}
+
+func TestProcessSnapshotterRead_GrowsBuffer(t *testing.T) {
+	snapshots := &processSnapshotter{buf: make([]uint64, 1)}
+
+	procs, _, err := snapshots.read()
+
+	require.NoError(t, err)
+	assert.NotEmpty(t, procs)
+	assert.Greater(t, len(snapshots.buf), 1)
+}
+
+func TestProcessSnapshotterSnapshot_SharesRecentRead(t *testing.T) {
+	snapshots := &processSnapshotter{}
+
+	_, first, err := snapshots.snapshot()
+	require.NoError(t, err)
+
+	_, second, err := snapshots.snapshot()
+	require.NoError(t, err)
+
+	assert.Equal(t, first, second)
+}
+
+func TestProcessDescendants_FindsChildAndGrandchild(t *testing.T) {
+	cmd := exec.Command("cmd.exe", "/c", "ping", "-n", "3", "127.0.0.1")
+	require.NoError(t, cmd.Start())
+
+	t.Cleanup(func() {
+		_ = cmd.Wait()
+	})
+
+	cmdPID := uint32(cmd.Process.Pid)
+
+	assert.Eventually(t, func() bool {
+		procs, _, err := (&processSnapshotter{}).read()
+		if err != nil {
+			return false
+		}
+
+		tree, found := processDescendants(procs, uint32(os.Getpid()))
+		if !found {
+			return false
+		}
+
+		var hasCmd, hasPing bool
+
+		for _, p := range tree {
+			hasCmd = hasCmd || p.PID == cmdPID
+			hasPing = hasPing || p.ParentPID == cmdPID
+		}
+
+		return hasCmd && hasPing
+	}, 5*time.Second, 100*time.Millisecond, "cmd.exe and the ping it runs must be in the test process's tree")
+}