0.2.78

server/internal/runtimestats/workers.go

@@ -0,0 +1,150 @@
+// Package runtimestats answers the one question the console's Process card exists for: is
+// the gateway healthy, and if something is abnormal, which area should be looked at?
+//
+// A bare goroutine count cannot answer it. Twenty-five means nothing on its own — an
+// operator cannot tell from it what those goroutines are doing, which part of Memby they
+// belong to, whether the number is normal, or whether it has been climbing all week. This
+// package supplies the three things that make the number readable, and keeps them apart by
+// what they cost:
+//
+//   - The registry here, which is free. A long-running Memby worker says its own name when
+//     it starts, so "Library ingest" and "Emby health probe" are named rather than inferred
+//     from a stack.
+//   - The sampler in sample.go, which is a handful of counters on a slow tick. A single
+//     instantaneous figure cannot show a leak; a trend can.
+//   - The stack breakdown in goroutines.go, which is genuinely expensive and is therefore
+//     collected only when an operator asks for it.
+//
+// Nothing here recovers a panic. A worker that dies must die exactly as it always did —
+// this package reports, and reporting must never change what it is reporting on.
+package runtimestats
+
+import (
+	"sort"
+	"sync"
+	"time"
+)
+
+// WorkerState is what became of a tracked worker. A worker that ends is not a fault —
+// several of the gateway's background jobs are one-shot startup work — but "still running"
+// and "finished" are different answers and the console must not merge them.
+type WorkerState string
+
+const (
+	WorkerRunning  WorkerState = "running"
+	WorkerFinished WorkerState = "finished"
+)
+
+// Worker is one named background goroutine.
+type Worker struct {
+	Name      string      `json:"name"`
+	Component string      `json:"component"`
+	State     WorkerState `json:"state"`
+	Started   time.Time   `json:"started"`
+	Stopped   time.Time   `json:"stopped,omitempty"`
+	// Starts counts how many times this name has been launched. It is on the record
+	// because a worker that is being restarted in a loop reads exactly like a healthy one
+	// from a single snapshot, and does not from this number.
+	Starts int `json:"starts"`
+}
+
+type workerEntry struct {
+	component string
+	running   int
+	starts    int
+	started   time.Time
+	stopped   time.Time
+}
+
+var (
+	workersMu sync.Mutex
+	workers   = map[string]*workerEntry{}
+)
+
+// Go starts fn on its own goroutine and records it under a name an operator can read. The
+// name is the worker's identity across restarts, so it must be stable and must not carry a
+// count or an address in it.
+func Go(name, component string, fn func()) {
+	begin(name, component)
+	go func() {
+		defer end(name)
+		fn()
+	}()
+}
+
+// Track records a worker whose goroutine something else owns. The returned function marks
+// it finished, and is safe to call more than once.
+func Track(name, component string) (done func()) {
+	begin(name, component)
+	var once sync.Once
+	return func() { once.Do(func() { end(name) }) }
+}
+
+func begin(name, component string) {
+	now := time.Now()
+	workersMu.Lock()
+	defer workersMu.Unlock()
+	entry := workers[name]
+	if entry == nil {
+		entry = &workerEntry{}
+		workers[name] = entry
+	}
+	entry.component = component
+	entry.running++
+	entry.starts++
+	entry.started = now
+	entry.stopped = time.Time{}
+}
+
+func end(name string) {
+	now := time.Now()
+	workersMu.Lock()
+	defer workersMu.Unlock()
+	entry := workers[name]
+	if entry == nil {
+		return
+	}
+	if entry.running > 0 {
+		entry.running--
+	}
+	if entry.running == 0 {
+		entry.stopped = now
+	}
+}
+
+// Workers lists what has been registered, running first and then by name, so the table
+// does not reorder itself under the operator on every poll.
+func Workers() []Worker {
+	workersMu.Lock()
+	list := make([]Worker, 0, len(workers))
+	for name, entry := range workers {
+		worker := Worker{
+			Name:      name,
+			Component: entry.component,
+			State:     WorkerFinished,
+			Started:   entry.started,
+			Stopped:   entry.stopped,
+			Starts:    entry.starts,
+		}
+		if entry.running > 0 {
+			worker.State = WorkerRunning
+			worker.Stopped = time.Time{}
+		}
+		list = append(list, worker)
+	}
+	workersMu.Unlock()
+	sort.Slice(list, func(a, b int) bool {
+		if (list[a].State == WorkerRunning) != (list[b].State == WorkerRunning) {
+			return list[a].State == WorkerRunning
+		}
+		return list[a].Name < list[b].Name
+	})
+	return list
+}
+
+// resetWorkers exists for the tests; the registry is process-wide by design.
+func resetWorkers() {
+	workersMu.Lock()
+	workers = map[string]*workerEntry{}
+	workersMu.Unlock()
+}