memby/server/internal/runtimestats/runtimestats_test.go

package runtimestats

import (
	"strings"
	"sync"
	"testing"
	"time"
)

// A real dump, trimmed. The parser is the one part of this package that reads text a Go
// release produces, so the fixture is kept verbatim rather than idealised — the argument
// lists, the "+0x" offsets and the "created by … in goroutine N" suffix are all things the
// parser has to survive.
const sampleDump = `goroutine 1 [chan receive, 940 minutes]:
main.run(0xc000122000)
	/app/cmd/memby-server/main.go:430 +0x8f4
main.main()
	/app/cmd/memby-server/main.go:41 +0x1f

goroutine 18 [select]:
github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run(0xc0001a2000, {0x1a4c0d0, 0xc0000a2000})
	/app/internal/library/ingest.go:118 +0x145
created by github.com/ponzischeme89/memby/server/internal/runtimestats.Go in goroutine 1

goroutine 40 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7f2c1c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:351 +0x85
net/http.(*conn).serve(0xc000310000, {0x1a4c0d0, 0xc0003a0000})
	/usr/local/go/src/net/http/server.go:2092 +0x5db
created by net/http.(*Server).Serve in goroutine 55

goroutine 41 [GC worker (idle)]:
runtime.gopark(0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/runtime/proc.go:402 +0xce
runtime.gcBgMarkWorker(0xc0000581c0)
	/usr/local/go/src/runtime/mgc.go:1310 +0xe5
created by runtime.gcBgMarkStartWorkers in goroutine 1

goroutine 77 [semacquire, 3 minutes, locked to thread]:
github.com/jackc/puddle/v2.(*Pool).acquire(0xc00019c000)
	/root/go/pkg/mod/github.com/jackc/puddle/v2@v2.2.2/pool.go:481 +0x10a
created by github.com/jackc/pgx/v5/pgxpool.NewWithConfig in goroutine 1
`

func TestParseGoroutinesReadsHeaderStateAndFrames(t *testing.T) {
	list := parseGoroutines(sampleDump)
	if len(list) != 5 {
		t.Fatalf("parsed %d goroutines, want 5", len(list))
	}

	first := list[0]
	if first.ID != 1 || first.State != "chan receive" || first.WaitMinutes != 940 {
		t.Fatalf("first = %+v", first)
	}
	if first.Frames[0].Function != "main.run" {
		t.Fatalf("first frame function = %q", first.Frames[0].Function)
	}
	// The offset is stripped so two goroutines in the same place group together rather
	// than differing by an address.
	if first.Frames[0].File != "/app/cmd/memby-server/main.go:430" {
		t.Fatalf("first frame file = %q", first.Frames[0].File)
	}

	if list[1].CreatedBy == "" {
		t.Fatal("created-by line was dropped")
	}
	// A state with trailing detail after a second comma keeps its state and its wait.
	last := list[4]
	if last.State != "semacquire" || last.WaitMinutes != 3 {
		t.Fatalf("last = %+v", last)
	}
}

// The categories have to partition the total or the breakdown is misleading, which is
// worse than not offering one.
func TestBuildReportPartitionsAndAttributes(t *testing.T) {
	report := buildReport(sampleDump, time.Now())
	if report.Total != 5 {
		t.Fatalf("total = %d", report.Total)
	}
	counted := 0
	for _, category := range report.Categories {
		counted += category.Count
	}
	if counted != report.Total {
		t.Fatalf("categories sum to %d, total %d", counted, report.Total)
	}
	attributed := 0
	for _, component := range report.Components {
		attributed += component.Count
	}
	if attributed != report.Total {
		t.Fatalf("components sum to %d, total %d", attributed, report.Total)
	}

	got := map[string]string{}
	for _, group := range report.Groups {
		got[group.Component] = group.Function
	}
	// Memby's own packages win over the libraries they call, and the frame reported is
	// the one that names the work rather than the runtime plumbing every parked goroutine
	// shares.
	if got["Library sync"] != "github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run" {
		t.Fatalf("library group = %q", got["Library sync"])
	}
	if got["HTTP server"] != "net/http.(*conn).serve" {
		t.Fatalf("http group = %q", got["HTTP server"])
	}
	if got["Database pool"] != "github.com/jackc/puddle/v2.(*Pool).acquire" {
		t.Fatalf("database group = %q", got["Database pool"])
	}
}

func TestCategoryForKnownStates(t *testing.T) {
	cases := map[string]Category{
		"running":          CategoryRunning,
		"runnable":         CategoryRunning,
		"syscall":          CategoryRunning,
		"IO wait":          CategoryIO,
		"select":           CategoryWaiting,
		"chan receive":     CategoryWaiting,
		"semacquire":       CategoryWaiting,
		"sync.Mutex.Lock":  CategoryWaiting,
		"sleep":            CategoryTimers,
		"timer goroutine":  CategoryTimers,
		"GC worker (idle)": CategoryRuntime,
		"force gc (idle)":  CategoryRuntime,
		"finalizer wait":   CategoryRuntime,
		// The runtime's own wait outranks the "wait" rule, or the collector would be
		// reported as a component sitting on a channel.
		"wait for GC cycle": CategoryRuntime,
		"something new":     CategoryOther,
	}
	for state, want := range cases {
		if got := categoryFor(state); got != want {
			t.Errorf("categoryFor(%q) = %q, want %q", state, got, want)
		}
	}
}

// A live report has to hold together too: this is the one test that would notice a Go
// release changing the dump format under the parser.
func TestCollectGoroutinesAgreesWithTheRuntime(t *testing.T) {
	blocked := make(chan struct{})
	var ready sync.WaitGroup
	ready.Add(1)
	go func() {
		ready.Done()
		<-blocked
	}()
	ready.Wait()
	defer close(blocked)

	report := CollectGoroutines()
	if report.Total < 2 {
		t.Fatalf("total = %d", report.Total)
	}
	counted := 0
	for _, category := range report.Categories {
		counted += category.Count
	}
	if counted != report.Total {
		t.Fatalf("categories sum to %d, total %d", counted, report.Total)
	}
	if report.CollectedInMs < 0 {
		t.Fatalf("collectedInMs = %v", report.CollectedInMs)
	}
	if !strings.Contains(StackDump(), "goroutine ") {
		t.Fatal("stack dump does not look like a stack dump")
	}
}

/* ---------- trends ---------- */

func series(values []float64, every time.Duration) ([]float64, []time.Time) {
	at := make([]time.Time, len(values))
	base := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
	for index := range values {
		at[index] = base.Add(time.Duration(index) * every)
	}
	return values, at
}

func TestTrendRefusesToAnswerWithoutEnoughHistory(t *testing.T) {
	// Four readings a minute apart is a gateway that restarted three minutes ago, not a
	// trend, and "steady" would be a claim the data does not support.
	points, at := series([]float64{20, 24, 30, 44}, time.Minute)
	if got := trendOf(points, at, 5); got.Direction != DirectionUnknown {
		t.Fatalf("direction = %q, want unknown", got.Direction)
	}
}

func TestTrendIgnoresABurstAndNamesASustainedRise(t *testing.T) {
	// A burst in the middle — a library import, a household arriving home — must not be
	// reported as a rise; medians are what make that true where a fitted line would not.
	burst := []float64{40, 41, 40, 42, 41, 300, 290, 40, 41, 40, 42, 41}
	points, at := series(burst, 5*time.Minute)
	if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
		t.Fatalf("burst direction = %q (%+v), want steady", got.Direction, got)
	}

	leak := make([]float64, 12)
	for index := range leak {
		leak[index] = float64(40 + index*5)
	}
	points, at = series(leak, 5*time.Minute)
	rising := trendOf(points, at, 5)
	if rising.Direction != DirectionRising {
		t.Fatalf("leak direction = %q (%+v), want rising", rising.Direction, rising)
	}
	if rising.PerHour <= 0 {
		t.Fatalf("perHour = %v, want a positive rate", rising.PerHour)
	}
	if rising.Latest != 95 || rising.First != 40 {
		t.Fatalf("first/latest = %v/%v", rising.First, rising.Latest)
	}
}

func TestTrendFloorKeepsAQuietGatewaySteady(t *testing.T) {
	// One extra goroutine over an hour is not a leak, and without the absolute floor the
	// proportional threshold would call it one on a gateway holding a handful.
	points, at := series([]float64{8, 8, 9, 9, 9, 9, 9, 10}, 10*time.Minute)
	if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
		t.Fatalf("direction = %q (%+v), want steady", got.Direction, got)
	}
}

/* ---------- the verdict ---------- */

func risingTrend(first, latest, perHour float64) Trend {
	return Trend{
		Direction: DirectionRising, PerHour: perHour, First: first, Latest: latest,
		SpanSeconds: 3600, Points: 60,
	}
}

func TestAssessSaysNothingWhenThereIsNothingToSay(t *testing.T) {
	health := assess(Snapshot{
		Memory:         Memory{MemoryLimit: 384 << 20, HeapInuse: 16 << 20, HeapShare: 0.04},
		GoroutineTrend: Trend{Direction: DirectionSteady, SpanSeconds: 7200},
		HeapTrend:      Trend{Direction: DirectionSteady},
	})
	if health.Level != LevelOK || len(health.Notes) != 0 {
		t.Fatalf("health = %+v", health)
	}
	if !strings.Contains(health.Summary, "Nothing abnormal") {
		t.Fatalf("summary = %q", health.Summary)
	}
}

func TestAssessNamesTheAreaToInvestigate(t *testing.T) {
	health := assess(Snapshot{
		Memory:         Memory{MemoryLimit: 384 << 20, HeapInuse: 350 << 20, HeapShare: 0.92},
		GoroutineTrend: risingTrend(60, 400, 340),
		HeapTrend:      Trend{Direction: DirectionSteady},
	})
	if health.Level != LevelBad {
		t.Fatalf("level = %q", health.Level)
	}
	areas := strings.Join(health.Areas, ",")
	if !strings.Contains(areas, "Memory") || !strings.Contains(areas, "Goroutines") {
		t.Fatalf("areas = %v", health.Areas)
	}
	// A verdict with no area is the number that could not be acted on, which is the whole
	// defect this replaced.
	for _, note := range health.Notes {
		if note.Area == "" {
			t.Fatalf("note without an area: %+v", note)
		}
	}
}

func TestAssessSeparatesADriftFromAFlood(t *testing.T) {
	drift := assess(Snapshot{GoroutineTrend: risingTrend(40, 60, 20)})
	if drift.Level != LevelWatch {
		t.Fatalf("drift level = %q", drift.Level)
	}
	flood := assess(Snapshot{GoroutineTrend: risingTrend(40, 400, 360)})
	if flood.Level != LevelBad {
		t.Fatalf("flood level = %q", flood.Level)
	}
	// A rise slower than the leak threshold is ordinary breathing and must not be a note
	// at all, or the card cries wolf on every busy evening.
	quiet := assess(Snapshot{GoroutineTrend: risingTrend(40, 44, 4)})
	if quiet.Level != LevelOK || len(quiet.Notes) != 0 {
		t.Fatalf("quiet = %+v", quiet)
	}
}

func TestAssessNoticesAWorkerThatKeepsRestarting(t *testing.T) {
	health := assess(Snapshot{Workers: []Worker{
		{Name: "Library ingest", Component: "Library sync", State: WorkerRunning, Starts: 5},
		{Name: "Startup library sync", Component: "Library sync", State: WorkerFinished, Starts: 1},
	}})
	if health.Level != LevelWatch || len(health.Notes) != 1 {
		t.Fatalf("health = %+v", health)
	}
	if !strings.Contains(health.Notes[0].Message, "Library ingest") {
		t.Fatalf("note = %q", health.Notes[0].Message)
	}
}

/* ---------- the registry ---------- */

func TestWorkersRecordRunningAndFinished(t *testing.T) {
	resetWorkers()
	t.Cleanup(resetWorkers)

	release := make(chan struct{})
	var running sync.WaitGroup
	running.Add(1)
	Go("Library ingest", "Library sync", func() {
		running.Done()
		<-release
	})
	running.Wait()

	var done sync.WaitGroup
	done.Add(1)
	Go("Startup library sync", "Library sync", func() { done.Done() })
	done.Wait()
	// The finished worker's goroutine has run its body; give the deferred end() the moment
	// it needs to land before reading the registry.
	waitFor(t, func() bool { return stateOf(t, "Startup library sync") == WorkerFinished })

	list := Workers()
	if len(list) != 2 {
		t.Fatalf("workers = %+v", list)
	}
	// Running first, so the table does not bury the thing that is still working under the
	// startup jobs that are not.
	if list[0].Name != "Library ingest" || list[0].State != WorkerRunning {
		t.Fatalf("first = %+v", list[0])
	}
	if list[1].State != WorkerFinished || list[1].Stopped.IsZero() {
		t.Fatalf("second = %+v", list[1])
	}
	close(release)
	waitFor(t, func() bool { return stateOf(t, "Library ingest") == WorkerFinished })

	// A restart is counted rather than replacing the record: a worker restarted in a loop
	// reads exactly like a healthy one from a single snapshot and does not from Starts.
	Go("Library ingest", "Library sync", func() {})
	waitFor(t, func() bool { return startsOf(t, "Library ingest") == 2 })
}

func stateOf(t *testing.T, name string) WorkerState {
	t.Helper()
	for _, worker := range Workers() {
		if worker.Name == name {
			return worker.State
		}
	}
	return ""
}

func startsOf(t *testing.T, name string) int {
	t.Helper()
	for _, worker := range Workers() {
		if worker.Name == name {
			return worker.Starts
		}
	}
	return 0
}

func waitFor(t *testing.T, done func() bool) {
	t.Helper()
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if done() {
			return
		}
		time.Sleep(time.Millisecond)
	}
	t.Fatal("condition not reached")
}