admin/memby
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.2.78

Browse Source
This commit is contained in:
ponzischeme89
2026-08-19 14:25:44 +12:00
parent 2b43b9ef12
commit 590e069366
83 changed files with 8948 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/runtimestats/goroutines.go
+467
View File
@@ -0,0 +1,467 @@
package runtimestats
import (
"runtime"
"sort"
"strconv"
"strings"
"time"
)
/* The expensive half, and the reason it is a separate route.
Reading every goroutine's stack means stopping the world for as long as it takes to walk
them all. That is measured in milliseconds rather than seconds, but it is not something
to do on a poll every open console tab makes every thirty seconds — so this is collected
only when an operator asks, and the answer says when it was collected and what it cost.
What comes back is a text dump, and parsing text a runtime produced is exactly the sort
of thing that quietly stops working. Two rules keep that honest: parseGoroutines is pure
and pinned by tests carrying real dump text, and every unrecognised state or frame falls
into a named "other" bucket rather than being dropped — a breakdown whose parts do not
add up to the total is worse than no breakdown, so the total is always the count of
goroutines parsed and the categories always partition it. */
// Category is the readable grouping a goroutine state falls into. These are the words an
// operator is being asked to reason with, so there are six of them and none is Go jargon.
type Category string
const (
CategoryRunning Category = "running"
CategoryIO Category = "io"
CategoryWaiting Category = "waiting"
CategoryTimers Category = "timers"
CategoryRuntime Category = "runtime"
CategoryOther Category = "other"
)
// categoryLabels is the console's wording. Sent from here, the stance every label the
// gateway prints takes: a category added later reads correctly on a console that predates
// it, because the console prints what it was handed.
var categoryLabels = map[Category]string{
CategoryRunning: "Running",
CategoryIO: "Network / I/O",
CategoryWaiting: "Waiting / idle",
CategoryTimers: "Timers / scheduled work",
CategoryRuntime: "Go runtime / collection",
CategoryOther: "Other",
}
// categoryOrder is the order they are presented in: busiest kind of work first, then the
// two that mean the goroutine is doing nothing, then the runtime's own housekeeping. It is
// fixed rather than sorted by count so the table does not reorder between two collections.
var categoryOrder = []Category{
CategoryRunning, CategoryIO, CategoryWaiting, CategoryTimers, CategoryRuntime, CategoryOther,
}
// categoryDescriptions say what the operator is looking at, because "semacquire" means
// nothing to somebody who does not write Go and the whole point of this page is that they
// should not have to.
var categoryDescriptions = map[Category]string{
CategoryRunning: "On a processor now, or queued for one.",
CategoryIO: "Waiting on a network read or write — Emby, Postgres, Redis, or a television.",
CategoryWaiting: "Parked waiting for work or for a lock. Idle, and costing almost nothing.",
CategoryTimers: "Asleep until a scheduled time.",
CategoryRuntime: "The Go runtime's own housekeeping. This handful is always present.",
CategoryOther: "A state this build does not have a category for.",
}
// categoryFor maps a runtime state string onto one of the six. The GC states are tested
// first because several of them contain words the later rules would otherwise claim —
// "wait for GC cycle" is the runtime's own, not a component waiting on a channel.
func categoryFor(state string) Category {
state = strings.TrimSpace(state)
lower := strings.ToLower(state)
switch {
case strings.Contains(lower, "gc "), strings.HasPrefix(lower, "gc"),
lower == "finalizer wait", strings.HasPrefix(lower, "trace reader"),
strings.HasPrefix(lower, "dumping heap"), strings.HasPrefix(lower, "stopping the world"),
strings.HasPrefix(lower, "idle"):
return CategoryRuntime
case lower == "running", lower == "runnable", strings.HasPrefix(lower, "syscall"):
return CategoryRunning
case strings.HasPrefix(lower, "io wait"), strings.HasPrefix(lower, "netpoll"):
return CategoryIO
case strings.HasPrefix(lower, "sleep"), strings.HasPrefix(lower, "timer"):
return CategoryTimers
case strings.HasPrefix(lower, "select"), strings.HasPrefix(lower, "chan "),
strings.HasPrefix(lower, "semacquire"), strings.HasPrefix(lower, "sync."),
strings.HasPrefix(lower, "wait"), strings.HasPrefix(lower, "preempted"):
return CategoryWaiting
default:
return CategoryOther
}
}
/* ---------- component attribution ---------- */
type componentRule struct {
component string
// patterns are matched against every frame of the stack — the function names and the
// file paths alike — plus the "created by" line, which is often the only frame that
// still names the component a long-parked goroutine belongs to.
patterns []string
}
// componentRules is ordered, and the order is the whole of it: Memby's own packages are
// tested before the libraries they call, or every database query in the gateway would be
// filed under "Database pool" and nothing would be attributable to the feature that made
// it. First match wins.
var componentRules = []componentRule{
{"Library sync", []string{"/internal/library"}},
{"Scheduled jobs", []string{"/internal/scheduler"}},
{"Credits detection", []string{"/internal/credits"}},
{"Recommendations", []string{"/internal/recommend", "/internal/foryou", "/internal/tracearr"}},
{"Notifications", []string{"/internal/notify"}},
{"Integrations", []string{"/internal/integrations", "/internal/adminevents"}},
{"Subtitles", []string{"/internal/bazarr", "/internal/opensubtitles", "/internal/subsync"}},
{"Sonarr and Radarr", []string{"/internal/sonarr", "/internal/radarr"}},
{"Ratings", []string{"/internal/mdblist"}},
{"Emby", []string{"/internal/emby", "/internal/trickplay"}},
{"Gateway API", []string{"/internal/api", "/internal/store", "/internal/cache", "/internal/logging"}},
{"Database pool", []string{"jackc/pgx", "jackc/puddle", "database/sql"}},
{"Redis", []string{"redis/go-redis"}},
{"HTTP server", []string{"net/http.(*conn).serve", "net/http.(*Server)", "net/http.(*connReader)"}},
{"HTTP client", []string{"net/http.(*persistConn)", "net/http.(*Transport)"}},
{"Go runtime", []string{"runtime.", "runtime/pprof"}},
}
const componentUnattributed = "Unattributed"
func componentFor(info goroutineInfo) string {
haystack := info.searchText()
for _, rule := range componentRules {
for _, pattern := range rule.patterns {
if strings.Contains(haystack, pattern) {
return rule.component
}
}
}
return componentUnattributed
}
/* ---------- parsing ---------- */
type frame struct {
Function string
File string
}
type goroutineInfo struct {
ID int
State string
WaitMinutes int
Frames []frame
CreatedBy string
}
func (info goroutineInfo) searchText() string {
var builder strings.Builder
for _, one := range info.Frames {
builder.WriteString(one.Function)
builder.WriteByte('\n')
builder.WriteString(one.File)
builder.WriteByte('\n')
}
builder.WriteString(info.CreatedBy)
return builder.String()
}
// topFrame is the function the goroutine is actually sitting in, skipping the runtime
// plumbing that every parked goroutine shares. Without the skip, half the table reads
// "runtime.gopark", which is true of every waiting goroutine in the process and therefore
// tells nobody anything.
func (info goroutineInfo) topFrame() frame {
for _, one := range info.Frames {
function := one.Function
if strings.HasPrefix(function, "runtime.") || strings.HasPrefix(function, "internal/poll.runtime_") ||
strings.HasPrefix(function, "sync.runtime_") || strings.HasPrefix(function, "sync.(*") ||
strings.HasPrefix(function, "time.Sleep") {
continue
}
return one
}
if len(info.Frames) > 0 {
return info.Frames[0]
}
return frame{}
}
// parseGoroutines reads a runtime.Stack dump. It is pure so the shape of the dump can be
// pinned by a test rather than discovered when a Go release changes it.
func parseGoroutines(dump string) []goroutineInfo {
var out []goroutineInfo
var current *goroutineInfo
flush := func() {
if current != nil {
out = append(out, *current)
current = nil
}
}
for _, line := range strings.Split(dump, "\n") {
switch {
case strings.HasPrefix(line, "goroutine "):
flush()
info, ok := parseHeader(line)
if !ok {
continue
}
current = &info
case current == nil:
continue
case strings.HasPrefix(line, "created by "):
current.CreatedBy = strings.TrimPrefix(line, "created by ")
case strings.HasPrefix(line, "\t"):
// A file line belongs to the function line above it.
if len(current.Frames) > 0 {
path := strings.TrimSpace(line)
if cut := strings.LastIndex(path, " +0x"); cut > 0 {
path = path[:cut]
}
current.Frames[len(current.Frames)-1].File = path
}
case strings.TrimSpace(line) == "":
flush()
default:
current.Frames = append(current.Frames, frame{Function: functionName(line)})
}
}
flush()
return out
}
// parseHeader reads `goroutine 42 [select, 5 minutes]:`.
func parseHeader(line string) (goroutineInfo, bool) {
open := strings.Index(line, "[")
shut := strings.LastIndex(line, "]")
if open < 0 || shut < open {
return goroutineInfo{}, false
}
id, _ := strconv.Atoi(strings.TrimSpace(line[len("goroutine "):open]))
inside := line[open+1 : shut]
state := inside
minutes := 0
// The state itself can contain a comma — "chan receive, 5 minutes" and "GC worker
// (idle)" both do not, but "semacquire, 3 minutes, locked to thread" does, so only a
// part that parses as a duration is taken as one.
parts := strings.Split(inside, ", ")
state = parts[0]
for _, part := range parts[1:] {
if value, ok := parseMinutes(part); ok {
minutes = value
}
}
return goroutineInfo{ID: id, State: state, WaitMinutes: minutes}, true
}
func parseMinutes(part string) (int, bool) {
part = strings.TrimSpace(part)
if !strings.HasSuffix(part, " minutes") && !strings.HasSuffix(part, " minute") {
return 0, false
}
value, err := strconv.Atoi(strings.Fields(part)[0])
if err != nil {
return 0, false
}
return value, true
}
// functionName strips the argument list a stack dump prints after the function, since the
// arguments are addresses and would make every otherwise-identical goroutine its own group.
func functionName(line string) string {
line = strings.TrimSpace(line)
if open := strings.Index(line, "("); open > 0 {
// Method receivers are themselves parenthesised — `pkg.(*Type).Method(0x1)` — so
// the argument list is the last balanced group rather than the first bracket.
if last := strings.LastIndex(line, "("); last > 0 && !strings.HasSuffix(line[:last], ".") {
return line[:last]
}
return line[:open]
}
return line
}
/* ---------- the report ---------- */
type CategoryCount struct {
Category Category `json:"category"`
Label string `json:"label"`
Description string `json:"description"`
Count int `json:"count"`
States []StateCount `json:"states"`
}
type StateCount struct {
State string `json:"state"`
Count int `json:"count"`
}
type ComponentCount struct {
Component string `json:"component"`
Count int `json:"count"`
// LongestWaitMinutes is the age of the oldest goroutine attributed here. A component
// whose count is climbing and whose oldest is hours old is the shape of a leak; one
// that is busy and young is the shape of a busy evening.
LongestWaitMinutes int `json:"longestWaitMinutes"`
}
type StackGroup struct {
Count int `json:"count"`
Component string `json:"component"`
Category Category `json:"category"`
State string `json:"state"`
Function string `json:"function"`
File string `json:"file"`
CreatedBy string `json:"createdBy,omitempty"`
LongestWaitMinutes int `json:"longestWaitMinutes"`
}
type GoroutineReport struct {
At time.Time `json:"at"`
Total int `json:"total"`
CollectedInMs float64 `json:"collectedInMs"`
DumpBytes int `json:"dumpBytes"`
Categories []CategoryCount `json:"categories"`
Components []ComponentCount `json:"components"`
Groups []StackGroup `json:"groups"`
// GroupsTotal is how many distinct groups there were before the cap below; a gateway
// with thousands of goroutines has a long tail that would be a page nobody reads.
GroupsTotal int `json:"groupsTotal"`
Workers []Worker `json:"workers"`
}
// maxGroups caps the table. The groups are sorted by count, so what is dropped is the tail
// of one-off goroutines, which is exactly the part that says nothing.
const maxGroups = 40
// CollectGoroutines takes the dump and builds the report. This is the expensive call.
func CollectGoroutines() GoroutineReport {
begin := time.Now()
dump := stackDump()
report := buildReport(dump, time.Now())
report.CollectedInMs = float64(time.Since(begin).Microseconds()) / 1000
report.Workers = Workers()
return report
}
// StackDump is the raw text, for an operator who wants the whole thing rather than the
// summary — the on-demand snapshot that replaces any temptation to leave a profiler
// endpoint permanently open.
func StackDump() string { return stackDump() }
// stackDump grows its buffer until the whole dump fits. runtime.Stack truncates silently
// at the length of the buffer it is given, and a truncated dump would produce a breakdown
// that is quietly missing whichever goroutines came last — which on a leak is the ones
// worth seeing.
func stackDump() string {
size := 1 << 20
for {
buffer := make([]byte, size)
written := runtime.Stack(buffer, true)
if written < len(buffer) {
return string(buffer[:written])
}
if size >= 1<<26 {
return string(buffer[:written])
}
size *= 2
}
}
func buildReport(dump string, at time.Time) GoroutineReport {
list := parseGoroutines(dump)
report := GoroutineReport{At: at, Total: len(list), DumpBytes: len(dump)}
states := map[Category]map[string]int{}
counts := map[Category]int{}
components := map[string]*ComponentCount{}
groups := map[string]*StackGroup{}
for _, info := range list {
category := categoryFor(info.State)
counts[category]++
if states[category] == nil {
states[category] = map[string]int{}
}
states[category][info.State]++
component := componentFor(info)
entry := components[component]
if entry == nil {
entry = &ComponentCount{Component: component}
components[component] = entry
}
entry.Count++
if info.WaitMinutes > entry.LongestWaitMinutes {
entry.LongestWaitMinutes = info.WaitMinutes
}
top := info.topFrame()
key := component + "\x00" + info.State + "\x00" + top.Function
group := groups[key]
if group == nil {
group = &StackGroup{
Component: component, Category: category, State: info.State,
Function: top.Function, File: top.File, CreatedBy: info.CreatedBy,
}
groups[key] = group
}
group.Count++
if info.WaitMinutes > group.LongestWaitMinutes {
group.LongestWaitMinutes = info.WaitMinutes
}
}
for _, category := range categoryOrder {
count := counts[category]
if count == 0 {
continue
}
report.Categories = append(report.Categories, CategoryCount{
Category: category,
Label: categoryLabels[category],
Description: categoryDescriptions[category],
Count: count,
States: sortedStates(states[category]),
})
}
for _, entry := range components {
report.Components = append(report.Components, *entry)
}
sort.Slice(report.Components, func(a, b int) bool {
if report.Components[a].Count != report.Components[b].Count {
return report.Components[a].Count > report.Components[b].Count
}
return report.Components[a].Component < report.Components[b].Component
})
for _, group := range groups {
report.Groups = append(report.Groups, *group)
}
sort.Slice(report.Groups, func(a, b int) bool {
if report.Groups[a].Count != report.Groups[b].Count {
return report.Groups[a].Count > report.Groups[b].Count
}
return report.Groups[a].Function < report.Groups[b].Function
})
report.GroupsTotal = len(report.Groups)
if len(report.Groups) > maxGroups {
report.Groups = report.Groups[:maxGroups]
}
return report
}
func sortedStates(counts map[string]int) []StateCount {
out := make([]StateCount, 0, len(counts))
for state, count := range counts {
out = append(out, StateCount{State: state, Count: count})
}
sort.Slice(out, func(a, b int) bool {
if out[a].Count != out[b].Count {
return out[a].Count > out[b].Count
}
return out[a].State < out[b].State
})
return out
}
server/internal/runtimestats/health.go
+257
View File
@@ -0,0 +1,257 @@
package runtimestats
import (
"fmt"
"math"
"sort"
)
// Health is the answer the Process card exists to give: is the gateway healthy, and if
// something is abnormal, which area should be looked at?
//
// It is stated on the server rather than derived in the console for the same reason every
// schedule label is the gateway's wording — the thresholds and the sentence explaining
// them belong together, and an older console must not be the thing deciding what "watch"
// means. It is a pure function of a snapshot so the thresholds can be tested rather than
// discovered in production.
type Health struct {
Level Level `json:"level"`
Summary string `json:"summary"`
Notes []Note `json:"notes"`
Areas []string `json:"areas,omitempty"`
}
type Level string
const (
// LevelOK is nothing to do. LevelWatch is something that is not wrong yet and would
// be if it continued — which is the whole reason trends are collected. LevelBad is
// something an operator should act on now.
LevelOK Level = "ok"
LevelWatch Level = "watch"
LevelBad Level = "bad"
)
type Note struct {
Level Level `json:"level"`
// Area names the part of Memby to look at. It is the note's most useful field: a
// number that has moved is only actionable once it points somewhere.
Area string `json:"area"`
Message string `json:"message"`
}
// The thresholds. Heap is judged against the configured limit because that is the number
// the container is actually killed at; the watch line is deliberately well below it, since
// the point of a watch is to arrive before the incident rather than during it.
const (
heapShareBad = 0.90
heapShareWatch = 0.75
// goroutineLeakPerHour is what separates a gateway that has picked up work from one
// that is not letting go of it. A household arriving home adds goroutines and gives
// them back; a leak does not, so this is only ever consulted alongside a rising trend
// measured over the trend window.
goroutineLeakPerHour = 10
goroutineFloodPerHour = 60
// pauseWatchMs is a collection pause long enough for a television to notice. The
// gateway answers a status poll every ten seconds from every open set.
pauseWatchMs = 50
pauseBadMs = 250
// restartsWatch: a worker that has been launched this many times is one that keeps
// dying. The registry counts starts for exactly this.
restartsWatch = 3
)
func assess(snapshot Snapshot) Health {
var notes []Note
memory := snapshot.Memory
switch {
case memory.HeapShare >= heapShareBad:
notes = append(notes, Note{
Level: LevelBad, Area: "Memory",
Message: fmt.Sprintf(
"The heap is using %s of the %s limit. Past the limit the container is stopped, so this needs attention now.",
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
),
})
case memory.HeapShare >= heapShareWatch:
notes = append(notes, Note{
Level: LevelWatch, Area: "Memory",
Message: fmt.Sprintf(
"The heap is using %s of the %s limit. There is room, but less than usual.",
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
),
})
}
if snapshot.HeapTrend.Direction == DirectionRising {
notes = append(notes, Note{
Level: LevelWatch, Area: "Memory",
Message: fmt.Sprintf(
"Heap in use has risen by about %s an hour over the last %s. If it does not fall after a collection, something is holding on to it.",
bytes(uint64(math.Max(0, snapshot.HeapTrend.PerHour))), duration(snapshot.HeapTrend.SpanSeconds),
),
})
}
if snapshot.GoroutineTrend.Direction == DirectionRising {
rate := snapshot.GoroutineTrend.PerHour
level := LevelOK
switch {
case rate >= goroutineFloodPerHour:
level = LevelBad
case rate >= goroutineLeakPerHour:
level = LevelWatch
}
if level != LevelOK {
notes = append(notes, Note{
Level: level, Area: "Goroutines",
Message: fmt.Sprintf(
"Goroutines have risen by about %.0f an hour over the last %s, from %.0f to %.0f. Open the breakdown to see which component they belong to.",
rate, duration(snapshot.GoroutineTrend.SpanSeconds),
snapshot.GoroutineTrend.First, snapshot.GoroutineTrend.Latest,
),
})
}
}
switch {
case memory.PauseRecentMs >= pauseBadMs:
notes = append(notes, Note{
Level: LevelBad, Area: "Collection",
Message: fmt.Sprintf(
"Recent collections have paused the gateway for %.0f ms on average. Everything the televisions ask for waits that long.",
memory.PauseRecentMs,
),
})
case memory.PauseRecentMs >= pauseWatchMs:
notes = append(notes, Note{
Level: LevelWatch, Area: "Collection",
Message: fmt.Sprintf(
"Recent collections have paused the gateway for %.0f ms on average.", memory.PauseRecentMs,
),
})
}
for _, worker := range snapshot.Workers {
if worker.State == WorkerRunning && worker.Starts >= restartsWatch {
notes = append(notes, Note{
Level: LevelWatch, Area: worker.Component,
Message: fmt.Sprintf(
"%s has been started %d times. A background worker that keeps restarting is failing at something.",
worker.Name, worker.Starts,
),
})
}
}
return Health{
Level: worst(notes),
Summary: summarise(snapshot, notes),
Notes: notes,
Areas: areasOf(notes),
}
}
func worst(notes []Note) Level {
level := LevelOK
for _, note := range notes {
if note.Level == LevelBad {
return LevelBad
}
if note.Level == LevelWatch {
level = LevelWatch
}
}
return level
}
// summarise is the one line the overview card prints. When there is nothing to say it says
// so plainly rather than reciting figures the operator can already see above it — and it
// says whether the verdict is based on enough history to be worth anything, because "no
// trouble" from four minutes of readings is a weaker claim than the same words from four
// hours.
func summarise(snapshot Snapshot, notes []Note) string {
if len(notes) == 0 {
if snapshot.GoroutineTrend.Direction == DirectionUnknown {
return "Nothing abnormal. Still gathering history — trends need about fifteen minutes."
}
return fmt.Sprintf(
"Nothing abnormal. Goroutines and memory have been %s over the last %s.",
steadyWord(snapshot), duration(snapshot.GoroutineTrend.SpanSeconds),
)
}
areas := areasOf(notes)
if len(notes) == 1 {
return notes[0].Message
}
return fmt.Sprintf("%d things to look at, in %s.", len(notes), joinAreas(areas))
}
func steadyWord(snapshot Snapshot) string {
if snapshot.GoroutineTrend.Direction == DirectionFalling || snapshot.HeapTrend.Direction == DirectionFalling {
return "steady or falling"
}
return "steady"
}
func areasOf(notes []Note) []string {
seen := map[string]bool{}
var areas []string
for _, note := range notes {
if note.Area == "" || seen[note.Area] {
continue
}
seen[note.Area] = true
areas = append(areas, note.Area)
}
sort.Strings(areas)
return areas
}
func joinAreas(areas []string) string {
switch len(areas) {
case 0:
return "the gateway"
case 1:
return areas[0]
case 2:
return areas[0] + " and " + areas[1]
}
return fmt.Sprintf("%s and %d other areas", areas[0], len(areas)-1)
}
/* ---------- wording helpers ----------
These exist because the sentences above are the gateway's own wording, the stance every
label the console prints takes: a console built before a threshold existed still reads
correctly, and the figure in the sentence cannot disagree with the threshold that
produced it. */
func percent(share float64) string {
return fmt.Sprintf("%.1f%%", share*100)
}
func bytes(value uint64) string {
const unit = 1024
if value < unit {
return fmt.Sprintf("%d B", value)
}
div, exponent := uint64(unit), 0
for size := value / unit; size >= unit; size /= unit {
div *= unit
exponent++
}
return fmt.Sprintf("%.1f %cB", float64(value)/float64(div), "KMGT"[exponent])
}
func duration(seconds float64) string {
switch {
case seconds < 90:
return fmt.Sprintf("%.0f seconds", seconds)
case seconds < 5400:
return fmt.Sprintf("%.0f minutes", seconds/60)
default:
return fmt.Sprintf("%.1f hours", seconds/3600)
}
}
server/internal/runtimestats/process.go
+170
View File
@@ -0,0 +1,170 @@
package runtimestats
import (
"os"
"runtime/pprof"
"strconv"
"strings"
"sync"
"time"
)
// Process is what the operating system knows about the container that the Go runtime does
// not. Every field of it is optional: this is read from /proc, which exists in the
// container the gateway is deployed in and does not exist on a developer's machine, so an
// unavailable figure is omitted rather than reported as zero — nought open sockets and
// "could not look" are different answers and the console must not print the first for the
// second.
type Process struct {
PID int `json:"pid"`
// CPUSeconds is cumulative processor time, and CPUPercent is the share of one
// processor used since the previous read. A hundred per cent is one core saturated,
// not the machine: with GOMAXPROCS processors available the ceiling is that times a
// hundred, which is why the console prints both.
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
CPUPercent float64 `json:"cpuPercent,omitempty"`
CPUKnown bool `json:"cpuKnown"`
OpenFiles int `json:"openFiles,omitempty"`
OpenSockets int `json:"openSockets,omitempty"`
FileLimit int `json:"fileLimit,omitempty"`
FilesKnown bool `json:"filesKnown"`
}
// threadCount is operating-system threads, which is a different number from goroutines and
// is the one that matters when goroutines are blocked in syscalls: the runtime creates a
// thread per blocked call, and a thread costs far more than a goroutine does.
func threadCount() int {
profile := pprof.Lookup("threadcreate")
if profile == nil {
return 0
}
return profile.Count()
}
// configuredMemoryLimit reports what the container was started with rather than what the
// runtime currently holds, so the console can say whether the limit is deliberate.
func configuredMemoryLimit() string {
return strings.TrimSpace(os.Getenv("GOMEMLIMIT"))
}
var (
cpuMu sync.Mutex
lastCPUAt time.Time
lastCPUSecs float64
lastPercent float64
)
func readProcess() Process {
process := Process{PID: os.Getpid()}
if seconds, ok := processCPUSeconds(); ok {
process.CPUSeconds = seconds
process.CPUKnown = true
process.CPUPercent = cpuPercentSince(seconds, time.Now())
}
if files, sockets, ok := openDescriptors(); ok {
process.OpenFiles = files
process.OpenSockets = sockets
process.FilesKnown = true
process.FileLimit = descriptorLimit()
}
return process
}
// cpuPercentSince needs two readings, so the first call after start-up reports nothing
// rather than dividing by the life of the process — an average over four hours would hide
// exactly the spike somebody opened the page to find. A read that arrives too soon after
// the last one repeats the previous answer instead of amplifying rounding into a spike.
func cpuPercentSince(seconds float64, now time.Time) float64 {
const minInterval = 5 * time.Second
cpuMu.Lock()
defer cpuMu.Unlock()
if lastCPUAt.IsZero() {
lastCPUAt, lastCPUSecs = now, seconds
return 0
}
elapsed := now.Sub(lastCPUAt)
if elapsed < minInterval {
return lastPercent
}
used := seconds - lastCPUSecs
lastCPUAt, lastCPUSecs = now, seconds
if used < 0 {
lastPercent = 0
return 0
}
lastPercent = used / elapsed.Seconds() * 100
return lastPercent
}
// processCPUSeconds prefers /proc/self/schedstat, whose first field is cumulative time on
// a processor in nanoseconds — exact, where /proc/self/stat is in clock ticks whose length
// cannot be read without cgo and has to be assumed to be the usual hundredth of a second.
func processCPUSeconds() (float64, bool) {
if raw, err := os.ReadFile("/proc/self/schedstat"); err == nil {
fields := strings.Fields(string(raw))
if len(fields) > 0 {
if nanos, err := strconv.ParseFloat(fields[0], 64); err == nil {
return nanos / 1e9, true
}
}
}
raw, err := os.ReadFile("/proc/self/stat")
if err != nil {
return 0, false
}
// The second field is the command name in brackets and may itself contain spaces, so
// the fields after it are counted from the closing bracket rather than from the start.
shut := strings.LastIndex(string(raw), ")")
if shut < 0 {
return 0, false
}
fields := strings.Fields(string(raw)[shut+1:])
// After the bracket, field 1 is the state; utime and stime are fields 12 and 13.
if len(fields) < 13 {
return 0, false
}
user, userErr := strconv.ParseFloat(fields[11], 64)
system, systemErr := strconv.ParseFloat(fields[12], 64)
if userErr != nil || systemErr != nil {
return 0, false
}
const ticksPerSecond = 100
return (user + system) / ticksPerSecond, true
}
func openDescriptors() (files, sockets int, ok bool) {
entries, err := os.ReadDir("/proc/self/fd")
if err != nil {
return 0, 0, false
}
for _, entry := range entries {
files++
target, err := os.Readlink("/proc/self/fd/" + entry.Name())
if err == nil && strings.HasPrefix(target, "socket:") {
sockets++
}
}
return files, sockets, true
}
func descriptorLimit() int {
raw, err := os.ReadFile("/proc/self/limits")
if err != nil {
return 0
}
for _, line := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(line, "Max open files") {
continue
}
fields := strings.Fields(strings.TrimPrefix(line, "Max open files"))
if len(fields) == 0 {
return 0
}
limit, err := strconv.Atoi(fields[0])
if err != nil {
return 0
}
return limit
}
return 0
}
server/internal/runtimestats/runtimestats_test.go
+377
View File
@@ -0,0 +1,377 @@
package runtimestats
import (
"strings"
"sync"
"testing"
"time"
)
// A real dump, trimmed. The parser is the one part of this package that reads text a Go
// release produces, so the fixture is kept verbatim rather than idealised — the argument
// lists, the "+0x" offsets and the "created by … in goroutine N" suffix are all things the
// parser has to survive.
const sampleDump = `goroutine 1 [chan receive, 940 minutes]:
main.run(0xc000122000)
/app/cmd/memby-server/main.go:430 +0x8f4
main.main()
/app/cmd/memby-server/main.go:41 +0x1f
goroutine 18 [select]:
github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run(0xc0001a2000, {0x1a4c0d0, 0xc0000a2000})
/app/internal/library/ingest.go:118 +0x145
created by github.com/ponzischeme89/memby/server/internal/runtimestats.Go in goroutine 1
goroutine 40 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7f2c1c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:351 +0x85
net/http.(*conn).serve(0xc000310000, {0x1a4c0d0, 0xc0003a0000})
/usr/local/go/src/net/http/server.go:2092 +0x5db
created by net/http.(*Server).Serve in goroutine 55
goroutine 41 [GC worker (idle)]:
runtime.gopark(0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/proc.go:402 +0xce
runtime.gcBgMarkWorker(0xc0000581c0)
/usr/local/go/src/runtime/mgc.go:1310 +0xe5
created by runtime.gcBgMarkStartWorkers in goroutine 1
goroutine 77 [semacquire, 3 minutes, locked to thread]:
github.com/jackc/puddle/v2.(*Pool).acquire(0xc00019c000)
/root/go/pkg/mod/github.com/jackc/puddle/v2@v2.2.2/pool.go:481 +0x10a
created by github.com/jackc/pgx/v5/pgxpool.NewWithConfig in goroutine 1
`
func TestParseGoroutinesReadsHeaderStateAndFrames(t *testing.T) {
list := parseGoroutines(sampleDump)
if len(list) != 5 {
t.Fatalf("parsed %d goroutines, want 5", len(list))
}
first := list[0]
if first.ID != 1 || first.State != "chan receive" || first.WaitMinutes != 940 {
t.Fatalf("first = %+v", first)
}
if first.Frames[0].Function != "main.run" {
t.Fatalf("first frame function = %q", first.Frames[0].Function)
}
// The offset is stripped so two goroutines in the same place group together rather
// than differing by an address.
if first.Frames[0].File != "/app/cmd/memby-server/main.go:430" {
t.Fatalf("first frame file = %q", first.Frames[0].File)
}
if list[1].CreatedBy == "" {
t.Fatal("created-by line was dropped")
}
// A state with trailing detail after a second comma keeps its state and its wait.
last := list[4]
if last.State != "semacquire" || last.WaitMinutes != 3 {
t.Fatalf("last = %+v", last)
}
}
// The categories have to partition the total or the breakdown is misleading, which is
// worse than not offering one.
func TestBuildReportPartitionsAndAttributes(t *testing.T) {
report := buildReport(sampleDump, time.Now())
if report.Total != 5 {
t.Fatalf("total = %d", report.Total)
}
counted := 0
for _, category := range report.Categories {
counted += category.Count
}
if counted != report.Total {
t.Fatalf("categories sum to %d, total %d", counted, report.Total)
}
attributed := 0
for _, component := range report.Components {
attributed += component.Count
}
if attributed != report.Total {
t.Fatalf("components sum to %d, total %d", attributed, report.Total)
}
got := map[string]string{}
for _, group := range report.Groups {
got[group.Component] = group.Function
}
// Memby's own packages win over the libraries they call, and the frame reported is
// the one that names the work rather than the runtime plumbing every parked goroutine
// shares.
if got["Library sync"] != "github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run" {
t.Fatalf("library group = %q", got["Library sync"])
}
if got["HTTP server"] != "net/http.(*conn).serve" {
t.Fatalf("http group = %q", got["HTTP server"])
}
if got["Database pool"] != "github.com/jackc/puddle/v2.(*Pool).acquire" {
t.Fatalf("database group = %q", got["Database pool"])
}
}
func TestCategoryForKnownStates(t *testing.T) {
cases := map[string]Category{
"running": CategoryRunning,
"runnable": CategoryRunning,
"syscall": CategoryRunning,
"IO wait": CategoryIO,
"select": CategoryWaiting,
"chan receive": CategoryWaiting,
"semacquire": CategoryWaiting,
"sync.Mutex.Lock": CategoryWaiting,
"sleep": CategoryTimers,
"timer goroutine": CategoryTimers,
"GC worker (idle)": CategoryRuntime,
"force gc (idle)": CategoryRuntime,
"finalizer wait": CategoryRuntime,
// The runtime's own wait outranks the "wait" rule, or the collector would be
// reported as a component sitting on a channel.
"wait for GC cycle": CategoryRuntime,
"something new": CategoryOther,
}
for state, want := range cases {
if got := categoryFor(state); got != want {
t.Errorf("categoryFor(%q) = %q, want %q", state, got, want)
}
}
}
// A live report has to hold together too: this is the one test that would notice a Go
// release changing the dump format under the parser.
func TestCollectGoroutinesAgreesWithTheRuntime(t *testing.T) {
blocked := make(chan struct{})
var ready sync.WaitGroup
ready.Add(1)
go func() {
ready.Done()
<-blocked
}()
ready.Wait()
defer close(blocked)
report := CollectGoroutines()
if report.Total < 2 {
t.Fatalf("total = %d", report.Total)
}
counted := 0
for _, category := range report.Categories {
counted += category.Count
}
if counted != report.Total {
t.Fatalf("categories sum to %d, total %d", counted, report.Total)
}
if report.CollectedInMs < 0 {
t.Fatalf("collectedInMs = %v", report.CollectedInMs)
}
if !strings.Contains(StackDump(), "goroutine ") {
t.Fatal("stack dump does not look like a stack dump")
}
}
/* ---------- trends ---------- */
func series(values []float64, every time.Duration) ([]float64, []time.Time) {
at := make([]time.Time, len(values))
base := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
for index := range values {
at[index] = base.Add(time.Duration(index) * every)
}
return values, at
}
func TestTrendRefusesToAnswerWithoutEnoughHistory(t *testing.T) {
// Four readings a minute apart is a gateway that restarted three minutes ago, not a
// trend, and "steady" would be a claim the data does not support.
points, at := series([]float64{20, 24, 30, 44}, time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionUnknown {
t.Fatalf("direction = %q, want unknown", got.Direction)
}
}
func TestTrendIgnoresABurstAndNamesASustainedRise(t *testing.T) {
// A burst in the middle — a library import, a household arriving home — must not be
// reported as a rise; medians are what make that true where a fitted line would not.
burst := []float64{40, 41, 40, 42, 41, 300, 290, 40, 41, 40, 42, 41}
points, at := series(burst, 5*time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
t.Fatalf("burst direction = %q (%+v), want steady", got.Direction, got)
}
leak := make([]float64, 12)
for index := range leak {
leak[index] = float64(40 + index*5)
}
points, at = series(leak, 5*time.Minute)
rising := trendOf(points, at, 5)
if rising.Direction != DirectionRising {
t.Fatalf("leak direction = %q (%+v), want rising", rising.Direction, rising)
}
if rising.PerHour <= 0 {
t.Fatalf("perHour = %v, want a positive rate", rising.PerHour)
}
if rising.Latest != 95 || rising.First != 40 {
t.Fatalf("first/latest = %v/%v", rising.First, rising.Latest)
}
}
func TestTrendFloorKeepsAQuietGatewaySteady(t *testing.T) {
// One extra goroutine over an hour is not a leak, and without the absolute floor the
// proportional threshold would call it one on a gateway holding a handful.
points, at := series([]float64{8, 8, 9, 9, 9, 9, 9, 10}, 10*time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
t.Fatalf("direction = %q (%+v), want steady", got.Direction, got)
}
}
/* ---------- the verdict ---------- */
func risingTrend(first, latest, perHour float64) Trend {
return Trend{
Direction: DirectionRising, PerHour: perHour, First: first, Latest: latest,
SpanSeconds: 3600, Points: 60,
}
}
func TestAssessSaysNothingWhenThereIsNothingToSay(t *testing.T) {
health := assess(Snapshot{
Memory: Memory{MemoryLimit: 384 << 20, HeapInuse: 16 << 20, HeapShare: 0.04},
GoroutineTrend: Trend{Direction: DirectionSteady, SpanSeconds: 7200},
HeapTrend: Trend{Direction: DirectionSteady},
})
if health.Level != LevelOK || len(health.Notes) != 0 {
t.Fatalf("health = %+v", health)
}
if !strings.Contains(health.Summary, "Nothing abnormal") {
t.Fatalf("summary = %q", health.Summary)
}
}
func TestAssessNamesTheAreaToInvestigate(t *testing.T) {
health := assess(Snapshot{
Memory: Memory{MemoryLimit: 384 << 20, HeapInuse: 350 << 20, HeapShare: 0.92},
GoroutineTrend: risingTrend(60, 400, 340),
HeapTrend: Trend{Direction: DirectionSteady},
})
if health.Level != LevelBad {
t.Fatalf("level = %q", health.Level)
}
areas := strings.Join(health.Areas, ",")
if !strings.Contains(areas, "Memory") || !strings.Contains(areas, "Goroutines") {
t.Fatalf("areas = %v", health.Areas)
}
// A verdict with no area is the number that could not be acted on, which is the whole
// defect this replaced.
for _, note := range health.Notes {
if note.Area == "" {
t.Fatalf("note without an area: %+v", note)
}
}
}
func TestAssessSeparatesADriftFromAFlood(t *testing.T) {
drift := assess(Snapshot{GoroutineTrend: risingTrend(40, 60, 20)})
if drift.Level != LevelWatch {
t.Fatalf("drift level = %q", drift.Level)
}
flood := assess(Snapshot{GoroutineTrend: risingTrend(40, 400, 360)})
if flood.Level != LevelBad {
t.Fatalf("flood level = %q", flood.Level)
}
// A rise slower than the leak threshold is ordinary breathing and must not be a note
// at all, or the card cries wolf on every busy evening.
quiet := assess(Snapshot{GoroutineTrend: risingTrend(40, 44, 4)})
if quiet.Level != LevelOK || len(quiet.Notes) != 0 {
t.Fatalf("quiet = %+v", quiet)
}
}
func TestAssessNoticesAWorkerThatKeepsRestarting(t *testing.T) {
health := assess(Snapshot{Workers: []Worker{
{Name: "Library ingest", Component: "Library sync", State: WorkerRunning, Starts: 5},
{Name: "Startup library sync", Component: "Library sync", State: WorkerFinished, Starts: 1},
}})
if health.Level != LevelWatch || len(health.Notes) != 1 {
t.Fatalf("health = %+v", health)
}
if !strings.Contains(health.Notes[0].Message, "Library ingest") {
t.Fatalf("note = %q", health.Notes[0].Message)
}
}
/* ---------- the registry ---------- */
func TestWorkersRecordRunningAndFinished(t *testing.T) {
resetWorkers()
t.Cleanup(resetWorkers)
release := make(chan struct{})
var running sync.WaitGroup
running.Add(1)
Go("Library ingest", "Library sync", func() {
running.Done()
<-release
})
running.Wait()
var done sync.WaitGroup
done.Add(1)
Go("Startup library sync", "Library sync", func() { done.Done() })
done.Wait()
// The finished worker's goroutine has run its body; give the deferred end() the moment
// it needs to land before reading the registry.
waitFor(t, func() bool { return stateOf(t, "Startup library sync") == WorkerFinished })
list := Workers()
if len(list) != 2 {
t.Fatalf("workers = %+v", list)
}
// Running first, so the table does not bury the thing that is still working under the
// startup jobs that are not.
if list[0].Name != "Library ingest" || list[0].State != WorkerRunning {
t.Fatalf("first = %+v", list[0])
}
if list[1].State != WorkerFinished || list[1].Stopped.IsZero() {
t.Fatalf("second = %+v", list[1])
}
close(release)
waitFor(t, func() bool { return stateOf(t, "Library ingest") == WorkerFinished })
// A restart is counted rather than replacing the record: a worker restarted in a loop
// reads exactly like a healthy one from a single snapshot and does not from Starts.
Go("Library ingest", "Library sync", func() {})
waitFor(t, func() bool { return startsOf(t, "Library ingest") == 2 })
}
func stateOf(t *testing.T, name string) WorkerState {
t.Helper()
for _, worker := range Workers() {
if worker.Name == name {
return worker.State
}
}
return ""
}
func startsOf(t *testing.T, name string) int {
t.Helper()
for _, worker := range Workers() {
if worker.Name == name {
return worker.Starts
}
}
return 0
}
func waitFor(t *testing.T, done func() bool) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if done() {
return
}
time.Sleep(time.Millisecond)
}
t.Fatal("condition not reached")
}
server/internal/runtimestats/sample.go
+337
View File
@@ -0,0 +1,337 @@
package runtimestats
import (
"context"
"math"
"runtime"
"runtime/debug"
"sort"
"sync"
"time"
)
// SampleInterval is how often the trend ring records a reading, and the ring's capacity is
// how far back it can therefore see. A minute apart is far cheaper than the 30s poll the
// console already makes, and four hours is long enough for a slow leak to become a line
// rather than noise while costing a few kilobytes.
const (
SampleInterval = time.Minute
trendCapacity = 240
)
// Sample is one reading. Deliberately the same source as the headline figures — both come
// from one ReadMemStats — because a trend drawn from a different counter than the number
// printed above it is a trend an operator cannot check.
type Sample struct {
At time.Time `json:"at"`
Goroutines int `json:"goroutines"`
HeapInuse uint64 `json:"heapInuse"`
Sys uint64 `json:"sys"`
NumGC uint32 `json:"numGc"`
}
var (
samplesMu sync.Mutex
samples []Sample
started = time.Now()
)
// StartSampling records a reading every SampleInterval until ctx ends. It takes the first
// reading immediately, so a console opened a minute after a deployment has a baseline
// rather than an empty chart.
func StartSampling(ctx context.Context) {
record(readSample())
ticker := time.NewTicker(SampleInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
record(readSample())
}
}
}
func readSample() Sample {
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
return Sample{
At: time.Now(),
Goroutines: runtime.NumGoroutine(),
HeapInuse: memory.HeapInuse,
Sys: memory.Sys,
NumGC: memory.NumGC,
}
}
func record(sample Sample) {
samplesMu.Lock()
defer samplesMu.Unlock()
samples = append(samples, sample)
if len(samples) > trendCapacity {
samples = append(samples[:0], samples[len(samples)-trendCapacity:]...)
}
}
// Samples returns the trend ring oldest first.
func Samples() []Sample {
samplesMu.Lock()
defer samplesMu.Unlock()
out := make([]Sample, len(samples))
copy(out, samples)
return out
}
/* ---------- trends ---------- */
// Direction is a trend's verdict. "unknown" is a real answer and the common one for the
// first few minutes after a restart: claiming "steady" from two readings a minute apart
// would be a claim the data does not support.
type Direction string
const (
DirectionUnknown Direction = "unknown"
DirectionSteady Direction = "steady"
DirectionRising Direction = "rising"
DirectionFalling Direction = "falling"
)
type Trend struct {
Direction Direction `json:"direction"`
// PerHour is the observed rate of change in the value's own units. It is what makes
// "rising" actionable: two goroutines an hour is a leak worth naming, two a minute is
// one worth investigating tonight.
PerHour float64 `json:"perHour"`
First float64 `json:"first"`
Latest float64 `json:"latest"`
Min float64 `json:"min"`
Max float64 `json:"max"`
SpanSeconds float64 `json:"spanSeconds"`
Points int `json:"points"`
}
// minTrendPoints and minTrendSpan are what a trend has to earn before it is stated at all.
// A leak is a claim about a slope, and a slope read off three readings inside five minutes
// is indistinguishable from a household that happened to start watching something.
const (
minTrendPoints = 5
minTrendSpan = 15 * time.Minute
)
// trendOf is the whole rule and it is pure. It compares the median of the oldest quarter
// of the window with the median of the newest quarter rather than fitting a line, because
// a single burst — a library import, a household arriving home — is exactly the shape
// least-squares reports as a trend and medians ignore.
//
// The threshold is proportional with an absolute floor: without the proportion a busy
// gateway is permanently "rising", and without the floor a quiet one calls one extra
// goroutine a trend.
func trendOf(points []float64, at []time.Time, floor float64) Trend {
trend := Trend{Direction: DirectionUnknown, Points: len(points)}
if len(points) == 0 || len(at) != len(points) {
return trend
}
trend.First, trend.Latest = points[0], points[len(points)-1]
trend.Min, trend.Max = points[0], points[0]
for _, value := range points {
trend.Min = math.Min(trend.Min, value)
trend.Max = math.Max(trend.Max, value)
}
span := at[len(at)-1].Sub(at[0])
trend.SpanSeconds = span.Seconds()
if len(points) < minTrendPoints || span < minTrendSpan {
return trend
}
quarter := len(points) / 4
if quarter < 1 {
quarter = 1
}
early := median(points[:quarter])
late := median(points[len(points)-quarter:])
change := late - early
trend.PerHour = change / span.Hours()
threshold := math.Max(floor, early*0.15)
switch {
case change > threshold:
trend.Direction = DirectionRising
case change < -threshold:
trend.Direction = DirectionFalling
default:
trend.Direction = DirectionSteady
}
return trend
}
func median(values []float64) float64 {
if len(values) == 0 {
return 0
}
sorted := make([]float64, len(values))
copy(sorted, values)
sort.Float64s(sorted)
middle := len(sorted) / 2
if len(sorted)%2 == 1 {
return sorted[middle]
}
return (sorted[middle-1] + sorted[middle]) / 2
}
func trendsFrom(list []Sample) (goroutines, heap, reserved Trend) {
at := make([]time.Time, len(list))
goroutinePoints := make([]float64, len(list))
heapPoints := make([]float64, len(list))
reservedPoints := make([]float64, len(list))
for index, sample := range list {
at[index] = sample.At
goroutinePoints[index] = float64(sample.Goroutines)
heapPoints[index] = float64(sample.HeapInuse)
reservedPoints[index] = float64(sample.Sys)
}
// The floors: five goroutines, and four megabytes of memory. Below those, movement is
// the ordinary breathing of a server answering requests.
return trendOf(goroutinePoints, at, 5),
trendOf(heapPoints, at, 4<<20),
trendOf(reservedPoints, at, 4<<20)
}
/* ---------- the snapshot the console polls ---------- */
// Memory separates the three figures an operator keeps conflating: what the program is
// actually holding, what the Go runtime has reserved from the operating system on its
// behalf, and the ceiling it has been given. Reserved is always the largest and is not a
// leak; the ratio to the limit is the number worth watching.
type Memory struct {
HeapAlloc uint64 `json:"heapAlloc"`
HeapInuse uint64 `json:"heapInuse"`
HeapIdle uint64 `json:"heapIdle"`
HeapReleased uint64 `json:"heapReleased"`
StackInuse uint64 `json:"stackInuse"`
Sys uint64 `json:"sys"`
NextGC uint64 `json:"nextGc"`
NumGC uint32 `json:"numGc"`
// PauseTotalMs and PauseRecentMs are how much of the life of the gateway has been
// spent stopped for collection. A rising heap that is being collected without pausing
// is a working cache; one that pauses for longer every cycle is not.
PauseTotalMs float64 `json:"pauseTotalMs"`
PauseRecentMs float64 `json:"pauseRecentMs"`
MemoryLimit int64 `json:"memoryLimit"`
ConfiguredLim string `json:"configuredLimit,omitempty"`
// HeapShare is heap in use as a fraction of the limit, or 0 when there is no limit.
// Computed here rather than in the console so the health verdict and the figure the
// operator reads cannot disagree.
HeapShare float64 `json:"heapShare"`
// GCPerHour is measured over the trend window rather than over the whole life of the
// process, because "collections have become more frequent" is the useful form of it.
GCPerHour float64 `json:"gcPerHour"`
}
type Snapshot struct {
At time.Time `json:"at"`
UptimeSeconds float64 `json:"uptimeSeconds"`
Goroutines int `json:"goroutines"`
GOMAXPROCS int `json:"gomaxprocs"`
Threads int `json:"threads"`
GoVersion string `json:"goVersion"`
Memory Memory `json:"memory"`
Workers []Worker `json:"workers"`
Process Process `json:"process"`
Samples []Sample `json:"samples"`
GoroutineTrend Trend `json:"goroutineTrend"`
HeapTrend Trend `json:"heapTrend"`
ReservedTrend Trend `json:"reservedTrend"`
SampleEverySecs float64 `json:"sampleEverySeconds"`
Health Health `json:"health"`
}
// Read builds the cheap snapshot. Nothing in here walks a stack or stops the world for
// longer than a ReadMemStats, so it is safe on the 30-second poll every open console tab
// makes.
func Read() Snapshot {
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
now := time.Now()
limit := debug.SetMemoryLimit(-1)
share := 0.0
if limit > 0 && limit < math.MaxInt64 {
share = float64(memory.HeapInuse) / float64(limit)
}
list := Samples()
goroutineTrend, heapTrend, reservedTrend := trendsFrom(list)
snapshot := Snapshot{
At: now,
UptimeSeconds: now.Sub(started).Seconds(),
Goroutines: runtime.NumGoroutine(),
GOMAXPROCS: runtime.GOMAXPROCS(0),
Threads: threadCount(),
GoVersion: runtime.Version(),
Memory: Memory{
HeapAlloc: memory.HeapAlloc,
HeapInuse: memory.HeapInuse,
HeapIdle: memory.HeapIdle,
HeapReleased: memory.HeapReleased,
StackInuse: memory.StackInuse,
Sys: memory.Sys,
NextGC: memory.NextGC,
NumGC: memory.NumGC,
PauseTotalMs: float64(memory.PauseTotalNs) / 1e6,
PauseRecentMs: recentPauseMs(&memory),
MemoryLimit: limit,
ConfiguredLim: configuredMemoryLimit(),
HeapShare: share,
GCPerHour: gcPerHour(list),
},
Workers: Workers(),
Process: readProcess(),
Samples: list,
GoroutineTrend: goroutineTrend,
HeapTrend: heapTrend,
ReservedTrend: reservedTrend,
SampleEverySecs: SampleInterval.Seconds(),
}
snapshot.Health = assess(snapshot)
return snapshot
}
// recentPauseMs averages the last few recorded pauses. Go keeps a circular buffer of the
// most recent 256, and the running total on its own cannot distinguish a process that
// paused badly an hour ago from one pausing badly now.
func recentPauseMs(memory *runtime.MemStats) float64 {
const window = 16
if memory.NumGC == 0 {
return 0
}
count := 0
total := uint64(0)
for index := 0; index < window; index++ {
slot := int(memory.NumGC) - 1 - index
if slot < 0 {
break
}
total += memory.PauseNs[slot%256]
count++
}
if count == 0 {
return 0
}
return float64(total) / float64(count) / 1e6
}
func gcPerHour(list []Sample) float64 {
if len(list) < 2 {
return 0
}
span := list[len(list)-1].At.Sub(list[0].At).Hours()
if span <= 0 {
return 0
}
return float64(list[len(list)-1].NumGC-list[0].NumGC) / span
}
server/internal/runtimestats/workers.go
+150
View File
@@ -0,0 +1,150 @@
// Package runtimestats answers the one question the console's Process card exists for: is
// the gateway healthy, and if something is abnormal, which area should be looked at?
//
// A bare goroutine count cannot answer it. Twenty-five means nothing on its own — an
// operator cannot tell from it what those goroutines are doing, which part of Memby they
// belong to, whether the number is normal, or whether it has been climbing all week. This
// package supplies the three things that make the number readable, and keeps them apart by
// what they cost:
//
// - The registry here, which is free. A long-running Memby worker says its own name when
// it starts, so "Library ingest" and "Emby health probe" are named rather than inferred
// from a stack.
// - The sampler in sample.go, which is a handful of counters on a slow tick. A single
// instantaneous figure cannot show a leak; a trend can.
// - The stack breakdown in goroutines.go, which is genuinely expensive and is therefore
// collected only when an operator asks for it.
//
// Nothing here recovers a panic. A worker that dies must die exactly as it always did —
// this package reports, and reporting must never change what it is reporting on.
package runtimestats
import (
"sort"
"sync"
"time"
)
// WorkerState is what became of a tracked worker. A worker that ends is not a fault —
// several of the gateway's background jobs are one-shot startup work — but "still running"
// and "finished" are different answers and the console must not merge them.
type WorkerState string
const (
WorkerRunning WorkerState = "running"
WorkerFinished WorkerState = "finished"
)
// Worker is one named background goroutine.
type Worker struct {
Name string `json:"name"`
Component string `json:"component"`
State WorkerState `json:"state"`
Started time.Time `json:"started"`
Stopped time.Time `json:"stopped,omitempty"`
// Starts counts how many times this name has been launched. It is on the record
// because a worker that is being restarted in a loop reads exactly like a healthy one
// from a single snapshot, and does not from this number.
Starts int `json:"starts"`
}
type workerEntry struct {
component string
running int
starts int
started time.Time
stopped time.Time
}
var (
workersMu sync.Mutex
workers = map[string]*workerEntry{}
)
// Go starts fn on its own goroutine and records it under a name an operator can read. The
// name is the worker's identity across restarts, so it must be stable and must not carry a
// count or an address in it.
func Go(name, component string, fn func()) {
begin(name, component)
go func() {
defer end(name)
fn()
}()
}
// Track records a worker whose goroutine something else owns. The returned function marks
// it finished, and is safe to call more than once.
func Track(name, component string) (done func()) {
begin(name, component)
var once sync.Once
return func() { once.Do(func() { end(name) }) }
}
func begin(name, component string) {
now := time.Now()
workersMu.Lock()
defer workersMu.Unlock()
entry := workers[name]
if entry == nil {
entry = &workerEntry{}
workers[name] = entry
}
entry.component = component
entry.running++
entry.starts++
entry.started = now
entry.stopped = time.Time{}
}
func end(name string) {
now := time.Now()
workersMu.Lock()
defer workersMu.Unlock()
entry := workers[name]
if entry == nil {
return
}
if entry.running > 0 {
entry.running--
}
if entry.running == 0 {
entry.stopped = now
}
}
// Workers lists what has been registered, running first and then by name, so the table
// does not reorder itself under the operator on every poll.
func Workers() []Worker {
workersMu.Lock()
list := make([]Worker, 0, len(workers))
for name, entry := range workers {
worker := Worker{
Name: name,
Component: entry.component,
State: WorkerFinished,
Started: entry.started,
Stopped: entry.stopped,
Starts: entry.starts,
}
if entry.running > 0 {
worker.State = WorkerRunning
worker.Stopped = time.Time{}
}
list = append(list, worker)
}
workersMu.Unlock()
sort.Slice(list, func(a, b int) bool {
if (list[a].State == WorkerRunning) != (list[b].State == WorkerRunning) {
return list[a].State == WorkerRunning
}
return list[a].Name < list[b].Name
})
return list
}
// resetWorkers exists for the tests; the registry is process-wide by design.
func resetWorkers() {
workersMu.Lock()
workers = map[string]*workerEntry{}
workersMu.Unlock()
}