338 lines
10 KiB
Go
338 lines
10 KiB
Go
package runtimestats
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"runtime"
|
|
"runtime/debug"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// SampleInterval is how often the trend ring records a reading, and the ring's capacity is
|
|
// how far back it can therefore see. A minute apart is far cheaper than the 30s poll the
|
|
// console already makes, and four hours is long enough for a slow leak to become a line
|
|
// rather than noise while costing a few kilobytes.
|
|
const (
|
|
SampleInterval = time.Minute
|
|
trendCapacity = 240
|
|
)
|
|
|
|
// Sample is one reading. Deliberately the same source as the headline figures — both come
|
|
// from one ReadMemStats — because a trend drawn from a different counter than the number
|
|
// printed above it is a trend an operator cannot check.
|
|
type Sample struct {
|
|
At time.Time `json:"at"`
|
|
Goroutines int `json:"goroutines"`
|
|
HeapInuse uint64 `json:"heapInuse"`
|
|
Sys uint64 `json:"sys"`
|
|
NumGC uint32 `json:"numGc"`
|
|
}
|
|
|
|
var (
|
|
samplesMu sync.Mutex
|
|
samples []Sample
|
|
started = time.Now()
|
|
)
|
|
|
|
// StartSampling records a reading every SampleInterval until ctx ends. It takes the first
|
|
// reading immediately, so a console opened a minute after a deployment has a baseline
|
|
// rather than an empty chart.
|
|
func StartSampling(ctx context.Context) {
|
|
record(readSample())
|
|
ticker := time.NewTicker(SampleInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
record(readSample())
|
|
}
|
|
}
|
|
}
|
|
|
|
func readSample() Sample {
|
|
var memory runtime.MemStats
|
|
runtime.ReadMemStats(&memory)
|
|
return Sample{
|
|
At: time.Now(),
|
|
Goroutines: runtime.NumGoroutine(),
|
|
HeapInuse: memory.HeapInuse,
|
|
Sys: memory.Sys,
|
|
NumGC: memory.NumGC,
|
|
}
|
|
}
|
|
|
|
func record(sample Sample) {
|
|
samplesMu.Lock()
|
|
defer samplesMu.Unlock()
|
|
samples = append(samples, sample)
|
|
if len(samples) > trendCapacity {
|
|
samples = append(samples[:0], samples[len(samples)-trendCapacity:]...)
|
|
}
|
|
}
|
|
|
|
// Samples returns the trend ring oldest first.
|
|
func Samples() []Sample {
|
|
samplesMu.Lock()
|
|
defer samplesMu.Unlock()
|
|
out := make([]Sample, len(samples))
|
|
copy(out, samples)
|
|
return out
|
|
}
|
|
|
|
/* ---------- trends ---------- */
|
|
|
|
// Direction is a trend's verdict. "unknown" is a real answer and the common one for the
|
|
// first few minutes after a restart: claiming "steady" from two readings a minute apart
|
|
// would be a claim the data does not support.
|
|
type Direction string
|
|
|
|
const (
|
|
DirectionUnknown Direction = "unknown"
|
|
DirectionSteady Direction = "steady"
|
|
DirectionRising Direction = "rising"
|
|
DirectionFalling Direction = "falling"
|
|
)
|
|
|
|
type Trend struct {
|
|
Direction Direction `json:"direction"`
|
|
// PerHour is the observed rate of change in the value's own units. It is what makes
|
|
// "rising" actionable: two goroutines an hour is a leak worth naming, two a minute is
|
|
// one worth investigating tonight.
|
|
PerHour float64 `json:"perHour"`
|
|
First float64 `json:"first"`
|
|
Latest float64 `json:"latest"`
|
|
Min float64 `json:"min"`
|
|
Max float64 `json:"max"`
|
|
SpanSeconds float64 `json:"spanSeconds"`
|
|
Points int `json:"points"`
|
|
}
|
|
|
|
// minTrendPoints and minTrendSpan are what a trend has to earn before it is stated at all.
|
|
// A leak is a claim about a slope, and a slope read off three readings inside five minutes
|
|
// is indistinguishable from a household that happened to start watching something.
|
|
const (
|
|
minTrendPoints = 5
|
|
minTrendSpan = 15 * time.Minute
|
|
)
|
|
|
|
// trendOf is the whole rule and it is pure. It compares the median of the oldest quarter
|
|
// of the window with the median of the newest quarter rather than fitting a line, because
|
|
// a single burst — a library import, a household arriving home — is exactly the shape
|
|
// least-squares reports as a trend and medians ignore.
|
|
//
|
|
// The threshold is proportional with an absolute floor: without the proportion a busy
|
|
// gateway is permanently "rising", and without the floor a quiet one calls one extra
|
|
// goroutine a trend.
|
|
func trendOf(points []float64, at []time.Time, floor float64) Trend {
|
|
trend := Trend{Direction: DirectionUnknown, Points: len(points)}
|
|
if len(points) == 0 || len(at) != len(points) {
|
|
return trend
|
|
}
|
|
trend.First, trend.Latest = points[0], points[len(points)-1]
|
|
trend.Min, trend.Max = points[0], points[0]
|
|
for _, value := range points {
|
|
trend.Min = math.Min(trend.Min, value)
|
|
trend.Max = math.Max(trend.Max, value)
|
|
}
|
|
span := at[len(at)-1].Sub(at[0])
|
|
trend.SpanSeconds = span.Seconds()
|
|
if len(points) < minTrendPoints || span < minTrendSpan {
|
|
return trend
|
|
}
|
|
|
|
quarter := len(points) / 4
|
|
if quarter < 1 {
|
|
quarter = 1
|
|
}
|
|
early := median(points[:quarter])
|
|
late := median(points[len(points)-quarter:])
|
|
change := late - early
|
|
trend.PerHour = change / span.Hours()
|
|
|
|
threshold := math.Max(floor, early*0.15)
|
|
switch {
|
|
case change > threshold:
|
|
trend.Direction = DirectionRising
|
|
case change < -threshold:
|
|
trend.Direction = DirectionFalling
|
|
default:
|
|
trend.Direction = DirectionSteady
|
|
}
|
|
return trend
|
|
}
|
|
|
|
func median(values []float64) float64 {
|
|
if len(values) == 0 {
|
|
return 0
|
|
}
|
|
sorted := make([]float64, len(values))
|
|
copy(sorted, values)
|
|
sort.Float64s(sorted)
|
|
middle := len(sorted) / 2
|
|
if len(sorted)%2 == 1 {
|
|
return sorted[middle]
|
|
}
|
|
return (sorted[middle-1] + sorted[middle]) / 2
|
|
}
|
|
|
|
func trendsFrom(list []Sample) (goroutines, heap, reserved Trend) {
|
|
at := make([]time.Time, len(list))
|
|
goroutinePoints := make([]float64, len(list))
|
|
heapPoints := make([]float64, len(list))
|
|
reservedPoints := make([]float64, len(list))
|
|
for index, sample := range list {
|
|
at[index] = sample.At
|
|
goroutinePoints[index] = float64(sample.Goroutines)
|
|
heapPoints[index] = float64(sample.HeapInuse)
|
|
reservedPoints[index] = float64(sample.Sys)
|
|
}
|
|
// The floors: five goroutines, and four megabytes of memory. Below those, movement is
|
|
// the ordinary breathing of a server answering requests.
|
|
return trendOf(goroutinePoints, at, 5),
|
|
trendOf(heapPoints, at, 4<<20),
|
|
trendOf(reservedPoints, at, 4<<20)
|
|
}
|
|
|
|
/* ---------- the snapshot the console polls ---------- */
|
|
|
|
// Memory separates the three figures an operator keeps conflating: what the program is
|
|
// actually holding, what the Go runtime has reserved from the operating system on its
|
|
// behalf, and the ceiling it has been given. Reserved is always the largest and is not a
|
|
// leak; the ratio to the limit is the number worth watching.
|
|
type Memory struct {
|
|
HeapAlloc uint64 `json:"heapAlloc"`
|
|
HeapInuse uint64 `json:"heapInuse"`
|
|
HeapIdle uint64 `json:"heapIdle"`
|
|
HeapReleased uint64 `json:"heapReleased"`
|
|
StackInuse uint64 `json:"stackInuse"`
|
|
Sys uint64 `json:"sys"`
|
|
NextGC uint64 `json:"nextGc"`
|
|
NumGC uint32 `json:"numGc"`
|
|
// PauseTotalMs and PauseRecentMs are how much of the life of the gateway has been
|
|
// spent stopped for collection. A rising heap that is being collected without pausing
|
|
// is a working cache; one that pauses for longer every cycle is not.
|
|
PauseTotalMs float64 `json:"pauseTotalMs"`
|
|
PauseRecentMs float64 `json:"pauseRecentMs"`
|
|
MemoryLimit int64 `json:"memoryLimit"`
|
|
ConfiguredLim string `json:"configuredLimit,omitempty"`
|
|
// HeapShare is heap in use as a fraction of the limit, or 0 when there is no limit.
|
|
// Computed here rather than in the console so the health verdict and the figure the
|
|
// operator reads cannot disagree.
|
|
HeapShare float64 `json:"heapShare"`
|
|
// GCPerHour is measured over the trend window rather than over the whole life of the
|
|
// process, because "collections have become more frequent" is the useful form of it.
|
|
GCPerHour float64 `json:"gcPerHour"`
|
|
}
|
|
|
|
type Snapshot struct {
|
|
At time.Time `json:"at"`
|
|
UptimeSeconds float64 `json:"uptimeSeconds"`
|
|
Goroutines int `json:"goroutines"`
|
|
GOMAXPROCS int `json:"gomaxprocs"`
|
|
Threads int `json:"threads"`
|
|
GoVersion string `json:"goVersion"`
|
|
|
|
Memory Memory `json:"memory"`
|
|
Workers []Worker `json:"workers"`
|
|
Process Process `json:"process"`
|
|
|
|
Samples []Sample `json:"samples"`
|
|
GoroutineTrend Trend `json:"goroutineTrend"`
|
|
HeapTrend Trend `json:"heapTrend"`
|
|
ReservedTrend Trend `json:"reservedTrend"`
|
|
SampleEverySecs float64 `json:"sampleEverySeconds"`
|
|
|
|
Health Health `json:"health"`
|
|
}
|
|
|
|
// Read builds the cheap snapshot. Nothing in here walks a stack or stops the world for
|
|
// longer than a ReadMemStats, so it is safe on the 30-second poll every open console tab
|
|
// makes.
|
|
func Read() Snapshot {
|
|
var memory runtime.MemStats
|
|
runtime.ReadMemStats(&memory)
|
|
now := time.Now()
|
|
|
|
limit := debug.SetMemoryLimit(-1)
|
|
share := 0.0
|
|
if limit > 0 && limit < math.MaxInt64 {
|
|
share = float64(memory.HeapInuse) / float64(limit)
|
|
}
|
|
|
|
list := Samples()
|
|
goroutineTrend, heapTrend, reservedTrend := trendsFrom(list)
|
|
|
|
snapshot := Snapshot{
|
|
At: now,
|
|
UptimeSeconds: now.Sub(started).Seconds(),
|
|
Goroutines: runtime.NumGoroutine(),
|
|
GOMAXPROCS: runtime.GOMAXPROCS(0),
|
|
Threads: threadCount(),
|
|
GoVersion: runtime.Version(),
|
|
Memory: Memory{
|
|
HeapAlloc: memory.HeapAlloc,
|
|
HeapInuse: memory.HeapInuse,
|
|
HeapIdle: memory.HeapIdle,
|
|
HeapReleased: memory.HeapReleased,
|
|
StackInuse: memory.StackInuse,
|
|
Sys: memory.Sys,
|
|
NextGC: memory.NextGC,
|
|
NumGC: memory.NumGC,
|
|
PauseTotalMs: float64(memory.PauseTotalNs) / 1e6,
|
|
PauseRecentMs: recentPauseMs(&memory),
|
|
MemoryLimit: limit,
|
|
ConfiguredLim: configuredMemoryLimit(),
|
|
HeapShare: share,
|
|
GCPerHour: gcPerHour(list),
|
|
},
|
|
Workers: Workers(),
|
|
Process: readProcess(),
|
|
Samples: list,
|
|
GoroutineTrend: goroutineTrend,
|
|
HeapTrend: heapTrend,
|
|
ReservedTrend: reservedTrend,
|
|
SampleEverySecs: SampleInterval.Seconds(),
|
|
}
|
|
snapshot.Health = assess(snapshot)
|
|
return snapshot
|
|
}
|
|
|
|
// recentPauseMs averages the last few recorded pauses. Go keeps a circular buffer of the
|
|
// most recent 256, and the running total on its own cannot distinguish a process that
|
|
// paused badly an hour ago from one pausing badly now.
|
|
func recentPauseMs(memory *runtime.MemStats) float64 {
|
|
const window = 16
|
|
if memory.NumGC == 0 {
|
|
return 0
|
|
}
|
|
count := 0
|
|
total := uint64(0)
|
|
for index := 0; index < window; index++ {
|
|
slot := int(memory.NumGC) - 1 - index
|
|
if slot < 0 {
|
|
break
|
|
}
|
|
total += memory.PauseNs[slot%256]
|
|
count++
|
|
}
|
|
if count == 0 {
|
|
return 0
|
|
}
|
|
return float64(total) / float64(count) / 1e6
|
|
}
|
|
|
|
func gcPerHour(list []Sample) float64 {
|
|
if len(list) < 2 {
|
|
return 0
|
|
}
|
|
span := list[len(list)-1].At.Sub(list[0].At).Hours()
|
|
if span <= 0 {
|
|
return 0
|
|
}
|
|
return float64(list[len(list)-1].NumGC-list[0].NumGC) / span
|
|
}
|