package runtimestats import ( "context" "math" "runtime" "runtime/debug" "sort" "sync" "time" ) // SampleInterval is how often the trend ring records a reading, and the ring's capacity is // how far back it can therefore see. A minute apart is far cheaper than the 30s poll the // console already makes, and four hours is long enough for a slow leak to become a line // rather than noise while costing a few kilobytes. const ( SampleInterval = time.Minute trendCapacity = 240 ) // Sample is one reading. Deliberately the same source as the headline figures — both come // from one ReadMemStats — because a trend drawn from a different counter than the number // printed above it is a trend an operator cannot check. type Sample struct { At time.Time `json:"at"` Goroutines int `json:"goroutines"` HeapInuse uint64 `json:"heapInuse"` Sys uint64 `json:"sys"` NumGC uint32 `json:"numGc"` } var ( samplesMu sync.Mutex samples []Sample started = time.Now() ) // StartSampling records a reading every SampleInterval until ctx ends. It takes the first // reading immediately, so a console opened a minute after a deployment has a baseline // rather than an empty chart. func StartSampling(ctx context.Context) { record(readSample()) ticker := time.NewTicker(SampleInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: record(readSample()) } } } func readSample() Sample { var memory runtime.MemStats runtime.ReadMemStats(&memory) return Sample{ At: time.Now(), Goroutines: runtime.NumGoroutine(), HeapInuse: memory.HeapInuse, Sys: memory.Sys, NumGC: memory.NumGC, } } func record(sample Sample) { samplesMu.Lock() defer samplesMu.Unlock() samples = append(samples, sample) if len(samples) > trendCapacity { samples = append(samples[:0], samples[len(samples)-trendCapacity:]...) } } // Samples returns the trend ring oldest first. func Samples() []Sample { samplesMu.Lock() defer samplesMu.Unlock() out := make([]Sample, len(samples)) copy(out, samples) return out } /* ---------- trends ---------- */ // Direction is a trend's verdict. "unknown" is a real answer and the common one for the // first few minutes after a restart: claiming "steady" from two readings a minute apart // would be a claim the data does not support. type Direction string const ( DirectionUnknown Direction = "unknown" DirectionSteady Direction = "steady" DirectionRising Direction = "rising" DirectionFalling Direction = "falling" ) type Trend struct { Direction Direction `json:"direction"` // PerHour is the observed rate of change in the value's own units. It is what makes // "rising" actionable: two goroutines an hour is a leak worth naming, two a minute is // one worth investigating tonight. PerHour float64 `json:"perHour"` First float64 `json:"first"` Latest float64 `json:"latest"` Min float64 `json:"min"` Max float64 `json:"max"` SpanSeconds float64 `json:"spanSeconds"` Points int `json:"points"` } // minTrendPoints and minTrendSpan are what a trend has to earn before it is stated at all. // A leak is a claim about a slope, and a slope read off three readings inside five minutes // is indistinguishable from a household that happened to start watching something. const ( minTrendPoints = 5 minTrendSpan = 15 * time.Minute ) // trendOf is the whole rule and it is pure. It compares the median of the oldest quarter // of the window with the median of the newest quarter rather than fitting a line, because // a single burst — a library import, a household arriving home — is exactly the shape // least-squares reports as a trend and medians ignore. // // The threshold is proportional with an absolute floor: without the proportion a busy // gateway is permanently "rising", and without the floor a quiet one calls one extra // goroutine a trend. func trendOf(points []float64, at []time.Time, floor float64) Trend { trend := Trend{Direction: DirectionUnknown, Points: len(points)} if len(points) == 0 || len(at) != len(points) { return trend } trend.First, trend.Latest = points[0], points[len(points)-1] trend.Min, trend.Max = points[0], points[0] for _, value := range points { trend.Min = math.Min(trend.Min, value) trend.Max = math.Max(trend.Max, value) } span := at[len(at)-1].Sub(at[0]) trend.SpanSeconds = span.Seconds() if len(points) < minTrendPoints || span < minTrendSpan { return trend } quarter := len(points) / 4 if quarter < 1 { quarter = 1 } early := median(points[:quarter]) late := median(points[len(points)-quarter:]) change := late - early trend.PerHour = change / span.Hours() threshold := math.Max(floor, early*0.15) switch { case change > threshold: trend.Direction = DirectionRising case change < -threshold: trend.Direction = DirectionFalling default: trend.Direction = DirectionSteady } return trend } func median(values []float64) float64 { if len(values) == 0 { return 0 } sorted := make([]float64, len(values)) copy(sorted, values) sort.Float64s(sorted) middle := len(sorted) / 2 if len(sorted)%2 == 1 { return sorted[middle] } return (sorted[middle-1] + sorted[middle]) / 2 } func trendsFrom(list []Sample) (goroutines, heap, reserved Trend) { at := make([]time.Time, len(list)) goroutinePoints := make([]float64, len(list)) heapPoints := make([]float64, len(list)) reservedPoints := make([]float64, len(list)) for index, sample := range list { at[index] = sample.At goroutinePoints[index] = float64(sample.Goroutines) heapPoints[index] = float64(sample.HeapInuse) reservedPoints[index] = float64(sample.Sys) } // The floors: five goroutines, and four megabytes of memory. Below those, movement is // the ordinary breathing of a server answering requests. return trendOf(goroutinePoints, at, 5), trendOf(heapPoints, at, 4<<20), trendOf(reservedPoints, at, 4<<20) } /* ---------- the snapshot the console polls ---------- */ // Memory separates the three figures an operator keeps conflating: what the program is // actually holding, what the Go runtime has reserved from the operating system on its // behalf, and the ceiling it has been given. Reserved is always the largest and is not a // leak; the ratio to the limit is the number worth watching. type Memory struct { HeapAlloc uint64 `json:"heapAlloc"` HeapInuse uint64 `json:"heapInuse"` HeapIdle uint64 `json:"heapIdle"` HeapReleased uint64 `json:"heapReleased"` StackInuse uint64 `json:"stackInuse"` Sys uint64 `json:"sys"` NextGC uint64 `json:"nextGc"` NumGC uint32 `json:"numGc"` // PauseTotalMs and PauseRecentMs are how much of the life of the gateway has been // spent stopped for collection. A rising heap that is being collected without pausing // is a working cache; one that pauses for longer every cycle is not. PauseTotalMs float64 `json:"pauseTotalMs"` PauseRecentMs float64 `json:"pauseRecentMs"` MemoryLimit int64 `json:"memoryLimit"` ConfiguredLim string `json:"configuredLimit,omitempty"` // HeapShare is heap in use as a fraction of the limit, or 0 when there is no limit. // Computed here rather than in the console so the health verdict and the figure the // operator reads cannot disagree. HeapShare float64 `json:"heapShare"` // GCPerHour is measured over the trend window rather than over the whole life of the // process, because "collections have become more frequent" is the useful form of it. GCPerHour float64 `json:"gcPerHour"` } type Snapshot struct { At time.Time `json:"at"` UptimeSeconds float64 `json:"uptimeSeconds"` Goroutines int `json:"goroutines"` GOMAXPROCS int `json:"gomaxprocs"` Threads int `json:"threads"` GoVersion string `json:"goVersion"` Memory Memory `json:"memory"` Workers []Worker `json:"workers"` Process Process `json:"process"` Samples []Sample `json:"samples"` GoroutineTrend Trend `json:"goroutineTrend"` HeapTrend Trend `json:"heapTrend"` ReservedTrend Trend `json:"reservedTrend"` SampleEverySecs float64 `json:"sampleEverySeconds"` Health Health `json:"health"` } // Read builds the cheap snapshot. Nothing in here walks a stack or stops the world for // longer than a ReadMemStats, so it is safe on the 30-second poll every open console tab // makes. func Read() Snapshot { var memory runtime.MemStats runtime.ReadMemStats(&memory) now := time.Now() limit := debug.SetMemoryLimit(-1) share := 0.0 if limit > 0 && limit < math.MaxInt64 { share = float64(memory.HeapInuse) / float64(limit) } list := Samples() goroutineTrend, heapTrend, reservedTrend := trendsFrom(list) snapshot := Snapshot{ At: now, UptimeSeconds: now.Sub(started).Seconds(), Goroutines: runtime.NumGoroutine(), GOMAXPROCS: runtime.GOMAXPROCS(0), Threads: threadCount(), GoVersion: runtime.Version(), Memory: Memory{ HeapAlloc: memory.HeapAlloc, HeapInuse: memory.HeapInuse, HeapIdle: memory.HeapIdle, HeapReleased: memory.HeapReleased, StackInuse: memory.StackInuse, Sys: memory.Sys, NextGC: memory.NextGC, NumGC: memory.NumGC, PauseTotalMs: float64(memory.PauseTotalNs) / 1e6, PauseRecentMs: recentPauseMs(&memory), MemoryLimit: limit, ConfiguredLim: configuredMemoryLimit(), HeapShare: share, GCPerHour: gcPerHour(list), }, Workers: Workers(), Process: readProcess(), Samples: list, GoroutineTrend: goroutineTrend, HeapTrend: heapTrend, ReservedTrend: reservedTrend, SampleEverySecs: SampleInterval.Seconds(), } snapshot.Health = assess(snapshot) return snapshot } // recentPauseMs averages the last few recorded pauses. Go keeps a circular buffer of the // most recent 256, and the running total on its own cannot distinguish a process that // paused badly an hour ago from one pausing badly now. func recentPauseMs(memory *runtime.MemStats) float64 { const window = 16 if memory.NumGC == 0 { return 0 } count := 0 total := uint64(0) for index := 0; index < window; index++ { slot := int(memory.NumGC) - 1 - index if slot < 0 { break } total += memory.PauseNs[slot%256] count++ } if count == 0 { return 0 } return float64(total) / float64(count) / 1e6 } func gcPerHour(list []Sample) float64 { if len(list) < 2 { return 0 } span := list[len(list)-1].At.Sub(list[0].At).Hours() if span <= 0 { return 0 } return float64(list[len(list)-1].NumGC-list[0].NumGC) / span }