0.2.78
This commit is contained in:
ponzischeme89
@@ -0,0 +1,257 @@
|
||||
package runtimestats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Health is the answer the Process card exists to give: is the gateway healthy, and if
|
||||
// something is abnormal, which area should be looked at?
|
||||
//
|
||||
// It is stated on the server rather than derived in the console for the same reason every
|
||||
// schedule label is the gateway's wording — the thresholds and the sentence explaining
|
||||
// them belong together, and an older console must not be the thing deciding what "watch"
|
||||
// means. It is a pure function of a snapshot so the thresholds can be tested rather than
|
||||
// discovered in production.
|
||||
type Health struct {
|
||||
Level Level `json:"level"`
|
||||
Summary string `json:"summary"`
|
||||
Notes []Note `json:"notes"`
|
||||
Areas []string `json:"areas,omitempty"`
|
||||
}
|
||||
|
||||
type Level string
|
||||
|
||||
const (
|
||||
// LevelOK is nothing to do. LevelWatch is something that is not wrong yet and would
|
||||
// be if it continued — which is the whole reason trends are collected. LevelBad is
|
||||
// something an operator should act on now.
|
||||
LevelOK Level = "ok"
|
||||
LevelWatch Level = "watch"
|
||||
LevelBad Level = "bad"
|
||||
)
|
||||
|
||||
type Note struct {
|
||||
Level Level `json:"level"`
|
||||
// Area names the part of Memby to look at. It is the note's most useful field: a
|
||||
// number that has moved is only actionable once it points somewhere.
|
||||
Area string `json:"area"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// The thresholds. Heap is judged against the configured limit because that is the number
|
||||
// the container is actually killed at; the watch line is deliberately well below it, since
|
||||
// the point of a watch is to arrive before the incident rather than during it.
|
||||
const (
|
||||
heapShareBad = 0.90
|
||||
heapShareWatch = 0.75
|
||||
// goroutineLeakPerHour is what separates a gateway that has picked up work from one
|
||||
// that is not letting go of it. A household arriving home adds goroutines and gives
|
||||
// them back; a leak does not, so this is only ever consulted alongside a rising trend
|
||||
// measured over the trend window.
|
||||
goroutineLeakPerHour = 10
|
||||
goroutineFloodPerHour = 60
|
||||
// pauseWatchMs is a collection pause long enough for a television to notice. The
|
||||
// gateway answers a status poll every ten seconds from every open set.
|
||||
pauseWatchMs = 50
|
||||
pauseBadMs = 250
|
||||
// restartsWatch: a worker that has been launched this many times is one that keeps
|
||||
// dying. The registry counts starts for exactly this.
|
||||
restartsWatch = 3
|
||||
)
|
||||
|
||||
func assess(snapshot Snapshot) Health {
|
||||
var notes []Note
|
||||
|
||||
memory := snapshot.Memory
|
||||
switch {
|
||||
case memory.HeapShare >= heapShareBad:
|
||||
notes = append(notes, Note{
|
||||
Level: LevelBad, Area: "Memory",
|
||||
Message: fmt.Sprintf(
|
||||
"The heap is using %s of the %s limit. Past the limit the container is stopped, so this needs attention now.",
|
||||
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
|
||||
),
|
||||
})
|
||||
case memory.HeapShare >= heapShareWatch:
|
||||
notes = append(notes, Note{
|
||||
Level: LevelWatch, Area: "Memory",
|
||||
Message: fmt.Sprintf(
|
||||
"The heap is using %s of the %s limit. There is room, but less than usual.",
|
||||
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
if snapshot.HeapTrend.Direction == DirectionRising {
|
||||
notes = append(notes, Note{
|
||||
Level: LevelWatch, Area: "Memory",
|
||||
Message: fmt.Sprintf(
|
||||
"Heap in use has risen by about %s an hour over the last %s. If it does not fall after a collection, something is holding on to it.",
|
||||
bytes(uint64(math.Max(0, snapshot.HeapTrend.PerHour))), duration(snapshot.HeapTrend.SpanSeconds),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
if snapshot.GoroutineTrend.Direction == DirectionRising {
|
||||
rate := snapshot.GoroutineTrend.PerHour
|
||||
level := LevelOK
|
||||
switch {
|
||||
case rate >= goroutineFloodPerHour:
|
||||
level = LevelBad
|
||||
case rate >= goroutineLeakPerHour:
|
||||
level = LevelWatch
|
||||
}
|
||||
if level != LevelOK {
|
||||
notes = append(notes, Note{
|
||||
Level: level, Area: "Goroutines",
|
||||
Message: fmt.Sprintf(
|
||||
"Goroutines have risen by about %.0f an hour over the last %s, from %.0f to %.0f. Open the breakdown to see which component they belong to.",
|
||||
rate, duration(snapshot.GoroutineTrend.SpanSeconds),
|
||||
snapshot.GoroutineTrend.First, snapshot.GoroutineTrend.Latest,
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case memory.PauseRecentMs >= pauseBadMs:
|
||||
notes = append(notes, Note{
|
||||
Level: LevelBad, Area: "Collection",
|
||||
Message: fmt.Sprintf(
|
||||
"Recent collections have paused the gateway for %.0f ms on average. Everything the televisions ask for waits that long.",
|
||||
memory.PauseRecentMs,
|
||||
),
|
||||
})
|
||||
case memory.PauseRecentMs >= pauseWatchMs:
|
||||
notes = append(notes, Note{
|
||||
Level: LevelWatch, Area: "Collection",
|
||||
Message: fmt.Sprintf(
|
||||
"Recent collections have paused the gateway for %.0f ms on average.", memory.PauseRecentMs,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
for _, worker := range snapshot.Workers {
|
||||
if worker.State == WorkerRunning && worker.Starts >= restartsWatch {
|
||||
notes = append(notes, Note{
|
||||
Level: LevelWatch, Area: worker.Component,
|
||||
Message: fmt.Sprintf(
|
||||
"%s has been started %d times. A background worker that keeps restarting is failing at something.",
|
||||
worker.Name, worker.Starts,
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return Health{
|
||||
Level: worst(notes),
|
||||
Summary: summarise(snapshot, notes),
|
||||
Notes: notes,
|
||||
Areas: areasOf(notes),
|
||||
}
|
||||
}
|
||||
|
||||
func worst(notes []Note) Level {
|
||||
level := LevelOK
|
||||
for _, note := range notes {
|
||||
if note.Level == LevelBad {
|
||||
return LevelBad
|
||||
}
|
||||
if note.Level == LevelWatch {
|
||||
level = LevelWatch
|
||||
}
|
||||
}
|
||||
return level
|
||||
}
|
||||
|
||||
// summarise is the one line the overview card prints. When there is nothing to say it says
|
||||
// so plainly rather than reciting figures the operator can already see above it — and it
|
||||
// says whether the verdict is based on enough history to be worth anything, because "no
|
||||
// trouble" from four minutes of readings is a weaker claim than the same words from four
|
||||
// hours.
|
||||
func summarise(snapshot Snapshot, notes []Note) string {
|
||||
if len(notes) == 0 {
|
||||
if snapshot.GoroutineTrend.Direction == DirectionUnknown {
|
||||
return "Nothing abnormal. Still gathering history — trends need about fifteen minutes."
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"Nothing abnormal. Goroutines and memory have been %s over the last %s.",
|
||||
steadyWord(snapshot), duration(snapshot.GoroutineTrend.SpanSeconds),
|
||||
)
|
||||
}
|
||||
areas := areasOf(notes)
|
||||
if len(notes) == 1 {
|
||||
return notes[0].Message
|
||||
}
|
||||
return fmt.Sprintf("%d things to look at, in %s.", len(notes), joinAreas(areas))
|
||||
}
|
||||
|
||||
func steadyWord(snapshot Snapshot) string {
|
||||
if snapshot.GoroutineTrend.Direction == DirectionFalling || snapshot.HeapTrend.Direction == DirectionFalling {
|
||||
return "steady or falling"
|
||||
}
|
||||
return "steady"
|
||||
}
|
||||
|
||||
func areasOf(notes []Note) []string {
|
||||
seen := map[string]bool{}
|
||||
var areas []string
|
||||
for _, note := range notes {
|
||||
if note.Area == "" || seen[note.Area] {
|
||||
continue
|
||||
}
|
||||
seen[note.Area] = true
|
||||
areas = append(areas, note.Area)
|
||||
}
|
||||
sort.Strings(areas)
|
||||
return areas
|
||||
}
|
||||
|
||||
func joinAreas(areas []string) string {
|
||||
switch len(areas) {
|
||||
case 0:
|
||||
return "the gateway"
|
||||
case 1:
|
||||
return areas[0]
|
||||
case 2:
|
||||
return areas[0] + " and " + areas[1]
|
||||
}
|
||||
return fmt.Sprintf("%s and %d other areas", areas[0], len(areas)-1)
|
||||
}
|
||||
|
||||
/* ---------- wording helpers ----------
|
||||
|
||||
These exist because the sentences above are the gateway's own wording, the stance every
|
||||
label the console prints takes: a console built before a threshold existed still reads
|
||||
correctly, and the figure in the sentence cannot disagree with the threshold that
|
||||
produced it. */
|
||||
|
||||
func percent(share float64) string {
|
||||
return fmt.Sprintf("%.1f%%", share*100)
|
||||
}
|
||||
|
||||
func bytes(value uint64) string {
|
||||
const unit = 1024
|
||||
if value < unit {
|
||||
return fmt.Sprintf("%d B", value)
|
||||
}
|
||||
div, exponent := uint64(unit), 0
|
||||
for size := value / unit; size >= unit; size /= unit {
|
||||
div *= unit
|
||||
exponent++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(value)/float64(div), "KMGT"[exponent])
|
||||
}
|
||||
|
||||
func duration(seconds float64) string {
|
||||
switch {
|
||||
case seconds < 90:
|
||||
return fmt.Sprintf("%.0f seconds", seconds)
|
||||
case seconds < 5400:
|
||||
return fmt.Sprintf("%.0f minutes", seconds/60)
|
||||
default:
|
||||
return fmt.Sprintf("%.1f hours", seconds/3600)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user