package runtimestats import ( "fmt" "math" "sort" ) // Health is the answer the Process card exists to give: is the gateway healthy, and if // something is abnormal, which area should be looked at? // // It is stated on the server rather than derived in the console for the same reason every // schedule label is the gateway's wording — the thresholds and the sentence explaining // them belong together, and an older console must not be the thing deciding what "watch" // means. It is a pure function of a snapshot so the thresholds can be tested rather than // discovered in production. type Health struct { Level Level `json:"level"` Summary string `json:"summary"` Notes []Note `json:"notes"` Areas []string `json:"areas,omitempty"` } type Level string const ( // LevelOK is nothing to do. LevelWatch is something that is not wrong yet and would // be if it continued — which is the whole reason trends are collected. LevelBad is // something an operator should act on now. LevelOK Level = "ok" LevelWatch Level = "watch" LevelBad Level = "bad" ) type Note struct { Level Level `json:"level"` // Area names the part of Memby to look at. It is the note's most useful field: a // number that has moved is only actionable once it points somewhere. Area string `json:"area"` Message string `json:"message"` } // The thresholds. Heap is judged against the configured limit because that is the number // the container is actually killed at; the watch line is deliberately well below it, since // the point of a watch is to arrive before the incident rather than during it. const ( heapShareBad = 0.90 heapShareWatch = 0.75 // goroutineLeakPerHour is what separates a gateway that has picked up work from one // that is not letting go of it. A household arriving home adds goroutines and gives // them back; a leak does not, so this is only ever consulted alongside a rising trend // measured over the trend window. goroutineLeakPerHour = 10 goroutineFloodPerHour = 60 // pauseWatchMs is a collection pause long enough for a television to notice. The // gateway answers a status poll every ten seconds from every open set. pauseWatchMs = 50 pauseBadMs = 250 // restartsWatch: a worker that has been launched this many times is one that keeps // dying. The registry counts starts for exactly this. restartsWatch = 3 ) func assess(snapshot Snapshot) Health { var notes []Note memory := snapshot.Memory switch { case memory.HeapShare >= heapShareBad: notes = append(notes, Note{ Level: LevelBad, Area: "Memory", Message: fmt.Sprintf( "The heap is using %s of the %s limit. Past the limit the container is stopped, so this needs attention now.", percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)), ), }) case memory.HeapShare >= heapShareWatch: notes = append(notes, Note{ Level: LevelWatch, Area: "Memory", Message: fmt.Sprintf( "The heap is using %s of the %s limit. There is room, but less than usual.", percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)), ), }) } if snapshot.HeapTrend.Direction == DirectionRising { notes = append(notes, Note{ Level: LevelWatch, Area: "Memory", Message: fmt.Sprintf( "Heap in use has risen by about %s an hour over the last %s. If it does not fall after a collection, something is holding on to it.", bytes(uint64(math.Max(0, snapshot.HeapTrend.PerHour))), duration(snapshot.HeapTrend.SpanSeconds), ), }) } if snapshot.GoroutineTrend.Direction == DirectionRising { rate := snapshot.GoroutineTrend.PerHour level := LevelOK switch { case rate >= goroutineFloodPerHour: level = LevelBad case rate >= goroutineLeakPerHour: level = LevelWatch } if level != LevelOK { notes = append(notes, Note{ Level: level, Area: "Goroutines", Message: fmt.Sprintf( "Goroutines have risen by about %.0f an hour over the last %s, from %.0f to %.0f. Open the breakdown to see which component they belong to.", rate, duration(snapshot.GoroutineTrend.SpanSeconds), snapshot.GoroutineTrend.First, snapshot.GoroutineTrend.Latest, ), }) } } switch { case memory.PauseRecentMs >= pauseBadMs: notes = append(notes, Note{ Level: LevelBad, Area: "Collection", Message: fmt.Sprintf( "Recent collections have paused the gateway for %.0f ms on average. Everything the televisions ask for waits that long.", memory.PauseRecentMs, ), }) case memory.PauseRecentMs >= pauseWatchMs: notes = append(notes, Note{ Level: LevelWatch, Area: "Collection", Message: fmt.Sprintf( "Recent collections have paused the gateway for %.0f ms on average.", memory.PauseRecentMs, ), }) } for _, worker := range snapshot.Workers { if worker.State == WorkerRunning && worker.Starts >= restartsWatch { notes = append(notes, Note{ Level: LevelWatch, Area: worker.Component, Message: fmt.Sprintf( "%s has been started %d times. A background worker that keeps restarting is failing at something.", worker.Name, worker.Starts, ), }) } } return Health{ Level: worst(notes), Summary: summarise(snapshot, notes), Notes: notes, Areas: areasOf(notes), } } func worst(notes []Note) Level { level := LevelOK for _, note := range notes { if note.Level == LevelBad { return LevelBad } if note.Level == LevelWatch { level = LevelWatch } } return level } // summarise is the one line the overview card prints. When there is nothing to say it says // so plainly rather than reciting figures the operator can already see above it — and it // says whether the verdict is based on enough history to be worth anything, because "no // trouble" from four minutes of readings is a weaker claim than the same words from four // hours. func summarise(snapshot Snapshot, notes []Note) string { if len(notes) == 0 { if snapshot.GoroutineTrend.Direction == DirectionUnknown { return "Nothing abnormal. Still gathering history — trends need about fifteen minutes." } return fmt.Sprintf( "Nothing abnormal. Goroutines and memory have been %s over the last %s.", steadyWord(snapshot), duration(snapshot.GoroutineTrend.SpanSeconds), ) } areas := areasOf(notes) if len(notes) == 1 { return notes[0].Message } return fmt.Sprintf("%d things to look at, in %s.", len(notes), joinAreas(areas)) } func steadyWord(snapshot Snapshot) string { if snapshot.GoroutineTrend.Direction == DirectionFalling || snapshot.HeapTrend.Direction == DirectionFalling { return "steady or falling" } return "steady" } func areasOf(notes []Note) []string { seen := map[string]bool{} var areas []string for _, note := range notes { if note.Area == "" || seen[note.Area] { continue } seen[note.Area] = true areas = append(areas, note.Area) } sort.Strings(areas) return areas } func joinAreas(areas []string) string { switch len(areas) { case 0: return "the gateway" case 1: return areas[0] case 2: return areas[0] + " and " + areas[1] } return fmt.Sprintf("%s and %d other areas", areas[0], len(areas)-1) } /* ---------- wording helpers ---------- These exist because the sentences above are the gateway's own wording, the stance every label the console prints takes: a console built before a threshold existed still reads correctly, and the figure in the sentence cannot disagree with the threshold that produced it. */ func percent(share float64) string { return fmt.Sprintf("%.1f%%", share*100) } func bytes(value uint64) string { const unit = 1024 if value < unit { return fmt.Sprintf("%d B", value) } div, exponent := uint64(unit), 0 for size := value / unit; size >= unit; size /= unit { div *= unit exponent++ } return fmt.Sprintf("%.1f %cB", float64(value)/float64(div), "KMGT"[exponent]) } func duration(seconds float64) string { switch { case seconds < 90: return fmt.Sprintf("%.0f seconds", seconds) case seconds < 5400: return fmt.Sprintf("%.0f minutes", seconds/60) default: return fmt.Sprintf("%.1f hours", seconds/3600) } }