memby/server/internal/runtimestats/health.go

package runtimestats

import (
	"fmt"
	"math"
	"sort"
)

// Health is the answer the Process card exists to give: is the gateway healthy, and if
// something is abnormal, which area should be looked at?
//
// It is stated on the server rather than derived in the console for the same reason every
// schedule label is the gateway's wording — the thresholds and the sentence explaining
// them belong together, and an older console must not be the thing deciding what "watch"
// means. It is a pure function of a snapshot so the thresholds can be tested rather than
// discovered in production.
type Health struct {
	Level   Level    `json:"level"`
	Summary string   `json:"summary"`
	Notes   []Note   `json:"notes"`
	Areas   []string `json:"areas,omitempty"`
}

type Level string

const (
	// LevelOK is nothing to do. LevelWatch is something that is not wrong yet and would
	// be if it continued — which is the whole reason trends are collected. LevelBad is
	// something an operator should act on now.
	LevelOK    Level = "ok"
	LevelWatch Level = "watch"
	LevelBad   Level = "bad"
)

type Note struct {
	Level Level `json:"level"`
	// Area names the part of Memby to look at. It is the note's most useful field: a
	// number that has moved is only actionable once it points somewhere.
	Area    string `json:"area"`
	Message string `json:"message"`
}

// The thresholds. Heap is judged against the configured limit because that is the number
// the container is actually killed at; the watch line is deliberately well below it, since
// the point of a watch is to arrive before the incident rather than during it.
const (
	heapShareBad   = 0.90
	heapShareWatch = 0.75
	// goroutineLeakPerHour is what separates a gateway that has picked up work from one
	// that is not letting go of it. A household arriving home adds goroutines and gives
	// them back; a leak does not, so this is only ever consulted alongside a rising trend
	// measured over the trend window.
	goroutineLeakPerHour  = 10
	goroutineFloodPerHour = 60
	// pauseWatchMs is a collection pause long enough for a television to notice. The
	// gateway answers a status poll every ten seconds from every open set.
	pauseWatchMs = 50
	pauseBadMs   = 250
	// restartsWatch: a worker that has been launched this many times is one that keeps
	// dying. The registry counts starts for exactly this.
	restartsWatch = 3
)

func assess(snapshot Snapshot) Health {
	var notes []Note

	memory := snapshot.Memory
	switch {
	case memory.HeapShare >= heapShareBad:
		notes = append(notes, Note{
			Level: LevelBad, Area: "Memory",
			Message: fmt.Sprintf(
				"The heap is using %s of the %s limit. Past the limit the container is stopped, so this needs attention now.",
				percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
			),
		})
	case memory.HeapShare >= heapShareWatch:
		notes = append(notes, Note{
			Level: LevelWatch, Area: "Memory",
			Message: fmt.Sprintf(
				"The heap is using %s of the %s limit. There is room, but less than usual.",
				percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
			),
		})
	}

	if snapshot.HeapTrend.Direction == DirectionRising {
		notes = append(notes, Note{
			Level: LevelWatch, Area: "Memory",
			Message: fmt.Sprintf(
				"Heap in use has risen by about %s an hour over the last %s. If it does not fall after a collection, something is holding on to it.",
				bytes(uint64(math.Max(0, snapshot.HeapTrend.PerHour))), duration(snapshot.HeapTrend.SpanSeconds),
			),
		})
	}

	if snapshot.GoroutineTrend.Direction == DirectionRising {
		rate := snapshot.GoroutineTrend.PerHour
		level := LevelOK
		switch {
		case rate >= goroutineFloodPerHour:
			level = LevelBad
		case rate >= goroutineLeakPerHour:
			level = LevelWatch
		}
		if level != LevelOK {
			notes = append(notes, Note{
				Level: level, Area: "Goroutines",
				Message: fmt.Sprintf(
					"Goroutines have risen by about %.0f an hour over the last %s, from %.0f to %.0f. Open the breakdown to see which component they belong to.",
					rate, duration(snapshot.GoroutineTrend.SpanSeconds),
					snapshot.GoroutineTrend.First, snapshot.GoroutineTrend.Latest,
				),
			})
		}
	}

	switch {
	case memory.PauseRecentMs >= pauseBadMs:
		notes = append(notes, Note{
			Level: LevelBad, Area: "Collection",
			Message: fmt.Sprintf(
				"Recent collections have paused the gateway for %.0f ms on average. Everything the televisions ask for waits that long.",
				memory.PauseRecentMs,
			),
		})
	case memory.PauseRecentMs >= pauseWatchMs:
		notes = append(notes, Note{
			Level: LevelWatch, Area: "Collection",
			Message: fmt.Sprintf(
				"Recent collections have paused the gateway for %.0f ms on average.", memory.PauseRecentMs,
			),
		})
	}

	for _, worker := range snapshot.Workers {
		if worker.State == WorkerRunning && worker.Starts >= restartsWatch {
			notes = append(notes, Note{
				Level: LevelWatch, Area: worker.Component,
				Message: fmt.Sprintf(
					"%s has been started %d times. A background worker that keeps restarting is failing at something.",
					worker.Name, worker.Starts,
				),
			})
		}
	}

	return Health{
		Level:   worst(notes),
		Summary: summarise(snapshot, notes),
		Notes:   notes,
		Areas:   areasOf(notes),
	}
}

func worst(notes []Note) Level {
	level := LevelOK
	for _, note := range notes {
		if note.Level == LevelBad {
			return LevelBad
		}
		if note.Level == LevelWatch {
			level = LevelWatch
		}
	}
	return level
}

// summarise is the one line the overview card prints. When there is nothing to say it says
// so plainly rather than reciting figures the operator can already see above it — and it
// says whether the verdict is based on enough history to be worth anything, because "no
// trouble" from four minutes of readings is a weaker claim than the same words from four
// hours.
func summarise(snapshot Snapshot, notes []Note) string {
	if len(notes) == 0 {
		if snapshot.GoroutineTrend.Direction == DirectionUnknown {
			return "Nothing abnormal. Still gathering history — trends need about fifteen minutes."
		}
		return fmt.Sprintf(
			"Nothing abnormal. Goroutines and memory have been %s over the last %s.",
			steadyWord(snapshot), duration(snapshot.GoroutineTrend.SpanSeconds),
		)
	}
	areas := areasOf(notes)
	if len(notes) == 1 {
		return notes[0].Message
	}
	return fmt.Sprintf("%d things to look at, in %s.", len(notes), joinAreas(areas))
}

func steadyWord(snapshot Snapshot) string {
	if snapshot.GoroutineTrend.Direction == DirectionFalling || snapshot.HeapTrend.Direction == DirectionFalling {
		return "steady or falling"
	}
	return "steady"
}

func areasOf(notes []Note) []string {
	seen := map[string]bool{}
	var areas []string
	for _, note := range notes {
		if note.Area == "" || seen[note.Area] {
			continue
		}
		seen[note.Area] = true
		areas = append(areas, note.Area)
	}
	sort.Strings(areas)
	return areas
}

func joinAreas(areas []string) string {
	switch len(areas) {
	case 0:
		return "the gateway"
	case 1:
		return areas[0]
	case 2:
		return areas[0] + " and " + areas[1]
	}
	return fmt.Sprintf("%s and %d other areas", areas[0], len(areas)-1)
}

/* ---------- wording helpers ----------

   These exist because the sentences above are the gateway's own wording, the stance every
   label the console prints takes: a console built before a threshold existed still reads
   correctly, and the figure in the sentence cannot disagree with the threshold that
   produced it. */

func percent(share float64) string {
	return fmt.Sprintf("%.1f%%", share*100)
}

func bytes(value uint64) string {
	const unit = 1024
	if value < unit {
		return fmt.Sprintf("%d B", value)
	}
	div, exponent := uint64(unit), 0
	for size := value / unit; size >= unit; size /= unit {
		div *= unit
		exponent++
	}
	return fmt.Sprintf("%.1f %cB", float64(value)/float64(div), "KMGT"[exponent])
}

func duration(seconds float64) string {
	switch {
	case seconds < 90:
		return fmt.Sprintf("%.0f seconds", seconds)
	case seconds < 5400:
		return fmt.Sprintf("%.0f minutes", seconds/60)
	default:
		return fmt.Sprintf("%.1f hours", seconds/3600)
	}
}