memby/server/internal/runtimestats/process.go

package runtimestats

import (
	"os"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Process is what the operating system knows about the container that the Go runtime does
// not. Every field of it is optional: this is read from /proc, which exists in the
// container the gateway is deployed in and does not exist on a developer's machine, so an
// unavailable figure is omitted rather than reported as zero — nought open sockets and
// "could not look" are different answers and the console must not print the first for the
// second.
type Process struct {
	PID int `json:"pid"`
	// CPUSeconds is cumulative processor time, and CPUPercent is the share of one
	// processor used since the previous read. A hundred per cent is one core saturated,
	// not the machine: with GOMAXPROCS processors available the ceiling is that times a
	// hundred, which is why the console prints both.
	CPUSeconds  float64 `json:"cpuSeconds,omitempty"`
	CPUPercent  float64 `json:"cpuPercent,omitempty"`
	CPUKnown    bool    `json:"cpuKnown"`
	OpenFiles   int     `json:"openFiles,omitempty"`
	OpenSockets int     `json:"openSockets,omitempty"`
	FileLimit   int     `json:"fileLimit,omitempty"`
	FilesKnown  bool    `json:"filesKnown"`
}

// threadCount is operating-system threads, which is a different number from goroutines and
// is the one that matters when goroutines are blocked in syscalls: the runtime creates a
// thread per blocked call, and a thread costs far more than a goroutine does.
func threadCount() int {
	profile := pprof.Lookup("threadcreate")
	if profile == nil {
		return 0
	}
	return profile.Count()
}

// configuredMemoryLimit reports what the container was started with rather than what the
// runtime currently holds, so the console can say whether the limit is deliberate.
func configuredMemoryLimit() string {
	return strings.TrimSpace(os.Getenv("GOMEMLIMIT"))
}

var (
	cpuMu       sync.Mutex
	lastCPUAt   time.Time
	lastCPUSecs float64
	lastPercent float64
)

func readProcess() Process {
	process := Process{PID: os.Getpid()}
	if seconds, ok := processCPUSeconds(); ok {
		process.CPUSeconds = seconds
		process.CPUKnown = true
		process.CPUPercent = cpuPercentSince(seconds, time.Now())
	}
	if files, sockets, ok := openDescriptors(); ok {
		process.OpenFiles = files
		process.OpenSockets = sockets
		process.FilesKnown = true
		process.FileLimit = descriptorLimit()
	}
	return process
}

// cpuPercentSince needs two readings, so the first call after start-up reports nothing
// rather than dividing by the life of the process — an average over four hours would hide
// exactly the spike somebody opened the page to find. A read that arrives too soon after
// the last one repeats the previous answer instead of amplifying rounding into a spike.
func cpuPercentSince(seconds float64, now time.Time) float64 {
	const minInterval = 5 * time.Second
	cpuMu.Lock()
	defer cpuMu.Unlock()
	if lastCPUAt.IsZero() {
		lastCPUAt, lastCPUSecs = now, seconds
		return 0
	}
	elapsed := now.Sub(lastCPUAt)
	if elapsed < minInterval {
		return lastPercent
	}
	used := seconds - lastCPUSecs
	lastCPUAt, lastCPUSecs = now, seconds
	if used < 0 {
		lastPercent = 0
		return 0
	}
	lastPercent = used / elapsed.Seconds() * 100
	return lastPercent
}

// processCPUSeconds prefers /proc/self/schedstat, whose first field is cumulative time on
// a processor in nanoseconds — exact, where /proc/self/stat is in clock ticks whose length
// cannot be read without cgo and has to be assumed to be the usual hundredth of a second.
func processCPUSeconds() (float64, bool) {
	if raw, err := os.ReadFile("/proc/self/schedstat"); err == nil {
		fields := strings.Fields(string(raw))
		if len(fields) > 0 {
			if nanos, err := strconv.ParseFloat(fields[0], 64); err == nil {
				return nanos / 1e9, true
			}
		}
	}
	raw, err := os.ReadFile("/proc/self/stat")
	if err != nil {
		return 0, false
	}
	// The second field is the command name in brackets and may itself contain spaces, so
	// the fields after it are counted from the closing bracket rather than from the start.
	shut := strings.LastIndex(string(raw), ")")
	if shut < 0 {
		return 0, false
	}
	fields := strings.Fields(string(raw)[shut+1:])
	// After the bracket, field 1 is the state; utime and stime are fields 12 and 13.
	if len(fields) < 13 {
		return 0, false
	}
	user, userErr := strconv.ParseFloat(fields[11], 64)
	system, systemErr := strconv.ParseFloat(fields[12], 64)
	if userErr != nil || systemErr != nil {
		return 0, false
	}
	const ticksPerSecond = 100
	return (user + system) / ticksPerSecond, true
}

func openDescriptors() (files, sockets int, ok bool) {
	entries, err := os.ReadDir("/proc/self/fd")
	if err != nil {
		return 0, 0, false
	}
	for _, entry := range entries {
		files++
		target, err := os.Readlink("/proc/self/fd/" + entry.Name())
		if err == nil && strings.HasPrefix(target, "socket:") {
			sockets++
		}
	}
	return files, sockets, true
}

func descriptorLimit() int {
	raw, err := os.ReadFile("/proc/self/limits")
	if err != nil {
		return 0
	}
	for _, line := range strings.Split(string(raw), "\n") {
		if !strings.HasPrefix(line, "Max open files") {
			continue
		}
		fields := strings.Fields(strings.TrimPrefix(line, "Max open files"))
		if len(fields) == 0 {
			return 0
		}
		limit, err := strconv.Atoi(fields[0])
		if err != nil {
			return 0
		}
		return limit
	}
	return 0
}