admin/memby
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.2.78

Browse Source
This commit is contained in:
ponzischeme89
2026-08-19 14:25:44 +12:00
parent 2b43b9ef12
commit 590e069366
83 changed files with 8948 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cmd/memby-server/main.go
+50 -39
View File
@@ -32,6 +32,7 @@ import (
"github.com/ponzischeme89/memby/server/internal/notify"
"github.com/ponzischeme89/memby/server/internal/radarr"
"github.com/ponzischeme89/memby/server/internal/recommend"
"github.com/ponzischeme89/memby/server/internal/runtimestats"
"github.com/ponzischeme89/memby/server/internal/scheduler"
"github.com/ponzischeme89/memby/server/internal/sonarr"
"github.com/ponzischeme89/memby/server/internal/store"
@@ -299,11 +300,17 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
return err
}
sched.SetPaused(server.ActivityPaused)
// The ranker calls Tracearr directly on the rebuild path, so it needs the operator's
// switch too — read at call time, because an integration switched off in the console
// must stop being called without a restart. It is wired here rather than where the
// engine is built because the switch belongs to the server, which does not exist yet
// at that point.
recommender.TracearrAllowed = server.TracearrEnabled
dispatcher.SetPaused(server.ActivityPaused)
dispatcher.Start(ctx)
if creditsService != nil {
creditsService.SetPaused(server.ActivityPaused)
go creditsService.Run(ctx)
runtimestats.Go("Credits detection", "Credits detection", func() { creditsService.Run(ctx) })
}
if ingester != nil {
// Quiet time is honoured here rather than at the hook: the webhook is recorded
@@ -313,7 +320,7 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
// is there rather than that it is coming. Installed here for the same reason
// SetAfterSync is: library stays ignorant of what an alert is.
ingester.Announce = server.AnnounceLibraryIngest
go ingester.Run(ctx)
runtimestats.Go("Library ingest", "Library sync", func() { ingester.Run(ctx) })
}
// Registration is separate from construction so the task list reads as a declaration
@@ -321,6 +328,12 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
server.RegisterHousekeeping(sched)
server.RegisterCreditsTasks(sched)
server.RegisterWatchTimeTasks(sched)
server.RegisterRequestTasks(sched)
// Sonarr's daily catalogue reading, Radarr's film refresh, the Tracearr import and the
// For You rebuild all used to be goroutines with tickers of their own here. They are
// scheduler tasks now, which is what gives the console's integrations area a run
// history without a second store — and what lets an operator run any of them by hand.
server.RegisterIntegrationTasks(sched)
sched.Start(ctx)
// Installed after the server exists, because both halves of a finished import are
@@ -344,62 +357,60 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
if err := server.LoadGatewaySettings(ctx); err != nil {
return err
}
go server.WatchGatewaySettings(ctx, 30*time.Second)
go server.WatchMaintenance(ctx, 30*time.Second)
go server.WatchQuietTime(ctx, 30*time.Second)
// Every long-running worker below is started through runtimestats.Go rather than with
// a bare `go`, so the admin console can name what is running instead of reporting a
// count of goroutines nobody can interpret. The name is the worker's identity across
// restarts and appears verbatim in the console, so it is wording rather than a symbol.
runtimestats.Go("Gateway settings watcher", "Gateway settings", func() {
server.WatchGatewaySettings(ctx, 30*time.Second)
})
runtimestats.Go("Maintenance watcher", "Gateway settings", func() {
server.WatchMaintenance(ctx, 30*time.Second)
})
runtimestats.Go("Quiet time watcher", "Gateway settings", func() {
server.WatchQuietTime(ctx, 30*time.Second)
})
// One probe per gateway, not per TV: the answer is the same for the whole house.
go server.WatchEmbyReachability(ctx)
// One Sonarr catalogue reading per day records lifecycle changes for the household and
// materialises cancellation notifications for every known viewer.
go server.WatchSonarrLifecycle(ctx, 24*time.Hour)
runtimestats.Go("Emby health probe", "Emby", func() { server.WatchEmbyReachability(ctx) })
// The trend ring behind the console's runtime page. A single instantaneous goroutine
// or heap figure cannot show a leak; this is what makes one visible.
runtimestats.Go("Runtime sampler", "Gateway API", func() { runtimestats.StartSampling(ctx) })
if err := server.LoadUpdatePolicy(ctx); err != nil {
return err
}
go server.WatchUpdatePolicy(ctx, 60*time.Second)
runtimestats.Go("App update policy watcher", "Gateway settings", func() {
server.WatchUpdatePolicy(ctx, 60*time.Second)
})
go syncer.Schedule(ctx, server.LibrarySyncInterval, server.ActivityPaused)
runtimestats.Go("Library sync schedule", "Library sync", func() {
syncer.Schedule(ctx, server.LibrarySyncInterval, server.ActivityPaused)
})
if cfg.SyncOnStart {
go func() {
runtimestats.Go("Startup library sync", "Library sync", func() {
if server.ActivityPaused() {
return
}
if _, err := syncer.Sync(ctx, "incremental", "startup"); err != nil {
log.Warn("startup sync failed", "error", err)
}
}()
})
}
if forYouService != nil {
go forYouService.Schedule(
ctx,
cfg.TracearrSyncInterval,
cfg.TracearrFullInterval,
cfg.ForYouRebuildHour,
server.ActivityPaused,
)
go func() {
// The import and the daily rebuild are scheduler tasks now — see
// RegisterIntegrationTasks — so what is left here is the one piece of start-up work
// neither of them is: profiles produced by an older algorithm. Only a version change
// warrants it, and normal dirty profiles still wait for the daily off-peak rebuild.
runtimestats.Go("Startup For You migration", "Recommendations", func() {
if server.ActivityPaused() {
return
}
importCtx, cancel := context.WithTimeout(ctx, cfg.SyncTimeout)
// Only if one is actually owed. An unconditional startup import made every
// redeploy or container bounce a fresh pass over Tracearr's history.
if _, _, err := forYouService.ImportIfDue(
importCtx, cfg.TracearrSyncInterval, cfg.TracearrFullInterval,
); err != nil {
cancel()
log.Warn("startup Tracearr import failed", "error", err)
return
}
cancel()
// Only an algorithm-version change warrants startup work. Normal dirty
// profiles wait for the daily off-peak rebuild.
rebuildCtx, rebuildCancel := context.WithTimeout(ctx, cfg.SyncTimeout)
defer rebuildCancel()
rebuildCtx, cancel := context.WithTimeout(ctx, cfg.SyncTimeout)
defer cancel()
if err := forYouService.RebuildOutdated(rebuildCtx); err != nil {
log.Warn("startup outdated For You rebuild failed", "error", err)
}
}()
})
}
httpServer := &http.Server{
Addr: cfg.ListenAddr,
@@ -410,7 +421,7 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
}
errCh := make(chan error, 1)
go func() {
runtimestats.Go("HTTP listener", "HTTP server", func() {
log.Info("gateway ready",
"listen", cfg.ListenAddr,
"emby", cfg.EmbyURL,
@@ -423,7 +434,7 @@ func run(log *slog.Logger, events *logging.Buffer, logLevel *slog.LevelVar) erro
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
errCh <- err
}
}()
})
select {
case err := <-errCh:
server/internal/api/admin.go
+43 -28
View File
@@ -5,10 +5,8 @@ import (
"crypto/subtle"
_ "embed"
"encoding/json"
"io"
"net/http"
"os"
"runtime"
"runtime/debug"
"strconv"
"strings"
"time"
@@ -16,6 +14,7 @@ import (
"github.com/ponzischeme89/memby/server/internal/appupdate"
"github.com/ponzischeme89/memby/server/internal/buildinfo"
"github.com/ponzischeme89/memby/server/internal/library"
"github.com/ponzischeme89/memby/server/internal/runtimestats"
"github.com/ponzischeme89/memby/server/internal/store"
)
@@ -54,6 +53,9 @@ func (s *Server) adminRoutes() http.Handler {
mux.Handle("GET /admin/api/searches", s.adminAuth(s.handleAdminSearches))
mux.Handle("GET /admin/api/events", s.adminAuth(s.handleAdminEvents))
mux.Handle("GET /admin/api/runtime", s.adminAuth(s.handleAdminRuntime))
// The breakdown is its own route because it costs its own money: walking every
// goroutine's stack stops the world, so it is asked for rather than polled.
mux.Handle("GET /admin/api/runtime/goroutines", s.adminAuth(s.handleAdminGoroutines))
mux.Handle("POST /admin/api/sync", s.adminAuth(s.handleAdminSync))
mux.Handle("GET /admin/api/ingest", s.adminAuth(s.handleAdminIngest))
mux.Handle("POST /admin/api/for-you", s.adminAuth(s.handleAdminForYou))
@@ -101,6 +103,18 @@ func (s *Server) adminRoutes() http.Handler {
// abandoned tab's session alive.
mux.Handle("GET /admin/api/notifications/stream", s.adminAuth(s.handleAdminNotificationStream))
// The Integrations area. Deliberately a longer path than /admin/api/integrations,
// which is the older, narrower question — the webhook destinations administrative
// events are posted to. These are the external services Memby depends on.
mux.Handle("GET /admin/api/integrations/services",
s.adminAuth(s.handleAdminIntegrationServices))
mux.Handle("GET /admin/api/integrations/services/{integrationID}",
s.adminAuth(s.handleAdminIntegrationService))
mux.Handle("POST /admin/api/integrations/services/{integrationID}/enabled",
s.adminAuth(s.handleAdminIntegrationEnabled))
mux.Handle("POST /admin/api/integrations/services/{integrationID}/test",
s.adminAuth(s.handleAdminIntegrationTest))
mux.Handle("GET /admin/api/integrations", s.adminAuth(s.handleAdminIntegrations))
mux.Handle("POST /admin/api/integrations", s.adminAuth(s.handleAdminSaveIntegration))
mux.Handle("DELETE /admin/api/integrations/{integrationID}", s.adminAuth(s.handleAdminDeleteIntegration))
@@ -134,33 +148,34 @@ func (s *Server) adminRoutes() http.Handler {
return mux
}
type adminRuntimeStatus struct {
Goroutines int `json:"goroutines"`
GOMAXPROCS int `json:"gomaxprocs"`
HeapAlloc uint64 `json:"heapAlloc"`
HeapInuse uint64 `json:"heapInuse"`
HeapIdle uint64 `json:"heapIdle"`
HeapReleased uint64 `json:"heapReleased"`
StackInuse uint64 `json:"stackInuse"`
Sys uint64 `json:"sys"`
NextGC uint64 `json:"nextGc"`
NumGC uint32 `json:"numGc"`
MemoryLimit int64 `json:"memoryLimit"`
ConfiguredLim string `json:"configuredLimit,omitempty"`
// The console's Process card used to be handed a bare goroutine count, which is a number
// nobody can act on: it says nothing about what those goroutines are doing, which part of
// Memby they belong to, or whether it has been climbing all week. Everything behind these
// two routes lives in internal/runtimestats, including the health verdict, so the
// thresholds and the sentences explaining them sit beside each other and can be tested.
func (s *Server) handleAdminRuntime(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, runtimestats.Read())
}
func (s *Server) handleAdminRuntime(w http.ResponseWriter, _ *http.Request) {
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
// handleAdminGoroutines is the on-demand snapshot. `?format=text` returns the raw dump for
// an operator who wants the whole thing — which is the alternative to leaving a profiling
// endpoint permanently mounted, and unlike one it is behind the admin guard and produces
// nothing until somebody asks.
func (s *Server) handleAdminGoroutines(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, http.StatusOK, adminRuntimeStatus{
Goroutines: runtime.NumGoroutine(), GOMAXPROCS: runtime.GOMAXPROCS(0),
HeapAlloc: memory.HeapAlloc, HeapInuse: memory.HeapInuse,
HeapIdle: memory.HeapIdle, HeapReleased: memory.HeapReleased,
StackInuse: memory.StackInuse, Sys: memory.Sys,
NextGC: memory.NextGC, NumGC: memory.NumGC,
MemoryLimit: debug.SetMemoryLimit(-1), ConfiguredLim: os.Getenv("GOMEMLIMIT"),
})
log := s.loggerFor(r.Context())
if r.URL.Query().Get("format") == "text" {
log.Info("goroutine dump requested", "component", "admin")
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Content-Disposition", "attachment; filename=memby-goroutines.txt")
_, _ = io.WriteString(w, runtimestats.StackDump())
return
}
report := runtimestats.CollectGoroutines()
log.Info("goroutine breakdown collected",
"component", "admin", "goroutines", report.Total, "took_ms", report.CollectedInMs)
writeJSON(w, http.StatusOK, report)
}
func (s *Server) handleAdminEvents(w http.ResponseWriter, r *http.Request) {
@@ -686,7 +701,7 @@ func (s *Server) handleAdminForYou(w http.ResponseWriter, r *http.Request) {
return
}
}
if err := s.forYou.RebuildAll(ctx, true); err != nil {
if _, err := s.forYou.RebuildAll(ctx, true); err != nil {
s.log.Error("manual For You rebuild failed",
"component", "admin", "action", req.Action, "error", err)
}
server/internal/api/admin_integration_services.go
+369
View File
@@ -0,0 +1,369 @@
package api
import (
"context"
"encoding/json"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/ponzischeme89/memby/server/internal/adminevents"
"github.com/ponzischeme89/memby/server/internal/scheduler"
"github.com/ponzischeme89/memby/server/internal/store"
)
// The Integrations area's API.
//
// One service is one row, and the row is assembled from three things the gateway already
// holds: the catalogue (what this service is and where its switch lives), the scheduler
// (what work belongs to it, when it last ran and when it is next due) and the run table
// read along the integration axis (what that work actually did). Nothing here is stored
// for the console's benefit.
//
// The status word is the gateway's, not the console's — the stance every label Memby
// prints takes. A console that decided for itself what "unhealthy" meant would have an
// older build disagreeing with a newer one about the same server, and the threshold and
// the sentence explaining it belong together.
// Integration status words, in the order they outrank each other. See integrationStatus.
const (
integrationStatusUnconfigured = "unconfigured"
integrationStatusDisabled = "disabled"
integrationStatusRunning = "running"
integrationStatusError = "error"
integrationStatusHealthy = "healthy"
integrationStatusIdle = "idle"
)
// integrationView is one service as the console reads it.
type integrationView struct {
ID string `json:"id"`
Name string `json:"name"`
Summary string `json:"summary"`
Address string `json:"address,omitempty"`
Configured bool `json:"configured"`
Enabled bool `json:"enabled"`
// Status is one word and StatusLabel is what to print; Detail is the sentence behind
// it — the failure reason, or what is running, or why there is nothing to say.
Status string `json:"status"`
StatusLabel string `json:"statusLabel"`
Detail string `json:"detail,omitempty"`
Running bool `json:"running"`
// Powers is what stops working when this is switched off, so a dependency is stated
// rather than discovered by something silently not happening.
Powers []string `json:"powers"`
LastRun *store.TaskRun `json:"lastRun,omitempty"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
LastError string `json:"lastError,omitempty"`
NextRun *time.Time `json:"nextRun,omitempty"`
Runs int `json:"runs"`
Failures int `json:"failures"`
// Health is the last reachability probe, absent for a service that is not probed.
Health *integrationHealth `json:"health,omitempty"`
// Probed says whether this service can be probed at all, which is what stops the
// console drawing "never checked" beside MDBList for ever.
Probed bool `json:"probed"`
Tasks []scheduler.Status `json:"tasks"`
Facts []integrationFact `json:"facts,omitempty"`
}
// integrationStatus resolves one word per service, in priority order, because a status
// column with two answers in it is one nobody can scan.
//
// Not configured outranks everything: a service with no address has no switch, no work and
// no health, and drawing it as "off" would suggest turning it on is a thing an operator
// could do from this page. Switched off comes next for the same reason a disabled task
// does — yesterday's success beside a service nobody is running reads as one that is still
// working. A run in flight outranks its own history, and below those the service is
// described by whichever of its probe and its last run has something to say. Idle is the
// honest answer for a configured, enabled service that has not yet done anything: it is
// neither working nor broken, and claiming either would be a guess.
func integrationStatus(view integrationView) (string, string, string) {
switch {
case !view.Configured:
return integrationStatusUnconfigured, "Not configured",
"No address or credential is set for this service."
case !view.Enabled:
return integrationStatusDisabled, "Disabled",
"Switched off: Memby schedules no work for it and makes no requests to it."
case view.Running:
return integrationStatusRunning, "Running", runningDetail(view.Tasks)
}
if view.Health != nil && !view.Health.Reachable && !view.Health.CheckedAt.IsZero() {
return integrationStatusError, "Error", view.Health.Error
}
// A failure that has been followed by a success is history, not a verdict. Without
// that comparison one bad night would leave a service red until the retention job
// eventually removed the row.
if view.LastFailureAt != nil &&
(view.LastSuccessAt == nil || view.LastSuccessAt.Before(*view.LastFailureAt)) {
return integrationStatusError, "Error", view.LastError
}
if view.Health != nil && view.Health.Reachable {
return integrationStatusHealthy, "Healthy", ""
}
if view.LastSuccessAt != nil {
return integrationStatusHealthy, "Healthy", ""
}
return integrationStatusIdle, "Idle", "Nothing has run for this service yet."
}
func runningDetail(tasks []scheduler.Status) string {
for _, task := range tasks {
if task.Running {
return task.Name + " is running now."
}
}
return ""
}
// integrationViews assembles every service. One pass over the catalogue, one scheduler
// snapshot and one grouped query, whatever the catalogue's length.
func (s *Server) integrationViews(ctx context.Context) []integrationView {
summaries := map[string]store.IntegrationRunSummary{}
if s.store != nil {
if read, err := s.store.IntegrationRunSummaries(ctx); err == nil {
summaries = read
} else {
s.loggerFor(ctx).Warn("integration run summaries unavailable", "error", err)
}
}
tasksByIntegration := map[string][]scheduler.Status{}
if s.scheduler != nil {
for _, task := range s.scheduler.Snapshot() {
if task.Integration == "" {
continue
}
tasksByIntegration[task.Integration] = append(
tasksByIntegration[task.Integration], task)
}
}
views := make([]integrationView, 0, len(integrationCatalogue()))
for _, definition := range integrationCatalogue() {
view := integrationView{
ID: definition.ID,
Name: definition.Name,
Summary: definition.Summary,
Powers: definition.Powers,
Configured: definition.Configured(s),
Probed: definition.Probe != nil,
Tasks: tasksByIntegration[definition.ID],
}
if view.Tasks == nil {
view.Tasks = []scheduler.Status{}
}
if view.Configured {
view.Address = definition.Address(s)
view.Enabled = definition.Enabled(s, ctx)
}
if summary, ok := summaries[definition.ID]; ok {
view.LastSuccessAt = summary.LastSuccessAt
view.LastFailureAt = summary.LastFailureAt
view.LastError = summary.LastError
view.Runs, view.Failures = summary.Runs, summary.Failures
}
// The most recent run and the next due one come from the scheduler rather than
// from a query: it holds both, and the next run has no row anywhere to read.
for _, task := range view.Tasks {
if task.Running {
view.Running = true
}
if task.LastRun != nil &&
(view.LastRun == nil || task.LastRun.StartedAt.After(view.LastRun.StartedAt)) {
last := *task.LastRun
view.LastRun = &last
}
if task.NextRun != nil &&
(view.NextRun == nil || task.NextRun.Before(*view.NextRun)) {
next := *task.NextRun
view.NextRun = &next
}
}
// A switched-off service has no next run to promise, whatever the scheduler still
// holds: its tasks stand down as soon as they start.
if !view.Enabled {
view.NextRun = nil
}
if state, ok := s.integrationHealth.get(definition.ID); ok {
health := state
view.Health = &health
}
view.Status, view.StatusLabel, view.Detail = integrationStatus(view)
views = append(views, view)
}
// Anything wrong sorts to the top, the order the tasks table takes and for the same
// reason: a page read from the top down should not need a sort to find the one thing
// that is broken.
sort.SliceStable(views, func(i, j int) bool {
return integrationRank(views[i].Status) < integrationRank(views[j].Status)
})
return views
}
func integrationRank(status string) int {
switch status {
case integrationStatusError:
return 0
case integrationStatusRunning:
return 1
case integrationStatusIdle:
return 2
case integrationStatusHealthy:
return 3
case integrationStatusDisabled:
return 4
default:
return 5
}
}
func (s *Server) handleAdminIntegrationServices(w http.ResponseWriter, r *http.Request) {
views := s.integrationViews(r.Context())
runs, err := s.store.IntegrationRuns(r.Context(), "", integrationRunLimit(r))
if err != nil {
s.loggerFor(r.Context()).Warn("integration runs unavailable", "error", err)
runs = []store.TaskRun{}
}
writeJSON(w, http.StatusOK, map[string]any{
"services": views,
// Every service's runs interleaved, which is the view that shows two integrations
// getting in each other's way — the same reason the tasks page carries one.
"runs": runs,
})
}
func (s *Server) handleAdminIntegrationService(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("integrationID"))
definition, ok := integrationDefinitionFor(id)
if !ok {
writeError(w, http.StatusNotFound, "no such integration")
return
}
var view integrationView
for _, candidate := range s.integrationViews(r.Context()) {
if candidate.ID == id {
view = candidate
break
}
}
// Facts are read only on the detail page: several of them are database queries, and
// the overview draws four services at a poll.
if definition.Facts != nil && view.Configured {
view.Facts = definition.Facts(s, r.Context())
}
runs, err := s.store.IntegrationRuns(r.Context(), id, integrationRunLimit(r))
if err != nil {
s.loggerFor(r.Context()).Warn("integration runs unavailable",
"integration", id, "error", err)
runs = []store.TaskRun{}
}
writeJSON(w, http.StatusOK, map[string]any{"service": view, "runs": runs})
}
func integrationRunLimit(r *http.Request) int {
limit, err := strconv.Atoi(strings.TrimSpace(r.URL.Query().Get("limit")))
if err != nil || limit <= 0 {
return 50
}
return limit
}
func (s *Server) handleAdminIntegrationEnabled(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("integrationID"))
definition, ok := integrationDefinitionFor(id)
if !ok {
writeError(w, http.StatusNotFound, "no such integration")
return
}
if !definition.Configured(s) {
// A service with no address has nothing to switch, and recording a preference
// about one would leave a stored decision nothing reads.
writeError(w, http.StatusConflict,
definition.Name+" is not configured on this gateway")
return
}
var req struct {
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<10)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "malformed request body")
return
}
if err := definition.SetEnabled(s, r.Context(), req.Enabled); err != nil {
s.loggerFor(r.Context()).Error("could not change integration",
"integration", id, "error", err)
writeError(w, http.StatusInternalServerError, "could not save the integration setting")
return
}
if !req.Enabled {
// Its last health reading described a service Memby has stopped calling. Left in
// place it would sit under a disabled row claiming a verdict nothing is renewing.
s.integrationHealth.forget(id)
}
s.loggerFor(r.Context()).Info("integration switched",
"integration", id, "enabled", req.Enabled)
state := "disabled"
if req.Enabled {
state = "enabled"
}
s.publishAdmin(r.Context(), adminevents.Event{
Type: adminevents.TypeSettingsChanged,
Severity: adminevents.SeverityInfo,
Title: definition.Name + " " + state,
Summary: integrationSwitchSummary(definition, req.Enabled),
Target: id,
Link: "/admin/integrations/" + id,
Metadata: adminevents.Meta(map[string]any{"integration": id, "enabled": req.Enabled}),
})
writeJSON(w, http.StatusOK, map[string]any{"services": s.integrationViews(r.Context())})
}
// integrationSwitchSummary spells out the consequence rather than repeating the switch.
// "Sonarr disabled" in an activity feed says nothing an operator did not just do; what
// they may not have thought about is the calendar going with it.
func integrationSwitchSummary(definition integrationDefinition, enabled bool) string {
if enabled {
return definition.Name + " is back on: its scheduled work resumes."
}
if len(definition.Powers) == 0 {
return definition.Name + " will no longer be called."
}
return "Memby will stop calling " + definition.Name +
". This also stops: " + strings.Join(definition.Powers, "; ") + "."
}
// handleAdminIntegrationTest probes one service now.
//
// It takes the same path the scheduled probe does, deliberately: a test that asked a
// different question could report a service as working while the row beside it stayed red.
func (s *Server) handleAdminIntegrationTest(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(r.PathValue("integrationID"))
definition, ok := integrationDefinitionFor(id)
if !ok {
writeError(w, http.StatusNotFound, "no such integration")
return
}
if !definition.Configured(s) {
writeError(w, http.StatusConflict, definition.Name+" is not configured on this gateway")
return
}
if definition.Probe == nil {
// Stated rather than faked. MDBList is the case: a probe would spend a request of
// an allowance bought by the day, and answering "healthy" without asking would be
// the console making something up.
writeError(w, http.StatusNotImplemented,
definition.Name+" cannot be tested without spending part of its daily allowance")
return
}
state := s.probeIntegration(r.Context(), definition)
s.loggerFor(r.Context()).Info("integration probed",
"integration", id, "reachable", state.Reachable, "duration_ms", state.LatencyMS)
writeJSON(w, http.StatusOK, state)
}
server/internal/api/admin_preview_test.go
+123 -3
View File
@@ -116,9 +116,9 @@ func adminPreviewData() map[string]any {
}
return map[string]any{
"/admin/api/status": status,
"/admin/api/runtime": map[string]any{"goroutines": 84, "heapInuse": 41943040,
"sys": 92274688, "numGc": 311, "nextGc": 62914560, "gomaxprocs": 4},
"/admin/api/status": status,
"/admin/api/runtime": adminPreviewRuntime(now),
"/admin/api/runtime/goroutines": adminPreviewGoroutines(now),
"/admin/api/accounts": map[string]any{
"accounts": []any{
map[string]any{"userId": "u-1", "username": "matt", "onboardingCompleted": true,
@@ -191,3 +191,123 @@ func adminPreviewData() map[string]any {
},
}
}
// A gateway with a mild, deliberate upward drift in both goroutines and heap, so the
// preview shows what a trend that has earned a "watch" actually looks like — the flat case
// is the one that draws itself.
func adminPreviewRuntime(now time.Time) map[string]any {
const points = 90
samples := make([]any, 0, points)
for index := 0; index < points; index++ {
at := now.Add(-time.Duration(points-1-index) * time.Minute)
samples = append(samples, map[string]any{
"at": at.Format(time.RFC3339),
"goroutines": 62 + index/4 + index%3,
"heapInuse": 33_000_000 + index*90_000,
"sys": 92_274_688,
"numGc": 220 + index,
})
}
trend := func(direction string, perHour, first, latest, low, high float64) map[string]any {
return map[string]any{
"direction": direction, "perHour": perHour, "first": first, "latest": latest,
"min": low, "max": high, "spanSeconds": float64((points - 1) * 60), "points": points,
}
}
return map[string]any{
"at": now.Format(time.RFC3339), "uptimeSeconds": 61_240.0,
"goroutines": 84, "gomaxprocs": 4, "threads": 17, "goVersion": "go1.26",
"memory": map[string]any{
"heapAlloc": 39_000_000, "heapInuse": 41_943_040, "heapIdle": 24_000_000,
"heapReleased": 8_000_000, "stackInuse": 1_800_000, "sys": 223_100_000,
"nextGc": 62_914_560, "numGc": 311, "pauseTotalMs": 812.4, "pauseRecentMs": 1.9,
"memoryLimit": 402_653_184, "configuredLimit": "384MiB",
"heapShare": 0.104, "gcPerHour": 60.0,
},
"workers": []any{
map[string]any{"name": "HTTP listener", "component": "HTTP server",
"state": "running", "started": now.Add(-17 * time.Hour).Format(time.RFC3339), "starts": 1},
map[string]any{"name": "Emby health probe", "component": "Emby",
"state": "running", "started": now.Add(-17 * time.Hour).Format(time.RFC3339), "starts": 1},
map[string]any{"name": "Library ingest", "component": "Library sync",
"state": "running", "started": now.Add(-17 * time.Hour).Format(time.RFC3339), "starts": 1},
map[string]any{"name": "Task scheduler", "component": "Scheduled jobs",
"state": "running", "started": now.Add(-17 * time.Hour).Format(time.RFC3339), "starts": 1},
map[string]any{"name": "Startup library sync", "component": "Library sync",
"state": "finished", "started": now.Add(-17 * time.Hour).Format(time.RFC3339),
"stopped": now.Add(-17 * time.Hour).Add(4 * time.Minute).Format(time.RFC3339), "starts": 1},
},
"process": map[string]any{"pid": 1, "cpuSeconds": 431.8, "cpuPercent": 3.4,
"cpuKnown": true, "openFiles": 41, "openSockets": 22, "fileLimit": 1048576,
"filesKnown": true},
"samples": samples,
"goroutineTrend": trend("rising", 14, 62, 84, 62, 86),
"heapTrend": trend("rising", 5_400_000, 33_000_000, 41_943_040, 33_000_000, 41_943_040),
"reservedTrend": trend("steady", 0, 92_274_688, 92_274_688, 92_274_688, 92_274_688),
"sampleEverySeconds": 60.0,
"health": map[string]any{
"level": "watch",
"summary": "2 things to look at, in Goroutines and Memory.",
"areas": []any{"Goroutines", "Memory"},
"notes": []any{
map[string]any{"level": "watch", "area": "Memory",
"message": "Heap in use has risen by about 5.1 MB an hour over the last 1.5 hours. If it does not fall after a collection, something is holding on to it."},
map[string]any{"level": "watch", "area": "Goroutines",
"message": "Goroutines have risen by about 14 an hour over the last 1.5 hours, from 62 to 84. Open the breakdown to see which component they belong to."},
},
},
}
}
func adminPreviewGoroutines(now time.Time) map[string]any {
category := func(key, label, description string, count int, states []any) map[string]any {
return map[string]any{"category": key, "label": label, "description": description,
"count": count, "states": states}
}
state := func(name string, count int) any {
return map[string]any{"state": name, "count": count}
}
return map[string]any{
"at": now.Format(time.RFC3339), "total": 84, "collectedInMs": 6.2, "dumpBytes": 98_304,
"categories": []any{
category("running", "Running", "On a processor now, or queued for one.", 3,
[]any{state("running", 2), state("runnable", 1)}),
category("io", "Network / I/O",
"Waiting on a network read or write — Emby, Postgres, Redis, or a television.", 26,
[]any{state("IO wait", 26)}),
category("waiting", "Waiting / idle",
"Parked waiting for work or for a lock. Idle, and costing almost nothing.", 41,
[]any{state("select", 29), state("chan receive", 9), state("semacquire", 3)}),
category("timers", "Timers / scheduled work", "Asleep until a scheduled time.", 8,
[]any{state("sleep", 8)}),
category("runtime", "Go runtime / collection",
"The Go runtime's own housekeeping. This handful is always present.", 6,
[]any{state("GC worker (idle)", 4), state("finalizer wait", 1), state("force gc (idle)", 1)}),
},
"components": []any{
map[string]any{"component": "HTTP server", "count": 24, "longestWaitMinutes": 3},
map[string]any{"component": "Database pool", "count": 18, "longestWaitMinutes": 940},
map[string]any{"component": "Emby", "count": 11, "longestWaitMinutes": 2},
map[string]any{"component": "Gateway API", "count": 9, "longestWaitMinutes": 0},
map[string]any{"component": "Redis", "count": 8, "longestWaitMinutes": 940},
map[string]any{"component": "Library sync", "count": 6, "longestWaitMinutes": 940},
map[string]any{"component": "Go runtime", "count": 6, "longestWaitMinutes": 0},
map[string]any{"component": "Scheduled jobs", "count": 2, "longestWaitMinutes": 940},
},
"groups": []any{
map[string]any{"count": 24, "component": "HTTP server", "category": "io",
"state": "IO wait", "function": "net/http.(*conn).serve",
"file": "/usr/local/go/src/net/http/server.go:2092",
"createdBy": "net/http.(*Server).Serve", "longestWaitMinutes": 3},
map[string]any{"count": 18, "component": "Database pool", "category": "waiting",
"state": "select", "function": "github.com/jackc/puddle/v2.(*Pool).acquire",
"file": "/root/go/pkg/mod/github.com/jackc/puddle/v2/pool.go:481",
"createdBy": "github.com/jackc/pgx/v5/pgxpool.NewWithConfig", "longestWaitMinutes": 940},
map[string]any{"count": 6, "component": "Library sync", "category": "waiting",
"state": "chan receive", "function": "memby/server/internal/library.(*Ingester).Run",
"file": "/app/internal/library/ingest.go:118",
"createdBy": "main.run", "longestWaitMinutes": 940},
},
"groupsTotal": 21,
}
}
server/internal/api/admin_test.go
+42 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/ponzischeme89/memby/server/internal/config"
"github.com/ponzischeme89/memby/server/internal/recommend"
"github.com/ponzischeme89/memby/server/internal/runtimestats"
"github.com/ponzischeme89/memby/server/internal/store"
)
@@ -488,18 +489,57 @@ func TestAdminRuntimeMetricsAreProtectedAndReportHeap(t *testing.T) {
if rec.Code != http.StatusOK {
t.Fatalf("runtime status = %d: %s", rec.Code, rec.Body.String())
}
var body adminRuntimeStatus
var body runtimestats.Snapshot
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if body.Goroutines < 1 || body.HeapInuse == 0 || body.MemoryLimit <= 0 {
if body.Goroutines < 1 || body.Memory.HeapInuse == 0 || body.Memory.MemoryLimit <= 0 {
t.Fatalf("runtime metrics = %+v", body)
}
// The health verdict is the reason this route exists: a count with no verdict beside
// it is the number the console could not explain.
if body.Health.Level == "" || body.Health.Summary == "" {
t.Fatalf("runtime health = %+v", body.Health)
}
if rec.Header().Get("Cache-Control") != "no-store" {
t.Fatalf("cache control = %q", rec.Header().Get("Cache-Control"))
}
}
// The breakdown is the answer to "what are those goroutines doing", so the parts must add
// up to the whole: a category table that quietly drops a state is worse than no table.
func TestAdminGoroutineBreakdownPartitionsTheTotal(t *testing.T) {
server := testServer(config.Config{AdminToken: "secret"})
req := httptest.NewRequest(http.MethodGet, "/admin/api/runtime/goroutines", nil)
req.Header.Set("Authorization", "Bearer secret")
rec := httptest.NewRecorder()
server.adminRoutes().ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("goroutine breakdown = %d: %s", rec.Code, rec.Body.String())
}
var report runtimestats.GoroutineReport
if err := json.Unmarshal(rec.Body.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.Total < 1 {
t.Fatalf("total = %d", report.Total)
}
counted := 0
for _, category := range report.Categories {
counted += category.Count
}
if counted != report.Total {
t.Fatalf("categories total %d, report total %d", counted, report.Total)
}
components := 0
for _, component := range report.Components {
components += component.Count
}
if components != report.Total {
t.Fatalf("components total %d, report total %d", components, report.Total)
}
}
func TestAdminPageEstablishesPersistentCookie(t *testing.T) {
server := testServer(config.Config{
AdminToken: "secret", ReleasePublishToken: "release-secret",
server/internal/api/api.go
+4
View File
@@ -95,6 +95,10 @@ type Server struct {
// ratingsWarm fills and renews the durable rating cache behind the viewer, so a row
// never waits on MDBList and the operator's daily allowance is spent once per title.
ratingsWarm ratingsWarmer
// integrationHealth is what the last reachability probe found for each external
// service. Cached rather than probed on request because the console polls: every open
// tab would otherwise be its own request against somebody's Sonarr.
integrationHealth integrationHealthCache
// alertMu serialises the read-modify-write of the shared alert list. Its producers
// are events — a webhook, a finished sync, a health probe — none of them paced by
// this server, so two can land at once.
server/internal/api/arr_hooks.go
+9
View File
@@ -41,6 +41,15 @@ func (s *Server) handleSonarrWebhook(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnauthorized, "invalid webhook token")
return
}
// A switched-off integration records nothing. Answering 200 rather than refusing is
// deliberate: neither *arr re-delivers a rejection, so a failure here would look to the
// operator like Memby losing imports rather than like the switch they set. The library
// sweep is what reconciles whatever arrives while it is off.
if s.integrationSuppressed(r.Context(), integrationSonarr) {
s.loggerFor(r.Context()).Debug("sonarr webhook ignored: integration switched off")
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "ignored": true})
return
}
var payload library.SonarrWebhook
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&payload); err != nil {
server/internal/api/integration_tasks.go
+425
View File
@@ -0,0 +1,425 @@
package api
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/ponzischeme89/memby/server/internal/mdblist"
"github.com/ponzischeme89/memby/server/internal/scheduler"
"github.com/ponzischeme89/memby/server/internal/store"
)
// The work each external service does on a schedule.
//
// These are ordinary scheduler tasks carrying an Integration id, which is the whole of how
// the integrations area gets a run history: a run is filed against the service as well as
// against the job, and the console reads the scheduler's own table along the other axis.
// There is no second store, no second retention job and no second place a piece of work
// can be recorded as having failed — which was the point of asking whether the existing
// infrastructure could answer before building anything.
//
// Every one of them begins by asking whether its integration is switched on, and reports
// a *skipped* run rather than an error when it is not. That distinction is the feature:
// "Memby did not run this" and "Memby ran this and it failed" look identical from a page
// that only records failures, and an operator who has switched something off is entitled
// to see the schedule quietly standing down rather than a red row every hour.
//
// The counters are the four in store.RunCounts and they mean the same thing everywhere:
// processed is what the run looked at, changed is what it wrote, skipped is what it
// deliberately passed over, failed is what went wrong without stopping the run.
// ratingsRefreshBatch bounds one MDBList refresh run.
//
// It is a cost decision rather than a throughput one: MDBList is bought by the day, this
// runs hourly, and the batch times the cadence is what the household spends on renewals
// before a single television has asked for anything. Forty an hour renews a thousand-title
// library about once a day while leaving most of the allowance for titles somebody is
// actually looking at.
const ratingsRefreshBatch = 40
// RegisterIntegrationTasks declares the background work belonging to external services.
func (s *Server) RegisterIntegrationTasks(sched *scheduler.Scheduler) {
if sched == nil {
return
}
sched.Register(scheduler.Task{
ID: "sonarr-lifecycle",
Name: "Series lifecycle scan",
Group: "Sonarr and Radarr",
Integration: integrationSonarr,
Description: "Reads Sonarr's catalogue and records which shows have been added, " +
"have returned or have been cancelled since the last reading.",
Interval: 24 * time.Hour,
Timeout: 10 * time.Minute,
// It ran on start-up before it was a task and still should: the history it keeps
// is a record of transitions, and a gateway that was down for a week has a week of
// catching up to do before it can tell one from a first sighting.
RunOnStart: true,
Work: s.runSonarrLifecycleScan,
})
sched.Register(scheduler.Task{
ID: "radarr-catalogue",
Name: "Film catalogue refresh",
Group: "Sonarr and Radarr",
Integration: integrationRadarr,
Description: "Re-reads Radarr's film catalogue, which is what the upcoming releases " +
"row, the request pages and the Radarr-only film pages are all served from.",
Interval: 6 * time.Hour,
Timeout: 5 * time.Minute,
Work: s.runRadarrCatalogueRefresh,
})
sched.Register(scheduler.Task{
ID: "tracearr-import",
Name: "Watch history import",
Group: "Recommendations",
Integration: integrationTracearr,
Description: "Brings the household's watch history across from Tracearr, which is " +
"what For You rows and watch-time summaries are built from.",
// The cadence here only decides how often the question is asked. What is actually
// due is decided by the persisted import stamps inside ImportIfDue, so a container
// restarted three times in an evening still imports on the configured schedule
// rather than three times over.
Interval: 15 * time.Minute,
Timeout: 30 * time.Minute,
Work: s.runTracearrImport,
})
sched.Register(scheduler.Task{
ID: "for-you-rebuild",
Name: "Recommendation rebuild",
Group: "Recommendations",
Integration: integrationTracearr,
Description: "Rebuilds every viewer's prepared For You rows from the imported " +
"watch history. Runs once a day at the household's configured hour.",
// Daily, and the hour is the operator's: the interval decides only how often the
// question is asked, and the rebuild is skipped unless the household's chosen hour
// has come round since the last one. Rebuilding at whatever time the container
// happened to start is what it did before, which on a redeploy meant the heaviest
// job in the gateway ran in the middle of the evening.
Interval: time.Hour,
Timeout: 30 * time.Minute,
Work: s.runForYouRebuild,
})
sched.Register(scheduler.Task{
ID: "mdblist-ratings-refresh",
Name: "Ratings refresh",
Group: "Ratings",
Integration: integrationMDBList,
Description: fmt.Sprintf(
"Renews up to %d of the oldest stored review scores. Titles nobody has looked "+
"at yet are fetched by the warmer on demand rather than here.",
ratingsRefreshBatch),
Interval: time.Hour,
Timeout: 10 * time.Minute,
Work: s.runRatingsRefresh,
})
// Not filed against any integration, deliberately. It touches all of them, so a run
// per probe cycle under each service's history would bury the runs that say what that
// service actually did under twelve reachability checks an hour.
sched.Register(scheduler.Task{
ID: "integration-health",
Name: "Integration health check",
Group: "System",
Description: "Asks each configured external service whether it is answering.",
Interval: integrationProbeInterval,
Timeout: 2 * time.Minute,
RunOnStart: true,
Work: s.runIntegrationHealthCheck,
})
}
// integrationOff is the outcome every integration task returns when its switch is off.
//
// A skipped run rather than an error or silence: silence is indistinguishable from a
// scheduler that has stopped, and an error would put a red row in the console for a state
// the operator chose. The sentence names the service so the row reads correctly in the
// all-tasks table, where the integration column is not necessarily beside it.
func integrationOff(name string) (scheduler.Outcome, error) {
return scheduler.Outcome{Detail: name + " is switched off"}, nil
}
func (s *Server) runSonarrLifecycleScan(ctx context.Context) (scheduler.Outcome, error) {
if !s.integrationEnabled(ctx, integrationSonarr) {
return integrationOff("Sonarr")
}
result, err := s.scanSonarrLifecycle(ctx)
if err != nil {
return scheduler.Outcome{}, err
}
outcome := scheduler.Outcome{RunCounts: store.RunCounts{
Processed: result.Series,
Changed: result.Changes,
}}
// Silent when nothing moved, which on a settled household is most days. See
// scheduler.announce: a job that reports itself every day is one nobody reads.
if result.Changes == 0 {
return outcome, nil
}
parts := []string{plural(result.Changes, "change", "changes")}
if result.Added > 0 {
parts = append(parts, fmt.Sprintf("%d added", result.Added))
}
if result.Cancelled > 0 {
parts = append(parts, fmt.Sprintf("%d cancelled", result.Cancelled))
}
if result.Notifications > 0 {
parts = append(parts, plural(result.Notifications, "notification", "notifications"))
}
outcome.Detail = fmt.Sprintf("%s checked · %s",
plural(result.Series, "series", "series"), strings.Join(parts, ", "))
return outcome, nil
}
func (s *Server) runRadarrCatalogueRefresh(ctx context.Context) (scheduler.Outcome, error) {
if !s.integrationEnabled(ctx, integrationRadarr) {
return integrationOff("Radarr")
}
// Straight to Radarr rather than through radarrMovieCatalogue: that reads the shared
// cache first, and a refresh whose whole job is to replace the cache must not be
// satisfied by it.
movies, err := s.radarr.Movies(ctx)
if err != nil {
return scheduler.Outcome{}, fmt.Errorf("read Radarr catalogue: %w", err)
}
held, upcoming := 0, 0
now := time.Now()
for _, movie := range movies {
if movie.HasFile {
held++
continue
}
// Upcoming is "monitored and not yet available", which is what the launcher's
// releases row draws from — an unmonitored film Radarr is not chasing is not on
// its way to anybody.
if movie.Monitored {
if movie.DigitalRelease == nil || movie.DigitalRelease.After(now) {
upcoming++
}
}
}
s.cacheRadarrMovies(ctx, movies)
return scheduler.Outcome{
Detail: fmt.Sprintf("%s checked · %d in the library, %d still to come",
plural(len(movies), "film", "films"), held, upcoming),
RunCounts: store.RunCounts{Processed: len(movies), Changed: held, Skipped: upcoming},
}, nil
}
func (s *Server) runTracearrImport(ctx context.Context) (scheduler.Outcome, error) {
if !s.integrationEnabled(ctx, integrationTracearr) {
return integrationOff("Tracearr")
}
if s.forYou.Running() {
// A rebuild started by an operator is already using it. Skipping is the right
// answer rather than queueing: the next tick is fifteen minutes away and the
// import is idempotent.
return scheduler.Outcome{Detail: "a For You rebuild is already running"}, nil
}
result, imported, err := s.forYou.ImportIfDue(
ctx, s.cfg.TracearrSyncInterval, s.cfg.TracearrFullInterval)
if err != nil {
return scheduler.Outcome{}, fmt.Errorf("Tracearr import: %w", err)
}
if !imported {
// Not due. Silent, because this is asked four times an hour and answered "no"
// almost every time.
return scheduler.Outcome{}, nil
}
outcome := scheduler.Outcome{RunCounts: store.RunCounts{
Processed: result.Seen,
Changed: result.Changed,
}}
if result.Seen == 0 && result.Changed == 0 {
return outcome, nil
}
outcome.Detail = fmt.Sprintf("%s import · %s read, %d changed",
result.Kind, plural(result.Seen, "session", "sessions"), result.Changed)
return outcome, nil
}
// forYouRebuildDue reports whether the household's daily rebuild hour has come round
// since the last one.
//
// Pure so the boundary can be tested: the two ways to get this wrong are both invisible
// from a log — never rebuilding, and rebuilding on every tick — and both look like a
// working scheduler from outside. A rebuild that has never happened is due, so a fresh
// household does not wait until tomorrow evening for its first rows.
func forYouRebuildDue(last *time.Time, now time.Time, hour int) bool {
if hour < 0 || hour > 23 {
hour = 0
}
if now.Hour() < hour {
return false
}
if last == nil {
return true
}
// The comparison is against the local calendar day rather than "24 hours ago", the
// rule every household-local boundary in Memby follows: a day containing a
// daylight-saving change is 23 or 25 hours long, and subtracting hours would skip or
// repeat a rebuild twice a year.
previous := last.In(now.Location())
sameDay := previous.Year() == now.Year() && previous.YearDay() == now.YearDay()
return !sameDay
}
func (s *Server) runForYouRebuild(ctx context.Context) (scheduler.Outcome, error) {
if !s.integrationEnabled(ctx, integrationTracearr) {
return integrationOff("Tracearr")
}
if s.forYou.Running() {
return scheduler.Outcome{Detail: "a For You rebuild is already running"}, nil
}
now := time.Now().In(s.householdLocation())
state, err := s.store.TracearrImportState(ctx)
if err != nil {
return scheduler.Outcome{}, fmt.Errorf("read For You rebuild state: %w", err)
}
if !forYouRebuildDue(state.LastRebuildAt, now, s.cfg.ForYouRebuildHour) {
return scheduler.Outcome{}, nil
}
result, err := s.forYou.RebuildAll(ctx, true)
if err != nil {
return scheduler.Outcome{}, fmt.Errorf("For You rebuild: %w", err)
}
if markErr := s.store.MarkForYouRebuild(ctx, now); markErr != nil {
// The rebuild happened; failing the run over the stamp would have the console
// report a failure for work that succeeded. It does mean the next tick tries
// again, which is the safe direction to be wrong in.
s.loggerFor(ctx).Warn("could not record For You rebuild", "error", markErr)
}
outcome := scheduler.Outcome{RunCounts: store.RunCounts{
Processed: result.Users, Changed: result.Built, Failed: result.Failed,
}}
if result.Users == 0 {
return outcome, nil
}
outcome.Detail = fmt.Sprintf("%s processed · %d rebuilt",
plural(result.Users, "viewer", "viewers"), result.Built)
if result.Failed > 0 {
outcome.Detail += fmt.Sprintf(", %d failed", result.Failed)
}
return outcome, nil
}
func (s *Server) runRatingsRefresh(ctx context.Context) (scheduler.Outcome, error) {
settings, enabled := s.mdblistSettings(ctx)
if !enabled || !s.integrationEnabled(ctx, integrationMDBList) {
return integrationOff("MDBList")
}
keys, err := s.store.StaleRatingKeys(
ctx, time.Now().Add(-ratingsRefreshInterval), ratingsRefreshBatch)
if err != nil {
return scheduler.Outcome{}, fmt.Errorf("read stale ratings: %w", err)
}
if len(keys) == 0 {
return scheduler.Outcome{}, nil
}
counts := store.RunCounts{}
var lastErr error
for _, key := range keys {
if ctx.Err() != nil {
break
}
// The same daily allowance the on-demand warmer spends from, claimed the same way.
// Two counters would let a quiet evening of browsing and a refresh run each spend a
// full day's worth between them.
if !s.claimRatingsBudget(time.Now()) {
counts.Skipped++
continue
}
counts.Processed++
ratings, fetchErr := s.fetchAndStoreRatings(ctx, settings.APIKey, key)
if fetchErr != nil {
counts.Failed++
lastErr = fetchErr
var apiErr *mdblist.APIError
if errors.As(fetchErr, &apiErr) &&
(apiErr.StatusCode == 429 || apiErr.StatusCode == 402) {
// The allowance is exhausted. Stopping the whole run is the point: the
// remaining titles are still stale and will be the oldest next hour, and
// hammering a provider that has just said no is how a key gets withdrawn.
s.blockRatingsWarming(time.Now().Add(ratingsWarmBackoff))
break
}
continue
}
if len(ratings) == 0 {
// A title MDBList has nothing for. Recorded rather than counted as changed:
// the stamp moved so it is not re-fetched immediately, but nothing on a
// television will look different.
counts.Skipped++
continue
}
counts.Changed++
}
outcome := scheduler.Outcome{RunCounts: counts}
if counts.Processed == 0 && counts.Skipped == 0 {
return outcome, nil
}
outcome.Detail = fmt.Sprintf("%s requested · %d updated, %d unavailable",
plural(counts.Processed, "title", "titles"), counts.Changed, counts.Skipped)
// A run that failed every title it asked for is a failed run, not a quiet one: that is
// a wrong key or a provider that is down, and it belongs in the notification feed. A
// run that lost a few titles among many is ordinary and stays a detail.
if counts.Failed > 0 && counts.Failed == counts.Processed && lastErr != nil {
return outcome, fmt.Errorf("every MDBList request failed: %w", lastErr)
}
if counts.Failed > 0 {
outcome.Detail += fmt.Sprintf(", %d failed", counts.Failed)
}
return outcome, nil
}
// runIntegrationHealthCheck probes every configured service.
//
// A probe failure is not this task's failure: the whole job is to find out, and a red row
// on the health check would say the gateway's own scheduler is broken when what is
// actually broken is somebody's Sonarr. The verdict lives on the integration's row
// instead, where it names the service.
func (s *Server) runIntegrationHealthCheck(ctx context.Context) (scheduler.Outcome, error) {
counts := store.RunCounts{}
unreachable := []string{}
for _, definition := range integrationCatalogue() {
if definition.Probe == nil || !definition.Configured(s) {
continue
}
if !definition.Enabled(s, ctx) {
// A switched-off service is not probed at all. Probing one would be the
// gateway continuing to call an API the operator has told it to stop calling,
// which is exactly what the switch promises not to do.
s.integrationHealth.forget(definition.ID)
counts.Skipped++
continue
}
counts.Processed++
if state := s.probeIntegration(ctx, definition); !state.Reachable {
counts.Failed++
unreachable = append(unreachable, definition.Name)
}
}
outcome := scheduler.Outcome{RunCounts: counts}
if len(unreachable) > 0 {
outcome.Detail = "not answering: " + strings.Join(unreachable, ", ")
}
return outcome, nil
}
// plural is the "3 films" / "1 film" the run details are written in. The console prints
// these sentences verbatim, so getting it wrong is visible on every row.
func plural(count int, singular, many string) string {
if count == 1 {
return "1 " + singular
}
return fmt.Sprintf("%d %s", count, many)
}
server/internal/api/integrations_catalogue.go
+427
View File
@@ -0,0 +1,427 @@
package api
import (
"context"
"fmt"
"net/url"
"strings"
"sync"
"time"
"github.com/ponzischeme89/memby/server/internal/store"
)
// The gateway's external services, as the console's Integrations area reads them.
//
// One catalogue in Go, the shape featureCatalogue and preferenceCatalogue already take:
// what an integration *is* — what it powers, whether it can be probed, where its switch is
// stored — belongs to the code, and only the operator's choices belong in the database.
// Adding a fifth service is an entry here plus its scheduler task, and the console page
// needs no change at all.
//
// Two decisions in here are worth stating because they look like omissions otherwise.
//
// **The switch is stored where the integration's other configuration already lives.**
// Sonarr and Radarr keep theirs in the arr integration policy, MDBList keeps its own in
// the ratings settings, and Tracearr — which had nowhere — uses the integration policy
// document. Consolidating them would mean a migration and, for the length of it, two
// documents that could disagree about whether a service was on. What the console needs is
// one reader and one writer, which is what integrationEnabled and setIntegrationEnabled
// are; which document answers is nobody else's business.
//
// **MDBList has no live probe.** Every other service here is a machine in the house and
// pinging it costs nothing. MDBList is an allowance bought by the day, and spending a
// request of it to draw a green dot on a page that polls would be the console competing
// with the televisions for the thing it is reporting on. Its health comes from its own run
// history instead, which is the honest answer and the cheaper one.
const (
integrationSonarr = "sonarr"
integrationRadarr = "radarr"
integrationTracearr = "tracearr"
integrationMDBList = "mdblist"
)
// integrationDefinition is one external service.
type integrationDefinition struct {
ID string
Name string
// Summary is what the service is for, in one sentence, from Memby's point of view
// rather than the vendor's.
Summary string
// Powers names the parts of Memby that stop working when this is switched off. It is
// the "make the dependency clear rather than silently failing" half: an operator
// turning Sonarr off is entitled to know the television calendar goes with it.
Powers []string
// Configured reports whether this deployment has an address and a credential for the
// service at all. A service that is not configured has nothing to switch.
Configured func(s *Server) bool
// Address is the service's location, for the console to print. Never a credential —
// the API key is part of neither the URL nor this string.
Address func(s *Server) string
// Enabled and SetEnabled are the operator's global switch. See the note above about
// where each one is stored.
Enabled func(s *Server, ctx context.Context) bool
SetEnabled func(s *Server, ctx context.Context, enabled bool) error
// Probe asks the service whether it is answering. Nil where asking costs something
// that should not be spent on a status page.
Probe func(s *Server, ctx context.Context) error
// Facts are the configuration lines the detail page prints. Deliberately a function
// of the live server rather than stored text, so a page cannot describe a deployment
// that has since been reconfigured.
Facts func(s *Server, ctx context.Context) []integrationFact
}
// integrationFact is one label-and-value line of an integration's configuration.
type integrationFact struct {
Label string `json:"label"`
Value string `json:"value"`
// Tone lets a fact carry a verdict where it has one — an unset API key is not
// neutral. Empty is the ordinary, judgement-free case.
Tone string `json:"tone,omitempty"`
}
func integrationCatalogue() []integrationDefinition {
return []integrationDefinition{
{
ID: integrationSonarr,
Name: "Sonarr",
Summary: "Follows television: what has been imported, what is still to air, and which shows have ended.",
Powers: []string{
"The television calendar and the launcher's airing-soon row",
"Series requests from a television",
"Cancellation and returning-show notifications",
"Import announcements for newly downloaded episodes",
},
Configured: func(s *Server) bool { return s.sonarr != nil },
Address: func(s *Server) string { return serviceAddress(s.cfg.SonarrURL) },
Enabled: func(s *Server, ctx context.Context) bool {
policy, err := s.store.ArrIntegrationPolicy(ctx)
if err != nil {
return false
}
return policy.SonarrEnabled
},
SetEnabled: func(s *Server, ctx context.Context, enabled bool) error {
policy, err := s.store.ArrIntegrationPolicy(ctx)
if err != nil {
policy = store.DefaultArrIntegrationPolicy()
}
policy.SonarrEnabled = enabled
return s.store.SetArrIntegrationPolicy(ctx, policy)
},
Probe: func(s *Server, ctx context.Context) error {
// Quality profiles rather than the series list: it is a handful of rows on
// any household, where the catalogue is thousands and would make the
// health check the most expensive thing on the page.
_, err := s.sonarr.QualityProfiles(ctx)
return err
},
Facts: func(s *Server, ctx context.Context) []integrationFact {
return []integrationFact{
{Label: "Address", Value: serviceAddress(s.cfg.SonarrURL)},
{Label: "API key", Value: credentialState(s.cfg.SonarrAPIKey),
Tone: credentialTone(s.cfg.SonarrAPIKey)},
{Label: "Import webhook", Value: credentialState(s.cfg.SonarrWebhookToken),
Tone: credentialTone(s.cfg.SonarrWebhookToken)},
}
},
},
{
ID: integrationRadarr,
Name: "Radarr",
Summary: "Follows films: what the household holds, what is on the way, and when a release becomes watchable.",
Powers: []string{
"The launcher's upcoming releases row and the Radarr-only film pages",
"Film requests from a television",
"Digital release dates, which the home hero is ranked by",
"Import announcements for newly downloaded films",
},
Configured: func(s *Server) bool { return s.radarr != nil },
Address: func(s *Server) string { return serviceAddress(s.cfg.RadarrURL) },
Enabled: func(s *Server, ctx context.Context) bool {
policy, err := s.store.ArrIntegrationPolicy(ctx)
if err != nil {
return false
}
return policy.RadarrEnabled
},
SetEnabled: func(s *Server, ctx context.Context, enabled bool) error {
policy, err := s.store.ArrIntegrationPolicy(ctx)
if err != nil {
policy = store.DefaultArrIntegrationPolicy()
}
policy.RadarrEnabled = enabled
return s.store.SetArrIntegrationPolicy(ctx, policy)
},
Probe: func(s *Server, ctx context.Context) error {
_, err := s.radarr.QualityProfiles(ctx)
return err
},
Facts: func(s *Server, ctx context.Context) []integrationFact {
return []integrationFact{
{Label: "Address", Value: serviceAddress(s.cfg.RadarrURL)},
{Label: "API key", Value: credentialState(s.cfg.RadarrAPIKey),
Tone: credentialTone(s.cfg.RadarrAPIKey)},
{Label: "Import webhook", Value: credentialState(s.cfg.RadarrWebhookToken),
Tone: credentialTone(s.cfg.RadarrWebhookToken)},
}
},
},
{
ID: integrationTracearr,
Name: "Tracearr",
Summary: "The household's real watch history, which is what personalised rows and watch-time summaries are built from.",
Powers: []string{
"For You rows and the recommendation profiles behind them",
"Weekly and monthly watch-time summaries",
"Watch time on the console's user pages",
},
Configured: func(s *Server) bool { return s.forYou != nil },
Address: func(s *Server) string { return serviceAddress(s.cfg.TracearrURL) },
Enabled: func(s *Server, ctx context.Context) bool {
policy, err := s.store.IntegrationPolicy(ctx)
if err != nil {
return false
}
return policy.Enabled(integrationTracearr)
},
SetEnabled: func(s *Server, ctx context.Context, enabled bool) error {
return s.store.SetIntegrationEnabled(ctx, integrationTracearr, enabled)
},
Probe: func(s *Server, ctx context.Context) error { return s.forYou.Ping(ctx) },
Facts: func(s *Server, ctx context.Context) []integrationFact {
facts := []integrationFact{
{Label: "Address", Value: serviceAddress(s.cfg.TracearrURL)},
{Label: "API key", Value: credentialState(s.cfg.TracearrAPIKey),
Tone: credentialTone(s.cfg.TracearrAPIKey)},
}
if s.cfg.TracearrServerID != "" {
facts = append(facts,
integrationFact{Label: "Server", Value: s.cfg.TracearrServerID})
}
if state, err := s.store.TracearrImportState(ctx); err == nil {
facts = append(facts, integrationFact{
Label: "Last full import", Value: importStamp(state.LastFullAt),
}, integrationFact{
Label: "Last incremental import", Value: importStamp(state.LastIncrementalAt),
})
}
return facts
},
},
{
ID: integrationMDBList,
Name: "MDBList",
Summary: "External review scores. Bought by the day, so a title is fetched once and kept.",
Powers: []string{
"The ratings strip on detail pages and cards",
"Score weighting in the home hero's ranking",
},
Configured: func(s *Server) bool { return s.mdblist != nil },
Address: func(s *Server) string { return "mdblist.com" },
Enabled: func(s *Server, ctx context.Context) bool {
settings, err := s.store.MDBListSettings(ctx)
if err != nil {
return false
}
return settings.Enabled
},
SetEnabled: func(s *Server, ctx context.Context, enabled bool) error {
settings, err := s.store.MDBListSettings(ctx)
if err != nil {
settings = store.DefaultMDBListSettings()
}
settings.Enabled = enabled
if err := s.store.SetMDBListSettings(ctx, settings); err != nil {
return err
}
// The ratings path caches this document for thirty seconds; without this
// the switch appears not to have worked for half a minute.
s.forgetMDBListSettings()
return nil
},
// Deliberately no probe. See the note at the top of this file.
Facts: func(s *Server, ctx context.Context) []integrationFact {
settings, err := s.store.MDBListSettings(ctx)
if err != nil {
settings = store.DefaultMDBListSettings()
}
facts := []integrationFact{
{Label: "API key", Value: credentialState(settings.APIKey),
Tone: credentialTone(settings.APIKey)},
{Label: "Sources shown", Value: fmt.Sprint(len(settings.Sources))},
}
total, stale, statsErr := s.store.MediaRatingsStats(
ctx, time.Now().Add(-ratingsRefreshInterval))
if statsErr == nil {
facts = append(facts,
integrationFact{Label: "Titles stored", Value: fmt.Sprint(total)},
integrationFact{Label: "Due to be re-checked", Value: fmt.Sprint(stale)})
}
return facts
},
},
}
}
func integrationDefinitionFor(id string) (integrationDefinition, bool) {
for _, definition := range integrationCatalogue() {
if definition.ID == id {
return definition, true
}
}
return integrationDefinition{}, false
}
// integrationEnabled is the single reader of an integration's global switch.
//
// It answers false for a service this deployment has not configured, which is what every
// caller means by the question: "may I use Sonarr" and "is Sonarr switched on" are only
// different questions to somebody looking at the console.
func (s *Server) integrationEnabled(ctx context.Context, id string) bool {
definition, ok := integrationDefinitionFor(id)
if !ok || s.store == nil || !definition.Configured(s) {
return false
}
return definition.Enabled(s, ctx)
}
// integrationSuppressed reports that the operator has explicitly switched a service off.
//
// Deliberately not the negation of integrationEnabled, which is also false for a service
// this deployment never configured. The difference matters at the import webhooks: a
// household can perfectly well point Sonarr's notification at Memby without giving Memby
// Sonarr's API key, and reading "no API credentials" as "the operator turned this off"
// would silently stop recording their imports.
func (s *Server) integrationSuppressed(ctx context.Context, id string) bool {
definition, ok := integrationDefinitionFor(id)
if !ok || s.store == nil {
return false
}
return !definition.Enabled(s, ctx)
}
// TracearrEnabled is the Tracearr switch, exported for the pieces of the gateway that are
// built outside this package and still have to honour it — today the recommendation
// engine, which calls Tracearr on the rebuild path.
func (s *Server) TracearrEnabled(ctx context.Context) bool {
return s.integrationEnabled(ctx, integrationTracearr)
}
// serviceAddress is a service's location with anything credential-shaped removed.
//
// An *arr address is ordinarily a bare host and port, but nothing stops an operator
// putting one behind basic auth, and a console that printed the URL verbatim would put
// that password on a page. Scheme, host and path only.
func serviceAddress(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
parsed, err := url.Parse(raw)
if err != nil || parsed.Host == "" {
return raw
}
parsed.User = nil
parsed.RawQuery = ""
parsed.Fragment = ""
return strings.TrimSuffix(parsed.String(), "/")
}
// credentialState says whether a credential is set and never what it is. The console has
// no use for the value and every reason not to hold it — the stance the MDBList key and
// the OpenSubtitles login already take.
func credentialState(value string) string {
if strings.TrimSpace(value) == "" {
return "not set"
}
return "saved"
}
func credentialTone(value string) string {
if strings.TrimSpace(value) == "" {
return "warn"
}
return "ok"
}
func importStamp(at *time.Time) string {
if at == nil {
return "never"
}
return at.UTC().Format(time.RFC3339)
}
// --- health -----------------------------------------------------------------------
// integrationProbeInterval is how often the health task asks each service whether it is
// answering. Five minutes is chosen against what a probe is worth rather than against what
// it costs: a service that has just gone away is news, and being up to five minutes late
// with it is invisible next to the hourly jobs that would otherwise be the first thing to
// notice. It is also the reason the console does not probe on page load — every open tab
// would be its own request against somebody's Sonarr.
const integrationProbeInterval = 5 * time.Minute
// integrationHealth is what the last probe found.
type integrationHealth struct {
Reachable bool `json:"reachable"`
CheckedAt time.Time `json:"checkedAt"`
Error string `json:"error,omitempty"`
// LatencyMS is how long the service took to answer, which is the difference between
// "working" and "working, and it is why the launcher is slow".
LatencyMS int64 `json:"latencyMs"`
}
type integrationHealthCache struct {
mu sync.RWMutex
states map[string]integrationHealth
}
func (c *integrationHealthCache) get(id string) (integrationHealth, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
state, ok := c.states[id]
return state, ok
}
func (c *integrationHealthCache) set(id string, state integrationHealth) {
c.mu.Lock()
defer c.mu.Unlock()
if c.states == nil {
c.states = map[string]integrationHealth{}
}
c.states[id] = state
}
func (c *integrationHealthCache) forget(id string) {
c.mu.Lock()
defer c.mu.Unlock()
delete(c.states, id)
}
// probeIntegration asks one service whether it is answering and records what it found.
//
// It is also what the console's Test button calls, which is deliberate: a test that took a
// different path from the scheduled probe could report a service as working while the page
// beside it stayed red.
func (s *Server) probeIntegration(
ctx context.Context, definition integrationDefinition,
) integrationHealth {
if definition.Probe == nil || !definition.Configured(s) {
return integrationHealth{}
}
probeCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
started := time.Now()
err := definition.Probe(s, probeCtx)
state := integrationHealth{
CheckedAt: time.Now().UTC(),
LatencyMS: time.Since(started).Milliseconds(),
Reachable: err == nil,
}
if err != nil {
state.Error = err.Error()
}
s.integrationHealth.set(definition.ID, state)
return state
}
server/internal/api/integrations_status_test.go
+145
View File
@@ -0,0 +1,145 @@
package api
import (
"testing"
"time"
)
// The two rules in the integrations area that are pure, and therefore the two that can be
// pinned without a database, a Sonarr or a clock. Both are the kind of judgement whose
// failures are invisible from outside: a status word that is merely wrong still renders,
// and a rebuild that never becomes due looks exactly like a working scheduler.
func TestIntegrationStatusResolvesInPriorityOrder(t *testing.T) {
past := time.Now().Add(-time.Hour)
recent := time.Now().Add(-time.Minute)
cases := []struct {
name string
view integrationView
want string
}{
{
// Not configured outranks everything, including a switch left on from a
// deployment that used to have an address for it.
name: "unconfigured outranks enabled",
view: integrationView{Configured: false, Enabled: true},
want: integrationStatusUnconfigured,
},
{
// Switched off outranks its own history: yesterday's success beside a service
// nobody is running reads as one that is still working.
name: "disabled outranks a past failure",
view: integrationView{
Configured: true, Enabled: false, LastFailureAt: &past, LastError: "boom",
},
want: integrationStatusDisabled,
},
{
name: "running outranks a past failure",
view: integrationView{
Configured: true, Enabled: true, Running: true, LastFailureAt: &past,
},
want: integrationStatusRunning,
},
{
// A probe that could not reach the service is the most current evidence there
// is, and outranks a run that happened to succeed before it went away.
name: "an unreachable probe beats an older success",
view: integrationView{
Configured: true, Enabled: true, LastSuccessAt: &past,
Health: &integrationHealth{CheckedAt: recent, Error: "connection refused"},
},
want: integrationStatusError,
},
{
// The comparison that stops one bad night leaving a service red for a month.
name: "a failure followed by a success is history",
view: integrationView{
Configured: true, Enabled: true,
LastFailureAt: &past, LastSuccessAt: &recent,
},
want: integrationStatusHealthy,
},
{
name: "a failure with no success since is the verdict",
view: integrationView{
Configured: true, Enabled: true,
LastSuccessAt: &past, LastFailureAt: &recent, LastError: "401",
},
want: integrationStatusError,
},
{
// Neither working nor broken. Claiming either would be a guess, and "healthy"
// on a service that has never done anything is the more damaging guess.
name: "nothing has run yet",
view: integrationView{Configured: true, Enabled: true},
want: integrationStatusIdle,
},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
status, label, _ := integrationStatus(testCase.view)
if status != testCase.want {
t.Fatalf("status = %q, want %q", status, testCase.want)
}
if label == "" {
t.Fatal("every status must carry something to print")
}
})
}
}
func TestIntegrationStatusExplainsEveryUnhappyAnswer(t *testing.T) {
// The console prints this sentence and nothing else says why. An unconfigured or
// disabled row with no explanation is one an operator has to guess at.
for _, view := range []integrationView{
{Configured: false},
{Configured: true, Enabled: false},
{Configured: true, Enabled: true},
} {
if _, _, detail := integrationStatus(view); detail == "" {
t.Fatalf("no detail for %+v", view)
}
}
}
func TestForYouRebuildDue(t *testing.T) {
location := time.UTC
at := func(day, hour int) time.Time {
return time.Date(2026, time.August, day, hour, 30, 0, 0, location)
}
// A household that has never rebuilt does not wait until tomorrow evening for its
// first rows — but it does wait for the hour it was told to use.
if forYouRebuildDue(nil, at(19, 2), 4) {
t.Fatal("rebuilt before the household's hour")
}
if !forYouRebuildDue(nil, at(19, 4), 4) {
t.Fatal("a household that has never rebuilt is due at its hour")
}
// Once today's rebuild has happened it is not due again today, however many times the
// hourly task asks. This is the whole of what stops the heaviest job in the gateway
// running twenty times an evening.
today := at(19, 4)
if forYouRebuildDue(&today, at(19, 23), 4) {
t.Fatal("rebuilt twice in one day")
}
tomorrow := at(20, 4)
if !forYouRebuildDue(&today, tomorrow, 4) {
t.Fatal("not due on the next day")
}
// The boundary is the local calendar day, not "24 hours ago". A rebuild at 04:30 and
// a tick at 04:00 the next morning are 23.5 hours apart and are still two days.
if !forYouRebuildDue(&today, at(20, 4), 4) {
t.Fatal("an interval shorter than 24 hours must still be a new day")
}
// An hour outside the clock is read as midnight rather than as a reason never to run.
if !forYouRebuildDue(nil, at(19, 0), 99) {
t.Fatal("an out-of-range hour must not strand the rebuild")
}
}
server/internal/api/radarr.go
+16 -5
View File
@@ -331,14 +331,25 @@ func (s *Server) radarrMovieCatalogue(ctx context.Context) ([]radarr.Movie, erro
if err != nil {
return nil, err
}
if body, marshalErr := json.Marshal(movies); marshalErr == nil {
if cacheErr := s.cache.Set(ctx, radarrMovieCacheKey, body, s.cfg.RadarrTTL); cacheErr != nil {
s.loggerFor(ctx).Warn("radarr movie cache write failed", "error", cacheErr)
}
}
s.cacheRadarrMovies(ctx, movies)
return movies, nil
}
// cacheRadarrMovies stores the household's shared copy of Radarr's catalogue.
//
// Split out because the scheduled refresh writes it too: that task reads Radarr directly
// — a refresh satisfied by the cache it exists to replace would do nothing — and would
// otherwise need its own copy of the key, the TTL and the failure handling.
func (s *Server) cacheRadarrMovies(ctx context.Context, movies []radarr.Movie) {
body, err := json.Marshal(movies)
if err != nil {
return
}
if cacheErr := s.cache.Set(ctx, radarrMovieCacheKey, body, s.cfg.RadarrTTL); cacheErr != nil {
s.loggerFor(ctx).Warn("radarr movie cache write failed", "error", cacheErr)
}
}
func (s *Server) cachedRadarrMovies(ctx context.Context) []radarr.Movie {
raw, err := s.cache.Get(ctx, radarrMovieCacheKey)
if err != nil {
server/internal/api/radarr_alerts.go
+9
View File
@@ -75,6 +75,15 @@ func (s *Server) handleRadarrWebhook(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusUnauthorized, "invalid webhook token")
return
}
// A switched-off integration records nothing. Answering 200 rather than refusing is
// deliberate: neither *arr re-delivers a rejection, so a failure here would look to the
// operator like Memby losing imports rather than like the switch they set. The library
// sweep is what reconciles whatever arrives while it is off.
if s.integrationSuppressed(r.Context(), integrationRadarr) {
s.loggerFor(r.Context()).Debug("radarr webhook ignored: integration switched off")
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "ignored": true})
return
}
var payload radarrWebhookPayload
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&payload); err != nil {
server/internal/api/requests.go
+30 -10
View File
@@ -259,7 +259,7 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request, sess stor
// is not the same as them never having asked for it, and their page is the
// only place that distinction is kept.
s.recordMediaRequest(r.Context(), sess, req, movie.Year,
radarrCoverURL(movie.Images, "poster"))
radarrCoverURL(movie.Images, "poster"), openingRequestStatus(movie.HasFile))
s.logMediaRequest(r.Context(), req, "already added", nil)
writeJSON(w, http.StatusOK, map[string]any{"status": "already_added", "title": movie.Title})
return
@@ -282,7 +282,7 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request, sess stor
req.Title = added.Title
s.logRadarrRequest(r.Context(), sess, req, added, added.ID, "successful", profileName, requestOptions.QualityProfileID, rootFolder, requestOptions.SearchImmediately, nil)
s.recordMediaRequest(r.Context(), sess, req, added.Year,
radarrCoverURL(added.Images, "poster"))
radarrCoverURL(added.Images, "poster"), openingRequestStatus(false))
s.logMediaRequest(r.Context(), req, "successful", nil)
writeJSON(w, http.StatusCreated, map[string]any{"status": "requested", "title": added.Title})
return
@@ -306,7 +306,7 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request, sess stor
if show.ID > 0 {
req.Title = show.Title
s.recordMediaRequest(r.Context(), sess, req, show.Year,
sonarrCoverURL(show.Images, "poster"))
sonarrCoverURL(show.Images, "poster"), openingRequestStatus(false))
s.logMediaRequest(r.Context(), req, "already added", nil)
writeJSON(w, http.StatusOK, map[string]any{"status": "already_added", "title": show.Title})
return
@@ -329,7 +329,7 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request, sess stor
req.Title = added.Title
s.logSonarrRequest(r.Context(), sess, req, added, added.ID, "successful", profileName, requestOptions.QualityProfileID, rootFolder, requestOptions.SearchImmediately, nil)
s.recordMediaRequest(r.Context(), sess, req, added.Year,
sonarrCoverURL(added.Images, "poster"))
sonarrCoverURL(added.Images, "poster"), openingRequestStatus(false))
s.logMediaRequest(r.Context(), req, "successful", nil)
writeJSON(w, http.StatusCreated, map[string]any{"status": "requested", "title": added.Title})
return
@@ -349,17 +349,18 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request, sess stor
// did. The cost of a lost write is that the ask is missing from their own page, which is
// recoverable by asking again — the cost of the opposite is a viewer requesting it twice.
func (s *Server) recordMediaRequest(
ctx context.Context, sess store.Session, req requestPayload, year int, posterURL string,
ctx context.Context, sess store.Session, req requestPayload, year int, posterURL, opening string,
) {
if s.store == nil || sess.EmbyUserID == "" {
return
}
err := s.store.SaveMediaRequest(ctx, sess.EmbyUserID, store.MediaRequest{
MediaType: req.MediaType,
ForeignID: req.ForeignID,
Title: req.Title,
Year: year,
PosterURL: posterURL,
MediaType: req.MediaType,
ForeignID: req.ForeignID,
Title: req.Title,
Year: year,
PosterURL: posterURL,
LastStatus: opening,
})
if err != nil {
s.loggerFor(ctx).Warn("media request not recorded",
@@ -367,6 +368,25 @@ func (s *Server) recordMediaRequest(
}
}
// openingRequestStatus is the state a request is born in, and it exists so that the ready
// sweep has something to compare against on its very first pass.
//
// Leaving it blank and letting the first sweep fill it in would lose exactly the arrivals
// worth announcing: a film that downloads in the three minutes between the ask and the first
// sweep would have its arrival recorded as its opening state, and nobody would ever be told.
// So the ask itself records what was true at the moment somebody pressed the button — which
// is the one moment the handler knows for certain, having just asked the *arr.
//
// A series is never born available. Sonarr's series list carries no file information, so
// "the household has this show" is a claim only the library can make, and the sweep is where
// it gets made.
func openingRequestStatus(hasFile bool) string {
if hasFile {
return RequestStatusAvailable
}
return RequestStatusSearching
}
func (s *Server) logMediaRequest(
ctx context.Context, req requestPayload, outcome string, err error,
) {
server/internal/api/requests_list.go
+90 -8
View File
@@ -3,11 +3,14 @@ package api
import (
"context"
"net/http"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/ponzischeme89/memby/server/internal/radarr"
"github.com/ponzischeme89/memby/server/internal/sonarr"
"github.com/ponzischeme89/memby/server/internal/store"
)
@@ -25,6 +28,15 @@ type myRequest struct {
Status string `json:"status"`
StatusLabel string `json:"statusLabel"`
StatusDetail string `json:"statusDetail"`
// Progress is whole percent and rides only a downloading card. It is omitted rather
// than sent as zero everywhere else, so the television draws a bar when it is given a
// figure and nothing at all when it is not — there is no "0%" state to confuse with a
// download that has not started.
Progress int `json:"progress,omitempty"`
// EstimatedReadySeconds is how long the download client says the bytes will take, and is
// omitted whenever nothing could say. That omission is the whole of "never fabricate an
// ETA": a card with no number here prints the plain wording in StatusDetail instead.
EstimatedReadySeconds int `json:"estimatedReadySeconds,omitempty"`
// EmbyItemID is set once Emby has imported the title, so the card can open the ordinary
// detail page instead of being a dead end at the moment it finally becomes watchable.
EmbyItemID string `json:"embyItemId,omitempty"`
@@ -49,21 +61,36 @@ func (s *Server) handleMyRequests(w http.ResponseWriter, r *http.Request, sess s
return
}
writeJSON(w, http.StatusOK, myRequestsResponse{
Requests: s.decorateRequests(r.Context(), stored),
Requests: s.decorateRequests(r.Context(), stored, requestProgressSupported(r)),
Allowed: true,
})
}
// requestProgressSupported asks whether this television can draw the download states.
//
// A capability rather than a version floor, because that is what the client already declares
// and what the console reports against — and because the answer is about what the app can
// *draw*, which is exactly what a capability token says. An older build is sent the collapsed
// vocabulary it understands (collapseRequestStatus) rather than four words it would file
// under "Nothing happening" and a percentage it has nowhere to put.
func requestProgressSupported(r *http.Request) bool {
return slices.Contains(clientCapabilities(r), "request_progress_v1")
}
// decorateRequests turns stored asks into cards by asking the two catalogues and the
// library what has become of each.
//
// The three lookups run concurrently and every one of them is allowed to fail: a request
// whose state cannot be established falls back to "requested", which is the honest answer —
// The lookups run concurrently and every one of them is allowed to fail: a request whose
// state cannot be established falls back to "requested", which is the honest answer —
// somebody asked and we cannot currently say more. A page that errored because Radarr was
// restarting would be a page that is broken exactly when a viewer wants to know why their
// title has not arrived.
// title has not arrived. The download queues degrade one step further and more gently
// still: losing them costs the percentage and the estimate, and the card falls back to
// "Searching", which is what the page said before there was a queue reader at all.
//
// progressAware is what this television can draw; see requestProgressSupported.
func (s *Server) decorateRequests(
ctx context.Context, stored []store.MediaRequest,
ctx context.Context, stored []store.MediaRequest, progressAware bool,
) []myRequest {
if len(stored) == 0 {
return []myRequest{}
@@ -85,6 +112,14 @@ func (s *Server) decorateRequests(
seriesInLibrary = map[int]bool{}
movieItemIDs = map[int]string{}
seriesItemIDs = map[int]string{}
// The two halves of the queue join, gathered separately and put together after the
// wait: a queue row names the *arr's own internal id, and only the catalogue can say
// which TMDb or TVDb id that is. Fetching them concurrently and joining afterwards is
// what keeps the queue read off the catalogue's critical path.
tmdbByMovieID = map[int]int{}
tvdbBySeriesID = map[int]int{}
movieQueue []radarr.QueueItem
seriesQueue []sonarr.QueueItem
)
now := time.Now()
@@ -101,6 +136,7 @@ func (s *Server) decorateRequests(
if movie.TMDBID == 0 {
continue
}
tmdbByMovieID[movie.ID] = movie.TMDBID
movies[movie.TMDBID] = requestCatalogueEntry{
tracked: true,
hasFile: movie.HasFile,
@@ -110,6 +146,16 @@ func (s *Server) decorateRequests(
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
queue, err := s.radarrQueue(ctx)
if err != nil {
s.loggerFor(ctx).Warn("radarr queue unavailable for requests", "error", err)
return
}
movieQueue = queue
}()
}
if len(seriesIDs) > 0 && s.sonarrEnabled(ctx) {
wg.Add(1)
@@ -124,6 +170,7 @@ func (s *Server) decorateRequests(
if show.TVDBID == 0 {
continue
}
tvdbBySeriesID[show.ID] = show.TVDBID
series[show.TVDBID] = requestCatalogueEntry{
tracked: true,
released: seriesReleased(show.Status, show.NextAiring, now),
@@ -132,6 +179,16 @@ func (s *Server) decorateRequests(
}
}
}()
wg.Add(1)
go func() {
defer wg.Done()
queue, err := s.sonarrQueue(ctx)
if err != nil {
s.loggerFor(ctx).Warn("sonarr queue unavailable for requests", "error", err)
return
}
seriesQueue = queue
}()
}
wg.Add(1)
go func() {
@@ -148,12 +205,18 @@ func (s *Server) decorateRequests(
}()
wg.Wait()
movieWork := groupMovieWork(movieQueue, tmdbByMovieID)
seriesWork := groupSeriesWork(seriesQueue, tvdbBySeriesID)
cards := make([]myRequest, 0, len(stored))
for _, req := range stored {
entry, inLibrary, itemID := movies[req.ForeignID], moviesInLibrary[req.ForeignID], movieItemIDs[req.ForeignID]
work := movieWork[req.ForeignID]
if req.MediaType == "series" {
entry, inLibrary, itemID = series[req.ForeignID], seriesInLibrary[req.ForeignID], seriesItemIDs[req.ForeignID]
work = seriesWork[req.ForeignID]
}
progress := downloadProgress(work)
status := RequestStatusRequested
// Only claim a state when something actually answered. With every catalogue down,
// "requested" is all that is known and is what the card must say.
@@ -163,13 +226,14 @@ func (s *Server) decorateRequests(
HasFile: entry.hasFile,
InLibrary: inLibrary,
Released: entry.released,
Progress: progress,
})
}
poster := req.PosterURL
if poster == "" {
poster = entry.poster
}
cards = append(cards, myRequest{
card := myRequest{
MediaType: req.MediaType,
ForeignID: req.ForeignID,
Title: req.Title,
@@ -179,9 +243,27 @@ func (s *Server) decorateRequests(
RequestedAt: req.RequestedAt.UTC().Format(time.RFC3339),
Status: status,
StatusLabel: requestStatusLabel(status),
StatusDetail: requestStatusDetail(status, req.MediaType),
StatusDetail: requestStatusDetail(status, req.MediaType, progress.EstimatedReadySeconds),
EmbyItemID: itemID,
})
}
// The figures belong to a download and to nothing else. A title that is available,
// pending or unavailable may still have a row in the queue — an upgrade, a stale
// entry — and pinning a percentage to it would put a progress bar under a card
// somebody can already press Play on.
if status == RequestStatusDownloading {
card.Progress = progress.Progress
card.EstimatedReadySeconds = progress.EstimatedReadySeconds
}
if !progressAware {
// Narrowed on the way out rather than on the way in, so the label and the
// detail are still computed from the truth and only the *slug* is generalised.
// That is the better half of the trade: an older television files the card under
// "On the way" as it always did, and still reads "Downloading" on the chip and
// the honest sentence underneath it. Only the bar and the number go.
card.Status = collapseRequestStatus(card.Status)
card.Progress, card.EstimatedReadySeconds = 0, 0
}
cards = append(cards, card)
}
return cards
}
server/internal/api/requests_progress.go
+262
View File
@@ -0,0 +1,262 @@
package api
import (
"math"
"strconv"
"strings"
"time"
)
// What the download client is doing with something somebody asked for, normalised into the
// few words a viewer can act on.
//
// The whole point of this file is that the television is never told about indexers, release
// profiles, trackers or import queues. Radarr and Sonarr describe one download in three
// overlapping vocabularies — see radarr.QueueItem — and between them they can produce
// something like twenty distinct states. A viewer standing in front of a card needs to know
// one of five things: nothing has been found yet, something has been found, it is coming
// down and how far through it is, it is being filed away, or it went wrong.
//
// Everything here is pure, because it is the half of the feature that has to be right and
// the half that is cheapest to be wrong about: an ETA is a promise, and a promise made from
// a misread field is worse than no promise at all.
// requestWork is one row of a download queue with the *arr it came from forgotten.
//
// Keeping it free of radarr and sonarr types is what lets one rule answer for both — a film
// and a season of a show are the same question about bytes — and lets the rule be tested
// without either service.
type requestWork struct {
// Size and SizeLeft are bytes. Both zero means the client has not said, which is
// different from a finished download and must not read as 100%.
Size float64
SizeLeft float64
// TimeLeft is the download client's own estimate as a .NET TimeSpan ("00:14:32",
// "1.02:03:04"), or empty when it will not say — which is the ordinary case for a
// queued or stalled item and exactly where an ETA must not be manufactured.
TimeLeft string
// Status is the download client's word, TrackedState is what the *arr will do with the
// bytes once they land, and TrackedStatus is the verdict over both.
Status string
TrackedState string
TrackedStatus string
}
// requestProgress is the normalised answer, and it is deliberately the shape of the wire.
//
// Progress and EstimatedReadySeconds are both zero when unknown, which is why they are
// omitempty on the response types that embed this: a card draws a figure it was given and
// says nothing at all when it was given none. Zero percent and "we have no idea" therefore
// look the same to the television, which is the correct conflation — neither is a number
// worth printing.
type requestProgress struct {
// Status is one of the request status slugs below, or empty when the queue had nothing
// to say about this title at all.
Status string
// Progress is whole percent, 0-100.
Progress int
// EstimatedReadySeconds is how long until the bytes have landed. It never includes the
// import that follows, because nothing measures that — see requestStatusDetail, where
// "a few minutes" is wording rather than an estimate.
EstimatedReadySeconds int
}
// Download-client states, in the order a request passes through them. These sit alongside
// the states in requests_status.go and share its vocabulary space; they are separate only
// because these four are the ones a queue can answer for.
const (
// RequestStatusSearching means monitored, released, nothing found yet — the honest
// reading of "the *arr holds this and the download client has never heard of it".
RequestStatusSearching = "searching"
// RequestStatusFound means a release has been grabbed and is waiting on the download
// client: queued, paused, held by a delay profile. There is something to wait for, but
// no bytes are moving.
RequestStatusFound = "found"
// RequestStatusDownloading means bytes are moving. This is the only state that carries
// a percentage.
RequestStatusDownloading = "downloading"
// RequestStatusFailed means the download failed and the *arr will look for another
// release. Deliberately not a dead end: it is the one state whose wording has to say
// that Memby is still trying.
RequestStatusFailed = "failed"
)
// workState is the per-row rule, and the order of its tests is the whole of it.
//
// A verdict of error outranks everything, because a row can look perfectly healthy —
// "completed", even — while the *arr has decided it cannot use what arrived. Importing
// outranks downloading next, since a row that has finished downloading is still reported by
// some clients with a downloading-ish status while the *arr moves the file. Only then does
// the download client's own word matter, and anything it says that is not "downloading" is
// a wait of some kind, which is what "found" means.
//
// A word this build has never seen falls through to found rather than to failed: a new
// vocabulary in a future Radarr must degrade to "something is happening" rather than
// telling a viewer their film is broken.
func workState(w requestWork) string {
status := strings.ToLower(strings.TrimSpace(w.Status))
tracked := strings.ToLower(strings.TrimSpace(w.TrackedState))
verdict := strings.ToLower(strings.TrimSpace(w.TrackedStatus))
switch {
case verdict == "error", status == "failed", tracked == "failed", tracked == "failedpending":
return RequestStatusFailed
case tracked == "importpending", tracked == "importing", tracked == "imported",
status == "completed":
return RequestStatusProcessing
case status == "downloading":
return RequestStatusDownloading
default:
return RequestStatusFound
}
}
// workRank orders the states by how much they deserve to be what a card says when one title
// has several rows — a season pack is a dozen episodes at a dozen different stages.
//
// Highest wins, and the ordering is "what is the most active thing happening to this": a
// show with one episode downloading and eleven already filed is downloading. Failed is
// lowest, so a single failed episode never overrides eleven healthy ones and a title only
// reads as failed when every row of it has.
func workRank(status string) int {
switch status {
case RequestStatusDownloading:
return 4
case RequestStatusFound:
return 3
case RequestStatusProcessing:
return 2
case RequestStatusFailed:
return 1
default:
return 0
}
}
// downloadProgress folds a title's queue rows into one answer.
//
// No rows is not an error and not a state: it is the caller's question to answer, since
// "nothing is downloading" means something different for a film that is already in the
// library, one nobody has released yet, and one the *arr has been searching for all
// afternoon. So this returns an empty status and requestStatusFor decides.
func downloadProgress(work []requestWork) requestProgress {
if len(work) == 0 {
return requestProgress{}
}
var (
best string
totalSize float64
totalLeft float64
// eta is the longest remaining time across the rows that have not landed yet: a
// season is ready when its slowest episode is, not its fastest.
eta time.Duration
// etaKnown starts true and is cleared by the first unfinished row that will not say.
// One silent row makes the total unknowable, and reporting the rest of the pack's
// time as the whole pack's would be an ETA that quietly expires and keeps going.
etaKnown = true
)
for _, row := range work {
state := workState(row)
if workRank(state) > workRank(best) {
best = state
}
if row.Size > 0 {
totalSize += row.Size
totalLeft += math.Min(math.Max(row.SizeLeft, 0), row.Size)
}
switch state {
case RequestStatusDownloading, RequestStatusFound:
remaining, ok := parseTimeLeft(row.TimeLeft)
if !ok {
etaKnown = false
continue
}
if remaining > eta {
eta = remaining
}
}
}
progress := requestProgress{Status: best}
if totalSize > 0 {
done := (totalSize - totalLeft) / totalSize * 100
progress.Progress = int(math.Round(math.Min(math.Max(done, 0), 100)))
}
// An estimate is only ever offered against moving bytes. A queue that is paused or
// waiting on a delay profile has a "time left" only in the sense that the download
// client is guessing, and an import has no measured duration at all.
if etaKnown && eta > 0 && best == RequestStatusDownloading {
progress.EstimatedReadySeconds = int(math.Round(eta.Seconds()))
}
return progress
}
// parseTimeLeft reads the .NET TimeSpan the *arrs send: "hh:mm:ss", with an optional
// "d." day part in front and an optional fractional-seconds part behind.
//
// It refuses anything it cannot read completely rather than salvaging a number from part of
// it. This is the one input that becomes a promise to a viewer, and a misparse here is how
// "ready in 14 minutes" becomes "ready in 14 hours".
func parseTimeLeft(value string) (time.Duration, bool) {
value = strings.TrimSpace(value)
if value == "" {
return 0, false
}
days := 0
// A day part is separated by a full stop, and so is a fractional second — so a leading
// "d." only exists when what follows still holds two colons.
if dot := strings.Index(value, "."); dot > 0 && strings.Count(value[dot+1:], ":") == 2 {
parsed, err := strconv.Atoi(value[:dot])
if err != nil || parsed < 0 {
return 0, false
}
days = parsed
value = value[dot+1:]
}
if dot := strings.Index(value, "."); dot >= 0 {
value = value[:dot] // drop fractional seconds; nobody counts a film in milliseconds
}
parts := strings.Split(value, ":")
if len(parts) != 3 {
return 0, false
}
units := []time.Duration{time.Hour, time.Minute, time.Second}
total := time.Duration(days) * 24 * time.Hour
for index, part := range parts {
number, err := strconv.Atoi(strings.TrimSpace(part))
if err != nil || number < 0 {
return 0, false
}
total += time.Duration(number) * units[index]
}
return total, true
}
// estimatedReadyLabel is the second half of the download line: the sentence under
// "Downloading - 68%".
//
// It is worded in the coarsest unit that is still useful, because the precision the
// download client reports is not precision anybody has: a client that says 14 minutes 3
// seconds is guessing at the minutes, so printing the seconds claims an accuracy the number
// does not have. An estimate of zero is not an estimate and produces nothing at all, which
// is what makes "we will let you know when it is ready" reachable.
func estimatedReadyLabel(seconds int) string {
if seconds <= 0 {
return ""
}
remaining := time.Duration(seconds) * time.Second
switch {
case remaining < 90*time.Second:
return "Estimated ready in under a minute"
case remaining < time.Hour:
return "Estimated ready in ~" + strconv.Itoa(int(math.Round(remaining.Minutes()))) + " minutes"
case remaining < 2*time.Hour:
return "Estimated ready in about an hour"
case remaining < 24*time.Hour:
return "Estimated ready in ~" + strconv.Itoa(int(remaining.Hours())) + " hours"
default:
// Past a day the number stops being an estimate and starts being a warning that
// something is wrong with the release, so it is deliberately vague.
return "Estimated ready in over a day"
}
}
server/internal/api/requests_progress_test.go
+255
View File
@@ -0,0 +1,255 @@
package api
import (
"testing"
"time"
"github.com/ponzischeme89/memby/server/internal/radarr"
)
func TestWorkStateReadsTheThreeVocabulariesInOrder(t *testing.T) {
cases := []struct {
name string
work requestWork
want string
}{
{
// The case the ordering exists for: everything about this row looks finished,
// and the *arr has decided it cannot use what arrived.
name: "an error verdict outranks a completed download",
work: requestWork{Status: "completed", TrackedState: "importPending", TrackedStatus: "error"},
want: RequestStatusFailed,
},
{
name: "importing is processing even while the client still says downloading",
work: requestWork{Status: "downloading", TrackedState: "importing", TrackedStatus: "ok"},
want: RequestStatusProcessing,
},
{
name: "moving bytes",
work: requestWork{Status: "downloading", TrackedState: "downloading", TrackedStatus: "ok"},
want: RequestStatusDownloading,
},
{
name: "queued is a wait, not a download",
work: requestWork{Status: "queued", TrackedState: "downloading", TrackedStatus: "ok"},
want: RequestStatusFound,
},
{
name: "a delay profile is a wait",
work: requestWork{Status: "delay", TrackedStatus: "ok"},
want: RequestStatusFound,
},
{
// A future Radarr inventing a word must not tell a viewer their film is broken.
name: "an unknown word degrades to found",
work: requestWork{Status: "reticulatingSplines", TrackedState: "somethingNew"},
want: RequestStatusFound,
},
{
// A warning is not a failure. Radarr flags a stalled download this way and will
// carry on with it.
name: "a warning verdict is not a failure",
work: requestWork{Status: "downloading", TrackedStatus: "warning"},
want: RequestStatusDownloading,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := workState(tc.work); got != tc.want {
t.Fatalf("expected %q, got %q", tc.want, got)
}
})
}
}
func TestDownloadProgressSaysNothingAboutAnEmptyQueue(t *testing.T) {
// The absence of a queue row is the caller's question, not a state: it means something
// different for a film already on the shelf, one nobody has released, and one that has
// been searched for all afternoon.
got := downloadProgress(nil)
if got.Status != "" {
t.Fatalf("an empty queue must claim no state, got %q", got.Status)
}
if got.Progress != 0 || got.EstimatedReadySeconds != 0 {
t.Fatal("an empty queue must carry no figures")
}
}
func TestDownloadProgressReportsPercentAndEstimate(t *testing.T) {
got := downloadProgress([]requestWork{{
Size: 1000, SizeLeft: 320, TimeLeft: "00:14:00",
Status: "downloading", TrackedState: "downloading", TrackedStatus: "ok",
}})
if got.Status != RequestStatusDownloading {
t.Fatalf("expected %q, got %q", RequestStatusDownloading, got.Status)
}
if got.Progress != 68 {
t.Fatalf("expected 68%%, got %d%%", got.Progress)
}
if got.EstimatedReadySeconds != 840 {
t.Fatalf("expected 840 seconds, got %d", got.EstimatedReadySeconds)
}
}
func TestASeasonIsOneAnswerOverManyRows(t *testing.T) {
// A season pack is one row per episode at a dozen different stages. What the card says
// is the most active of them, and the percentage is over the whole pack rather than
// whichever episode happens to be first.
work := []requestWork{
{Size: 100, SizeLeft: 0, Status: "completed", TrackedState: "imported"},
{Size: 100, SizeLeft: 50, TimeLeft: "00:05:00", Status: "downloading", TrackedState: "downloading"},
{Size: 100, SizeLeft: 100, TimeLeft: "00:20:00", Status: "queued", TrackedState: "downloading"},
}
got := downloadProgress(work)
if got.Status != RequestStatusDownloading {
t.Fatalf("a pack with one episode downloading is downloading, got %q", got.Status)
}
if got.Progress != 50 {
t.Fatalf("expected 50%% over the whole pack, got %d%%", got.Progress)
}
// The pack is ready when its slowest episode is, not its fastest.
if got.EstimatedReadySeconds != 1200 {
t.Fatalf("expected the longest remaining time, got %d", got.EstimatedReadySeconds)
}
}
func TestOneFailedEpisodeDoesNotFailTheWholeShow(t *testing.T) {
work := []requestWork{
{Size: 100, SizeLeft: 100, Status: "failed", TrackedStatus: "error"},
{Size: 100, SizeLeft: 40, TimeLeft: "00:03:00", Status: "downloading"},
}
if got := downloadProgress(work).Status; got != RequestStatusDownloading {
t.Fatalf("expected %q, got %q", RequestStatusDownloading, got)
}
// With nothing else happening, though, failed is the honest answer.
only := []requestWork{{Size: 100, SizeLeft: 100, Status: "failed", TrackedStatus: "error"}}
if got := downloadProgress(only).Status; got != RequestStatusFailed {
t.Fatalf("expected %q, got %q", RequestStatusFailed, got)
}
}
func TestAnEstimateIsNeverManufactured(t *testing.T) {
// A download client that will not say how long it has left leaves the estimate absent.
// This is the acceptance criterion the feature is judged on, so it is asserted from
// several directions.
silent := downloadProgress([]requestWork{{Size: 100, SizeLeft: 60, Status: "downloading"}})
if silent.EstimatedReadySeconds != 0 {
t.Fatalf("no time left reported must produce no estimate, got %d", silent.EstimatedReadySeconds)
}
if silent.Progress != 40 {
t.Fatalf("a missing estimate must not cost the percentage, got %d%%", silent.Progress)
}
// One silent row makes the whole pack unknowable: reporting the rest of it as the whole
// would be an estimate that quietly expires while the download carries on.
partly := downloadProgress([]requestWork{
{Size: 100, SizeLeft: 50, TimeLeft: "00:05:00", Status: "downloading"},
{Size: 100, SizeLeft: 90, Status: "downloading"},
})
if partly.EstimatedReadySeconds != 0 {
t.Fatalf("one silent row must make the pack unknowable, got %d", partly.EstimatedReadySeconds)
}
// An import has no measured duration, so its "few minutes" is wording rather than a
// number — see requestStatusDetail.
importing := downloadProgress([]requestWork{
{Size: 100, SizeLeft: 0, TimeLeft: "00:00:30", Status: "completed", TrackedState: "importing"},
})
if importing.EstimatedReadySeconds != 0 {
t.Fatalf("an import must not carry an estimate, got %d", importing.EstimatedReadySeconds)
}
}
func TestSizeIsOnlyReadWhenTheClientGaveOne(t *testing.T) {
// Zero size and zero left is a client that has not said, and reading it as
// (0-0)/0 finished would draw a full bar over a download that has not started.
got := downloadProgress([]requestWork{{Status: "downloading"}})
if got.Progress != 0 {
t.Fatalf("an unsized download must report no percentage, got %d%%", got.Progress)
}
// A client reporting more left than the whole is clamped rather than trusted into a
// negative percentage.
odd := downloadProgress([]requestWork{{Size: 100, SizeLeft: 400, Status: "downloading"}})
if odd.Progress != 0 {
t.Fatalf("expected a clamped 0%%, got %d%%", odd.Progress)
}
}
func TestParseTimeLeftReadsTheArrsFormat(t *testing.T) {
cases := []struct {
in string
want time.Duration
ok bool
}{
{"00:14:32", 14*time.Minute + 32*time.Second, true},
{"01:00:00", time.Hour, true},
{"1.02:03:04", 26*time.Hour + 3*time.Minute + 4*time.Second, true},
{"00:00:30.5000000", 30 * time.Second, true},
// Everything it cannot read completely is refused rather than salvaged. This is the
// one input that becomes a promise to a viewer.
{"", 0, false},
{"14:32", 0, false},
{"soon", 0, false},
{"00:xx:32", 0, false},
{"-1.00:00:01", 0, false},
}
for _, tc := range cases {
got, ok := parseTimeLeft(tc.in)
if ok != tc.ok {
t.Fatalf("%q: expected ok=%v, got %v", tc.in, tc.ok, ok)
}
if ok && got != tc.want {
t.Fatalf("%q: expected %s, got %s", tc.in, tc.want, got)
}
}
}
func TestEstimatedReadyLabelIsCoarserThanTheNumberItWasGiven(t *testing.T) {
// The download client's seconds are not precision anybody has, so printing them would
// claim an accuracy the estimate does not carry.
cases := []struct {
seconds int
want string
}{
{0, ""},
{-5, ""},
{45, "Estimated ready in under a minute"},
{840, "Estimated ready in ~14 minutes"},
{4500, "Estimated ready in about an hour"},
{5 * 3600, "Estimated ready in ~5 hours"},
{50 * 3600, "Estimated ready in over a day"},
}
for _, tc := range cases {
if got := estimatedReadyLabel(tc.seconds); got != tc.want {
t.Fatalf("%d seconds: expected %q, got %q", tc.seconds, tc.want, got)
}
}
}
func TestQueueIsGroupedByTheIdTheRequestWasRecordedAgainst(t *testing.T) {
// A queue row names the *arr's own internal id, which nothing outside it has seen. The
// translation to TMDb is what lets a stored request find its own download.
work := groupMovieWork(
[]radarr.QueueItem{
{MovieID: 7, Size: 100, Sizeleft: 25, Status: "downloading"},
// Radarr id 99 is not in the catalogue map: a film added since it was cached.
{MovieID: 99, Size: 100, Sizeleft: 100, Status: "queued"},
},
map[int]int{7: 550, 8: 603},
)
if len(work[550]) != 1 {
t.Fatalf("expected one row against tmdb 550, got %d", len(work[550]))
}
if work[550][0].Status != "downloading" {
t.Fatalf("the row lost its status: %+v", work[550][0])
}
// A download for a film added since the catalogue was cached cannot be translated, and
// is dropped rather than guessed at.
if _, ok := work[0]; ok {
t.Fatal("an untranslatable row must not be filed under a zero id")
}
if len(work) != 1 {
t.Fatalf("expected one translated title, got %d", len(work))
}
}
server/internal/api/requests_queue.go
+161
View File
@@ -0,0 +1,161 @@
package api
import (
"context"
"encoding/json"
"time"
"github.com/ponzischeme89/memby/server/internal/radarr"
"github.com/ponzischeme89/memby/server/internal/sonarr"
)
// Reading the download queues, which is the one part of the request page that cannot be
// cached for long and is read by every television in the house at once.
//
// The two *arr catalogues beside this are cached for the day: what Radarr holds and whether
// it has a file changes a handful of times a day, and a viewer finding out at midnight costs
// nothing. A queue is the opposite — it is the number moving on the card — so it carries its
// own short cache instead. requestQueueTTL is what stops four televisions on the launcher
// turning a page refresh into four requests apiece: within the window they share one answer,
// and past it the figure is stale by at most that window, which at this cadence is invisible
// against a download measured in minutes.
const (
radarrQueueCacheKey = "radarr:queue:v1"
sonarrQueueCacheKey = "sonarr:queue:v1"
requestQueueTTL = 15 * time.Second
)
// radarrQueue is what Radarr's download client is working on.
//
// Single-flighted behind the same mutex the catalogue uses, so a burst of polls costs one
// upstream read. A miss is not an error the page can be failed over: see decorateRequests,
// where a queue that will not answer costs the percentage and never the card.
func (s *Server) radarrQueue(ctx context.Context) ([]radarr.QueueItem, error) {
if !s.radarrEnabled(ctx) {
return nil, nil
}
if queue, ok := cachedQueue[radarr.QueueItem](ctx, s, radarrQueueCacheKey); ok {
return queue, nil
}
s.radarrMu.Lock()
defer s.radarrMu.Unlock()
if queue, ok := cachedQueue[radarr.QueueItem](ctx, s, radarrQueueCacheKey); ok {
return queue, nil
}
queue, err := s.radarr.Queue(ctx)
if err != nil {
return nil, err
}
s.cacheQueue(ctx, radarrQueueCacheKey, queue)
return queue, nil
}
// sonarrQueue is what Sonarr's download client is working on.
func (s *Server) sonarrQueue(ctx context.Context) ([]sonarr.QueueItem, error) {
if !s.sonarrEnabled(ctx) {
return nil, nil
}
if queue, ok := cachedQueue[sonarr.QueueItem](ctx, s, sonarrQueueCacheKey); ok {
return queue, nil
}
s.sonarrSeriesMu.Lock()
defer s.sonarrSeriesMu.Unlock()
if queue, ok := cachedQueue[sonarr.QueueItem](ctx, s, sonarrQueueCacheKey); ok {
return queue, nil
}
queue, err := s.sonarr.Queue(ctx)
if err != nil {
return nil, err
}
s.cacheQueue(ctx, sonarrQueueCacheKey, queue)
return queue, nil
}
// cachedQueue reads a stored queue. An empty queue is a real and common answer — most of the
// time the household is downloading nothing — so the second return distinguishes "nothing is
// stored" from "nothing is downloading", which a nil slice could not.
func cachedQueue[T any](ctx context.Context, s *Server, key string) ([]T, bool) {
if s.cache == nil {
return nil, false
}
raw, err := s.cache.Get(ctx, key)
if err != nil {
return nil, false
}
var queue []T
if err := json.Unmarshal(raw, &queue); err != nil {
return nil, false
}
if queue == nil {
queue = []T{}
}
return queue, true
}
func (s *Server) cacheQueue(ctx context.Context, key string, queue any) {
if s.cache == nil {
return
}
body, err := json.Marshal(queue)
if err != nil {
return
}
if err := s.cache.Set(ctx, key, body, requestQueueTTL); err != nil {
s.loggerFor(ctx).Warn("download queue cache write failed", "key", key, "error", err)
}
}
// groupMovieWork turns Radarr's queue into work per *TMDb* id.
//
// The translation is the point. A queue row names Radarr's own movie id, which is an
// internal number nothing outside Radarr has ever seen; a request is recorded against the
// TMDb id, which is what the catalogue, the library and Emby all agree on. The map comes
// from the catalogue the caller already has, so this costs no extra request — and a row
// whose film is not in that map is dropped rather than guessed at, which is what a download
// for something added since the catalogue was cached looks like.
func groupMovieWork(queue []radarr.QueueItem, tmdbByMovieID map[int]int) map[int][]requestWork {
work := map[int][]requestWork{}
for _, row := range queue {
tmdbID, ok := tmdbByMovieID[row.MovieID]
if !ok || tmdbID == 0 {
continue
}
work[tmdbID] = append(work[tmdbID], requestWork{
Size: row.Size,
SizeLeft: row.Sizeleft,
TimeLeft: row.Timeleft,
Status: row.Status,
TrackedState: row.TrackedDownloadState,
TrackedStatus: row.TrackedDownloadStatus,
})
}
return work
}
// groupSeriesWork turns Sonarr's queue into work per TVDb id.
//
// A show legitimately has many rows — a season pack is one row per episode — and they are
// kept as a list rather than reduced here, because downloadProgress is what decides how a
// dozen episodes at a dozen stages become one sentence.
func groupSeriesWork(queue []sonarr.QueueItem, tvdbBySeriesID map[int]int) map[int][]requestWork {
work := map[int][]requestWork{}
for _, row := range queue {
tvdbID, ok := tvdbBySeriesID[row.SeriesID]
if !ok || tvdbID == 0 {
continue
}
work[tvdbID] = append(work[tvdbID], requestWork{
Size: row.Size,
SizeLeft: row.Sizeleft,
TimeLeft: row.Timeleft,
Status: row.Status,
TrackedState: row.TrackedDownloadState,
TrackedStatus: row.TrackedDownloadStatus,
})
}
return work
}
server/internal/api/requests_ready.go
+192
View File
@@ -0,0 +1,192 @@
package api
import (
"context"
"fmt"
"strconv"
"time"
"github.com/ponzischeme89/memby/server/internal/notify"
"github.com/ponzischeme89/memby/server/internal/scheduler"
"github.com/ponzischeme89/memby/server/internal/store"
)
// Telling somebody the thing they asked for has arrived.
//
// This is the one part of the request feature that cannot be answered by looking: every
// other state on a card is derived per read from the *arrs and the library, but "it has just
// become ready" is a *difference between two observations*, and a viewer has to be told about
// it exactly once. media_requests.last_status is the memory that makes the difference
// visible; this sweep is what looks.
//
// It is a personal notification rather than a service alert, and that is the same distinction
// the watch-time digest makes: a service alert is the house being told something, and one
// person's request arriving is news for one person. It therefore lands in My Alerts and
// follows them to whichever television they sign into, rather than dropping a bar across
// somebody else's film.
const (
// requestReadyKind is what the television files the notification under. An app that
// predates it draws the fallback icon, which is why the wording has to stand on its own.
requestReadyKind = "request-ready"
// notifySourceRequestReady names this producer in the notification log, so an operator
// answering "did Memby tell them?" has one thing to filter on.
notifySourceRequestReady = "request-ready"
)
// RegisterRequestTasks declares the arrival sweep.
//
// Registered as an ordinary scheduler job for the reason the digest is: an operator can see
// when it last ran and can run one by hand. For a job whose whole output is silence on a
// quiet day, that is the difference between "nothing has arrived" and "it has not run".
func (s *Server) RegisterRequestTasks(sched *scheduler.Scheduler) {
if sched == nil {
return
}
sched.Register(scheduler.Task{
ID: "request-ready-scan",
Name: "Requested content arrivals",
Group: "Notifications",
Description: "Notices when something a viewer requested has become watchable and tells " +
"the person who asked for it. Needs Radarr or Sonarr.",
// Five minutes is chosen against what it is watching rather than against cost. An
// import is the last few seconds of a wait measured in tens of minutes, so being up
// to five minutes late with the news is invisible; being an hour late is the
// difference between a notification and a thing somebody had already found for
// themselves.
Interval: 5 * time.Minute,
Timeout: 2 * time.Minute,
Run: s.runRequestReadyScan,
})
}
func (s *Server) runRequestReadyScan(ctx context.Context) (string, error) {
if s.store == nil {
return "", nil
}
// Nothing to compare against with both catalogues gone: every card would read
// "requested", which is not a transition and must not be recorded as one.
if !s.radarrEnabled(ctx) && !s.sonarrEnabled(ctx) {
return "", nil
}
owned, err := s.store.AllMediaRequests(ctx, store.MediaRequestSweepLimit)
if err != nil {
return "", fmt.Errorf("request arrivals: read requests: %w", err)
}
if len(owned) == 0 {
return "", nil
}
stored := make([]store.MediaRequest, 0, len(owned))
for _, req := range owned {
stored = append(stored, req.MediaRequest)
}
// progressAware is true because nothing here is drawn: the sweep wants the truth, and
// collapsing it would make a download and a search indistinguishable in the memory this
// writes back.
cards := s.decorateRequests(ctx, stored, true)
if len(cards) != len(owned) {
// decorateRequests answers one card per stored ask, in order. If that ever stopped
// being true the zip below would attribute one person's arrival to another, which is
// the one mistake here that reaches somebody as a notification about a title they
// never asked for.
return "", fmt.Errorf("request arrivals: %d cards for %d requests", len(cards), len(owned))
}
announced, moved := 0, 0
for index, card := range cards {
req := owned[index]
if card.Status == req.LastStatus {
continue
}
// A state that could not be established is not a transition. With Radarr restarting,
// every card falls back to "requested", and recording that would throw away the
// memory of what each request was actually doing — so the next sweep, once Radarr is
// back, would read an arrival as a move from "requested" and announce titles that had
// been on the shelf for a week.
if card.Status == RequestStatusRequested {
continue
}
if card.Status == RequestStatusAvailable && req.LastStatus != RequestStatusAvailable {
if s.announceRequestReady(ctx, req, card) {
announced++
}
}
if err := s.store.SetMediaRequestStatus(
ctx, req.UserID, req.MediaType, req.ForeignID, card.Status,
); err != nil {
s.loggerFor(ctx).Warn("request status not recorded",
"user_id", req.UserID, "type", req.MediaType,
"foreign_id", req.ForeignID, "error", err)
continue
}
moved++
}
// An empty detail keeps a job that runs every five minutes out of the operator's feed
// every five minutes; see scheduler.announce. Movement without an arrival is real work
// and worth reporting, because it is the evidence that the sweep is watching anything
// at all on a household where nothing has finished downloading yet.
switch {
case announced == 1:
return "1 request ready", nil
case announced > 1:
return fmt.Sprintf("%d requests ready", announced), nil
case moved > 0:
return fmt.Sprintf("%d requests moved on", moved), nil
default:
return "", nil
}
}
// announceRequestReady tells one viewer their title has landed.
//
// It carries the Emby item id whenever the library has one, which is what lets pressing the
// notification open the title's own detail page rather than a page about nothing. A title the
// *arr reports a file for but Emby has not imported yet legitimately has none — the news is
// still true and still worth sending, and the notification simply is not a link.
//
// There is deliberately no preference check. The other personal notifications are things
// Memby decided to send somebody; this one is the answer to a question they asked by pressing
// a button, and a viewer who has requested a film has said as clearly as they can that they
// want to know when it arrives.
func (s *Server) announceRequestReady(
ctx context.Context, req store.OwnedMediaRequest, card myRequest,
) bool {
title := card.Title
if title == "" {
title = req.Title
}
thing := "film"
if req.MediaType == "series" {
thing = "series"
}
eventAt := time.Now()
sent := s.notifyUser(ctx, notify.Notification{
Kind: requestReadyKind,
Source: notifySourceRequestReady,
UserID: req.UserID,
Title: "Your " + thing + " is ready",
Body: title + " is now available to watch.",
ItemID: card.EmbyItemID,
// The key names the request rather than the moment, so the arrival is announced once
// however many times the sweep runs — and a viewer who deletes the request, asks
// again and waits through a second download is a second key only if the title left
// the library in between, which is the case where the news is genuinely new.
SourceKey: "request-ready:" + req.MediaType + ":" + strconv.Itoa(req.ForeignID) +
":" + req.UserID,
EventAt: &eventAt,
Metadata: map[string]any{
"mediaType": req.MediaType,
"foreignId": req.ForeignID,
},
})
if sent {
s.loggerFor(ctx).Info("requested title is ready",
"user_id", req.UserID, "type", req.MediaType,
"title", clientLogValue(title), "foreign_id", req.ForeignID,
"item_id", card.EmbyItemID)
}
return sent
}
server/internal/api/requests_status.go
+76 -4
View File
@@ -16,7 +16,11 @@ import (
const (
// The household has it. Either Emby has imported it or the *arr reports a file.
RequestStatusAvailable = "available"
// Accepted and being worked on: released, monitored, no file yet.
// Being filed away: the bytes have landed and the *arr is importing them. This used to
// mean "released, monitored, no file yet" — everything between the ask and the arrival —
// and now means only the last step of it, with searching, found and downloading (in
// requests_progress.go) covering the rest. A television that predates those three is
// sent this word for all four; see collapseRequestStatus.
RequestStatusProcessing = "processing"
// Accepted, but there is nothing to fetch yet — unreleased, or still only in cinemas.
RequestStatusPending = "pending"
@@ -68,6 +72,12 @@ type RequestSubject struct {
// Released is whether there is anything to fetch yet. A film not yet on digital and a
// series whose first episode has not aired are both false.
Released bool
// Progress is what the download client is doing with it, already folded by
// downloadProgress. It is passed in rather than computed here so that the caller — which
// has the queue rows and has to send the percentage and the ETA anyway — folds them
// once. An empty Status means the queue had nothing to say, which is the ordinary case
// and not a state of its own.
Progress requestProgress
}
// requestStatusFor is the whole rule, and it is ordered by how much each signal is worth.
@@ -77,16 +87,48 @@ type RequestSubject struct {
// recorded. Only then does absence from the *arr mean removal — checked before the release
// state, because an untracked title's release date says nothing about a request nobody is
// working on any more.
//
// The download client outranks the release state, and that ordering is deliberate. Radarr's
// minimum-availability setting is a policy about when to *start looking*, not a fact about
// whether bytes are moving — a household that fetches on the cinema date has films that are
// "not released" and 40% downloaded at the same time, and the honest thing to tell somebody
// watching that card is the 40%.
//
// With nothing in the queue and nothing on disk, a tracked and released title is being
// searched for. That is the state this feature exists to make visible: before it, the whole
// span from "somebody asked" to "the file landed" was one word, so a viewer could not tell a
// request nothing had been found for from one that was minutes away.
func requestStatusFor(subject RequestSubject) string {
switch {
case subject.InLibrary || subject.HasFile:
return RequestStatusAvailable
case !subject.Tracked:
return RequestStatusUnavailable
case subject.Progress.Status != "":
return subject.Progress.Status
case !subject.Released:
return RequestStatusPending
default:
return RequestStatusSearching
}
}
// collapseRequestStatus is what a television that predates the download states is told.
//
// The four new words all live inside the span the old "processing" covered, so an older app
// is sent that one word and reads it exactly as it always did — "Searching for a copy",
// filed under On the way. It loses the percentage and the estimate, which it has nowhere to
// draw anyway, and it gains nothing wrong.
//
// This is a *narrowing*, never a translation in the other direction: a build that declares
// request_progress_v1 is sent the truth. See requestProgressSupported.
func collapseRequestStatus(status string) string {
switch status {
case RequestStatusSearching, RequestStatusFound,
RequestStatusDownloading, RequestStatusFailed:
return RequestStatusProcessing
default:
return status
}
}
@@ -131,11 +173,23 @@ func seriesReleased(status string, nextAiring *time.Time, now time.Time) bool {
func requestStatusLabel(status string) string {
switch status {
case RequestStatusAvailable:
return "Available"
return "Ready to watch"
case RequestStatusSearching:
return "Searching"
case RequestStatusFound:
return "Found"
// Deliberately just the word. The percentage is a number that moves every ten seconds
// and the wording does not, so the television composes "Downloading - 43%" from this
// label and the progress field rather than the server sending a sentence that is stale
// before it is drawn.
case RequestStatusDownloading:
return "Downloading"
case RequestStatusProcessing:
return "Processing"
case RequestStatusPending:
return "Pending"
case RequestStatusFailed:
return "Unable to download"
case RequestStatusUnavailable:
return "Unavailable"
case RequestStatusRequestable:
@@ -147,7 +201,14 @@ func requestStatusLabel(status string) string {
// requestStatusDetail is the quiet second line: what the state means for the viewer, in
// plain language, rather than a repeat of the word above it.
func requestStatusDetail(status, mediaType string) string {
//
// The download states are where this line earns its place, because the word above it is not
// the news. "Downloading" is not what somebody is standing there wanting to know — when it
// will be ready is — and this is the only line that can say so. The estimate is passed in
// rather than recomputed so that the sentence and the estimatedReadySeconds field on the
// wire can never disagree; when there is no estimate the line says so plainly instead of
// reaching for a vaguer number, which is the whole of "never manufacture an ETA".
func requestStatusDetail(status, mediaType string, estimateSeconds int) string {
thing := "film"
if mediaType == "series" {
thing = "series"
@@ -155,8 +216,19 @@ func requestStatusDetail(status, mediaType string) string {
switch status {
case RequestStatusAvailable:
return "Ready to watch now"
case RequestStatusSearching:
return "We haven't found a suitable release yet"
case RequestStatusFound:
return "Preparing the download"
case RequestStatusDownloading:
if label := estimatedReadyLabel(estimateSeconds); label != "" {
return label
}
return "We'll let you know when it's ready"
case RequestStatusProcessing:
return "Searching for a copy"
return "Should be ready in a few minutes"
case RequestStatusFailed:
return "Memby will keep looking for another release"
case RequestStatusPending:
if thing == "series" {
return "Waiting for it to air"
server/internal/api/requests_status_test.go
+78 -8
View File
@@ -1,6 +1,7 @@
package api
import (
"strings"
"testing"
"time"
)
@@ -38,14 +39,62 @@ func TestRequestStatusReportsRemovalBeforeReleaseState(t *testing.T) {
}
}
func TestRequestStatusSeparatesPendingFromProcessing(t *testing.T) {
func TestRequestStatusSeparatesPendingFromSearching(t *testing.T) {
pending := requestStatusFor(RequestSubject{Tracked: true, Released: false})
if pending != RequestStatusPending {
t.Fatalf("expected %q, got %q", RequestStatusPending, pending)
}
processing := requestStatusFor(RequestSubject{Tracked: true, Released: true})
if processing != RequestStatusProcessing {
t.Fatalf("expected %q, got %q", RequestStatusProcessing, processing)
// Tracked, released, and the download client has never heard of it: nothing has been
// found. This is the distinction the whole feature rests on — before it, this case and
// a download two minutes from finishing were the same word.
searching := requestStatusFor(RequestSubject{Tracked: true, Released: true})
if searching != RequestStatusSearching {
t.Fatalf("expected %q, got %q", RequestStatusSearching, searching)
}
}
func TestQueueOutranksTheReleaseState(t *testing.T) {
// Radarr's minimum-availability setting decides when to start looking, not whether bytes
// are moving. A household that fetches on the cinema date has films that are "not
// released" and half downloaded at once, and the honest answer is the half.
got := requestStatusFor(RequestSubject{
Tracked: true,
Released: false,
Progress: requestProgress{Status: RequestStatusDownloading, Progress: 50},
})
if got != RequestStatusDownloading {
t.Fatalf("expected %q, got %q", RequestStatusDownloading, got)
}
// But nothing outranks having it. A queue row against a title already on the shelf is an
// upgrade, and a card somebody can press Play on must not read as downloading.
got = requestStatusFor(RequestSubject{
Tracked: true,
InLibrary: true,
Progress: requestProgress{Status: RequestStatusDownloading, Progress: 50},
})
if got != RequestStatusAvailable {
t.Fatalf("expected %q, got %q", RequestStatusAvailable, got)
}
}
func TestCollapseNarrowsOnlyTheDownloadStates(t *testing.T) {
// An older television is told "processing" for the whole span it used to cover, and
// nothing else it understands is disturbed.
for _, state := range []string{
RequestStatusSearching, RequestStatusFound,
RequestStatusDownloading, RequestStatusFailed,
} {
if got := collapseRequestStatus(state); got != RequestStatusProcessing {
t.Fatalf("%q should collapse to %q, got %q", state, RequestStatusProcessing, got)
}
}
for _, state := range []string{
RequestStatusAvailable, RequestStatusPending, RequestStatusRequested,
RequestStatusUnavailable, RequestStatusRequestable, RequestStatusProcessing,
} {
if got := collapseRequestStatus(state); got != state {
t.Fatalf("%q must survive collapsing, got %q", state, got)
}
}
}
@@ -114,8 +163,10 @@ func TestRequestStatusLabelAndDetailCoverEveryState(t *testing.T) {
// a state that fell through to the default would show "Requested" on a card that is
// actually available.
states := []string{
RequestStatusAvailable, RequestStatusProcessing, RequestStatusPending,
RequestStatusUnavailable, RequestStatusRequestable, RequestStatusRequested,
RequestStatusAvailable, RequestStatusSearching, RequestStatusFound,
RequestStatusDownloading, RequestStatusProcessing, RequestStatusPending,
RequestStatusFailed, RequestStatusUnavailable, RequestStatusRequestable,
RequestStatusRequested,
}
seen := map[string]bool{}
for _, state := range states {
@@ -128,8 +179,27 @@ func TestRequestStatusLabelAndDetailCoverEveryState(t *testing.T) {
}
seen[label] = true
}
if requestStatusDetail(RequestStatusPending, "series") ==
requestStatusDetail(RequestStatusPending, "movie") {
for _, state := range states {
if requestStatusDetail(state, "movie", 0) == "" {
t.Fatalf("state %q has no detail line", state)
}
}
if requestStatusDetail(RequestStatusPending, "series", 0) ==
requestStatusDetail(RequestStatusPending, "movie", 0) {
t.Fatal("a pending series and a pending film are waiting for different things")
}
// The one rule this feature is judged on: an estimate is offered when the download
// client gave one and never otherwise. Both sentences are legitimate; saying the second
// while holding a number, or the first while holding none, is not.
withEstimate := requestStatusDetail(RequestStatusDownloading, "movie", 840)
withoutEstimate := requestStatusDetail(RequestStatusDownloading, "movie", 0)
if withEstimate == withoutEstimate {
t.Fatal("a download with an estimate must not read the same as one without")
}
if !strings.Contains(withEstimate, "14 minutes") {
t.Fatalf("840 seconds should read as about 14 minutes, got %q", withEstimate)
}
if strings.Contains(withoutEstimate, "~") {
t.Fatalf("a download with no estimate must not print one, got %q", withoutEstimate)
}
}
server/internal/api/sonarr_status.go
+26 -32
View File
@@ -12,38 +12,28 @@ import (
"github.com/ponzischeme89/memby/server/internal/store"
)
// WatchSonarrLifecycle seeds the durable status history on startup, then refreshes it
// daily. History stores changes rather than identical daily snapshots: it still records
// the complete lifecycle while making an active-to-cancelled transition unambiguous.
func (s *Server) WatchSonarrLifecycle(ctx context.Context, interval time.Duration) {
if !s.sonarrEnabled(ctx) || interval <= 0 {
return
}
scan := func() {
if s.quietTimeActive() {
return
}
if err := s.scanSonarrLifecycle(ctx); err != nil && ctx.Err() == nil {
s.log.Warn("Sonarr lifecycle scan failed", "error", err)
}
}
scan()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
scan()
}
}
// sonarrLifecycleResult is what one scan looked at and what moved.
//
// Counted rather than only logged because these are the figures the integrations console
// prints beside the run: "412 series checked, 3 changed" is an answer, where "the scan
// finished" is a line in a log an operator has to go and find.
type sonarrLifecycleResult struct {
Series int
Changes int
Added int
Cancelled int
Notifications int
}
func (s *Server) scanSonarrLifecycle(ctx context.Context) error {
// scanSonarrLifecycle seeds the durable status history and records what changed since
// the last reading. History stores changes rather than identical daily snapshots: it still
// records the complete lifecycle while making an active-to-cancelled transition
// unambiguous.
func (s *Server) scanSonarrLifecycle(ctx context.Context) (sonarrLifecycleResult, error) {
result := sonarrLifecycleResult{}
series, err := s.sonarr.Series(ctx)
if err != nil {
return fmt.Errorf("read Sonarr series: %w", err)
return result, fmt.Errorf("read Sonarr series: %w", err)
}
now := time.Now().UTC()
observations := make([]store.SonarrSeriesStatus, 0, len(series))
@@ -57,10 +47,12 @@ func (s *Server) scanSonarrLifecycle(ctx context.Context) error {
Title: item.Title, Year: item.Year, Status: item.Status, ObservedAt: now,
})
}
result.Series = len(observations)
changes, err := s.store.RecordSonarrSeriesStatuses(ctx, observations)
if err != nil {
return err
return result, err
}
result.Changes = len(changes)
cancellations := make([]store.SonarrSeriesStatusChange, 0, len(changes))
additions := make([]store.SonarrSeriesStatusChange, 0, len(changes))
for _, change := range changes {
@@ -71,14 +63,15 @@ func (s *Server) scanSonarrLifecycle(ctx context.Context) error {
cancellations = append(cancellations, change)
}
}
result.Added, result.Cancelled = len(additions), len(cancellations)
if len(cancellations) == 0 && len(additions) == 0 {
s.log.Info("Sonarr lifecycle scan complete", "series", len(observations), "changes", len(changes))
return nil
return result, nil
}
users, err := s.store.KnownUsers(ctx)
if err != nil {
return err
return result, err
}
preferences := map[string]store.NotificationPreferences{}
preferenceErrors := map[string]bool{}
@@ -153,10 +146,11 @@ func (s *Server) scanSonarrLifecycle(ctx context.Context) error {
}
}
}
result.Notifications = notifications
s.log.Info("Sonarr lifecycle scan complete",
"series", len(observations), "changes", len(changes),
"added", len(additions), "cancelled", len(cancellations), "notifications", notifications)
return nil
return result, nil
}
func sonarrLifecycleNotificationKind(previous, current string) string {
server/internal/buildinfo/VERSION
+1 -1
View File
@@ -1 +1 @@
0.1.57
0.1.59
server/internal/foryou/service.go
+42 -63
View File
@@ -99,6 +99,20 @@ func (s *Service) Running() bool {
return s.importRunning || len(s.building) > 0
}
// Ping asks Tracearr for the smallest thing it will answer with.
//
// It exists so the integrations console can report whether Tracearr is reachable without
// reaching past this service for the client: the service owns the connection, and a
// console holding its own copy of the client would be a second place the address and key
// could be wrong. One user, one page — the answer is discarded and only the error matters.
func (s *Service) Ping(ctx context.Context) error {
if s == nil || s.tracearr == nil {
return errors.New("tracearr is not configured")
}
_, err := s.tracearr.Users(ctx, 1, 1)
return err
}
func (s *Service) Stats(ctx context.Context) (store.ForYouStats, error) {
return s.store.ForYouStats(ctx)
}
@@ -403,17 +417,35 @@ func (s *Service) RefreshAsync(sess store.Session, force bool) {
}()
}
func (s *Service) RebuildAll(ctx context.Context, force bool) error {
// RebuildResult is what one pass over the household did.
//
// Counted because a rebuild that quietly failed for every viewer and one that succeeded
// for every viewer are the same "no error" from outside: a per-user failure is logged and
// swallowed on purpose — one viewer's broken profile must not stop the rest being built —
// so the count is the only thing that can say it happened.
type RebuildResult struct {
Users int
Built int
Failed int
Skipped int
}
func (s *Service) RebuildAll(ctx context.Context, force bool) (RebuildResult, error) {
result := RebuildResult{}
users, err := s.recommendationUsers(ctx)
if err != nil {
return err
return result, err
}
result.Users = len(users)
for _, user := range users {
if err := s.Rebuild(ctx, user, force); err != nil {
result.Failed++
s.log.Warn("For You user rebuild failed", "user", user.EmbyUserID, "error", err)
continue
}
result.Built++
}
return nil
return result, nil
}
// RebuildOutdated refreshes only profiles produced by an older algorithm. This makes
@@ -767,66 +799,13 @@ func importDue(
return false, false
}
func (s *Service) Schedule(
ctx context.Context,
importEvery, fullEvery time.Duration,
rebuildHour int,
paused ...func() bool,
) {
// One ticker asks "is anything owed?"; the persisted stamps decide what and whether.
// Two independent tickers measured process uptime, which is what let a restart reset
// the cadence and a bounced container import far more often than configured.
checkEvery := importEvery
if checkEvery <= 0 || (fullEvery > 0 && fullEvery < checkEvery) {
checkEvery = fullEvery
}
var importC <-chan time.Time
if checkEvery > 0 {
importTicker := time.NewTicker(checkEvery)
importC = importTicker.C
defer importTicker.Stop()
s.log.Info("Tracearr auto-import scheduled",
"incremental", importEvery.String(), "full", fullEvery.String())
} else {
s.log.Info("Tracearr auto-import disabled")
}
nextRebuild := nextDailyRebuild(time.Now(), s.location, rebuildHour)
rebuildTimer := time.NewTimer(time.Until(nextRebuild))
defer rebuildTimer.Stop()
s.log.Info("For You daily rebuild scheduled", "next", nextRebuild)
for {
select {
case <-ctx.Done():
return
case <-importC:
if len(paused) > 0 && paused[0] != nil && paused[0]() {
continue
}
if _, _, err := s.ImportIfDue(ctx, importEvery, fullEvery); err != nil {
s.log.Warn("scheduled Tracearr import failed", "error", err)
}
case <-rebuildTimer.C:
if len(paused) == 0 || paused[0] == nil || !paused[0]() {
if err := s.RebuildAll(ctx, true); err != nil {
s.log.Warn("scheduled daily For You rebuild failed", "error", err)
}
}
rebuildTimer.Reset(time.Until(nextDailyRebuild(time.Now(), s.location, rebuildHour)))
}
}
}
func nextDailyRebuild(now time.Time, location *time.Location, hour int) time.Time {
if location == nil {
location = time.UTC
}
local := now.In(location)
next := time.Date(local.Year(), local.Month(), local.Day(), hour, 0, 0, 0, location)
if !next.After(local) {
next = next.AddDate(0, 0, 1)
}
return next
}
// Schedule used to own both the Tracearr import ticker and the daily rebuild timer, and
// nextDailyRebuild was when the household's off-peak hour next came round. Both are gone:
// the import and the rebuild are scheduler tasks now (see api.RegisterIntegrationTasks),
// because a ticker in here could report nothing to an operator, could not be started by
// hand, and — since the tasks carry an integration id — could not be counted towards
// Tracearr's run history. The "is the rebuild due" rule moved with the work, to
// api.forYouRebuildDue, and is still pure and still tested.
func (s *Service) beginBuild(userID string) bool {
s.mu.Lock()
server/internal/foryou/service_test.go
-11
View File
@@ -179,17 +179,6 @@ func TestImportDueRespectsDisabledIntervals(t *testing.T) {
full, due)
}
}
func TestNextDailyRebuildUsesConfiguredLocalHour(t *testing.T) {
location := time.FixedZone("NZST", 12*60*60)
now := time.Date(2026, 7, 31, 5, 30, 0, 0, location)
next := nextDailyRebuild(now, location, 4)
want := time.Date(2026, 8, 1, 4, 0, 0, 0, location)
if !next.Equal(want) {
t.Fatalf("next rebuild = %v, want %v", next, want)
}
}
func TestStoredResultCapsCandidatePoolAndPersistsAlgorithmVersion(t *testing.T) {
result := recommend.PreparedResult{
Candidates: make([]recommend.PreparedCandidate, maxPreparedCandidates+50),
server/internal/integrations/integrations.go
+6 -2
View File
@@ -19,6 +19,8 @@ import (
"sync"
"time"
"github.com/ponzischeme89/memby/server/internal/runtimestats"
"github.com/ponzischeme89/memby/server/internal/adminevents"
"github.com/ponzischeme89/memby/server/internal/notify"
"github.com/ponzischeme89/memby/server/internal/store"
@@ -108,7 +110,9 @@ func (d *Dispatcher) register(transport Transport) {
// of events a minute, and serialising them means a destination cannot be hit with four
// concurrent posts by a burst.
func (d *Dispatcher) Start(ctx context.Context) {
go func() {
// Named rather than launched bare, so the console can say this worker is running
// without having to infer it from a stack. See internal/runtimestats.
runtimestats.Go("Integrations dispatcher", "Integrations", func() {
for {
select {
case <-ctx.Done():
@@ -126,7 +130,7 @@ func (d *Dispatcher) Start(ctx context.Context) {
d.deliver(ctx, work)
}
}
}()
})
}
// DeliverAdminEvent implements adminevents.Sink. It only enqueues: the bus calls this
server/internal/radarr/client.go
+56
View File
@@ -285,3 +285,59 @@ func (c *Client) Movies(ctx context.Context) ([]Movie, error) {
}
return movies, nil
}
// QueueItem is one thing the download client is working on, in the narrow shape the
// request page needs: which film, how far through, and how it is going.
//
// Radarr describes a download in three overlapping words rather than one, and all three
// are needed. Status is the *download client's* view (queued, downloading, paused,
// completed, failed, warning, delay). TrackedDownloadState is Radarr's own view of what
// happens after the bytes land (downloading, importPending, importing, imported,
// failedPending, failed) — which is the only thing that separates "still coming down the
// wire" from "almost on the shelf". TrackedDownloadStatus is the verdict (ok, warning,
// error), and is what says an otherwise healthy-looking row has actually gone wrong.
type QueueItem struct {
ID int `json:"id"`
MovieID int `json:"movieId"`
// Size and Sizeleft are bytes, as floats — Radarr sends them that way, and a film is
// comfortably past what a 32-bit int holds.
Size float64 `json:"size"`
Sizeleft float64 `json:"sizeleft"`
// Timeleft is the download client's own estimate, formatted "00:14:32" or
// "1.02:03:04". It is absent for a queued or stalled item, which is exactly the case
// where Memby must not invent one.
Timeleft string `json:"timeleft"`
Status string `json:"status"`
TrackedDownloadState string `json:"trackedDownloadState"`
TrackedDownloadStatus string `json:"trackedDownloadStatus"`
ErrorMessage string `json:"errorMessage"`
}
// queuePageSize is what one read asks for. The queue is what the household is downloading
// right now, so it is small by nature; the cap exists so a download client that has wedged
// with a thousand rows cannot turn a request-page refresh into a large response.
const queuePageSize = 200
type queuePage struct {
Records []QueueItem `json:"records"`
}
// Queue returns what Radarr is currently working on.
//
// Unknown items are excluded: those are downloads in the client that Radarr cannot match
// to a film it tracks, so they can never be the answer to "what is happening to the thing
// I asked for" and would only be rows nothing could use.
func (c *Client) Queue(ctx context.Context) ([]QueueItem, error) {
req, err := c.request(ctx, "/api/v3/queue", url.Values{
"pageSize": {strconv.Itoa(queuePageSize)},
"includeUnknownMovieItems": {"false"},
})
if err != nil {
return nil, err
}
var page queuePage
if err := c.do(req, &page); err != nil {
return nil, err
}
return page.Records, nil
}
server/internal/recommend/engine.go
+22 -2
View File
@@ -94,6 +94,15 @@ type Engine struct {
Library LibrarySource
Tracearr TracearrSource
Behavior BehaviorSource
// TracearrAllowed is the operator's global switch for the Tracearr integration, read
// at call time rather than at construction: a switch consulted once at start-up is
// not a switch, and turning an integration off must stop the calls without a restart.
//
// It is a function rather than a bool for the same reason it is not a policy lookup
// inside the engine: what "switched off" means belongs to the gateway's integrations
// area, and the ranker's business is ranking. Nil means allowed, so every test and
// every caller that predates the switch behaves as it did.
TracearrAllowed func(ctx context.Context) bool
// MinRowItems is the shortest row worth showing. A two-item "Recommended" strip
// looks broken next to full rows, so short rows are dropped entirely.
@@ -144,7 +153,7 @@ func (e *Engine) BuildForYou(
var sessions []tracearr.Session
contextAffinity := NewContextAffinityProfile()
if e.Tracearr != nil {
if e.tracearrUsable(ctx) {
if fetched, traceErr := e.Tracearr.History(ctx, username, 300); traceErr != nil {
e.log.Warn("tracearr history unavailable; using emby signals", "error", traceErr)
} else {
@@ -217,6 +226,17 @@ func powDecay(base float64, position int) float64 {
return math.Pow(base, float64(position))
}
// tracearrUsable reports whether the engine may call Tracearr right now. Falling back to
// Emby's own signals is what happens when it may not, which is the same degradation an
// unreachable Tracearr already produces — so switching it off costs the extra signal and
// never the row.
func (e *Engine) tracearrUsable(ctx context.Context) bool {
if e.Tracearr == nil {
return false
}
return e.TracearrAllowed == nil || e.TracearrAllowed(ctx)
}
func (e *Engine) applyTracearrSignals(
profile *Profile,
history []Item,
@@ -614,7 +634,7 @@ func (e *Engine) BuildRowsForUser(
profile := BuildProfile(history, favorites)
contextAffinity := NewContextAffinityProfile()
if e.Tracearr != nil && strings.TrimSpace(username) != "" {
if e.tracearrUsable(ctx) && strings.TrimSpace(username) != "" {
if sessions, traceErr := e.Tracearr.History(ctx, username, 300); traceErr != nil {
e.log.Warn("tracearr signals unavailable for shelves", "error", traceErr)
} else {
server/internal/runtimestats/goroutines.go
+467
View File
@@ -0,0 +1,467 @@
package runtimestats
import (
"runtime"
"sort"
"strconv"
"strings"
"time"
)
/* The expensive half, and the reason it is a separate route.
Reading every goroutine's stack means stopping the world for as long as it takes to walk
them all. That is measured in milliseconds rather than seconds, but it is not something
to do on a poll every open console tab makes every thirty seconds — so this is collected
only when an operator asks, and the answer says when it was collected and what it cost.
What comes back is a text dump, and parsing text a runtime produced is exactly the sort
of thing that quietly stops working. Two rules keep that honest: parseGoroutines is pure
and pinned by tests carrying real dump text, and every unrecognised state or frame falls
into a named "other" bucket rather than being dropped — a breakdown whose parts do not
add up to the total is worse than no breakdown, so the total is always the count of
goroutines parsed and the categories always partition it. */
// Category is the readable grouping a goroutine state falls into. These are the words an
// operator is being asked to reason with, so there are six of them and none is Go jargon.
type Category string
const (
CategoryRunning Category = "running"
CategoryIO Category = "io"
CategoryWaiting Category = "waiting"
CategoryTimers Category = "timers"
CategoryRuntime Category = "runtime"
CategoryOther Category = "other"
)
// categoryLabels is the console's wording. Sent from here, the stance every label the
// gateway prints takes: a category added later reads correctly on a console that predates
// it, because the console prints what it was handed.
var categoryLabels = map[Category]string{
CategoryRunning: "Running",
CategoryIO: "Network / I/O",
CategoryWaiting: "Waiting / idle",
CategoryTimers: "Timers / scheduled work",
CategoryRuntime: "Go runtime / collection",
CategoryOther: "Other",
}
// categoryOrder is the order they are presented in: busiest kind of work first, then the
// two that mean the goroutine is doing nothing, then the runtime's own housekeeping. It is
// fixed rather than sorted by count so the table does not reorder between two collections.
var categoryOrder = []Category{
CategoryRunning, CategoryIO, CategoryWaiting, CategoryTimers, CategoryRuntime, CategoryOther,
}
// categoryDescriptions say what the operator is looking at, because "semacquire" means
// nothing to somebody who does not write Go and the whole point of this page is that they
// should not have to.
var categoryDescriptions = map[Category]string{
CategoryRunning: "On a processor now, or queued for one.",
CategoryIO: "Waiting on a network read or write — Emby, Postgres, Redis, or a television.",
CategoryWaiting: "Parked waiting for work or for a lock. Idle, and costing almost nothing.",
CategoryTimers: "Asleep until a scheduled time.",
CategoryRuntime: "The Go runtime's own housekeeping. This handful is always present.",
CategoryOther: "A state this build does not have a category for.",
}
// categoryFor maps a runtime state string onto one of the six. The GC states are tested
// first because several of them contain words the later rules would otherwise claim —
// "wait for GC cycle" is the runtime's own, not a component waiting on a channel.
func categoryFor(state string) Category {
state = strings.TrimSpace(state)
lower := strings.ToLower(state)
switch {
case strings.Contains(lower, "gc "), strings.HasPrefix(lower, "gc"),
lower == "finalizer wait", strings.HasPrefix(lower, "trace reader"),
strings.HasPrefix(lower, "dumping heap"), strings.HasPrefix(lower, "stopping the world"),
strings.HasPrefix(lower, "idle"):
return CategoryRuntime
case lower == "running", lower == "runnable", strings.HasPrefix(lower, "syscall"):
return CategoryRunning
case strings.HasPrefix(lower, "io wait"), strings.HasPrefix(lower, "netpoll"):
return CategoryIO
case strings.HasPrefix(lower, "sleep"), strings.HasPrefix(lower, "timer"):
return CategoryTimers
case strings.HasPrefix(lower, "select"), strings.HasPrefix(lower, "chan "),
strings.HasPrefix(lower, "semacquire"), strings.HasPrefix(lower, "sync."),
strings.HasPrefix(lower, "wait"), strings.HasPrefix(lower, "preempted"):
return CategoryWaiting
default:
return CategoryOther
}
}
/* ---------- component attribution ---------- */
type componentRule struct {
component string
// patterns are matched against every frame of the stack — the function names and the
// file paths alike — plus the "created by" line, which is often the only frame that
// still names the component a long-parked goroutine belongs to.
patterns []string
}
// componentRules is ordered, and the order is the whole of it: Memby's own packages are
// tested before the libraries they call, or every database query in the gateway would be
// filed under "Database pool" and nothing would be attributable to the feature that made
// it. First match wins.
var componentRules = []componentRule{
{"Library sync", []string{"/internal/library"}},
{"Scheduled jobs", []string{"/internal/scheduler"}},
{"Credits detection", []string{"/internal/credits"}},
{"Recommendations", []string{"/internal/recommend", "/internal/foryou", "/internal/tracearr"}},
{"Notifications", []string{"/internal/notify"}},
{"Integrations", []string{"/internal/integrations", "/internal/adminevents"}},
{"Subtitles", []string{"/internal/bazarr", "/internal/opensubtitles", "/internal/subsync"}},
{"Sonarr and Radarr", []string{"/internal/sonarr", "/internal/radarr"}},
{"Ratings", []string{"/internal/mdblist"}},
{"Emby", []string{"/internal/emby", "/internal/trickplay"}},
{"Gateway API", []string{"/internal/api", "/internal/store", "/internal/cache", "/internal/logging"}},
{"Database pool", []string{"jackc/pgx", "jackc/puddle", "database/sql"}},
{"Redis", []string{"redis/go-redis"}},
{"HTTP server", []string{"net/http.(*conn).serve", "net/http.(*Server)", "net/http.(*connReader)"}},
{"HTTP client", []string{"net/http.(*persistConn)", "net/http.(*Transport)"}},
{"Go runtime", []string{"runtime.", "runtime/pprof"}},
}
const componentUnattributed = "Unattributed"
func componentFor(info goroutineInfo) string {
haystack := info.searchText()
for _, rule := range componentRules {
for _, pattern := range rule.patterns {
if strings.Contains(haystack, pattern) {
return rule.component
}
}
}
return componentUnattributed
}
/* ---------- parsing ---------- */
type frame struct {
Function string
File string
}
type goroutineInfo struct {
ID int
State string
WaitMinutes int
Frames []frame
CreatedBy string
}
func (info goroutineInfo) searchText() string {
var builder strings.Builder
for _, one := range info.Frames {
builder.WriteString(one.Function)
builder.WriteByte('\n')
builder.WriteString(one.File)
builder.WriteByte('\n')
}
builder.WriteString(info.CreatedBy)
return builder.String()
}
// topFrame is the function the goroutine is actually sitting in, skipping the runtime
// plumbing that every parked goroutine shares. Without the skip, half the table reads
// "runtime.gopark", which is true of every waiting goroutine in the process and therefore
// tells nobody anything.
func (info goroutineInfo) topFrame() frame {
for _, one := range info.Frames {
function := one.Function
if strings.HasPrefix(function, "runtime.") || strings.HasPrefix(function, "internal/poll.runtime_") ||
strings.HasPrefix(function, "sync.runtime_") || strings.HasPrefix(function, "sync.(*") ||
strings.HasPrefix(function, "time.Sleep") {
continue
}
return one
}
if len(info.Frames) > 0 {
return info.Frames[0]
}
return frame{}
}
// parseGoroutines reads a runtime.Stack dump. It is pure so the shape of the dump can be
// pinned by a test rather than discovered when a Go release changes it.
func parseGoroutines(dump string) []goroutineInfo {
var out []goroutineInfo
var current *goroutineInfo
flush := func() {
if current != nil {
out = append(out, *current)
current = nil
}
}
for _, line := range strings.Split(dump, "\n") {
switch {
case strings.HasPrefix(line, "goroutine "):
flush()
info, ok := parseHeader(line)
if !ok {
continue
}
current = &info
case current == nil:
continue
case strings.HasPrefix(line, "created by "):
current.CreatedBy = strings.TrimPrefix(line, "created by ")
case strings.HasPrefix(line, "\t"):
// A file line belongs to the function line above it.
if len(current.Frames) > 0 {
path := strings.TrimSpace(line)
if cut := strings.LastIndex(path, " +0x"); cut > 0 {
path = path[:cut]
}
current.Frames[len(current.Frames)-1].File = path
}
case strings.TrimSpace(line) == "":
flush()
default:
current.Frames = append(current.Frames, frame{Function: functionName(line)})
}
}
flush()
return out
}
// parseHeader reads `goroutine 42 [select, 5 minutes]:`.
func parseHeader(line string) (goroutineInfo, bool) {
open := strings.Index(line, "[")
shut := strings.LastIndex(line, "]")
if open < 0 || shut < open {
return goroutineInfo{}, false
}
id, _ := strconv.Atoi(strings.TrimSpace(line[len("goroutine "):open]))
inside := line[open+1 : shut]
state := inside
minutes := 0
// The state itself can contain a comma — "chan receive, 5 minutes" and "GC worker
// (idle)" both do not, but "semacquire, 3 minutes, locked to thread" does, so only a
// part that parses as a duration is taken as one.
parts := strings.Split(inside, ", ")
state = parts[0]
for _, part := range parts[1:] {
if value, ok := parseMinutes(part); ok {
minutes = value
}
}
return goroutineInfo{ID: id, State: state, WaitMinutes: minutes}, true
}
func parseMinutes(part string) (int, bool) {
part = strings.TrimSpace(part)
if !strings.HasSuffix(part, " minutes") && !strings.HasSuffix(part, " minute") {
return 0, false
}
value, err := strconv.Atoi(strings.Fields(part)[0])
if err != nil {
return 0, false
}
return value, true
}
// functionName strips the argument list a stack dump prints after the function, since the
// arguments are addresses and would make every otherwise-identical goroutine its own group.
func functionName(line string) string {
line = strings.TrimSpace(line)
if open := strings.Index(line, "("); open > 0 {
// Method receivers are themselves parenthesised — `pkg.(*Type).Method(0x1)` — so
// the argument list is the last balanced group rather than the first bracket.
if last := strings.LastIndex(line, "("); last > 0 && !strings.HasSuffix(line[:last], ".") {
return line[:last]
}
return line[:open]
}
return line
}
/* ---------- the report ---------- */
type CategoryCount struct {
Category Category `json:"category"`
Label string `json:"label"`
Description string `json:"description"`
Count int `json:"count"`
States []StateCount `json:"states"`
}
type StateCount struct {
State string `json:"state"`
Count int `json:"count"`
}
type ComponentCount struct {
Component string `json:"component"`
Count int `json:"count"`
// LongestWaitMinutes is the age of the oldest goroutine attributed here. A component
// whose count is climbing and whose oldest is hours old is the shape of a leak; one
// that is busy and young is the shape of a busy evening.
LongestWaitMinutes int `json:"longestWaitMinutes"`
}
type StackGroup struct {
Count int `json:"count"`
Component string `json:"component"`
Category Category `json:"category"`
State string `json:"state"`
Function string `json:"function"`
File string `json:"file"`
CreatedBy string `json:"createdBy,omitempty"`
LongestWaitMinutes int `json:"longestWaitMinutes"`
}
type GoroutineReport struct {
At time.Time `json:"at"`
Total int `json:"total"`
CollectedInMs float64 `json:"collectedInMs"`
DumpBytes int `json:"dumpBytes"`
Categories []CategoryCount `json:"categories"`
Components []ComponentCount `json:"components"`
Groups []StackGroup `json:"groups"`
// GroupsTotal is how many distinct groups there were before the cap below; a gateway
// with thousands of goroutines has a long tail that would be a page nobody reads.
GroupsTotal int `json:"groupsTotal"`
Workers []Worker `json:"workers"`
}
// maxGroups caps the table. The groups are sorted by count, so what is dropped is the tail
// of one-off goroutines, which is exactly the part that says nothing.
const maxGroups = 40
// CollectGoroutines takes the dump and builds the report. This is the expensive call.
func CollectGoroutines() GoroutineReport {
begin := time.Now()
dump := stackDump()
report := buildReport(dump, time.Now())
report.CollectedInMs = float64(time.Since(begin).Microseconds()) / 1000
report.Workers = Workers()
return report
}
// StackDump is the raw text, for an operator who wants the whole thing rather than the
// summary — the on-demand snapshot that replaces any temptation to leave a profiler
// endpoint permanently open.
func StackDump() string { return stackDump() }
// stackDump grows its buffer until the whole dump fits. runtime.Stack truncates silently
// at the length of the buffer it is given, and a truncated dump would produce a breakdown
// that is quietly missing whichever goroutines came last — which on a leak is the ones
// worth seeing.
func stackDump() string {
size := 1 << 20
for {
buffer := make([]byte, size)
written := runtime.Stack(buffer, true)
if written < len(buffer) {
return string(buffer[:written])
}
if size >= 1<<26 {
return string(buffer[:written])
}
size *= 2
}
}
func buildReport(dump string, at time.Time) GoroutineReport {
list := parseGoroutines(dump)
report := GoroutineReport{At: at, Total: len(list), DumpBytes: len(dump)}
states := map[Category]map[string]int{}
counts := map[Category]int{}
components := map[string]*ComponentCount{}
groups := map[string]*StackGroup{}
for _, info := range list {
category := categoryFor(info.State)
counts[category]++
if states[category] == nil {
states[category] = map[string]int{}
}
states[category][info.State]++
component := componentFor(info)
entry := components[component]
if entry == nil {
entry = &ComponentCount{Component: component}
components[component] = entry
}
entry.Count++
if info.WaitMinutes > entry.LongestWaitMinutes {
entry.LongestWaitMinutes = info.WaitMinutes
}
top := info.topFrame()
key := component + "\x00" + info.State + "\x00" + top.Function
group := groups[key]
if group == nil {
group = &StackGroup{
Component: component, Category: category, State: info.State,
Function: top.Function, File: top.File, CreatedBy: info.CreatedBy,
}
groups[key] = group
}
group.Count++
if info.WaitMinutes > group.LongestWaitMinutes {
group.LongestWaitMinutes = info.WaitMinutes
}
}
for _, category := range categoryOrder {
count := counts[category]
if count == 0 {
continue
}
report.Categories = append(report.Categories, CategoryCount{
Category: category,
Label: categoryLabels[category],
Description: categoryDescriptions[category],
Count: count,
States: sortedStates(states[category]),
})
}
for _, entry := range components {
report.Components = append(report.Components, *entry)
}
sort.Slice(report.Components, func(a, b int) bool {
if report.Components[a].Count != report.Components[b].Count {
return report.Components[a].Count > report.Components[b].Count
}
return report.Components[a].Component < report.Components[b].Component
})
for _, group := range groups {
report.Groups = append(report.Groups, *group)
}
sort.Slice(report.Groups, func(a, b int) bool {
if report.Groups[a].Count != report.Groups[b].Count {
return report.Groups[a].Count > report.Groups[b].Count
}
return report.Groups[a].Function < report.Groups[b].Function
})
report.GroupsTotal = len(report.Groups)
if len(report.Groups) > maxGroups {
report.Groups = report.Groups[:maxGroups]
}
return report
}
func sortedStates(counts map[string]int) []StateCount {
out := make([]StateCount, 0, len(counts))
for state, count := range counts {
out = append(out, StateCount{State: state, Count: count})
}
sort.Slice(out, func(a, b int) bool {
if out[a].Count != out[b].Count {
return out[a].Count > out[b].Count
}
return out[a].State < out[b].State
})
return out
}
server/internal/runtimestats/health.go
+257
View File
@@ -0,0 +1,257 @@
package runtimestats
import (
"fmt"
"math"
"sort"
)
// Health is the answer the Process card exists to give: is the gateway healthy, and if
// something is abnormal, which area should be looked at?
//
// It is stated on the server rather than derived in the console for the same reason every
// schedule label is the gateway's wording — the thresholds and the sentence explaining
// them belong together, and an older console must not be the thing deciding what "watch"
// means. It is a pure function of a snapshot so the thresholds can be tested rather than
// discovered in production.
type Health struct {
Level Level `json:"level"`
Summary string `json:"summary"`
Notes []Note `json:"notes"`
Areas []string `json:"areas,omitempty"`
}
type Level string
const (
// LevelOK is nothing to do. LevelWatch is something that is not wrong yet and would
// be if it continued — which is the whole reason trends are collected. LevelBad is
// something an operator should act on now.
LevelOK Level = "ok"
LevelWatch Level = "watch"
LevelBad Level = "bad"
)
type Note struct {
Level Level `json:"level"`
// Area names the part of Memby to look at. It is the note's most useful field: a
// number that has moved is only actionable once it points somewhere.
Area string `json:"area"`
Message string `json:"message"`
}
// The thresholds. Heap is judged against the configured limit because that is the number
// the container is actually killed at; the watch line is deliberately well below it, since
// the point of a watch is to arrive before the incident rather than during it.
const (
heapShareBad = 0.90
heapShareWatch = 0.75
// goroutineLeakPerHour is what separates a gateway that has picked up work from one
// that is not letting go of it. A household arriving home adds goroutines and gives
// them back; a leak does not, so this is only ever consulted alongside a rising trend
// measured over the trend window.
goroutineLeakPerHour = 10
goroutineFloodPerHour = 60
// pauseWatchMs is a collection pause long enough for a television to notice. The
// gateway answers a status poll every ten seconds from every open set.
pauseWatchMs = 50
pauseBadMs = 250
// restartsWatch: a worker that has been launched this many times is one that keeps
// dying. The registry counts starts for exactly this.
restartsWatch = 3
)
func assess(snapshot Snapshot) Health {
var notes []Note
memory := snapshot.Memory
switch {
case memory.HeapShare >= heapShareBad:
notes = append(notes, Note{
Level: LevelBad, Area: "Memory",
Message: fmt.Sprintf(
"The heap is using %s of the %s limit. Past the limit the container is stopped, so this needs attention now.",
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
),
})
case memory.HeapShare >= heapShareWatch:
notes = append(notes, Note{
Level: LevelWatch, Area: "Memory",
Message: fmt.Sprintf(
"The heap is using %s of the %s limit. There is room, but less than usual.",
percent(memory.HeapShare), bytes(uint64(memory.MemoryLimit)),
),
})
}
if snapshot.HeapTrend.Direction == DirectionRising {
notes = append(notes, Note{
Level: LevelWatch, Area: "Memory",
Message: fmt.Sprintf(
"Heap in use has risen by about %s an hour over the last %s. If it does not fall after a collection, something is holding on to it.",
bytes(uint64(math.Max(0, snapshot.HeapTrend.PerHour))), duration(snapshot.HeapTrend.SpanSeconds),
),
})
}
if snapshot.GoroutineTrend.Direction == DirectionRising {
rate := snapshot.GoroutineTrend.PerHour
level := LevelOK
switch {
case rate >= goroutineFloodPerHour:
level = LevelBad
case rate >= goroutineLeakPerHour:
level = LevelWatch
}
if level != LevelOK {
notes = append(notes, Note{
Level: level, Area: "Goroutines",
Message: fmt.Sprintf(
"Goroutines have risen by about %.0f an hour over the last %s, from %.0f to %.0f. Open the breakdown to see which component they belong to.",
rate, duration(snapshot.GoroutineTrend.SpanSeconds),
snapshot.GoroutineTrend.First, snapshot.GoroutineTrend.Latest,
),
})
}
}
switch {
case memory.PauseRecentMs >= pauseBadMs:
notes = append(notes, Note{
Level: LevelBad, Area: "Collection",
Message: fmt.Sprintf(
"Recent collections have paused the gateway for %.0f ms on average. Everything the televisions ask for waits that long.",
memory.PauseRecentMs,
),
})
case memory.PauseRecentMs >= pauseWatchMs:
notes = append(notes, Note{
Level: LevelWatch, Area: "Collection",
Message: fmt.Sprintf(
"Recent collections have paused the gateway for %.0f ms on average.", memory.PauseRecentMs,
),
})
}
for _, worker := range snapshot.Workers {
if worker.State == WorkerRunning && worker.Starts >= restartsWatch {
notes = append(notes, Note{
Level: LevelWatch, Area: worker.Component,
Message: fmt.Sprintf(
"%s has been started %d times. A background worker that keeps restarting is failing at something.",
worker.Name, worker.Starts,
),
})
}
}
return Health{
Level: worst(notes),
Summary: summarise(snapshot, notes),
Notes: notes,
Areas: areasOf(notes),
}
}
func worst(notes []Note) Level {
level := LevelOK
for _, note := range notes {
if note.Level == LevelBad {
return LevelBad
}
if note.Level == LevelWatch {
level = LevelWatch
}
}
return level
}
// summarise is the one line the overview card prints. When there is nothing to say it says
// so plainly rather than reciting figures the operator can already see above it — and it
// says whether the verdict is based on enough history to be worth anything, because "no
// trouble" from four minutes of readings is a weaker claim than the same words from four
// hours.
func summarise(snapshot Snapshot, notes []Note) string {
if len(notes) == 0 {
if snapshot.GoroutineTrend.Direction == DirectionUnknown {
return "Nothing abnormal. Still gathering history — trends need about fifteen minutes."
}
return fmt.Sprintf(
"Nothing abnormal. Goroutines and memory have been %s over the last %s.",
steadyWord(snapshot), duration(snapshot.GoroutineTrend.SpanSeconds),
)
}
areas := areasOf(notes)
if len(notes) == 1 {
return notes[0].Message
}
return fmt.Sprintf("%d things to look at, in %s.", len(notes), joinAreas(areas))
}
func steadyWord(snapshot Snapshot) string {
if snapshot.GoroutineTrend.Direction == DirectionFalling || snapshot.HeapTrend.Direction == DirectionFalling {
return "steady or falling"
}
return "steady"
}
func areasOf(notes []Note) []string {
seen := map[string]bool{}
var areas []string
for _, note := range notes {
if note.Area == "" || seen[note.Area] {
continue
}
seen[note.Area] = true
areas = append(areas, note.Area)
}
sort.Strings(areas)
return areas
}
func joinAreas(areas []string) string {
switch len(areas) {
case 0:
return "the gateway"
case 1:
return areas[0]
case 2:
return areas[0] + " and " + areas[1]
}
return fmt.Sprintf("%s and %d other areas", areas[0], len(areas)-1)
}
/* ---------- wording helpers ----------
These exist because the sentences above are the gateway's own wording, the stance every
label the console prints takes: a console built before a threshold existed still reads
correctly, and the figure in the sentence cannot disagree with the threshold that
produced it. */
func percent(share float64) string {
return fmt.Sprintf("%.1f%%", share*100)
}
func bytes(value uint64) string {
const unit = 1024
if value < unit {
return fmt.Sprintf("%d B", value)
}
div, exponent := uint64(unit), 0
for size := value / unit; size >= unit; size /= unit {
div *= unit
exponent++
}
return fmt.Sprintf("%.1f %cB", float64(value)/float64(div), "KMGT"[exponent])
}
func duration(seconds float64) string {
switch {
case seconds < 90:
return fmt.Sprintf("%.0f seconds", seconds)
case seconds < 5400:
return fmt.Sprintf("%.0f minutes", seconds/60)
default:
return fmt.Sprintf("%.1f hours", seconds/3600)
}
}
server/internal/runtimestats/process.go
+170
View File
@@ -0,0 +1,170 @@
package runtimestats
import (
"os"
"runtime/pprof"
"strconv"
"strings"
"sync"
"time"
)
// Process is what the operating system knows about the container that the Go runtime does
// not. Every field of it is optional: this is read from /proc, which exists in the
// container the gateway is deployed in and does not exist on a developer's machine, so an
// unavailable figure is omitted rather than reported as zero — nought open sockets and
// "could not look" are different answers and the console must not print the first for the
// second.
type Process struct {
PID int `json:"pid"`
// CPUSeconds is cumulative processor time, and CPUPercent is the share of one
// processor used since the previous read. A hundred per cent is one core saturated,
// not the machine: with GOMAXPROCS processors available the ceiling is that times a
// hundred, which is why the console prints both.
CPUSeconds float64 `json:"cpuSeconds,omitempty"`
CPUPercent float64 `json:"cpuPercent,omitempty"`
CPUKnown bool `json:"cpuKnown"`
OpenFiles int `json:"openFiles,omitempty"`
OpenSockets int `json:"openSockets,omitempty"`
FileLimit int `json:"fileLimit,omitempty"`
FilesKnown bool `json:"filesKnown"`
}
// threadCount is operating-system threads, which is a different number from goroutines and
// is the one that matters when goroutines are blocked in syscalls: the runtime creates a
// thread per blocked call, and a thread costs far more than a goroutine does.
func threadCount() int {
profile := pprof.Lookup("threadcreate")
if profile == nil {
return 0
}
return profile.Count()
}
// configuredMemoryLimit reports what the container was started with rather than what the
// runtime currently holds, so the console can say whether the limit is deliberate.
func configuredMemoryLimit() string {
return strings.TrimSpace(os.Getenv("GOMEMLIMIT"))
}
var (
cpuMu sync.Mutex
lastCPUAt time.Time
lastCPUSecs float64
lastPercent float64
)
func readProcess() Process {
process := Process{PID: os.Getpid()}
if seconds, ok := processCPUSeconds(); ok {
process.CPUSeconds = seconds
process.CPUKnown = true
process.CPUPercent = cpuPercentSince(seconds, time.Now())
}
if files, sockets, ok := openDescriptors(); ok {
process.OpenFiles = files
process.OpenSockets = sockets
process.FilesKnown = true
process.FileLimit = descriptorLimit()
}
return process
}
// cpuPercentSince needs two readings, so the first call after start-up reports nothing
// rather than dividing by the life of the process — an average over four hours would hide
// exactly the spike somebody opened the page to find. A read that arrives too soon after
// the last one repeats the previous answer instead of amplifying rounding into a spike.
func cpuPercentSince(seconds float64, now time.Time) float64 {
const minInterval = 5 * time.Second
cpuMu.Lock()
defer cpuMu.Unlock()
if lastCPUAt.IsZero() {
lastCPUAt, lastCPUSecs = now, seconds
return 0
}
elapsed := now.Sub(lastCPUAt)
if elapsed < minInterval {
return lastPercent
}
used := seconds - lastCPUSecs
lastCPUAt, lastCPUSecs = now, seconds
if used < 0 {
lastPercent = 0
return 0
}
lastPercent = used / elapsed.Seconds() * 100
return lastPercent
}
// processCPUSeconds prefers /proc/self/schedstat, whose first field is cumulative time on
// a processor in nanoseconds — exact, where /proc/self/stat is in clock ticks whose length
// cannot be read without cgo and has to be assumed to be the usual hundredth of a second.
func processCPUSeconds() (float64, bool) {
if raw, err := os.ReadFile("/proc/self/schedstat"); err == nil {
fields := strings.Fields(string(raw))
if len(fields) > 0 {
if nanos, err := strconv.ParseFloat(fields[0], 64); err == nil {
return nanos / 1e9, true
}
}
}
raw, err := os.ReadFile("/proc/self/stat")
if err != nil {
return 0, false
}
// The second field is the command name in brackets and may itself contain spaces, so
// the fields after it are counted from the closing bracket rather than from the start.
shut := strings.LastIndex(string(raw), ")")
if shut < 0 {
return 0, false
}
fields := strings.Fields(string(raw)[shut+1:])
// After the bracket, field 1 is the state; utime and stime are fields 12 and 13.
if len(fields) < 13 {
return 0, false
}
user, userErr := strconv.ParseFloat(fields[11], 64)
system, systemErr := strconv.ParseFloat(fields[12], 64)
if userErr != nil || systemErr != nil {
return 0, false
}
const ticksPerSecond = 100
return (user + system) / ticksPerSecond, true
}
func openDescriptors() (files, sockets int, ok bool) {
entries, err := os.ReadDir("/proc/self/fd")
if err != nil {
return 0, 0, false
}
for _, entry := range entries {
files++
target, err := os.Readlink("/proc/self/fd/" + entry.Name())
if err == nil && strings.HasPrefix(target, "socket:") {
sockets++
}
}
return files, sockets, true
}
func descriptorLimit() int {
raw, err := os.ReadFile("/proc/self/limits")
if err != nil {
return 0
}
for _, line := range strings.Split(string(raw), "\n") {
if !strings.HasPrefix(line, "Max open files") {
continue
}
fields := strings.Fields(strings.TrimPrefix(line, "Max open files"))
if len(fields) == 0 {
return 0
}
limit, err := strconv.Atoi(fields[0])
if err != nil {
return 0
}
return limit
}
return 0
}
server/internal/runtimestats/runtimestats_test.go
+377
View File
@@ -0,0 +1,377 @@
package runtimestats
import (
"strings"
"sync"
"testing"
"time"
)
// A real dump, trimmed. The parser is the one part of this package that reads text a Go
// release produces, so the fixture is kept verbatim rather than idealised — the argument
// lists, the "+0x" offsets and the "created by … in goroutine N" suffix are all things the
// parser has to survive.
const sampleDump = `goroutine 1 [chan receive, 940 minutes]:
main.run(0xc000122000)
/app/cmd/memby-server/main.go:430 +0x8f4
main.main()
/app/cmd/memby-server/main.go:41 +0x1f
goroutine 18 [select]:
github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run(0xc0001a2000, {0x1a4c0d0, 0xc0000a2000})
/app/internal/library/ingest.go:118 +0x145
created by github.com/ponzischeme89/memby/server/internal/runtimestats.Go in goroutine 1
goroutine 40 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7f2c1c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:351 +0x85
net/http.(*conn).serve(0xc000310000, {0x1a4c0d0, 0xc0003a0000})
/usr/local/go/src/net/http/server.go:2092 +0x5db
created by net/http.(*Server).Serve in goroutine 55
goroutine 41 [GC worker (idle)]:
runtime.gopark(0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/proc.go:402 +0xce
runtime.gcBgMarkWorker(0xc0000581c0)
/usr/local/go/src/runtime/mgc.go:1310 +0xe5
created by runtime.gcBgMarkStartWorkers in goroutine 1
goroutine 77 [semacquire, 3 minutes, locked to thread]:
github.com/jackc/puddle/v2.(*Pool).acquire(0xc00019c000)
/root/go/pkg/mod/github.com/jackc/puddle/v2@v2.2.2/pool.go:481 +0x10a
created by github.com/jackc/pgx/v5/pgxpool.NewWithConfig in goroutine 1
`
func TestParseGoroutinesReadsHeaderStateAndFrames(t *testing.T) {
list := parseGoroutines(sampleDump)
if len(list) != 5 {
t.Fatalf("parsed %d goroutines, want 5", len(list))
}
first := list[0]
if first.ID != 1 || first.State != "chan receive" || first.WaitMinutes != 940 {
t.Fatalf("first = %+v", first)
}
if first.Frames[0].Function != "main.run" {
t.Fatalf("first frame function = %q", first.Frames[0].Function)
}
// The offset is stripped so two goroutines in the same place group together rather
// than differing by an address.
if first.Frames[0].File != "/app/cmd/memby-server/main.go:430" {
t.Fatalf("first frame file = %q", first.Frames[0].File)
}
if list[1].CreatedBy == "" {
t.Fatal("created-by line was dropped")
}
// A state with trailing detail after a second comma keeps its state and its wait.
last := list[4]
if last.State != "semacquire" || last.WaitMinutes != 3 {
t.Fatalf("last = %+v", last)
}
}
// The categories have to partition the total or the breakdown is misleading, which is
// worse than not offering one.
func TestBuildReportPartitionsAndAttributes(t *testing.T) {
report := buildReport(sampleDump, time.Now())
if report.Total != 5 {
t.Fatalf("total = %d", report.Total)
}
counted := 0
for _, category := range report.Categories {
counted += category.Count
}
if counted != report.Total {
t.Fatalf("categories sum to %d, total %d", counted, report.Total)
}
attributed := 0
for _, component := range report.Components {
attributed += component.Count
}
if attributed != report.Total {
t.Fatalf("components sum to %d, total %d", attributed, report.Total)
}
got := map[string]string{}
for _, group := range report.Groups {
got[group.Component] = group.Function
}
// Memby's own packages win over the libraries they call, and the frame reported is
// the one that names the work rather than the runtime plumbing every parked goroutine
// shares.
if got["Library sync"] != "github.com/ponzischeme89/memby/server/internal/library.(*Ingester).Run" {
t.Fatalf("library group = %q", got["Library sync"])
}
if got["HTTP server"] != "net/http.(*conn).serve" {
t.Fatalf("http group = %q", got["HTTP server"])
}
if got["Database pool"] != "github.com/jackc/puddle/v2.(*Pool).acquire" {
t.Fatalf("database group = %q", got["Database pool"])
}
}
func TestCategoryForKnownStates(t *testing.T) {
cases := map[string]Category{
"running": CategoryRunning,
"runnable": CategoryRunning,
"syscall": CategoryRunning,
"IO wait": CategoryIO,
"select": CategoryWaiting,
"chan receive": CategoryWaiting,
"semacquire": CategoryWaiting,
"sync.Mutex.Lock": CategoryWaiting,
"sleep": CategoryTimers,
"timer goroutine": CategoryTimers,
"GC worker (idle)": CategoryRuntime,
"force gc (idle)": CategoryRuntime,
"finalizer wait": CategoryRuntime,
// The runtime's own wait outranks the "wait" rule, or the collector would be
// reported as a component sitting on a channel.
"wait for GC cycle": CategoryRuntime,
"something new": CategoryOther,
}
for state, want := range cases {
if got := categoryFor(state); got != want {
t.Errorf("categoryFor(%q) = %q, want %q", state, got, want)
}
}
}
// A live report has to hold together too: this is the one test that would notice a Go
// release changing the dump format under the parser.
func TestCollectGoroutinesAgreesWithTheRuntime(t *testing.T) {
blocked := make(chan struct{})
var ready sync.WaitGroup
ready.Add(1)
go func() {
ready.Done()
<-blocked
}()
ready.Wait()
defer close(blocked)
report := CollectGoroutines()
if report.Total < 2 {
t.Fatalf("total = %d", report.Total)
}
counted := 0
for _, category := range report.Categories {
counted += category.Count
}
if counted != report.Total {
t.Fatalf("categories sum to %d, total %d", counted, report.Total)
}
if report.CollectedInMs < 0 {
t.Fatalf("collectedInMs = %v", report.CollectedInMs)
}
if !strings.Contains(StackDump(), "goroutine ") {
t.Fatal("stack dump does not look like a stack dump")
}
}
/* ---------- trends ---------- */
func series(values []float64, every time.Duration) ([]float64, []time.Time) {
at := make([]time.Time, len(values))
base := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC)
for index := range values {
at[index] = base.Add(time.Duration(index) * every)
}
return values, at
}
func TestTrendRefusesToAnswerWithoutEnoughHistory(t *testing.T) {
// Four readings a minute apart is a gateway that restarted three minutes ago, not a
// trend, and "steady" would be a claim the data does not support.
points, at := series([]float64{20, 24, 30, 44}, time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionUnknown {
t.Fatalf("direction = %q, want unknown", got.Direction)
}
}
func TestTrendIgnoresABurstAndNamesASustainedRise(t *testing.T) {
// A burst in the middle — a library import, a household arriving home — must not be
// reported as a rise; medians are what make that true where a fitted line would not.
burst := []float64{40, 41, 40, 42, 41, 300, 290, 40, 41, 40, 42, 41}
points, at := series(burst, 5*time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
t.Fatalf("burst direction = %q (%+v), want steady", got.Direction, got)
}
leak := make([]float64, 12)
for index := range leak {
leak[index] = float64(40 + index*5)
}
points, at = series(leak, 5*time.Minute)
rising := trendOf(points, at, 5)
if rising.Direction != DirectionRising {
t.Fatalf("leak direction = %q (%+v), want rising", rising.Direction, rising)
}
if rising.PerHour <= 0 {
t.Fatalf("perHour = %v, want a positive rate", rising.PerHour)
}
if rising.Latest != 95 || rising.First != 40 {
t.Fatalf("first/latest = %v/%v", rising.First, rising.Latest)
}
}
func TestTrendFloorKeepsAQuietGatewaySteady(t *testing.T) {
// One extra goroutine over an hour is not a leak, and without the absolute floor the
// proportional threshold would call it one on a gateway holding a handful.
points, at := series([]float64{8, 8, 9, 9, 9, 9, 9, 10}, 10*time.Minute)
if got := trendOf(points, at, 5); got.Direction != DirectionSteady {
t.Fatalf("direction = %q (%+v), want steady", got.Direction, got)
}
}
/* ---------- the verdict ---------- */
func risingTrend(first, latest, perHour float64) Trend {
return Trend{
Direction: DirectionRising, PerHour: perHour, First: first, Latest: latest,
SpanSeconds: 3600, Points: 60,
}
}
func TestAssessSaysNothingWhenThereIsNothingToSay(t *testing.T) {
health := assess(Snapshot{
Memory: Memory{MemoryLimit: 384 << 20, HeapInuse: 16 << 20, HeapShare: 0.04},
GoroutineTrend: Trend{Direction: DirectionSteady, SpanSeconds: 7200},
HeapTrend: Trend{Direction: DirectionSteady},
})
if health.Level != LevelOK || len(health.Notes) != 0 {
t.Fatalf("health = %+v", health)
}
if !strings.Contains(health.Summary, "Nothing abnormal") {
t.Fatalf("summary = %q", health.Summary)
}
}
func TestAssessNamesTheAreaToInvestigate(t *testing.T) {
health := assess(Snapshot{
Memory: Memory{MemoryLimit: 384 << 20, HeapInuse: 350 << 20, HeapShare: 0.92},
GoroutineTrend: risingTrend(60, 400, 340),
HeapTrend: Trend{Direction: DirectionSteady},
})
if health.Level != LevelBad {
t.Fatalf("level = %q", health.Level)
}
areas := strings.Join(health.Areas, ",")
if !strings.Contains(areas, "Memory") || !strings.Contains(areas, "Goroutines") {
t.Fatalf("areas = %v", health.Areas)
}
// A verdict with no area is the number that could not be acted on, which is the whole
// defect this replaced.
for _, note := range health.Notes {
if note.Area == "" {
t.Fatalf("note without an area: %+v", note)
}
}
}
func TestAssessSeparatesADriftFromAFlood(t *testing.T) {
drift := assess(Snapshot{GoroutineTrend: risingTrend(40, 60, 20)})
if drift.Level != LevelWatch {
t.Fatalf("drift level = %q", drift.Level)
}
flood := assess(Snapshot{GoroutineTrend: risingTrend(40, 400, 360)})
if flood.Level != LevelBad {
t.Fatalf("flood level = %q", flood.Level)
}
// A rise slower than the leak threshold is ordinary breathing and must not be a note
// at all, or the card cries wolf on every busy evening.
quiet := assess(Snapshot{GoroutineTrend: risingTrend(40, 44, 4)})
if quiet.Level != LevelOK || len(quiet.Notes) != 0 {
t.Fatalf("quiet = %+v", quiet)
}
}
func TestAssessNoticesAWorkerThatKeepsRestarting(t *testing.T) {
health := assess(Snapshot{Workers: []Worker{
{Name: "Library ingest", Component: "Library sync", State: WorkerRunning, Starts: 5},
{Name: "Startup library sync", Component: "Library sync", State: WorkerFinished, Starts: 1},
}})
if health.Level != LevelWatch || len(health.Notes) != 1 {
t.Fatalf("health = %+v", health)
}
if !strings.Contains(health.Notes[0].Message, "Library ingest") {
t.Fatalf("note = %q", health.Notes[0].Message)
}
}
/* ---------- the registry ---------- */
func TestWorkersRecordRunningAndFinished(t *testing.T) {
resetWorkers()
t.Cleanup(resetWorkers)
release := make(chan struct{})
var running sync.WaitGroup
running.Add(1)
Go("Library ingest", "Library sync", func() {
running.Done()
<-release
})
running.Wait()
var done sync.WaitGroup
done.Add(1)
Go("Startup library sync", "Library sync", func() { done.Done() })
done.Wait()
// The finished worker's goroutine has run its body; give the deferred end() the moment
// it needs to land before reading the registry.
waitFor(t, func() bool { return stateOf(t, "Startup library sync") == WorkerFinished })
list := Workers()
if len(list) != 2 {
t.Fatalf("workers = %+v", list)
}
// Running first, so the table does not bury the thing that is still working under the
// startup jobs that are not.
if list[0].Name != "Library ingest" || list[0].State != WorkerRunning {
t.Fatalf("first = %+v", list[0])
}
if list[1].State != WorkerFinished || list[1].Stopped.IsZero() {
t.Fatalf("second = %+v", list[1])
}
close(release)
waitFor(t, func() bool { return stateOf(t, "Library ingest") == WorkerFinished })
// A restart is counted rather than replacing the record: a worker restarted in a loop
// reads exactly like a healthy one from a single snapshot and does not from Starts.
Go("Library ingest", "Library sync", func() {})
waitFor(t, func() bool { return startsOf(t, "Library ingest") == 2 })
}
func stateOf(t *testing.T, name string) WorkerState {
t.Helper()
for _, worker := range Workers() {
if worker.Name == name {
return worker.State
}
}
return ""
}
func startsOf(t *testing.T, name string) int {
t.Helper()
for _, worker := range Workers() {
if worker.Name == name {
return worker.Starts
}
}
return 0
}
func waitFor(t *testing.T, done func() bool) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if done() {
return
}
time.Sleep(time.Millisecond)
}
t.Fatal("condition not reached")
}
server/internal/runtimestats/sample.go
+337
View File
@@ -0,0 +1,337 @@
package runtimestats
import (
"context"
"math"
"runtime"
"runtime/debug"
"sort"
"sync"
"time"
)
// SampleInterval is how often the trend ring records a reading, and the ring's capacity is
// how far back it can therefore see. A minute apart is far cheaper than the 30s poll the
// console already makes, and four hours is long enough for a slow leak to become a line
// rather than noise while costing a few kilobytes.
const (
SampleInterval = time.Minute
trendCapacity = 240
)
// Sample is one reading. Deliberately the same source as the headline figures — both come
// from one ReadMemStats — because a trend drawn from a different counter than the number
// printed above it is a trend an operator cannot check.
type Sample struct {
At time.Time `json:"at"`
Goroutines int `json:"goroutines"`
HeapInuse uint64 `json:"heapInuse"`
Sys uint64 `json:"sys"`
NumGC uint32 `json:"numGc"`
}
var (
samplesMu sync.Mutex
samples []Sample
started = time.Now()
)
// StartSampling records a reading every SampleInterval until ctx ends. It takes the first
// reading immediately, so a console opened a minute after a deployment has a baseline
// rather than an empty chart.
func StartSampling(ctx context.Context) {
record(readSample())
ticker := time.NewTicker(SampleInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
record(readSample())
}
}
}
func readSample() Sample {
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
return Sample{
At: time.Now(),
Goroutines: runtime.NumGoroutine(),
HeapInuse: memory.HeapInuse,
Sys: memory.Sys,
NumGC: memory.NumGC,
}
}
func record(sample Sample) {
samplesMu.Lock()
defer samplesMu.Unlock()
samples = append(samples, sample)
if len(samples) > trendCapacity {
samples = append(samples[:0], samples[len(samples)-trendCapacity:]...)
}
}
// Samples returns the trend ring oldest first.
func Samples() []Sample {
samplesMu.Lock()
defer samplesMu.Unlock()
out := make([]Sample, len(samples))
copy(out, samples)
return out
}
/* ---------- trends ---------- */
// Direction is a trend's verdict. "unknown" is a real answer and the common one for the
// first few minutes after a restart: claiming "steady" from two readings a minute apart
// would be a claim the data does not support.
type Direction string
const (
DirectionUnknown Direction = "unknown"
DirectionSteady Direction = "steady"
DirectionRising Direction = "rising"
DirectionFalling Direction = "falling"
)
type Trend struct {
Direction Direction `json:"direction"`
// PerHour is the observed rate of change in the value's own units. It is what makes
// "rising" actionable: two goroutines an hour is a leak worth naming, two a minute is
// one worth investigating tonight.
PerHour float64 `json:"perHour"`
First float64 `json:"first"`
Latest float64 `json:"latest"`
Min float64 `json:"min"`
Max float64 `json:"max"`
SpanSeconds float64 `json:"spanSeconds"`
Points int `json:"points"`
}
// minTrendPoints and minTrendSpan are what a trend has to earn before it is stated at all.
// A leak is a claim about a slope, and a slope read off three readings inside five minutes
// is indistinguishable from a household that happened to start watching something.
const (
minTrendPoints = 5
minTrendSpan = 15 * time.Minute
)
// trendOf is the whole rule and it is pure. It compares the median of the oldest quarter
// of the window with the median of the newest quarter rather than fitting a line, because
// a single burst — a library import, a household arriving home — is exactly the shape
// least-squares reports as a trend and medians ignore.
//
// The threshold is proportional with an absolute floor: without the proportion a busy
// gateway is permanently "rising", and without the floor a quiet one calls one extra
// goroutine a trend.
func trendOf(points []float64, at []time.Time, floor float64) Trend {
trend := Trend{Direction: DirectionUnknown, Points: len(points)}
if len(points) == 0 || len(at) != len(points) {
return trend
}
trend.First, trend.Latest = points[0], points[len(points)-1]
trend.Min, trend.Max = points[0], points[0]
for _, value := range points {
trend.Min = math.Min(trend.Min, value)
trend.Max = math.Max(trend.Max, value)
}
span := at[len(at)-1].Sub(at[0])
trend.SpanSeconds = span.Seconds()
if len(points) < minTrendPoints || span < minTrendSpan {
return trend
}
quarter := len(points) / 4
if quarter < 1 {
quarter = 1
}
early := median(points[:quarter])
late := median(points[len(points)-quarter:])
change := late - early
trend.PerHour = change / span.Hours()
threshold := math.Max(floor, early*0.15)
switch {
case change > threshold:
trend.Direction = DirectionRising
case change < -threshold:
trend.Direction = DirectionFalling
default:
trend.Direction = DirectionSteady
}
return trend
}
func median(values []float64) float64 {
if len(values) == 0 {
return 0
}
sorted := make([]float64, len(values))
copy(sorted, values)
sort.Float64s(sorted)
middle := len(sorted) / 2
if len(sorted)%2 == 1 {
return sorted[middle]
}
return (sorted[middle-1] + sorted[middle]) / 2
}
func trendsFrom(list []Sample) (goroutines, heap, reserved Trend) {
at := make([]time.Time, len(list))
goroutinePoints := make([]float64, len(list))
heapPoints := make([]float64, len(list))
reservedPoints := make([]float64, len(list))
for index, sample := range list {
at[index] = sample.At
goroutinePoints[index] = float64(sample.Goroutines)
heapPoints[index] = float64(sample.HeapInuse)
reservedPoints[index] = float64(sample.Sys)
}
// The floors: five goroutines, and four megabytes of memory. Below those, movement is
// the ordinary breathing of a server answering requests.
return trendOf(goroutinePoints, at, 5),
trendOf(heapPoints, at, 4<<20),
trendOf(reservedPoints, at, 4<<20)
}
/* ---------- the snapshot the console polls ---------- */
// Memory separates the three figures an operator keeps conflating: what the program is
// actually holding, what the Go runtime has reserved from the operating system on its
// behalf, and the ceiling it has been given. Reserved is always the largest and is not a
// leak; the ratio to the limit is the number worth watching.
type Memory struct {
HeapAlloc uint64 `json:"heapAlloc"`
HeapInuse uint64 `json:"heapInuse"`
HeapIdle uint64 `json:"heapIdle"`
HeapReleased uint64 `json:"heapReleased"`
StackInuse uint64 `json:"stackInuse"`
Sys uint64 `json:"sys"`
NextGC uint64 `json:"nextGc"`
NumGC uint32 `json:"numGc"`
// PauseTotalMs and PauseRecentMs are how much of the life of the gateway has been
// spent stopped for collection. A rising heap that is being collected without pausing
// is a working cache; one that pauses for longer every cycle is not.
PauseTotalMs float64 `json:"pauseTotalMs"`
PauseRecentMs float64 `json:"pauseRecentMs"`
MemoryLimit int64 `json:"memoryLimit"`
ConfiguredLim string `json:"configuredLimit,omitempty"`
// HeapShare is heap in use as a fraction of the limit, or 0 when there is no limit.
// Computed here rather than in the console so the health verdict and the figure the
// operator reads cannot disagree.
HeapShare float64 `json:"heapShare"`
// GCPerHour is measured over the trend window rather than over the whole life of the
// process, because "collections have become more frequent" is the useful form of it.
GCPerHour float64 `json:"gcPerHour"`
}
type Snapshot struct {
At time.Time `json:"at"`
UptimeSeconds float64 `json:"uptimeSeconds"`
Goroutines int `json:"goroutines"`
GOMAXPROCS int `json:"gomaxprocs"`
Threads int `json:"threads"`
GoVersion string `json:"goVersion"`
Memory Memory `json:"memory"`
Workers []Worker `json:"workers"`
Process Process `json:"process"`
Samples []Sample `json:"samples"`
GoroutineTrend Trend `json:"goroutineTrend"`
HeapTrend Trend `json:"heapTrend"`
ReservedTrend Trend `json:"reservedTrend"`
SampleEverySecs float64 `json:"sampleEverySeconds"`
Health Health `json:"health"`
}
// Read builds the cheap snapshot. Nothing in here walks a stack or stops the world for
// longer than a ReadMemStats, so it is safe on the 30-second poll every open console tab
// makes.
func Read() Snapshot {
var memory runtime.MemStats
runtime.ReadMemStats(&memory)
now := time.Now()
limit := debug.SetMemoryLimit(-1)
share := 0.0
if limit > 0 && limit < math.MaxInt64 {
share = float64(memory.HeapInuse) / float64(limit)
}
list := Samples()
goroutineTrend, heapTrend, reservedTrend := trendsFrom(list)
snapshot := Snapshot{
At: now,
UptimeSeconds: now.Sub(started).Seconds(),
Goroutines: runtime.NumGoroutine(),
GOMAXPROCS: runtime.GOMAXPROCS(0),
Threads: threadCount(),
GoVersion: runtime.Version(),
Memory: Memory{
HeapAlloc: memory.HeapAlloc,
HeapInuse: memory.HeapInuse,
HeapIdle: memory.HeapIdle,
HeapReleased: memory.HeapReleased,
StackInuse: memory.StackInuse,
Sys: memory.Sys,
NextGC: memory.NextGC,
NumGC: memory.NumGC,
PauseTotalMs: float64(memory.PauseTotalNs) / 1e6,
PauseRecentMs: recentPauseMs(&memory),
MemoryLimit: limit,
ConfiguredLim: configuredMemoryLimit(),
HeapShare: share,
GCPerHour: gcPerHour(list),
},
Workers: Workers(),
Process: readProcess(),
Samples: list,
GoroutineTrend: goroutineTrend,
HeapTrend: heapTrend,
ReservedTrend: reservedTrend,
SampleEverySecs: SampleInterval.Seconds(),
}
snapshot.Health = assess(snapshot)
return snapshot
}
// recentPauseMs averages the last few recorded pauses. Go keeps a circular buffer of the
// most recent 256, and the running total on its own cannot distinguish a process that
// paused badly an hour ago from one pausing badly now.
func recentPauseMs(memory *runtime.MemStats) float64 {
const window = 16
if memory.NumGC == 0 {
return 0
}
count := 0
total := uint64(0)
for index := 0; index < window; index++ {
slot := int(memory.NumGC) - 1 - index
if slot < 0 {
break
}
total += memory.PauseNs[slot%256]
count++
}
if count == 0 {
return 0
}
return float64(total) / float64(count) / 1e6
}
func gcPerHour(list []Sample) float64 {
if len(list) < 2 {
return 0
}
span := list[len(list)-1].At.Sub(list[0].At).Hours()
if span <= 0 {
return 0
}
return float64(list[len(list)-1].NumGC-list[0].NumGC) / span
}
server/internal/runtimestats/workers.go
+150
View File
@@ -0,0 +1,150 @@
// Package runtimestats answers the one question the console's Process card exists for: is
// the gateway healthy, and if something is abnormal, which area should be looked at?
//
// A bare goroutine count cannot answer it. Twenty-five means nothing on its own — an
// operator cannot tell from it what those goroutines are doing, which part of Memby they
// belong to, whether the number is normal, or whether it has been climbing all week. This
// package supplies the three things that make the number readable, and keeps them apart by
// what they cost:
//
// - The registry here, which is free. A long-running Memby worker says its own name when
// it starts, so "Library ingest" and "Emby health probe" are named rather than inferred
// from a stack.
// - The sampler in sample.go, which is a handful of counters on a slow tick. A single
// instantaneous figure cannot show a leak; a trend can.
// - The stack breakdown in goroutines.go, which is genuinely expensive and is therefore
// collected only when an operator asks for it.
//
// Nothing here recovers a panic. A worker that dies must die exactly as it always did —
// this package reports, and reporting must never change what it is reporting on.
package runtimestats
import (
"sort"
"sync"
"time"
)
// WorkerState is what became of a tracked worker. A worker that ends is not a fault —
// several of the gateway's background jobs are one-shot startup work — but "still running"
// and "finished" are different answers and the console must not merge them.
type WorkerState string
const (
WorkerRunning WorkerState = "running"
WorkerFinished WorkerState = "finished"
)
// Worker is one named background goroutine.
type Worker struct {
Name string `json:"name"`
Component string `json:"component"`
State WorkerState `json:"state"`
Started time.Time `json:"started"`
Stopped time.Time `json:"stopped,omitempty"`
// Starts counts how many times this name has been launched. It is on the record
// because a worker that is being restarted in a loop reads exactly like a healthy one
// from a single snapshot, and does not from this number.
Starts int `json:"starts"`
}
type workerEntry struct {
component string
running int
starts int
started time.Time
stopped time.Time
}
var (
workersMu sync.Mutex
workers = map[string]*workerEntry{}
)
// Go starts fn on its own goroutine and records it under a name an operator can read. The
// name is the worker's identity across restarts, so it must be stable and must not carry a
// count or an address in it.
func Go(name, component string, fn func()) {
begin(name, component)
go func() {
defer end(name)
fn()
}()
}
// Track records a worker whose goroutine something else owns. The returned function marks
// it finished, and is safe to call more than once.
func Track(name, component string) (done func()) {
begin(name, component)
var once sync.Once
return func() { once.Do(func() { end(name) }) }
}
func begin(name, component string) {
now := time.Now()
workersMu.Lock()
defer workersMu.Unlock()
entry := workers[name]
if entry == nil {
entry = &workerEntry{}
workers[name] = entry
}
entry.component = component
entry.running++
entry.starts++
entry.started = now
entry.stopped = time.Time{}
}
func end(name string) {
now := time.Now()
workersMu.Lock()
defer workersMu.Unlock()
entry := workers[name]
if entry == nil {
return
}
if entry.running > 0 {
entry.running--
}
if entry.running == 0 {
entry.stopped = now
}
}
// Workers lists what has been registered, running first and then by name, so the table
// does not reorder itself under the operator on every poll.
func Workers() []Worker {
workersMu.Lock()
list := make([]Worker, 0, len(workers))
for name, entry := range workers {
worker := Worker{
Name: name,
Component: entry.component,
State: WorkerFinished,
Started: entry.started,
Stopped: entry.stopped,
Starts: entry.starts,
}
if entry.running > 0 {
worker.State = WorkerRunning
worker.Stopped = time.Time{}
}
list = append(list, worker)
}
workersMu.Unlock()
sort.Slice(list, func(a, b int) bool {
if (list[a].State == WorkerRunning) != (list[b].State == WorkerRunning) {
return list[a].State == WorkerRunning
}
return list[a].Name < list[b].Name
})
return list
}
// resetWorkers exists for the tests; the registry is process-wide by design.
func resetWorkers() {
workersMu.Lock()
workers = map[string]*workerEntry{}
workersMu.Unlock()
}
server/internal/scheduler/scheduler.go
+47 -10
View File
@@ -17,6 +17,7 @@ import (
"time"
"github.com/ponzischeme89/memby/server/internal/adminevents"
"github.com/ponzischeme89/memby/server/internal/runtimestats"
"github.com/ponzischeme89/memby/server/internal/store"
)
@@ -31,6 +32,22 @@ const tick = 30 * time.Second
// minutes announcing itself into the notification bell every ten minutes.
type TaskFunc func(ctx context.Context) (detail string, err error)
// Outcome is the richer form: the sentence, plus what the run counted.
//
// It exists because "412 movies checked, 7 updated, 2 skipped" is the question an operator
// has about an integration and a sentence is a poor place to keep numbers — they cannot be
// compared between runs, sorted, or drawn as anything but prose. Most tasks count nothing
// and keep the plain TaskFunc; only work that processes a batch has anything to say here.
type Outcome struct {
// Detail is the one line the console prints beside the run, and is empty when nothing
// happened — the same rule TaskFunc's return follows, and for the same reason.
Detail string
store.RunCounts
}
// WorkFunc is a task that counts what it did. A task declares Run or Work, never both.
type WorkFunc func(ctx context.Context) (Outcome, error)
// Task is a declaration. Everything about it except the operator's overrides is code, so
// the registry is readable as a list of what the gateway does in the background.
type Task struct {
@@ -45,7 +62,14 @@ type Task struct {
// RunOnStart runs the task once shortly after boot regardless of when it last ran.
// For jobs whose cost is trivial and whose value is highest immediately.
RunOnStart bool
Run TaskFunc
// Integration names the external service this job belongs to, and is empty for the
// gateway's own work. It is what lets the integrations area read its operational
// history out of this registry rather than keeping a second one: a run is filed
// against the service as well as against the job, and nothing else is needed.
Integration string
Run TaskFunc
// Work is Run's counting form. Exactly one of the two must be set.
Work WorkFunc
}
// Status is one task as the console reads it: the declaration, the operator's overrides,
@@ -55,6 +79,9 @@ type Status struct {
Name string `json:"name"`
Description string `json:"description"`
Group string `json:"group"`
// Integration is the external service this job belongs to, empty for the gateway's
// own work. The integrations area lists a service's jobs by matching on it.
Integration string `json:"integration,omitempty"`
Interval int64 `json:"intervalSeconds"`
// DefaultInterval is the cadence declared in code, which Interval hides whenever an
// operator has overridden it. Both are sent because the console cannot otherwise tell
@@ -132,8 +159,8 @@ func (s *Scheduler) Register(task Task) {
if s == nil {
return
}
if task.ID == "" || task.Run == nil {
panic("scheduler: a task needs an id and a function")
if task.ID == "" || (task.Run == nil) == (task.Work == nil) {
panic("scheduler: a task needs an id and exactly one of Run or Work")
}
s.mu.Lock()
defer s.mu.Unlock()
@@ -166,7 +193,9 @@ func (s *Scheduler) Start(ctx context.Context) {
s.mu.Unlock()
s.restore(ctx)
go s.loop(ctx)
// Named rather than launched bare, so the console can say this worker is running
// without having to infer it from a stack. See internal/runtimestats.
runtimestats.Go("Task scheduler", "Scheduled jobs", func() { s.loop(ctx) })
}
func (s *Scheduler) restore(ctx context.Context) {
@@ -320,7 +349,7 @@ func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger stri
var runID int64
if s.store != nil {
id, err := s.store.BeginTaskRun(ctx, task.ID, trigger)
id, err := s.store.BeginTaskRun(ctx, task.ID, task.Integration, trigger)
if err != nil {
s.log.Warn("could not open task run", "task", task.ID, "error", err)
} else {
@@ -329,9 +358,10 @@ func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger stri
}
runCtx, cancel := context.WithTimeout(ctx, task.Timeout)
detail, err := s.safeRun(runCtx, task)
outcome, err := s.safeRun(runCtx, task)
cancel()
elapsed := time.Since(started)
detail := outcome.Detail
status := store.TaskSuccess
failure := ""
@@ -340,7 +370,7 @@ func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger stri
}
if s.store != nil && runID != 0 {
if closeErr := s.store.FinishTaskRun(
context.WithoutCancel(ctx), runID, status, detail, failure,
context.WithoutCancel(ctx), runID, status, detail, failure, outcome.RunCounts,
); closeErr != nil {
s.log.Warn("could not close task run", "task", task.ID, "error", closeErr)
}
@@ -349,9 +379,11 @@ func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger stri
finished := time.Now()
entry.mu.Lock()
entry.lastRun = &store.TaskRun{
ID: runID, TaskID: task.ID, Trigger: trigger, Status: status,
ID: runID, TaskID: task.ID, IntegrationID: task.Integration,
Trigger: trigger, Status: status,
StartedAt: started, FinishedAt: &finished,
DurationMS: elapsed.Milliseconds(), Detail: detail, Error: failure,
Counts: outcome.RunCounts,
}
entry.mu.Unlock()
@@ -362,13 +394,17 @@ func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger stri
// panic takes the whole process down for a reason nobody is watching for, and one
// housekeeping job with a nil map must not be able to stop the gateway serving
// television.
func (s *Scheduler) safeRun(ctx context.Context, task Task) (detail string, err error) {
func (s *Scheduler) safeRun(ctx context.Context, task Task) (outcome Outcome, err error) {
defer func() {
if recovered := recover(); recovered != nil {
err = fmt.Errorf("task panicked: %v", recovered)
}
}()
return task.Run(ctx)
if task.Work != nil {
return task.Work(ctx)
}
detail, err := task.Run(ctx)
return Outcome{Detail: detail}, err
}
// announce writes the log line and, when it is worth an operator's attention, publishes
@@ -489,6 +525,7 @@ func (s *Scheduler) Snapshot() []Status {
status := Status{
ID: entry.task.ID, Name: entry.task.Name,
Description: entry.task.Description, Group: entry.task.Group,
Integration: entry.task.Integration,
Interval: int64(entry.effectiveInterval() / time.Second),
DefaultInterval: int64(entry.task.Interval / time.Second),
Enabled: entry.enabled, Running: entry.running,
server/internal/sonarr/client.go
+43
View File
@@ -307,3 +307,46 @@ func (c *Client) do(req *http.Request, out any) error {
}
return nil
}
// QueueItem is one episode the download client is working on. See radarr.QueueItem for why
// three status words are carried rather than one; Sonarr describes a download in the same
// vocabulary.
//
// SeriesID is what the request page joins on: a viewer asks for a show, and what comes back
// is some number of episodes of it.
type QueueItem struct {
ID int `json:"id"`
SeriesID int `json:"seriesId"`
EpisodeID int `json:"episodeId"`
Size float64 `json:"size"`
Sizeleft float64 `json:"sizeleft"`
Timeleft string `json:"timeleft"`
Status string `json:"status"`
TrackedDownloadState string `json:"trackedDownloadState"`
TrackedDownloadStatus string `json:"trackedDownloadStatus"`
ErrorMessage string `json:"errorMessage"`
}
// queuePageSize is what one read asks for; see radarr's own note.
const queuePageSize = 200
type queuePage struct {
Records []QueueItem `json:"records"`
}
// Queue returns what Sonarr is currently working on, excluding downloads it cannot match to
// a series it tracks.
func (c *Client) Queue(ctx context.Context) ([]QueueItem, error) {
req, err := c.request(ctx, "/api/v3/queue", url.Values{
"pageSize": {strconv.Itoa(queuePageSize)},
"includeUnknownSeriesItems": {"false"},
})
if err != nil {
return nil, err
}
var page queuePage
if err := c.do(req, &page); err != nil {
return nil, err
}
return page.Records, nil
}
server/internal/store/for_you.go
+22
View File
@@ -64,6 +64,14 @@ type TracearrImportState struct {
LastIncrementalAt *time.Time `json:"lastIncrementalAt,omitempty"`
LastFullAt *time.Time `json:"lastFullAt,omitempty"`
LastError string `json:"lastError,omitempty"`
// LastRebuildAt is when the household's For You rows were last rebuilt in full.
//
// It lives in this document rather than in a table of its own because it is the same
// kind of fact as the two stamps above it — where the For You pipeline has got to —
// and because the daily rebuild is now a scheduled task, which means the alternative
// was inferring "did today's rebuild happen" from run history that also records the
// ticks on which it correctly declined to run.
LastRebuildAt *time.Time `json:"lastRebuildAt,omitempty"`
}
type RecommendationProfile struct {
@@ -407,6 +415,20 @@ func (s *Store) SetTracearrImportState(ctx context.Context, state TracearrImport
return nil
}
// MarkForYouRebuild records that the household's rows have just been rebuilt.
//
// Read-modify-write rather than a whole-document put, because the importer owns the other
// two stamps in this document and an import running beside a rebuild must not lose its own.
func (s *Store) MarkForYouRebuild(ctx context.Context, at time.Time) error {
state, err := s.TracearrImportState(ctx)
if err != nil {
return err
}
stamp := at.UTC()
state.LastRebuildAt = &stamp
return s.SetTracearrImportState(ctx, state)
}
func (s *Store) ActiveRecommendationUsers(ctx context.Context) ([]Session, error) {
rows, err := s.pool.Query(ctx, `
SELECT DISTINCT ON (emby_user_id)
server/internal/store/integration_policy.go
+98
View File
@@ -0,0 +1,98 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
// IntegrationPolicyKey is the operator's global on/off switch for external services that
// have no configuration document of their own.
//
// It is deliberately *not* a second copy of every integration's switch. Sonarr and Radarr
// already store theirs in the arr integration policy and MDBList stores its own in the
// ratings settings; moving those here would mean a migration and, worse, a window in which
// two documents disagreed about whether a service was on. The rule is one stored truth per
// integration, kept where that integration's other configuration already lives — and this
// document is that home for the ones that have nowhere else, which today is Tracearr.
//
// The API's integrationEnabled/setIntegrationEnabled pair is the single reader and writer,
// so the console never has to know which document answers for which service.
const IntegrationPolicyKey = "integration_policy"
// IntegrationPolicy is a map of integration id to whether it is switched on.
//
// Absence means on. A household that upgrades into this feature has every service it had
// configured still working, which is the only safe reading: the alternative silently turns
// off recommendation imports on the day the console gained a switch for them.
type IntegrationPolicy struct {
Disabled map[string]bool `json:"disabled"`
UpdatedAt time.Time `json:"updatedAt"`
}
// Enabled reports whether one integration is switched on. A service nobody has ever
// touched has no entry, and no entry is on.
func (p IntegrationPolicy) Enabled(id string) bool {
return !p.Disabled[strings.TrimSpace(id)]
}
func (s *Store) IntegrationPolicy(ctx context.Context) (IntegrationPolicy, error) {
empty := IntegrationPolicy{Disabled: map[string]bool{}}
var raw []byte
err := s.pool.QueryRow(ctx,
`SELECT value FROM app_settings WHERE key = $1`, IntegrationPolicyKey).Scan(&raw)
if errors.Is(err, pgx.ErrNoRows) {
return empty, nil
}
if err != nil {
return empty, fmt.Errorf("store: read integration policy: %w", err)
}
var policy IntegrationPolicy
if err := json.Unmarshal(raw, &policy); err != nil {
return empty, fmt.Errorf("store: decode integration policy: %w", err)
}
if policy.Disabled == nil {
policy.Disabled = map[string]bool{}
}
return policy, nil
}
// SetIntegrationEnabled records one service's switch, leaving every other entry alone.
//
// Read-modify-write rather than a whole-document put, because the console sends one
// switch at a time and a put would let a page rendered before another integration existed
// silently re-enable it.
func (s *Store) SetIntegrationEnabled(ctx context.Context, id string, enabled bool) error {
id = strings.TrimSpace(id)
if id == "" {
return fmt.Errorf("store: integration policy needs an id")
}
policy, err := s.IntegrationPolicy(ctx)
if err != nil {
return err
}
if enabled {
delete(policy.Disabled, id)
} else {
policy.Disabled[id] = true
}
policy.UpdatedAt = time.Now().UTC()
raw, err := json.Marshal(policy)
if err != nil {
return err
}
_, err = s.pool.Exec(ctx, `
INSERT INTO app_settings (key, value, updated_at)
VALUES ($1, $2::jsonb, now())
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`,
IntegrationPolicyKey, string(raw))
if err != nil {
return fmt.Errorf("store: write integration policy: %w", err)
}
return nil
}
server/internal/store/ratings.go
+34
View File
@@ -223,6 +223,40 @@ func (s *Store) MediaRatingsStats(ctx context.Context, staleBefore time.Time) (t
return total, stale, nil
}
// StaleRatingKeys is the oldest stored titles due to be re-checked, oldest first.
//
// Oldest first rather than by any measure of popularity, because the refresh is bounded
// per run: taking the oldest means every title comes round eventually, where taking the
// most-watched would leave the tail of the library permanently on scores from the year it
// was imported. The limit is what keeps a run's cost — and therefore the day's external
// allowance — a figure the operator can reason about.
func (s *Store) StaleRatingKeys(
ctx context.Context, staleBefore time.Time, limit int,
) ([]RatingKey, error) {
if limit <= 0 || limit > 500 {
limit = 100
}
rows, err := s.pool.Query(ctx, `
SELECT media_type, provider, provider_id
FROM external_media_ratings
WHERE fetched_at < $1
ORDER BY fetched_at ASC
LIMIT $2`, staleBefore, limit)
if err != nil {
return nil, fmt.Errorf("store: stale rating keys: %w", err)
}
defer rows.Close()
keys := []RatingKey{}
for rows.Next() {
var key RatingKey
if err := rows.Scan(&key.MediaType, &key.Provider, &key.ProviderID); err != nil {
return nil, fmt.Errorf("store: scan stale rating key: %w", err)
}
keys = append(keys, key)
}
return keys, rows.Err()
}
// SaveMediaRatings upserts a successfully loaded provider response.
func (s *Store) SaveMediaRatings(
ctx context.Context, mediaType, provider, providerID string, ratings json.RawMessage,
server/internal/store/requests.go
+85 -7
View File
@@ -7,8 +7,11 @@ import (
"time"
)
// MediaRequest is one viewer's ask, as recorded. It carries no status: see the schema
// comment on media_requests for why the state is derived per read rather than stored.
// MediaRequest is one viewer's ask, as recorded.
//
// The status a card shows is not here: it is derived per read from the *arrs and the
// library, for the reason the schema gives. The one exception is LastStatus, which is not
// the card's status but the memory of it — the only way to notice that something changed.
type MediaRequest struct {
MediaType string `json:"mediaType"`
ForeignID int `json:"foreignId"`
@@ -16,6 +19,19 @@ type MediaRequest struct {
Year int `json:"year,omitempty"`
PosterURL string `json:"posterUrl,omitempty"`
RequestedAt time.Time `json:"requestedAt"`
// LastStatus is the state this request was in the last time anything looked, and it is
// the only piece of request state that is stored. See the schema comment: an arrival is
// a difference between two observations rather than a property of one, and the viewer is
// told about it once.
LastStatus string `json:"-"`
}
// OwnedMediaRequest is a stored ask with the person who made it, which the per-viewer read
// does not need to carry because it was asked for by user. The ready sweep looks at the
// whole household in one query, so there it is the point.
type OwnedMediaRequest struct {
MediaRequest
UserID string
}
// RequestUsage is the operator-facing use of the request feature. A recorded request is
@@ -59,16 +75,20 @@ func (s *Store) SaveMediaRequest(ctx context.Context, userID string, req MediaRe
if userID == "" || req.ForeignID <= 0 {
return fmt.Errorf("store: media request needs a user and a foreign id")
}
// last_status is deliberately absent from the UPDATE. Asking again is somebody saying
// they still want it, not a reason to re-announce an arrival they were already told
// about — and re-seeding it here would make a second press of Request the way to make
// the gateway repeat itself.
_, err := s.pool.Exec(ctx, `
INSERT INTO media_requests
(emby_user_id, media_type, foreign_id, title, year, poster_url, requested_at)
VALUES ($1, $2, $3, $4, $5, $6, now())
(emby_user_id, media_type, foreign_id, title, year, poster_url, last_status, requested_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, now())
ON CONFLICT (emby_user_id, media_type, foreign_id) DO UPDATE
SET title = EXCLUDED.title,
year = EXCLUDED.year,
poster_url = EXCLUDED.poster_url,
requested_at = now()`,
userID, req.MediaType, req.ForeignID, req.Title, req.Year, req.PosterURL)
userID, req.MediaType, req.ForeignID, req.Title, req.Year, req.PosterURL, req.LastStatus)
if err != nil {
return fmt.Errorf("store: save media request: %w", err)
}
@@ -78,7 +98,7 @@ func (s *Store) SaveMediaRequest(ctx context.Context, userID string, req MediaRe
// MediaRequests returns one viewer's asks, most recent first.
func (s *Store) MediaRequests(ctx context.Context, userID string) ([]MediaRequest, error) {
rows, err := s.pool.Query(ctx, `
SELECT media_type, foreign_id, title, year, poster_url, requested_at
SELECT media_type, foreign_id, title, year, poster_url, last_status, requested_at
FROM media_requests
WHERE emby_user_id = $1
ORDER BY requested_at DESC
@@ -92,7 +112,8 @@ func (s *Store) MediaRequests(ctx context.Context, userID string) ([]MediaReques
for rows.Next() {
var req MediaRequest
if err := rows.Scan(
&req.MediaType, &req.ForeignID, &req.Title, &req.Year, &req.PosterURL, &req.RequestedAt,
&req.MediaType, &req.ForeignID, &req.Title, &req.Year, &req.PosterURL,
&req.LastStatus, &req.RequestedAt,
); err != nil {
return nil, fmt.Errorf("store: scan media request: %w", err)
}
@@ -118,3 +139,60 @@ func (s *Store) DeleteMediaRequest(
}
return nil
}
// AllMediaRequests reads the whole household's asks, newest first, with the person attached.
//
// The per-viewer read above is what a page needs; this is what the ready sweep needs, and
// the difference is worth one query rather than one per account: a household of six with
// eighty requests between them is one read, and the sweep has to look at all of them anyway
// because two people can be waiting for the same film.
//
// It is bounded like the per-viewer read. A sweep that fell behind on a household which had
// been asking for things for two years must not become an unbounded query on a timer.
func (s *Store) AllMediaRequests(ctx context.Context, limit int) ([]OwnedMediaRequest, error) {
if limit <= 0 {
limit = MediaRequestSweepLimit
}
rows, err := s.pool.Query(ctx, `
SELECT emby_user_id, media_type, foreign_id, title, year, poster_url, last_status, requested_at
FROM media_requests
ORDER BY requested_at DESC
LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("store: read all media requests: %w", err)
}
defer rows.Close()
requests := []OwnedMediaRequest{}
for rows.Next() {
var req OwnedMediaRequest
if err := rows.Scan(
&req.UserID, &req.MediaType, &req.ForeignID, &req.Title, &req.Year,
&req.PosterURL, &req.LastStatus, &req.RequestedAt,
); err != nil {
return nil, fmt.Errorf("store: scan media request: %w", err)
}
requests = append(requests, req)
}
return requests, rows.Err()
}
// MediaRequestSweepLimit caps what one pass of the ready sweep will look at.
const MediaRequestSweepLimit = 500
// SetMediaRequestStatus records what a request was last seen doing.
//
// Written only when the state actually moved, so a sweep over a household where nothing has
// changed — which is almost every sweep — costs no writes at all.
func (s *Store) SetMediaRequestStatus(
ctx context.Context, userID, mediaType string, foreignID int, status string,
) error {
_, err := s.pool.Exec(ctx, `
UPDATE media_requests SET last_status = $4
WHERE emby_user_id = $1 AND media_type = $2 AND foreign_id = $3`,
strings.TrimSpace(userID), mediaType, foreignID, status)
if err != nil {
return fmt.Errorf("store: set media request status: %w", err)
}
return nil
}
server/internal/store/scheduler.go
+151 -23
View File
@@ -29,17 +29,44 @@ const (
TriggerStartup = "startup"
)
// RunCounts is what a run did, in numbers.
//
// Four figures rather than a free map, because these are the four questions an operator
// asks of any piece of batch work — how much did it look at, how much did it change, how
// much did it decline, how much went wrong — and a schema-free bag would let two
// integrations answer them under different names. Anything an integration counts beyond
// these belongs in the run's own sentence.
//
// Processed is the load-bearing one: a run reporting zero processed counted nothing at
// all, and the console draws no figures rather than four zeroes.
type RunCounts struct {
Processed int `json:"processed"`
Changed int `json:"changed"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
}
// Counted reports whether this run counted anything worth printing.
func (c RunCounts) Counted() bool {
return c.Processed != 0 || c.Changed != 0 || c.Skipped != 0 || c.Failed != 0
}
// TaskRun is one execution.
type TaskRun struct {
ID int64 `json:"id"`
TaskID string `json:"taskId"`
Trigger string `json:"trigger"`
Status string `json:"status"`
StartedAt time.Time `json:"startedAt"`
FinishedAt *time.Time `json:"finishedAt,omitempty"`
DurationMS int64 `json:"durationMs"`
Detail string `json:"detail,omitempty"`
Error string `json:"error,omitempty"`
ID int64 `json:"id"`
TaskID string `json:"taskId"`
// IntegrationID names the external service this run belongs to, and is empty for the
// gateway's own housekeeping. It is what lets the integrations area read operational
// history out of the scheduler's own table instead of keeping a second one.
IntegrationID string `json:"integrationId,omitempty"`
Trigger string `json:"trigger"`
Status string `json:"status"`
StartedAt time.Time `json:"startedAt"`
FinishedAt *time.Time `json:"finishedAt,omitempty"`
DurationMS int64 `json:"durationMs"`
Detail string `json:"detail,omitempty"`
Error string `json:"error,omitempty"`
Counts RunCounts `json:"counts"`
}
// TaskSettings is an operator's override for one task. IntervalSeconds of 0 means "the
@@ -55,24 +82,31 @@ type TaskSettings struct {
// BeginTaskRun opens a run and returns its id. The row exists before the work starts so a
// task killed by a restart leaves evidence it began — which is the only way to tell a job
// that hangs from one that was never scheduled.
func (s *Store) BeginTaskRun(ctx context.Context, taskID, trigger string) (int64, error) {
func (s *Store) BeginTaskRun(
ctx context.Context, taskID, integrationID, trigger string,
) (int64, error) {
var id int64
err := s.pool.QueryRow(ctx, `
INSERT INTO scheduled_task_runs (task_id, trigger, status)
VALUES ($1, $2, $3) RETURNING id`, taskID, trigger, TaskRunning).Scan(&id)
INSERT INTO scheduled_task_runs (task_id, integration_id, trigger, status)
VALUES ($1, $2, $3, $4) RETURNING id`,
taskID, integrationID, trigger, TaskRunning).Scan(&id)
if err != nil {
return 0, fmt.Errorf("store: begin task run: %w", err)
}
return id, nil
}
// FinishTaskRun closes a run with its outcome.
func (s *Store) FinishTaskRun(ctx context.Context, id int64, status, detail, failure string) error {
// FinishTaskRun closes a run with its outcome and whatever it counted.
func (s *Store) FinishTaskRun(
ctx context.Context, id int64, status, detail, failure string, counts RunCounts,
) error {
_, err := s.pool.Exec(ctx, `
UPDATE scheduled_task_runs
SET status = $2, finished_at = now(), detail = $3, error = $4,
processed = $5, changed = $6, skipped = $7, failed = $8,
duration_ms = GREATEST(0, EXTRACT(EPOCH FROM (now() - started_at)) * 1000)::bigint
WHERE id = $1`, id, status, detail, failure)
WHERE id = $1`, id, status, detail, failure,
counts.Processed, counts.Changed, counts.Skipped, counts.Failed)
if err != nil {
return fmt.Errorf("store: finish task run: %w", err)
}
@@ -103,7 +137,8 @@ func (s *Store) AbandonRunningTasks(ctx context.Context) (int64, error) {
func (s *Store) LatestTaskRuns(ctx context.Context) (map[string]TaskRun, error) {
rows, err := s.pool.Query(ctx, `
SELECT DISTINCT ON (task_id)
id, task_id, trigger, status, started_at, finished_at, duration_ms, detail, error
id, task_id, integration_id, trigger, status, started_at, finished_at,
duration_ms, detail, error, processed, changed, skipped, failed
FROM scheduled_task_runs
ORDER BY task_id, started_at DESC, id DESC`)
if err != nil {
@@ -113,9 +148,10 @@ func (s *Store) LatestTaskRuns(ctx context.Context) (map[string]TaskRun, error)
latest := map[string]TaskRun{}
for rows.Next() {
var run TaskRun
if err := rows.Scan(&run.ID, &run.TaskID, &run.Trigger, &run.Status,
&run.StartedAt, &run.FinishedAt, &run.DurationMS, &run.Detail,
&run.Error); err != nil {
if err := rows.Scan(&run.ID, &run.TaskID, &run.IntegrationID, &run.Trigger,
&run.Status, &run.StartedAt, &run.FinishedAt, &run.DurationMS, &run.Detail,
&run.Error, &run.Counts.Processed, &run.Counts.Changed,
&run.Counts.Skipped, &run.Counts.Failed); err != nil {
return nil, fmt.Errorf("store: scan task run: %w", err)
}
latest[run.TaskID] = run
@@ -129,7 +165,8 @@ func (s *Store) TaskRuns(ctx context.Context, taskID string, limit int) ([]TaskR
limit = 50
}
rows, err := s.pool.Query(ctx, `
SELECT id, task_id, trigger, status, started_at, finished_at, duration_ms, detail, error
SELECT id, task_id, integration_id, trigger, status, started_at, finished_at,
duration_ms, detail, error, processed, changed, skipped, failed
FROM scheduled_task_runs
WHERE ($1 = '' OR task_id = $1)
ORDER BY started_at DESC, id DESC
@@ -141,9 +178,10 @@ func (s *Store) TaskRuns(ctx context.Context, taskID string, limit int) ([]TaskR
runs := []TaskRun{}
for rows.Next() {
var run TaskRun
if err := rows.Scan(&run.ID, &run.TaskID, &run.Trigger, &run.Status,
&run.StartedAt, &run.FinishedAt, &run.DurationMS, &run.Detail,
&run.Error); err != nil {
if err := rows.Scan(&run.ID, &run.TaskID, &run.IntegrationID, &run.Trigger,
&run.Status, &run.StartedAt, &run.FinishedAt, &run.DurationMS, &run.Detail,
&run.Error, &run.Counts.Processed, &run.Counts.Changed,
&run.Counts.Skipped, &run.Counts.Failed); err != nil {
return nil, fmt.Errorf("store: scan task run: %w", err)
}
runs = append(runs, run)
@@ -338,3 +376,93 @@ func (s *Store) PruneIntegrationDeliveries(ctx context.Context, keep int) (int64
}
return tag.RowsAffected(), nil
}
// --- integration run history ------------------------------------------------------
//
// The same rows as above, read with a different question in mind. An integration run *is*
// a scheduled task run: giving the integrations area a table of its own would mean two
// schedulers, two retention jobs and two places one piece of work could be recorded as
// having failed. What differs is only the axis — by service rather than by job.
// IntegrationRuns is the operational history for one external service, newest first.
func (s *Store) IntegrationRuns(
ctx context.Context, integrationID string, limit int,
) ([]TaskRun, error) {
if limit <= 0 || limit > 500 {
limit = 50
}
rows, err := s.pool.Query(ctx, `
SELECT id, task_id, integration_id, trigger, status, started_at, finished_at,
duration_ms, detail, error, processed, changed, skipped, failed
FROM scheduled_task_runs
WHERE integration_id <> '' AND ($1 = '' OR integration_id = $1)
ORDER BY started_at DESC, id DESC
LIMIT $2`, integrationID, limit)
if err != nil {
return nil, fmt.Errorf("store: list integration runs: %w", err)
}
defer rows.Close()
runs := []TaskRun{}
for rows.Next() {
var run TaskRun
if err := rows.Scan(&run.ID, &run.TaskID, &run.IntegrationID, &run.Trigger,
&run.Status, &run.StartedAt, &run.FinishedAt, &run.DurationMS, &run.Detail,
&run.Error, &run.Counts.Processed, &run.Counts.Changed,
&run.Counts.Skipped, &run.Counts.Failed); err != nil {
return nil, fmt.Errorf("store: scan integration run: %w", err)
}
runs = append(runs, run)
}
return runs, rows.Err()
}
// IntegrationRunSummary answers the overview's whole row for one service.
//
// Last success and last failure are both carried because they are different questions and
// the answer to one is not the absence of the other: a service that failed an hour ago and
// has worked since is healthy, and one that succeeded last week and has failed every hour
// since is not. Neither is derivable from a single "last run".
type IntegrationRunSummary struct {
IntegrationID string `json:"integrationId"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
LastFailureAt *time.Time `json:"lastFailureAt,omitempty"`
LastError string `json:"lastError,omitempty"`
Runs int `json:"runs"`
Failures int `json:"failures"`
}
// IntegrationRunSummaries is every service's summary in one query, because the overview
// draws one per row and a query per integration would grow with the catalogue.
func (s *Store) IntegrationRunSummaries(
ctx context.Context,
) (map[string]IntegrationRunSummary, error) {
rows, err := s.pool.Query(ctx, `
SELECT integration_id,
max(started_at) FILTER (WHERE status = $1),
max(started_at) FILTER (WHERE status = $2),
(array_remove(array_agg(error ORDER BY started_at DESC)
FILTER (WHERE status = $2), ''))[1],
count(*), count(*) FILTER (WHERE status = $2)
FROM scheduled_task_runs
WHERE integration_id <> ''
GROUP BY integration_id`, TaskSuccess, TaskFailed)
if err != nil {
return nil, fmt.Errorf("store: integration run summaries: %w", err)
}
defer rows.Close()
summaries := map[string]IntegrationRunSummary{}
for rows.Next() {
var summary IntegrationRunSummary
var lastError *string
if err := rows.Scan(&summary.IntegrationID, &summary.LastSuccessAt,
&summary.LastFailureAt, &lastError, &summary.Runs,
&summary.Failures); err != nil {
return nil, fmt.Errorf("store: scan integration run summary: %w", err)
}
if lastError != nil {
summary.LastError = *lastError
}
summaries[summary.IntegrationID] = summary
}
return summaries, rows.Err()
}
server/internal/store/schema.sql
+33
View File
@@ -577,6 +577,17 @@ CREATE TABLE IF NOT EXISTS media_requests (
PRIMARY KEY (emby_user_id, media_type, foreign_id)
);
-- last_status is the one piece of request state that *is* stored, and only because a
-- transition cannot be derived from a single read. Everything else on a request card is
-- computed per read from the *arrs and the library; "it has just become ready" is not a
-- property of the present, it is the difference between two observations, and the viewer
-- has to be told about it exactly once.
--
-- It is seeded when the request is recorded rather than left blank for a sweep to fill in,
-- because a film that downloads in the three minutes before the first sweep would otherwise
-- have its arrival recorded as its opening state and nobody would ever be told.
ALTER TABLE media_requests ADD COLUMN IF NOT EXISTS last_status TEXT NOT NULL DEFAULT '';
CREATE INDEX IF NOT EXISTS media_requests_user_requested_idx
ON media_requests (emby_user_id, requested_at DESC);
@@ -663,6 +674,28 @@ CREATE TABLE IF NOT EXISTS scheduled_task_runs (
error TEXT NOT NULL DEFAULT ''
);
-- Which integration a run belongs to, and what it actually did.
--
-- Deliberately more columns on this table rather than a second one: an integration run IS
-- a scheduled task run, read with a different question in mind. Operations history and
-- "what does the gateway do in the background" are the same rows; giving integrations
-- their own table would mean two schedulers, two retention jobs and two places a run can
-- be recorded as having failed.
--
-- The counters are nullable-by-default zeroes because most tasks count nothing: a
-- housekeeping prune has one number and it is already in `detail`. A run that counted
-- nothing is drawn without figures rather than as four zeroes, which is why the API sends
-- them only when `processed` is non-zero.
ALTER TABLE scheduled_task_runs ADD COLUMN IF NOT EXISTS integration_id TEXT NOT NULL DEFAULT '';
ALTER TABLE scheduled_task_runs ADD COLUMN IF NOT EXISTS processed INT NOT NULL DEFAULT 0;
ALTER TABLE scheduled_task_runs ADD COLUMN IF NOT EXISTS changed INT NOT NULL DEFAULT 0;
ALTER TABLE scheduled_task_runs ADD COLUMN IF NOT EXISTS skipped INT NOT NULL DEFAULT 0;
ALTER TABLE scheduled_task_runs ADD COLUMN IF NOT EXISTS failed INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS scheduled_task_runs_integration_idx
ON scheduled_task_runs (integration_id, started_at DESC)
WHERE integration_id <> '';
CREATE INDEX IF NOT EXISTS scheduled_task_runs_task_idx
ON scheduled_task_runs (task_id, started_at DESC);
CREATE INDEX IF NOT EXISTS scheduled_task_runs_time_idx