0.2.79 - Slow api fixes

server/internal/api/admin_gateway_settings.go

@@ -40,12 +40,24 @@ func (s *Server) gatewaySettingsResponse() adminGatewaySettingsResponse {
 			SonarrAlertMinutes: int(s.sonarrAlertWindow() / time.Minute),
 			RadarrAlertMinutes: int(s.radarrAlertWindow() / time.Minute),
 			EmbyHealthSeconds:  int(s.embyHealthInterval() / time.Second),
+			SlowRequestMillis:  effectiveSlowRequestMillis(s.slowRequestThreshold()),
+			LibrarySyncMinutes: int(s.LibrarySyncInterval() / time.Minute),
 		},
 		LogLevels: store.GatewayLogLevels,
 		Version:   buildinfo.Version(),
 	}
 }
 
+// effectiveSlowRequestMillis reports the switched-off threshold as zero rather than as
+// the impossible duration the reader uses, because the console draws the effective value
+// beside the field an operator cleared and "0" is what its own vocabulary means by off.
+func effectiveSlowRequestMillis(threshold time.Duration) int {
+	if threshold <= 0 || threshold > time.Hour {
+		return 0
+	}
+	return int(threshold / time.Millisecond)
+}
+
 func (s *Server) effectiveLogLevel() string {
 	if s.logLevel != nil {
 		return levelName(s.logLevel.Level())
@@ -121,6 +133,12 @@ func gatewaySettingsChanges(before, after store.GatewaySettings) string {
 	if before.EmbyHealthSeconds != after.EmbyHealthSeconds {
 		changes = append(changes, "Emby health probe")
 	}
+	if before.SlowRequestMillis != after.SlowRequestMillis {
+		changes = append(changes, "slow-request threshold")
+	}
+	if before.LibrarySyncMinutes != after.LibrarySyncMinutes {
+		changes = append(changes, "library sweep interval")
+	}
 	switch len(changes) {
 	case 0:
 		return ""

server/internal/api/api.go

@@ -40,6 +40,7 @@ import (
 	"github.com/ponzischeme89/memby/server/internal/scheduler"
 	"github.com/ponzischeme89/memby/server/internal/sonarr"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 type Server struct {
@@ -107,6 +108,19 @@ type Server struct {
 	// playbackTitles lets a progress or stop report, which carries only an item id, be
 	// logged by name.
 	playbackTitles playbackTitles
+	// featurePolicy keeps the operator's feature switches out of the request path. See
+	// features_cache.go.
+	featurePolicy featurePolicyCache
+	// upstream deduplicates concurrent cache misses for the same key, so two televisions
+	// asking for the same expensive answer at the same moment cost one upstream call
+	// rather than two. See coalesce.go.
+	upstream upstreamGroup
+	// household caches the completion scores every viewer's ranking reads and none of
+	// them can get a different answer to. See ranking.go.
+	household householdScores
+	// followChecks stops the automatic My Shows check repeating for the length of an
+	// episode. See playback_follow.go.
+	followChecks followChecks
 
 	// ingestRuns collapses a season pack's worth of finished scans into one banner.
 	ingestRuns ingestRuns
@@ -505,6 +519,13 @@ func (s *Server) withLogging(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
 		r, identity := withRequestIdentity(r)
+		// Every layer under this one — the Emby client, the *arr clients, Postgres,
+		// Redis — records against this. It is installed for every request rather than
+		// only for ones that turn out to be slow, because nothing knows a request is
+		// slow until it has finished and the evidence has to have been collected on the
+		// way through. What it costs when the request is fast is a map nobody reads.
+		ctx, trace := timing.New(r.Context())
+		r = r.WithContext(ctx)
 		w.Header().Set("X-Memby-Correlation", identity.correlation)
 		s.loggerFor(r.Context()).Log(r.Context(), serverlogging.LevelTrace, "request started",
 			"method", r.Method, "path", r.URL.Path, "query_keys", queryKeys(r))
@@ -539,6 +560,16 @@ func (s *Server) withLogging(next http.Handler) http.Handler {
 		if cached := rec.Header().Get("X-Memby-Cache"); cached != "" {
 			fields = append(fields, "cache", cached)
 		}
+		// A duration on its own says something is slow and nothing about why, which is
+		// the state every latency investigation here started from. The breakdown is
+		// attached only past the threshold: on a fast request it would be a second
+		// column of noise on every line, and on a slow one it is the answer.
+		if elapsed := time.Since(start); elapsed >= s.slowRequestThreshold() && !trace.Empty() {
+			fields = append(fields, "breakdown", trace.Breakdown())
+			if rest := trace.Unattributed(elapsed); rest > 0 {
+				fields = append(fields, "elsewhere", rest.Round(time.Millisecond))
+			}
+		}
 		s.log.Log(r.Context(), level, "request", fields...)
 	})
 }

server/internal/api/coalesce.go

@@ -0,0 +1,105 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"golang.org/x/sync/singleflight"
+
+	"github.com/ponzischeme89/memby/server/internal/timing"
+)
+
+// coalesceTimeout bounds a shared upstream read. It is longer than any single call the
+// gateway makes because it covers the whole miss — read, decorate, cache — and shorter
+// than a television's patience, so a joiner is never held past the point where the
+// answer would have been useful.
+const coalesceTimeout = 30 * time.Second
+
+// cachedRead is the shape of nearly every read-through route here: look in Redis, ask
+// upstream on a miss, store what came back. What it adds is the thing missing from every
+// hand-written copy of that shape — only one request at a time actually asks.
+//
+// The failure this exists for is a real one and it is visible in the log as a sequence:
+// the same person lookup answering in 2.2s, then 4.6s, then 7.7s within a few seconds.
+// Nothing was getting slower. Three televisions — or one television warming three cast
+// cards — missed the same key at the same moment, all three asked Emby, and Emby served
+// three copies of one expensive query while each of them waited for the other two's work
+// to finish. The cache could not help: nothing had been written yet when the second and
+// third arrived. Coalescing is what turns that into one upstream call and three fast
+// answers, and it gets better rather than worse as the household grows.
+//
+// Two properties are load-bearing:
+//
+//   - **The shared work does not inherit the caller's cancellation.** singleflight hands
+//     every joiner the first caller's result, so if that context is the one that goes
+//     away — a D-pad moving off a card, a television giving up — everybody waiting behind
+//     it is failed by a request none of them made. The work runs on a detached context
+//     with its own deadline for that reason.
+//   - **A joiner is counted.** `coalesced` on the breakdown is what distinguishes "this
+//     route is slow" from "this route is slow because it is queued behind itself", which
+//     are different problems with different fixes.
+func (s *Server) cachedRead(
+	ctx context.Context,
+	key string,
+	ttl time.Duration,
+	load func(context.Context) (json.RawMessage, error),
+) (json.RawMessage, bool, error) {
+	if raw, err := s.cache.Get(ctx, key); err == nil {
+		return raw, true, nil
+	}
+	body, err := s.buildCached(ctx, key, func(json.RawMessage) time.Duration { return ttl }, load)
+	return body, false, err
+}
+
+// buildCached is cachedRead without the lookup, for a route that has to do some of its
+// own work before it knows what to build — and for one whose time to live depends on
+// what came back. The lookup is not optional there, only earlier.
+func (s *Server) buildCached(
+	ctx context.Context,
+	key string,
+	ttlFor func(json.RawMessage) time.Duration,
+	load func(context.Context) (json.RawMessage, error),
+) (json.RawMessage, error) {
+	shared, err, joined := s.upstream.Do(key, func() (any, error) {
+		// Detached deliberately — see above. The parent's values (the trace, the request
+		// identity) are kept so the work is still attributable to the request that
+		// triggered it.
+		work, cancel := context.WithTimeout(context.WithoutCancel(ctx), coalesceTimeout)
+		defer cancel()
+		body, err := load(work)
+		if err != nil {
+			return nil, err
+		}
+		if err := s.cache.Set(work, key, body, ttlFor(body)); err != nil {
+			s.loggerFor(work).Warn("cache write failed", "key", key, "error", err)
+		}
+		return body, nil
+	})
+	if joined {
+		timing.Count(ctx, "coalesced")
+	}
+	if err != nil {
+		return nil, err
+	}
+	body, _ := shared.(json.RawMessage)
+	return body, nil
+}
+
+// upstreamGroup is the deduplicator itself. One per server: the keys are already the
+// cache keys, which are namespaced per user where the answer is per user, so nothing in
+// one household's traffic can join another's.
+type upstreamGroup = singleflight.Group
+
+// writeCached is the one place the cache header and the body are written together, so a
+// route cannot answer from cache and report a miss. Two lines, and it was wrong at least
+// once before it was one function.
+func writeCached(w http.ResponseWriter, hit bool, body json.RawMessage) {
+	if hit {
+		w.Header().Set("X-Memby-Cache", "hit")
+	} else {
+		w.Header().Set("X-Memby-Cache", "miss")
+	}
+	writeRaw(w, http.StatusOK, body)
+}

server/internal/api/extras.go

@@ -1,11 +1,15 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
+	"sync"
 
 	"github.com/ponzischeme89/memby/server/internal/cache"
+	"github.com/ponzischeme89/memby/server/internal/emby"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 // extrasResponse is the Extras tab: featurettes, deleted scenes, interviews and the local
@@ -31,48 +35,64 @@ func (s *Server) handleExtras(w http.ResponseWriter, r *http.Request, sess store
 		writeError(w, http.StatusBadRequest, "item id is required")
 		return
 	}
-	key := cache.UserKey(sess.EmbyUserID, "extras:v1:"+itemID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		w.Header().Set("X-Memby-Cache", "hit")
-		writeRaw(w, http.StatusOK, raw)
-		return
-	}
-
-	cred := credentials(sess)
-	// Neither list failing is fatal, and only both failing is a failure worth reporting: a
-	// title can perfectly well have a trailer and no special features, or the reverse, and
-	// an Extras tab withheld because one of two lookups was unwell is the worse outcome.
-	var items []json.RawMessage
-	trailerErr := error(nil)
-	if result, err := s.emby.LocalTrailers(ctx, cred, itemID); err == nil {
-		items = append(items, result.Items...)
-	} else {
-		trailerErr = err
-	}
-	featureErr := error(nil)
-	if result, err := s.emby.SpecialFeatures(ctx, cred, itemID); err == nil {
-		items = append(items, result.Items...)
-	} else {
-		featureErr = err
-	}
-	if trailerErr != nil && featureErr != nil {
-		s.writeUpstreamError(ctx, w, featureErr, "could not load extras")
-		return
-	}
-
-	items = dedupeExtras(items)
-	body, err := json.Marshal(extrasResponse{Items: nonNilRaws(items)})
+	// Cached for the household rather than per viewer, and outside the namespace a
+	// playback stop invalidates: what a film ships with is a fact about the film, and
+	// nothing in this response carries user data. Empty answers included — most of a
+	// library has no extras at all and every detail page asks, so the "no" is the entry
+	// worth keeping.
+	key := cache.MetadataKey("extras:v2:" + itemID)
+	body, hit, err := s.cachedRead(ctx, key, s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			return s.extras(ctx, credentials(sess), itemID)
+		})
 	if err != nil {
-		writeError(w, http.StatusInternalServerError, "could not encode extras")
+		s.writeUpstreamError(ctx, w, err, "could not load extras")
 		return
 	}
-	// Cached for the ordinary item lifetime, empty answers included. Most of a library has
-	// no extras at all and every detail page asks, so the "no" is the entry worth keeping.
-	if err := s.cache.Set(ctx, key, body, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("extras cache write failed", "error", err)
+	writeCached(w, hit, body)
+}
+
+// extras joins the two places Emby keeps them.
+//
+// The two lookups are independent and are made together. They used to run in turn, which
+// on a detail page open — where this is one of five requests the television makes at once
+// — was one round trip of pure waiting for no reason at all.
+//
+// Neither list failing is fatal, and only both failing is a failure worth reporting: a
+// title can perfectly well have a trailer and no special features, or the reverse, and an
+// Extras tab withheld because one of two lookups was unwell is the worse outcome.
+func (s *Server) extras(
+	ctx context.Context, cred emby.Credentials, itemID string,
+) (json.RawMessage, error) {
+	var (
+		trailers   *emby.ItemsResult
+		trailerErr error
+		features   *emby.ItemsResult
+		featureErr error
+		wg         sync.WaitGroup
+	)
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		trailers, trailerErr = s.emby.LocalTrailers(timing.WithLabel(ctx, "emby.trailers"), cred, itemID)
+	}()
+	go func() {
+		defer wg.Done()
+		features, featureErr = s.emby.SpecialFeatures(timing.WithLabel(ctx, "emby.features"), cred, itemID)
+	}()
+	wg.Wait()
+
+	if trailerErr != nil && featureErr != nil {
+		return nil, featureErr
 	}
-	w.Header().Set("X-Memby-Cache", "miss")
-	writeRaw(w, http.StatusOK, body)
+	var items []json.RawMessage
+	if trailerErr == nil && trailers != nil {
+		items = append(items, trailers.Items...)
+	}
+	if featureErr == nil && features != nil {
+		items = append(items, features.Items...)
+	}
+	return json.Marshal(extrasResponse{Items: nonNilRaws(dedupeExtras(items))})
 }
 
 // dedupeExtras drops the same file appearing in both lists, in first-seen order.

server/internal/api/features.go

@@ -195,16 +195,21 @@ func evaluateFeature(policy store.FeaturePolicy, definition featureDefinition, p
 	return evaluatedFeature{featureDefinition: definition, Enabled: enabled, Source: source, Compatible: compatible}
 }
 
+// currentFeaturePolicy is read on the request path from sixteen places and by the status
+// poll every open television makes, so it is cached rather than queried — see
+// featurePolicyCache for how long and why that is safe.
 func (s *Server) currentFeaturePolicy(ctx context.Context) store.FeaturePolicy {
 	if s.store == nil {
 		return store.DefaultFeaturePolicy()
 	}
-	policy, err := s.store.FeaturePolicy(ctx)
-	if err != nil {
-		s.loggerFor(ctx).Warn("feature policy unavailable; using safe defaults", "error", err)
-		return store.DefaultFeaturePolicy()
-	}
-	return policy
+	return s.featurePolicy.read(ctx, func(ctx context.Context) store.FeaturePolicy {
+		policy, err := s.store.FeaturePolicy(ctx)
+		if err != nil {
+			s.loggerFor(ctx).Warn("feature policy unavailable; using safe defaults", "error", err)
+			return store.DefaultFeaturePolicy()
+		}
+		return policy
+	})
 }
 
 func (s *Server) featureEnabled(ctx context.Context, key string) bool {
@@ -296,6 +301,7 @@ func (s *Server) handleAdminFeaturePolicy(w http.ResponseWriter, r *http.Request
 		writeError(w, http.StatusBadRequest, "unknown feature policy action")
 		return
 	}
+	s.featurePolicy.invalidate()
 	stored, err := s.store.SetFeaturePolicy(r.Context(), next, req.ExpectedRevision)
 	if err != nil {
 		if errors.Is(err, store.ErrFeaturePolicyConflict) {

server/internal/api/features_cache.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ponzischeme89/memby/server/internal/store"
+)
+
+// featurePolicyTTL is how stale the cached feature policy may be.
+//
+// The document is read on the request path from sixteen places and by /v1/status, which
+// every open television polls every ten seconds — so on a four-set household the policy
+// was fetched from Postgres somewhere over twenty times a minute to answer a question
+// whose answer changes when an operator presses a switch. Each read is a millisecond and
+// none of them was ever the reason a screen was slow; what they were is a standing draw
+// on a connection pool that the requests which *are* slow have to queue behind.
+//
+// Five seconds because the operator is the only writer and their own write clears this
+// instance's copy outright: the window is not "how long until my change takes effect" but
+// "how long until an instance that did not make the change notices", which is the same
+// question WatchMaintenance answers with thirty.
+const featurePolicyTTL = 5 * time.Second
+
+// featurePolicyCache is a read-through cache with a deliberately simple concurrency
+// story: a stale read takes the lock, refreshes, and every other caller waits for that
+// one refresh rather than starting its own. That is worth stating because the opposite —
+// releasing the lock to query — is what turns one expired entry into a thundering herd of
+// identical queries, which is the failure this exists to prevent.
+type featurePolicyCache struct {
+	mu      sync.Mutex
+	value   store.FeaturePolicy
+	valid   bool
+	fetched time.Time
+}
+
+// read returns the cached value, refreshing through load when it has expired. load is
+// only ever called with the lock held, so it must not itself read the cache.
+func (c *featurePolicyCache) read(
+	ctx context.Context, load func(context.Context) store.FeaturePolicy,
+) store.FeaturePolicy {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.valid && time.Since(c.fetched) < featurePolicyTTL {
+		return c.value
+	}
+	c.value = load(ctx)
+	c.valid = true
+	c.fetched = time.Now()
+	return c.value
+}
+
+// invalidate drops the copy so the next read goes to Postgres. Called by the operator's
+// own write: a console that showed the old answer back to the person who had just
+// changed it would read as a save that failed.
+func (c *featurePolicyCache) invalidate() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.valid = false
+}

server/internal/api/gateway_settings.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"log/slog"
+	"math"
 	"sync"
 	"time"
 
@@ -124,6 +125,19 @@ func (s *Server) radarrAlertWindow() time.Duration {
 		s.cfg.RadarrAlertWindow)
 }
 
+// slowRequestThreshold is the line between a request that logs a breakdown and one that
+// does not. Switched off it returns a duration no request can exceed rather than zero,
+// because zero would mean *every* request carried one — the opposite of what the operator
+// asked for.
+func (s *Server) slowRequestThreshold() time.Duration {
+	threshold := overrideWindow(s.gatewaySettings.get().SlowRequestMillis, time.Millisecond,
+		s.cfg.SlowRequestThreshold)
+	if threshold <= 0 {
+		return math.MaxInt64
+	}
+	return threshold
+}
+
 func (s *Server) embyHealthInterval() time.Duration {
 	return overrideWindow(s.gatewaySettings.get().EmbyHealthSeconds, time.Second,
 		s.cfg.EmbyHealthInterval)
@@ -162,6 +176,7 @@ type deployedGatewaySettings struct {
 	SonarrAlertMinutes int    `json:"sonarrAlertMinutes"`
 	RadarrAlertMinutes int    `json:"radarrAlertMinutes"`
 	EmbyHealthSeconds  int    `json:"embyHealthSeconds"`
+	SlowRequestMillis  int    `json:"slowRequestMillis"`
 	LibrarySyncMinutes int    `json:"librarySyncMinutes"`
 }
 
@@ -177,6 +192,7 @@ func (s *Server) deployedSettings() deployedGatewaySettings {
 		SonarrAlertMinutes: int(s.cfg.SonarrAlertWindow / time.Minute),
 		RadarrAlertMinutes: int(s.cfg.RadarrAlertWindow / time.Minute),
 		EmbyHealthSeconds:  int(s.cfg.EmbyHealthInterval / time.Second),
+		SlowRequestMillis:  int(s.cfg.SlowRequestThreshold / time.Millisecond),
 		LibrarySyncMinutes: int(s.cfg.SyncInterval / time.Minute),
 	}
 }

server/internal/api/home.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ponzischeme89/memby/server/internal/emby"
 	"github.com/ponzischeme89/memby/server/internal/recommend"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 // Field sets mirror what each TV row actually renders. Asking Emby for less is the
@@ -88,6 +89,11 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 
 	cred := credentials(sess)
+	// The fan-out's wall clock, which is a different number from the summed Emby time
+	// beside it and the more useful of the two: the sum says how much work Emby did, this
+	// says how long the launcher waited for it. They diverge exactly when the calls stop
+	// running concurrently, which is the failure this instrumentation exists to catch.
+	upstream := timing.Start(ctx, "fanout")
 	var (
 		mu             sync.Mutex
 		failures       int
@@ -97,33 +103,39 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 		forYouRows     []recommend.Row
 		forYouRowStale bool
 		seriesPlayed   map[string]time.Time
+		ranking        rankingInputs
+		rowStats       []store.RowStat
+		rowStatsOK     bool
 		wg             sync.WaitGroup
 	)
 
-	run := func(dest *[]json.RawMessage, fetch func() (*emby.ItemsResult, error)) {
+	// Each row names its own stage, so a slow launcher says which query it was waiting
+	// on rather than only that it was waiting on Emby. They are concurrent, so a single
+	// summed `emby=` would be the one number that could not answer that.
+	run := func(name string, dest *[]json.RawMessage, fetch func(context.Context) (*emby.ItemsResult, error)) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			result, err := fetch()
+			result, err := fetch(timing.WithLabel(ctx, "emby."+name))
 			mu.Lock()
 			defer mu.Unlock()
 			if err != nil {
 				failures++
-				s.loggerFor(ctx).Warn("home row failed", "error", err)
+				s.loggerFor(ctx).Warn("home row failed", "row", name, "error", err)
 				return
 			}
 			*dest = result.Items
 		}()
 	}
 
-	run(&out.ContinueWatching, func() (*emby.ItemsResult, error) {
+	run("resume", &out.ContinueWatching, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.ResumeItems(ctx, cred, rowParams(url.Values{
 			"Recursive":  {"true"},
 			"MediaTypes": {"Video"},
 			"Limit":      {itoa(limit)},
 		}, fieldsContinue))
 	})
-	run(&out.Favorites, func() (*emby.ItemsResult, error) {
+	run("favourites", &out.Favorites, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.Items(ctx, cred, rowParams(url.Values{
 			"Filters":          {"IsFavorite"},
 			"IncludeItemTypes": {"Movie,Series"},
@@ -133,7 +145,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 			"Limit":            {itoa(limit)},
 		}, fieldsRow))
 	})
-	run(&out.NextUp, func() (*emby.ItemsResult, error) {
+	run("nextup", &out.NextUp, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.NextUp(ctx, cred, rowParams(url.Values{
 			"Limit": {itoa(limit)},
 		}, fieldsContinue))
@@ -144,7 +156,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		played, err := s.recentlyPlayedSeries(ctx, cred)
+		played, err := s.recentlyPlayedSeries(timing.WithLabel(ctx, "emby.recent"), cred)
 		if err != nil {
 			s.loggerFor(ctx).Warn("recently played lookup failed", "error", err)
 			return
@@ -153,7 +165,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 		seriesPlayed = played
 		mu.Unlock()
 	}()
-	run(&out.LatestMovies, func() (*emby.ItemsResult, error) {
+	run("latest", &out.LatestMovies, func(ctx context.Context) (*emby.ItemsResult, error) {
 		newReleaseDays := s.weightedConfig().NewReleaseDays
 		if newReleaseDays < 1 {
 			newReleaseDays = recommend.DefaultWeightedConfig().NewReleaseDays
@@ -197,6 +209,30 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 			mu.Unlock()
 		}()
 	}
+	// The ranking evidence and the row-order preferences depend on the viewer and on
+	// nothing that is being fetched around them, so they are read *beside* the Emby
+	// fan-out rather than after it. They used to be the first two things the response did
+	// once every row had arrived — eight Postgres reads with nothing else in flight,
+	// entirely on the critical path, for answers that were available before the request
+	// asked Emby anything.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ranking = s.rankingInputs(ctx, sess.EmbyUserID)
+	}()
+	if s.store != nil {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			stats, err := s.store.UserRowStats(ctx, sess.EmbyUserID, time.Now().Add(-45*24*time.Hour))
+			if err != nil {
+				s.loggerFor(ctx).Warn("home row preferences unavailable",
+					"user", sess.EmbyUserID, "error", err)
+				return
+			}
+			rowStats, rowStatsOK = stats, true
+		}()
+	}
 	if s.forYou != nil {
 		// The prepared pool is one indexed PostgreSQL read. It runs beside the Emby
 		// calls and deliberately has no live-engine fallback, so Home can never inherit
@@ -226,6 +262,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 
 	wg.Wait()
+	upstream()
 
 	if failures == 4 {
 		writeError(w, http.StatusBadGateway, "could not reach the emby server")
@@ -249,6 +286,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	if recommendations == nil {
 		s.refreshRecommendationsInBackground(sess)
 	}
+	assemble := timing.Start(ctx, timing.StageRows)
 	rows := baseRows(out)
 	nearContinue := make([]recommend.Row, 0, len(forYouRows)+2)
 	if len(forYouRows) > 0 {
@@ -271,34 +309,38 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 	out.Rows = append(rows, recommendations...)
 	out.Rows = s.filterRecommendationPermissions(ctx, sess, out.Rows)
-	if stats, err := s.store.UserRowStats(
-		ctx,
-		sess.EmbyUserID,
-		time.Now().Add(-45*24*time.Hour),
-	); err != nil {
-		s.loggerFor(ctx).Warn("home row preferences unavailable", "user", sess.EmbyUserID, "error", err)
-	} else {
-		out.Rows = personalizeHomeRows(out.Rows, stats)
+	if rowStatsOK {
+		out.Rows = personalizeHomeRows(out.Rows, rowStats)
 	}
-	out.Rows = s.personalizeTitles(ctx, sess, out.Rows)
+	assemble()
+	rank := timing.Start(ctx, timing.StageRank)
+	out.Rows = s.personalizeTitlesWith(out.Rows, ranking)
 	out.Rows = personalizeRowsByTitleScores(selectPersonalizedRows(out.Rows))
 	out.Rows = deduplicateRows(out.Rows)
+	rank()
 	// Ratings ride on the cards themselves. Only what is already stored is attached, so
 	// the launcher pays one indexed read rather than a request per poster, and a card
 	// shows its scores as it is drawn instead of when focus reaches it.
+	decorate := timing.Start(ctx, timing.StageRatings)
 	s.decorateHomeRatings(ctx, &out)
+	decorate()
 	// The hero is composed last, from the finished rows, because that is the only point
 	// at which the ratings it ranks by are already attached. It is prepended rather than
 	// inserted: the television consumes this row instead of drawing it, so its position
 	// among the shelves means nothing, and being first is what lets an older reader that
 	// does draw it put it somewhere sensible.
 	if hero {
-		if row := s.heroRow(ctx, out.Rows, sess.EmbyUserID, time.Now()); row != nil {
+		compose := timing.Start(ctx, timing.StageHero)
+		row := s.heroRow(ctx, out.Rows, sess.EmbyUserID, time.Now())
+		compose()
+		if row != nil {
 			out.Rows = append([]recommend.Row{*row}, out.Rows...)
 		}
 	}
 
+	encode := timing.Start(ctx, timing.StageEncode)
 	body, err := json.Marshal(out)
+	encode()
 	if err != nil {
 		s.loggerFor(ctx).Error("home encode failed", "error", err)
 		writeError(w, http.StatusInternalServerError, "could not build the home payload")

server/internal/api/items.go

@@ -7,10 +7,12 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/ponzischeme89/memby/server/internal/cache"
 	"github.com/ponzischeme89/memby/server/internal/sonarr"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 type seriesEpisodesResponse struct {
@@ -75,23 +77,21 @@ func itemDetailKey(userID, itemID string) string {
 func (s *Server) detailItem(
 	ctx context.Context, sess store.Session, itemID string,
 ) (json.RawMessage, error) {
-	key := itemDetailKey(sess.EmbyUserID, itemID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		return raw, nil
-	}
-	item, err := s.emby.Item(ctx, credentials(sess), itemID, fieldsDetail)
-	if err != nil {
-		return nil, err
-	}
-	// A detail page can then draw its ratings with the rest of the hero rather than
-	// after a second request. Anything not yet stored still arrives on /ratings.
-	decorated := []json.RawMessage{item}
-	s.decorateItemRatings(ctx, decorated)
-	item = decorated[0]
-	if err := s.cache.Set(ctx, key, item, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("item cache write failed", "error", err)
-	}
-	return item, nil
+	item, _, err := s.cachedRead(ctx, itemDetailKey(sess.EmbyUserID, itemID), s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			raw, err := s.emby.Item(
+				timing.WithLabel(ctx, "emby.item"), credentials(sess), itemID, fieldsDetail)
+			if err != nil {
+				return nil, err
+			}
+			// A detail page can then draw its ratings with the rest of the hero rather
+			// than after a second request. Anything not yet stored still arrives on
+			// /ratings.
+			decorated := []json.RawMessage{raw}
+			s.decorateItemRatings(ctx, decorated)
+			return decorated[0], nil
+		})
+	return item, err
 }
 
 // handleSeasonFinale verifies an episode against Sonarr's complete season, including
@@ -109,92 +109,113 @@ func (s *Server) handleSeasonFinale(w http.ResponseWriter, r *http.Request, sess
 		writeJSON(w, http.StatusOK, empty)
 		return
 	}
-	key := cache.UserKey(sess.EmbyUserID, "season-finale:v1:"+itemID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		writeRaw(w, http.StatusOK, raw)
-		return
-	}
-
-	currentRaw, err := s.emby.Item(
-		ctx, credentials(sess), itemID,
-		"ProviderIds,SeriesId,SeriesName,ParentIndexNumber,IndexNumber",
-	)
+	// Cached for the household rather than per viewer, and outside the namespace a
+	// playback stop invalidates. Whether an episode closes its season is a fact about
+	// the season; it carries no user data, and it was previously being thrown away by
+	// the very event that most often precedes somebody asking for it — the stop report
+	// at the end of the episode before.
+	key := cache.MetadataKey("season-finale:v2:" + itemID)
+	body, hit, err := s.cachedRead(ctx, key, s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			return json.Marshal(s.seasonFinale(ctx, sess, itemID))
+		})
 	if err != nil {
 		s.writeUpstreamError(ctx, w, err, "could not inspect the playing episode")
 		return
 	}
+	writeCached(w, hit, body)
+}
+
+// seasonFinale is four upstream reads and only three of them are a chain.
+//
+// The episode has to be read before its series can be, and the series before Sonarr can
+// be searched for it — but Sonarr's series catalogue depends on none of that, and used
+// to be fetched fourth, in sequence, after both Emby lookups had returned. It is now
+// fetched beside them, which takes a leg off the wait for the ordinary case and all of
+// it for a household whose catalogue is still warm.
+//
+// Every failure below answers "not a finale" rather than an error. This decorates the
+// end of an episode; being unable to say is the same outcome as saying no, and a viewer
+// must never be shown a failure for it.
+func (s *Server) seasonFinale(
+	ctx context.Context, sess store.Session, itemID string,
+) seasonFinaleResponse {
+	empty := seasonFinaleResponse{}
+
+	var (
+		catalogue    []sonarr.Series
+		catalogueErr error
+		wg           sync.WaitGroup
+	)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		catalogue, catalogueErr = s.sonarrSeriesCatalogue(timing.WithLabel(ctx, "sonarr.series"))
+	}()
+	// Whatever happens below, the catalogue read has to be waited for: it is running on
+	// this request's context and returning without it would leave a goroutine writing to
+	// variables the caller has moved on from.
+	defer wg.Wait()
+
+	currentRaw, err := s.emby.Item(
+		timing.WithLabel(ctx, "emby.episode"), credentials(sess), itemID,
+		"ProviderIds,SeriesId,SeriesName,ParentIndexNumber,IndexNumber",
+	)
+	if err != nil {
+		s.loggerFor(ctx).Warn("season finale episode unavailable", "item", itemID, "error", err)
+		return empty
+	}
 	var current finaleEmbyItem
 	if json.Unmarshal(currentRaw, &current) != nil ||
 		!strings.EqualFold(current.Type, "Episode") || current.SeriesID == "" ||
 		current.ParentIndexNumber <= 0 || current.IndexNumber <= 0 {
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
 
-	seriesRaw, err := s.emby.Item(ctx, credentials(sess), current.SeriesID, "ProviderIds")
+	seriesRaw, err := s.emby.Item(
+		timing.WithLabel(ctx, "emby.series"), credentials(sess), current.SeriesID, "ProviderIds")
 	if err != nil {
 		s.loggerFor(ctx).Warn("season finale series metadata unavailable", "item", itemID, "error", err)
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
 	var seriesItem finaleEmbyItem
 	if json.Unmarshal(seriesRaw, &seriesItem) != nil {
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
 	tvdbID, err := strconv.Atoi(providerID(seriesItem.ProviderIDs, "tvdb"))
 	if err != nil || tvdbID <= 0 {
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
 
-	series, err := s.sonarrSeriesCatalogue(ctx)
-	if err != nil {
-		s.loggerFor(ctx).Warn("season finale Sonarr series unavailable", "item", itemID, "error", err)
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+	wg.Wait()
+	if catalogueErr != nil {
+		s.loggerFor(ctx).Warn("season finale Sonarr series unavailable", "item", itemID, "error", catalogueErr)
+		return empty
 	}
 	sonarrSeriesID := 0
-	for _, candidate := range series {
+	for _, candidate := range catalogue {
 		if candidate.TVDBID == tvdbID {
 			sonarrSeriesID = candidate.ID
 			break
 		}
 	}
 	if sonarrSeriesID == 0 {
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
-	episodes, err := s.sonarr.Episodes(ctx, sonarrSeriesID)
+	episodes, err := s.sonarr.Episodes(timing.WithLabel(ctx, "sonarr.episodes"), sonarrSeriesID)
 	if err != nil {
 		s.loggerFor(ctx).Warn("season finale Sonarr episodes unavailable", "item", itemID, "error", err)
-		s.writeSeasonFinaleResponse(ctx, key, empty, w)
-		return
+		return empty
 	}
-	result := seasonFinaleResponse{
-		SeasonFinale:  isSeasonFinale(current.ParentIndexNumber, current.IndexNumber, episodes),
+	if !isSeasonFinale(current.ParentIndexNumber, current.IndexNumber, episodes) {
+		return empty
+	}
+	return seasonFinaleResponse{
+		SeasonFinale:  true,
 		SeriesName:    current.SeriesName,
 		SeasonNumber:  current.ParentIndexNumber,
 		EpisodeNumber: current.IndexNumber,
 	}
-	if !result.SeasonFinale {
-		result = empty
-	}
-	s.writeSeasonFinaleResponse(ctx, key, result, w)
-}
-
-func (s *Server) writeSeasonFinaleResponse(
-	ctx context.Context, key string, result seasonFinaleResponse, w http.ResponseWriter,
-) {
-	body, err := json.Marshal(result)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, "could not encode finale status")
-		return
-	}
-	if err := s.cache.Set(ctx, key, body, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("season finale cache write failed", "error", err)
-	}
-	writeRaw(w, http.StatusOK, body)
 }
 
 func providerID(ids map[string]string, wanted string) string {
@@ -228,42 +249,42 @@ func (s *Server) handleSeriesEpisodes(w http.ResponseWriter, r *http.Request, se
 		writeError(w, http.StatusBadRequest, "series id is required")
 		return
 	}
+	// Per viewer, because the browser marks what has been watched — and coalesced,
+	// because this is the largest request the television makes and it is made twice for
+	// the same show as a matter of course: the launcher warms it while a Continue
+	// Watching card is focused, and the page asks again when somebody presses. A
+	// long-running show is a thousand records, so two of them is a real cost on the one
+	// press that must feel free.
 	key := cache.UserKey(sess.EmbyUserID, "series-episodes:"+seriesID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		w.Header().Set("X-Memby-Cache", "hit")
-		writeRaw(w, http.StatusOK, raw)
-		return
-	}
-
-	result, err := s.emby.Episodes(ctx, credentials(sess), seriesID, url.Values{
-		"UserId": {sess.EmbyUserID},
-		// PremiereDate is the one fact that tells two episodes of a list apart, so the
-		// episode browser asks for it by name — Emby does not return it otherwise.
-		"Fields":           {"Overview,RunTimeTicks,SeriesName,PremiereDate,PrimaryImageAspectRatio"},
-		"EnableUserData":   {"true"},
-		"EnableImages":     {"true"},
-		"EnableImageTypes": {"Primary,Thumb,Backdrop"},
-		"ImageTypeLimit":   {"1"},
-		"Limit":            {"1000"},
-	})
+	body, hit, err := s.cachedRead(ctx, key, s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			result, err := s.emby.Episodes(
+				timing.WithLabel(ctx, "emby.episodes"), credentials(sess), seriesID, url.Values{
+					"UserId": {sess.EmbyUserID},
+					// PremiereDate is the one fact that tells two episodes of a list apart,
+					// so the episode browser asks for it by name — Emby does not return it
+					// otherwise.
+					"Fields":           {"Overview,RunTimeTicks,SeriesName,PremiereDate,PrimaryImageAspectRatio"},
+					"EnableUserData":   {"true"},
+					"EnableImages":     {"true"},
+					"EnableImageTypes": {"Primary,Thumb,Backdrop"},
+					"ImageTypeLimit":   {"1"},
+					"Limit":            {"1000"},
+				})
+			if err != nil {
+				return nil, err
+			}
+			items := result.Items
+			if items == nil {
+				items = []json.RawMessage{}
+			}
+			return json.Marshal(seriesEpisodesResponse{Items: items})
+		})
 	if err != nil {
 		s.writeUpstreamError(ctx, w, err, "could not load series episodes")
 		return
 	}
-	items := result.Items
-	if items == nil {
-		items = []json.RawMessage{}
-	}
-	body, err := json.Marshal(seriesEpisodesResponse{Items: items})
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, "could not encode series episodes")
-		return
-	}
-	if err := s.cache.Set(ctx, key, body, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("series episodes cache write failed", "error", err)
-	}
-	w.Header().Set("X-Memby-Cache", "miss")
-	writeRaw(w, http.StatusOK, body)
+	writeCached(w, hit, body)
 }
 
 // handleTrailer answers with the item's first local trailer, or 404 when it has none.

server/internal/api/people.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"net/url"
@@ -15,6 +16,12 @@ type personFilmographyResponse struct {
 
 // handlePerson returns Emby's person item. PremiereDate and EndDate are the person's
 // birth and death dates; Overview is their biography.
+//
+// It is cached for the household rather than per viewer, and outside the namespace a
+// playback stop invalidates. Nothing about who Cillian Murphy is depends on who is
+// asking or on what they finished watching ten minutes ago, and filing it under the
+// viewer meant a cast page paid Emby again after every episode anybody in the house
+// completed.
 func (s *Server) handlePerson(w http.ResponseWriter, r *http.Request, sess store.Session) {
 	ctx := r.Context()
 	personID := r.PathValue("id")
@@ -22,31 +29,26 @@ func (s *Server) handlePerson(w http.ResponseWriter, r *http.Request, sess store
 		writeError(w, http.StatusBadRequest, "person id is required")
 		return
 	}
-	key := cache.UserKey(sess.EmbyUserID, "person:v1:"+personID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		w.Header().Set("X-Memby-Cache", "hit")
-		writeRaw(w, http.StatusOK, raw)
-		return
-	}
-	person, err := s.emby.Item(
-		ctx,
-		credentials(sess),
-		personID,
-		"Overview,Genres,PrimaryImageAspectRatio",
-	)
+	person, hit, err := s.cachedRead(ctx, cache.MetadataKey("person:v2:"+personID), s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			return s.emby.Item(ctx, credentials(sess), personID,
+				"Overview,Genres,PrimaryImageAspectRatio")
+		})
 	if err != nil {
 		s.writeUpstreamError(ctx, w, err, "could not load the person")
 		return
 	}
-	if err := s.cache.Set(ctx, key, person, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("person cache write failed", "error", err)
-	}
-	w.Header().Set("X-Memby-Cache", "miss")
-	writeRaw(w, http.StatusOK, person)
+	writeCached(w, hit, person)
 }
 
 // Filmography is kept separate from the biography so life dates can decorate the cast
 // row without also loading every cast member's credits.
+//
+// Unlike the biography this one is genuinely per viewer — it asks Emby for user data, so
+// the grid can mark what has been watched — and therefore stays where invalidation can
+// reach it. What it gains instead is coalescing: this is the most expensive query the
+// gateway makes of Emby, a recursive scan of the whole library filtered by person, and
+// three televisions opening the same cast card used to mean three of them.
 func (s *Server) handlePersonFilmography(w http.ResponseWriter, r *http.Request, sess store.Session) {
 	ctx := r.Context()
 	personID := r.PathValue("id")
@@ -55,40 +57,33 @@ func (s *Server) handlePersonFilmography(w http.ResponseWriter, r *http.Request,
 		return
 	}
 	key := cache.UserKey(sess.EmbyUserID, "person-filmography:v1:"+personID)
-	if raw, err := s.cache.Get(ctx, key); err == nil {
-		w.Header().Set("X-Memby-Cache", "hit")
-		writeRaw(w, http.StatusOK, raw)
-		return
-	}
-	result, err := s.emby.Items(ctx, credentials(sess), url.Values{
-		"PersonIds":        {personID},
-		"IncludeItemTypes": {"Movie,Series"},
-		"Recursive":        {"true"},
-		"SortBy":           {"ProductionYear,SortName"},
-		"SortOrder":        {"Descending"},
-		"Limit":            {"60"},
-		"Fields":           {"Overview,ProductionYear,PrimaryImageAspectRatio"},
-		"EnableImages":     {"true"},
-		"EnableImageTypes": {"Primary"},
-		"ImageTypeLimit":   {"1"},
-		"EnableUserData":   {"true"},
-	})
+	body, hit, err := s.cachedRead(ctx, key, s.cfg.ItemTTL,
+		func(ctx context.Context) (json.RawMessage, error) {
+			result, err := s.emby.Items(ctx, credentials(sess), url.Values{
+				"PersonIds":        {personID},
+				"IncludeItemTypes": {"Movie,Series"},
+				"Recursive":        {"true"},
+				"SortBy":           {"ProductionYear,SortName"},
+				"SortOrder":        {"Descending"},
+				"Limit":            {"60"},
+				"Fields":           {"Overview,ProductionYear,PrimaryImageAspectRatio"},
+				"EnableImages":     {"true"},
+				"EnableImageTypes": {"Primary"},
+				"ImageTypeLimit":   {"1"},
+				"EnableUserData":   {"true"},
+			})
+			if err != nil {
+				return nil, err
+			}
+			items := result.Items
+			if items == nil {
+				items = []json.RawMessage{}
+			}
+			return json.Marshal(personFilmographyResponse{Items: items})
+		})
 	if err != nil {
 		s.writeUpstreamError(ctx, w, err, "could not load the person's filmography")
 		return
 	}
-	items := result.Items
-	if items == nil {
-		items = []json.RawMessage{}
-	}
-	body, err := json.Marshal(personFilmographyResponse{Items: items})
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, "could not encode the filmography")
-		return
-	}
-	if err := s.cache.Set(ctx, key, body, s.cfg.ItemTTL); err != nil {
-		s.loggerFor(ctx).Warn("person filmography cache write failed", "error", err)
-	}
-	w.Header().Set("X-Memby-Cache", "miss")
-	writeRaw(w, http.StatusOK, body)
+	writeCached(w, hit, body)
 }

server/internal/api/playback.go

@@ -14,6 +14,7 @@ import (
 	serverlogging "github.com/ponzischeme89/memby/server/internal/logging"
 	"github.com/ponzischeme89/memby/server/internal/notify"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 const ticksPerMillisecond = 10_000
@@ -826,7 +827,8 @@ func (s *Server) handlePlaybackReport(w http.ResponseWriter, r *http.Request, se
 	)
 
 	err := s.emby.ReportPlayback(
-		r.Context(), credentials(sess), phase, report.ItemID, report.MediaSourceID,
+		timing.WithLabel(r.Context(), "emby.report"),
+		credentials(sess), phase, report.ItemID, report.MediaSourceID,
 		report.PlaySessionID, report.PlayMethod, report.EventName,
 		max64(report.PositionMs, 0)*ticksPerMillisecond, report.IsPaused,
 	)
@@ -873,18 +875,27 @@ func (s *Server) handlePlaybackReport(w http.ResponseWriter, r *http.Request, se
 		playbackSessionKey(sess.DeviceID, report.PlaySessionID))
 
 	if phase == "stopped" {
+		invalidate := timing.Start(r.Context(), "invalidate")
 		if err := s.cache.InvalidateUser(r.Context(), sess.EmbyUserID); err != nil {
 			s.loggerFor(r.Context()).Warn("cache invalidation failed", "error", err)
 		}
+		invalidate()
 		// Recommendation taste changes slowly. Tracearr marks this user's prepared
 		// profile dirty only when the session first becomes terminal; the daily builder
 		// then refreshes it without turning every player exit into catalogue-wide work.
 	}
 
 	response := playbackReportResponse{}
+	// The order of these three is the whole of what keeps a progress report cheap. The
+	// claim is a map lookup and comes first, so every report after the first for this
+	// episode stops here rather than at the durable insert four upstream calls later;
+	// the feature check is a cached read; and only then is anything asked of Emby.
 	if shouldAutoFollowShow(phase, report.PositionMs, report.DurationMs) &&
+		s.followChecks.claim(sess.EmbyUserID, report.ItemID) &&
 		s.featureEnabled(r.Context(), featureAutomaticMyShows) {
+		follow := timing.Start(r.Context(), "autofollow")
 		response.AutoFollowedShowTitle = s.autoFollowContinuingShow(r.Context(), sess, report.ItemID)
+		follow()
 	}
 	writeJSON(w, http.StatusOK, response)
 }

server/internal/api/playback_follow.go

@@ -0,0 +1,54 @@
+package api
+
+import "sync"
+
+// consideredFollows bounds the memory below. A household has a handful of episodes in
+// flight; this is generous enough that a whole evening's viewing fits and small enough
+// that nothing a client can do grows it into a leak.
+const consideredFollows = 512
+
+// followChecks remembers which (viewer, episode) pairs have already been put through
+// the automatic My Shows check.
+//
+// The check is hung off playback reports, and a report arrives every ten seconds for the
+// length of an episode. Its own guard — half watched — is true for every one of those
+// after the halfway point, so on a forty-minute episode the second half of the viewing
+// ran the whole check a hundred times: two serial Emby lookups, the Sonarr catalogue and
+// a Postgres write, all of it on the response path of a request the television makes
+// while somebody is watching something. Every run after the first reached the same
+// `inserted == false` and discarded its own work.
+//
+// Deduplicating on the durable insert is correct and was never the problem; what it
+// could not do is prevent the work leading up to it. This is deliberately in memory and
+// deliberately lossy, the playbackTitles arrangement: a restarted gateway does the check
+// once more per episode, which is the cost of one lookup rather than a schema change.
+type followChecks struct {
+	mu    sync.Mutex
+	seen  map[string]struct{}
+	order []string
+}
+
+// claim reports whether this is the first time the pair has been offered, and records it.
+// One call does both because two would be a race between the check and the record, and
+// the thing being protected is precisely a burst of concurrent reports.
+func (f *followChecks) claim(userID, itemID string) bool {
+	if userID == "" || itemID == "" {
+		return false
+	}
+	key := userID + "|" + itemID
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.seen == nil {
+		f.seen = make(map[string]struct{}, consideredFollows)
+	}
+	if _, known := f.seen[key]; known {
+		return false
+	}
+	f.seen[key] = struct{}{}
+	f.order = append(f.order, key)
+	if len(f.order) > consideredFollows {
+		delete(f.seen, f.order[0])
+		f.order = f.order[1:]
+	}
+	return true
+}

server/internal/api/playback_follow_test.go

@@ -0,0 +1,87 @@
+package api
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+)
+
+// The whole point of the claim is that it is true exactly once. A progress report arrives
+// every ten seconds for the length of an episode and its own guard — half watched — is
+// true for every one of those after the halfway mark, so without this the check behind it
+// ran two Emby lookups, a Sonarr catalogue read and a Postgres write a hundred times per
+// episode and discarded all but the first.
+func TestFollowCheckIsClaimedOnce(t *testing.T) {
+	var checks followChecks
+	if !checks.claim("user-1", "episode-1") {
+		t.Fatal("the first report should claim the check")
+	}
+	for range 10 {
+		if checks.claim("user-1", "episode-1") {
+			t.Fatal("a repeat report must not claim the check again")
+		}
+	}
+}
+
+// Two people watching the same episode on two televisions are two viewers, and each of
+// them has their own My Shows list.
+func TestFollowCheckIsPerViewerAndPerEpisode(t *testing.T) {
+	var checks followChecks
+	checks.claim("user-1", "episode-1")
+	if !checks.claim("user-2", "episode-1") {
+		t.Fatal("another viewer must get their own check")
+	}
+	if !checks.claim("user-1", "episode-2") {
+		t.Fatal("the next episode must get its own check")
+	}
+}
+
+// An empty id is not a claim to record. The alternative is one entry that swallows the
+// first real check for whichever viewer or episode arrives unnamed.
+func TestFollowCheckRefusesEmptyIdentifiers(t *testing.T) {
+	var checks followChecks
+	if checks.claim("", "episode-1") || checks.claim("user-1", "") {
+		t.Fatal("an unnamed viewer or episode is not a check to claim")
+	}
+}
+
+// The memory is bounded, and what falls out of it falls out oldest first. A gateway that
+// has been up for a month must not be holding every episode the house has ever watched.
+func TestFollowCheckForgetsTheOldestFirst(t *testing.T) {
+	var checks followChecks
+	for index := range consideredFollows + 1 {
+		checks.claim("user-1", fmt.Sprintf("episode-%d", index))
+	}
+	if !checks.claim("user-1", "episode-0") {
+		t.Fatal("the oldest entry should have been evicted")
+	}
+	if checks.claim("user-1", fmt.Sprintf("episode-%d", consideredFollows)) {
+		t.Fatal("the newest entry should still be held")
+	}
+}
+
+// Checking and recording are one call because two would be a race, and the thing being
+// protected is precisely a burst of concurrent reports for one playback.
+func TestFollowCheckClaimsOnceUnderConcurrency(t *testing.T) {
+	var (
+		checks  followChecks
+		claimed int
+		mu      sync.Mutex
+		wg      sync.WaitGroup
+	)
+	for range 32 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if checks.claim("user-1", "episode-1") {
+				mu.Lock()
+				claimed++
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	if claimed != 1 {
+		t.Fatalf("claimed %d times, want exactly 1", claimed)
+	}
+}

server/internal/api/ranking.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/ponzischeme89/memby/server/internal/recommend"
@@ -40,32 +41,60 @@ func (s *Server) filterRecommendationPermissions(
 	if len(ids) == 0 {
 		return rows
 	}
+	// The batches are independent questions about disjoint sets of ids, and there can be
+	// several of them on a launcher full of recommendations. Asking Emby them one after
+	// another put the whole of that latency on the tail of the home response; asking
+	// together costs Emby the same work and the viewer one batch's wait.
+	//
+	// There is deliberately no cap on how many run at once. The count is a hundredth of
+	// the recommendation candidate pool, which is itself bounded, and the Emby client's
+	// transport already keeps a warm pool of connections for them to share.
 	allowed := map[string]bool{}
 	cred := credentials(sess)
+	var (
+		mu     sync.Mutex
+		failed bool
+		wg     sync.WaitGroup
+	)
 	for start := 0; start < len(ids); start += 100 {
 		end := min(start+100, len(ids))
-		result, err := s.emby.Items(ctx, cred, rowParams(url.Values{
-			"Ids":              {strings.Join(ids[start:end], ",")},
-			"Recursive":        {"true"},
-			"IncludeItemTypes": {"Movie,Series"},
-			"Limit":            {itoa(end - start)},
-		}, fieldsRow))
-		if err != nil {
-			s.log.Warn("recommendation permission check failed; hiding candidates",
-				"user", sess.EmbyUserID, "error", err)
-			for index := range rows {
-				if recommendationRow(rows[index].ID) {
-					rows[index].Items = []json.RawMessage{}
+		batch := ids[start:end]
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			result, err := s.emby.Items(ctx, cred, rowParams(url.Values{
+				"Ids":              {strings.Join(batch, ",")},
+				"Recursive":        {"true"},
+				"IncludeItemTypes": {"Movie,Series"},
+				"Limit":            {itoa(len(batch))},
+			}, fieldsRow))
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				s.loggerFor(ctx).Warn("recommendation permission check failed; hiding candidates",
+					"user", sess.EmbyUserID, "error", err)
+				failed = true
+				return
+			}
+			for _, raw := range result.Items {
+				decoded := recommend.Decode([]json.RawMessage{raw})
+				if len(decoded) == 1 {
+					allowed[decoded[0].ID] = true
 				}
 			}
-			return rows
-		}
-		for _, raw := range result.Items {
-			decoded := recommend.Decode([]json.RawMessage{raw})
-			if len(decoded) == 1 {
-				allowed[decoded[0].ID] = true
+		}()
+	}
+	wg.Wait()
+	// One failed batch hides every candidate, exactly as it did when the loop returned
+	// early: this is Emby being the final authority on what a viewer may see, and a
+	// partial answer is not evidence that the rest is permitted.
+	if failed {
+		for index := range rows {
+			if recommendationRow(rows[index].ID) {
+				rows[index].Items = []json.RawMessage{}
 			}
 		}
+		return rows
 	}
 	for index := range rows {
 		if !recommendationRow(rows[index].ID) {
@@ -82,6 +111,22 @@ func (s *Server) filterRecommendationPermissions(
 	return rows
 }
 
+// rankingContext gathers everything the weighted ranker needs about one viewer: their
+// learned profile, what they have already been shown, and how the household as a whole
+// has been getting on with the library.
+//
+// It is seven Postgres reads and it used to make all seven one after another, on the tail
+// of the home response — after the Emby fan-out had finished, so nothing else was in
+// flight and every one of them was pure added latency. Six of the seven depend on nothing
+// but the viewer's id. They are now issued together, which turns the sum of seven round
+// trips into roughly the slowest one.
+//
+// The two that are genuinely a chain stay a chain — an onboarding document names the
+// items its ratings refer to, and the actions name theirs — but the two chains run beside
+// each other, and their results are applied to the profile afterwards in the order they
+// were applied before. Ordering the *writes* rather than the reads is the whole trick:
+// ApplyOnboarding and ApplyExplicitPreference are not commutative and this must not
+// become a place where the ranking depends on which query answered first.
 func (s *Server) rankingContext(
 	ctx context.Context,
 	userID string,
@@ -90,10 +135,90 @@ func (s *Server) rankingContext(
 	if s.store == nil {
 		return profile, nil, nil
 	}
-	if raw, err := s.store.WeightedRecommendationProfile(ctx, userID); err == nil {
-		_ = json.Unmarshal(raw, &profile)
-	} else {
-		s.log.Warn("weighted profile unavailable", "user", userID, "error", err)
+
+	var (
+		wg sync.WaitGroup
+
+		profileRaw json.RawMessage
+
+		onboarding      recommend.OnboardingPreferences
+		onboardingOK    bool
+		onboardingItems []json.RawMessage
+
+		actions     []store.RecommendationAction
+		actionItems []json.RawMessage
+
+		exposureValues []store.ItemExposureStat
+		household      map[string]float64
+	)
+
+	run := func(fn func()) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			fn()
+		}()
+	}
+
+	run(func() {
+		raw, err := s.store.WeightedRecommendationProfile(ctx, userID)
+		if err != nil {
+			s.loggerFor(ctx).Warn("weighted profile unavailable", "user", userID, "error", err)
+			return
+		}
+		profileRaw = raw
+	})
+	run(func() {
+		raw, err := s.store.RecommendationOnboarding(ctx, userID)
+		if err != nil {
+			return
+		}
+		var preferences recommend.OnboardingPreferences
+		if json.Unmarshal(raw, &preferences) != nil {
+			return
+		}
+		onboarding, onboardingOK = preferences, true
+		ids := make([]string, 0, len(preferences.Ratings))
+		for id, rating := range preferences.Ratings {
+			if strings.TrimSpace(id) != "" && rating >= 1 && rating <= 5 {
+				ids = append(ids, id)
+			}
+		}
+		if len(ids) == 0 {
+			return
+		}
+		if raws, err := s.store.LibraryItemsByID(ctx, ids); err == nil {
+			onboardingItems = raws
+		}
+	})
+	run(func() {
+		values, err := s.store.RecommendationActions(ctx, userID)
+		if err != nil {
+			return
+		}
+		actions = values
+		ids := make([]string, 0, len(values))
+		for _, action := range values {
+			ids = append(ids, action.ItemID)
+		}
+		if len(ids) == 0 {
+			return
+		}
+		if raws, err := s.store.LibraryItemsByID(ctx, ids); err == nil {
+			actionItems = raws
+		}
+	})
+	run(func() {
+		values, err := s.store.UserItemExposures(ctx, userID, time.Now().Add(-45*24*time.Hour))
+		if err == nil {
+			exposureValues = values
+		}
+	})
+	run(func() { household = s.householdCompletionScores(ctx) })
+	wg.Wait()
+
+	if len(profileRaw) > 0 {
+		_ = json.Unmarshal(profileRaw, &profile)
 	}
 	if profile.ExplicitPositive == nil {
 		profile.ExplicitPositive = map[string]bool{}
@@ -101,31 +226,16 @@ func (s *Server) rankingContext(
 	if profile.ExplicitNegative == nil {
 		profile.ExplicitNegative = map[string]bool{}
 	}
-	if raw, err := s.store.RecommendationOnboarding(ctx, userID); err == nil {
-		var preferences recommend.OnboardingPreferences
-		if json.Unmarshal(raw, &preferences) == nil {
-			profile.ApplyOnboarding(preferences, s.weightedConfig().MinimumEvidence)
-			ids := make([]string, 0, len(preferences.Ratings))
-			for id, rating := range preferences.Ratings {
-				if strings.TrimSpace(id) != "" && rating >= 1 && rating <= 5 {
-					ids = append(ids, id)
-				}
-			}
-			if raws, itemErr := s.store.LibraryItemsByID(ctx, ids); itemErr == nil {
-				for _, item := range recommend.Decode(raws) {
-					profile.ApplyOnboardingRating(
-						item, preferences.Ratings[item.ID],
-						s.weightedConfig().MinimumEvidence,
-					)
-				}
-			}
+	if onboardingOK {
+		evidence := s.weightedConfig().MinimumEvidence
+		profile.ApplyOnboarding(onboarding, evidence)
+		for _, item := range recommend.Decode(onboardingItems) {
+			profile.ApplyOnboardingRating(item, onboarding.Ratings[item.ID], evidence)
 		}
 	}
-	if actions, err := s.store.RecommendationActions(ctx, userID); err == nil {
-		ids := make([]string, 0, len(actions))
+	if len(actions) > 0 {
 		byID := make(map[string]string, len(actions))
 		for _, action := range actions {
-			ids = append(ids, action.ItemID)
 			byID[action.ItemID] = action.Action
 			switch action.Action {
 			case "more_like_this":
@@ -134,30 +244,82 @@ func (s *Server) rankingContext(
 				profile.ExplicitNegative[action.ItemID] = true
 			}
 		}
-		if raws, itemErr := s.store.LibraryItemsByID(ctx, ids); itemErr == nil {
-			for _, item := range recommend.Decode(raws) {
-				profile.ApplyExplicitPreference(item, byID[item.ID] == "more_like_this")
-			}
+		for _, item := range recommend.Decode(actionItems) {
+			profile.ApplyExplicitPreference(item, byID[item.ID] == "more_like_this")
 		}
 	}
-	exposures := map[string]recommend.ItemExposure{}
-	if values, err := s.store.UserItemExposures(
-		ctx, userID, time.Now().Add(-45*24*time.Hour),
-	); err == nil {
-		for _, value := range values {
-			exposures[value.ItemID] = recommend.ItemExposure{
-				Impressions: value.Impressions, Focuses: value.Focuses,
-				Selects: value.Selects, LastShown: value.LastShown,
-			}
+	exposures := make(map[string]recommend.ItemExposure, len(exposureValues))
+	for _, value := range exposureValues {
+		exposures[value.ItemID] = recommend.ItemExposure{
+			Impressions: value.Impressions, Focuses: value.Focuses,
+			Selects: value.Selects, LastShown: value.LastShown,
 		}
 	}
-	household, err := s.store.HouseholdCompletionScores(ctx, time.Now().Add(-180*24*time.Hour))
-	if err != nil {
+	if household == nil {
 		household = map[string]float64{}
 	}
 	return profile, exposures, household
 }
 
+// householdCompletionScoreTTL is how long one reading of the household's completion
+// scores is reused.
+//
+// The query aggregates six months of viewing across everybody, it is the heaviest read
+// in rankingContext, and its answer is *the same for every viewer in the house* — so
+// four televisions refreshing their launchers were running four copies of one
+// six-month aggregate, none of which could have produced a different number. A minute
+// is chosen against what it measures: this moves when somebody finishes something, and
+// nothing on a launcher is different for having learned that a minute sooner.
+const householdCompletionScoreTTL = time.Minute
+
+// householdScores caches that reading. Coalesced as well as cached, so the request that
+// finds it expired is the only one that pays for the refresh rather than the first of
+// several that all do.
+type householdScores struct {
+	mu      sync.Mutex
+	value   map[string]float64
+	fetched time.Time
+}
+
+func (s *Server) householdCompletionScores(ctx context.Context) map[string]float64 {
+	s.household.mu.Lock()
+	defer s.household.mu.Unlock()
+	if s.household.value != nil && time.Since(s.household.fetched) < householdCompletionScoreTTL {
+		return s.household.value
+	}
+	scores, err := s.store.HouseholdCompletionScores(ctx, time.Now().Add(-180*24*time.Hour))
+	if err != nil {
+		// A failed read must not be cached as an empty household: that would suppress
+		// the signal for a whole minute on the strength of one timeout. The previous
+		// reading is the better answer where there is one.
+		if s.household.value != nil {
+			return s.household.value
+		}
+		return map[string]float64{}
+	}
+	s.household.value = scores
+	s.household.fetched = time.Now()
+	return scores
+}
+
+// rankingInputs is one viewer's ranking evidence, gathered.
+//
+// It exists so the gathering can happen somewhere other than immediately before the
+// ranking. On the home path the evidence depends on nothing but the viewer's id, while
+// the rows it will be applied to take seconds to arrive from Emby — so it is read
+// beside them rather than after them, and by the time there is anything to rank it is
+// already in hand. See handleHome.
+type rankingInputs struct {
+	profile   recommend.WeightedProfile
+	exposures map[string]recommend.ItemExposure
+	household map[string]float64
+}
+
+func (s *Server) rankingInputs(ctx context.Context, userID string) rankingInputs {
+	profile, exposures, household := s.rankingContext(ctx, userID)
+	return rankingInputs{profile: profile, exposures: exposures, household: household}
+}
+
 func (s *Server) personalizeTitles(
 	ctx context.Context,
 	sess store.Session,
@@ -166,7 +328,18 @@ func (s *Server) personalizeTitles(
 	if len(rows) == 0 {
 		return rows
 	}
-	profile, exposures, household := s.rankingContext(ctx, sess.EmbyUserID)
+	return s.personalizeTitlesWith(rows, s.rankingInputs(ctx, sess.EmbyUserID))
+}
+
+// personalizeTitlesWith is the ranking itself, over evidence somebody else gathered.
+func (s *Server) personalizeTitlesWith(
+	rows []recommend.Row,
+	inputs rankingInputs,
+) []recommend.Row {
+	if len(rows) == 0 {
+		return rows
+	}
+	profile, exposures, household := inputs.profile, inputs.exposures, inputs.household
 	cfg := s.weightedConfig()
 	now := time.Now()
 	location := s.householdLocation()

server/internal/api/related.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"net/http"
@@ -10,6 +11,7 @@ import (
 	"github.com/ponzischeme89/memby/server/internal/emby"
 	"github.com/ponzischeme89/memby/server/internal/recommend"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 // fieldsRelated is the detail set. It once added Studios on its own account — the
@@ -50,7 +52,24 @@ func (s *Server) handleRelated(w http.ResponseWriter, r *http.Request, sess stor
 		return
 	}
 
-	reasons, related, err := s.recommender.RelatedTo(ctx, credentials(sess), item, relatedRowSize)
+	// Coalesced, because this route is asked for twice for the same title as a matter of
+	// course: the launcher warms it while a card is focused and the detail page asks for
+	// it again the moment somebody presses. Those two requests used to build the answer
+	// twice, and building it is the same Emby fan-out the home rows pay for.
+	body, err := s.buildCached(ctx, key, relatedTTL(s.cfg.ItemTTL),
+		func(ctx context.Context) (json.RawMessage, error) {
+			reasons, related, err := s.recommender.RelatedTo(
+				timing.WithLabel(ctx, "emby.related"), credentials(sess), item, relatedRowSize)
+			if err != nil {
+				return nil, err
+			}
+			items := nonNilRaws(recommend.Raws(related))
+			s.decorateItemRatings(ctx, items)
+			return json.Marshal(relatedResponse{
+				Reasons: nonNilStrings(reasons),
+				Items:   items,
+			})
+		})
 	if err != nil {
 		// RelatedTo degrades rather than failing, so the only error it returns is the
 		// viewer having navigated on. That is the television saving work, not a fault.
@@ -61,29 +80,20 @@ func (s *Server) handleRelated(w http.ResponseWriter, r *http.Request, sess stor
 		s.writeUpstreamError(ctx, w, err, "could not load related titles")
 		return
 	}
+	writeCached(w, false, body)
+}
 
-	items := nonNilRaws(recommend.Raws(related))
-	s.decorateItemRatings(ctx, items)
-	body, err := json.Marshal(relatedResponse{
-		Reasons: nonNilStrings(reasons),
-		Items:   items,
-	})
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, "could not encode related titles")
-		return
+// relatedTTL keeps an empty carousel briefly rather than for the item lifetime: it
+// usually means something upstream was unwell, and the ten-minute answer would otherwise
+// outlive the minute of trouble that produced it.
+func relatedTTL(itemTTL time.Duration) func(json.RawMessage) time.Duration {
+	return func(body json.RawMessage) time.Duration {
+		var decoded relatedResponse
+		if json.Unmarshal(body, &decoded) == nil && len(decoded.Items) == 0 {
+			return relatedEmptyTTL
+		}
+		return itemTTL
 	}
-	// An empty carousel is cached briefly rather than for the item lifetime: it usually
-	// means something upstream was unwell, and the ten-minute answer would otherwise
-	// outlive the minute of trouble that produced it.
-	ttl := s.cfg.ItemTTL
-	if len(items) == 0 {
-		ttl = relatedEmptyTTL
-	}
-	if err := s.cache.Set(ctx, key, body, ttl); err != nil {
-		s.loggerFor(ctx).Warn("related cache write failed", "error", err)
-	}
-	w.Header().Set("X-Memby-Cache", "miss")
-	writeRaw(w, http.StatusOK, body)
 }
 
 // relatedRowSize is a carousel's worth. The strip scrolls, but a viewer who reaches the