0.2.79 - Slow api fixes

server/internal/api/home.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ponzischeme89/memby/server/internal/emby"
 	"github.com/ponzischeme89/memby/server/internal/recommend"
 	"github.com/ponzischeme89/memby/server/internal/store"
+	"github.com/ponzischeme89/memby/server/internal/timing"
 )
 
 // Field sets mirror what each TV row actually renders. Asking Emby for less is the
@@ -88,6 +89,11 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 
 	cred := credentials(sess)
+	// The fan-out's wall clock, which is a different number from the summed Emby time
+	// beside it and the more useful of the two: the sum says how much work Emby did, this
+	// says how long the launcher waited for it. They diverge exactly when the calls stop
+	// running concurrently, which is the failure this instrumentation exists to catch.
+	upstream := timing.Start(ctx, "fanout")
 	var (
 		mu             sync.Mutex
 		failures       int
@@ -97,33 +103,39 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 		forYouRows     []recommend.Row
 		forYouRowStale bool
 		seriesPlayed   map[string]time.Time
+		ranking        rankingInputs
+		rowStats       []store.RowStat
+		rowStatsOK     bool
 		wg             sync.WaitGroup
 	)
 
-	run := func(dest *[]json.RawMessage, fetch func() (*emby.ItemsResult, error)) {
+	// Each row names its own stage, so a slow launcher says which query it was waiting
+	// on rather than only that it was waiting on Emby. They are concurrent, so a single
+	// summed `emby=` would be the one number that could not answer that.
+	run := func(name string, dest *[]json.RawMessage, fetch func(context.Context) (*emby.ItemsResult, error)) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			result, err := fetch()
+			result, err := fetch(timing.WithLabel(ctx, "emby."+name))
 			mu.Lock()
 			defer mu.Unlock()
 			if err != nil {
 				failures++
-				s.loggerFor(ctx).Warn("home row failed", "error", err)
+				s.loggerFor(ctx).Warn("home row failed", "row", name, "error", err)
 				return
 			}
 			*dest = result.Items
 		}()
 	}
 
-	run(&out.ContinueWatching, func() (*emby.ItemsResult, error) {
+	run("resume", &out.ContinueWatching, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.ResumeItems(ctx, cred, rowParams(url.Values{
 			"Recursive":  {"true"},
 			"MediaTypes": {"Video"},
 			"Limit":      {itoa(limit)},
 		}, fieldsContinue))
 	})
-	run(&out.Favorites, func() (*emby.ItemsResult, error) {
+	run("favourites", &out.Favorites, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.Items(ctx, cred, rowParams(url.Values{
 			"Filters":          {"IsFavorite"},
 			"IncludeItemTypes": {"Movie,Series"},
@@ -133,7 +145,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 			"Limit":            {itoa(limit)},
 		}, fieldsRow))
 	})
-	run(&out.NextUp, func() (*emby.ItemsResult, error) {
+	run("nextup", &out.NextUp, func(ctx context.Context) (*emby.ItemsResult, error) {
 		return s.emby.NextUp(ctx, cred, rowParams(url.Values{
 			"Limit": {itoa(limit)},
 		}, fieldsContinue))
@@ -144,7 +156,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		played, err := s.recentlyPlayedSeries(ctx, cred)
+		played, err := s.recentlyPlayedSeries(timing.WithLabel(ctx, "emby.recent"), cred)
 		if err != nil {
 			s.loggerFor(ctx).Warn("recently played lookup failed", "error", err)
 			return
@@ -153,7 +165,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 		seriesPlayed = played
 		mu.Unlock()
 	}()
-	run(&out.LatestMovies, func() (*emby.ItemsResult, error) {
+	run("latest", &out.LatestMovies, func(ctx context.Context) (*emby.ItemsResult, error) {
 		newReleaseDays := s.weightedConfig().NewReleaseDays
 		if newReleaseDays < 1 {
 			newReleaseDays = recommend.DefaultWeightedConfig().NewReleaseDays
@@ -197,6 +209,30 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 			mu.Unlock()
 		}()
 	}
+	// The ranking evidence and the row-order preferences depend on the viewer and on
+	// nothing that is being fetched around them, so they are read *beside* the Emby
+	// fan-out rather than after it. They used to be the first two things the response did
+	// once every row had arrived — eight Postgres reads with nothing else in flight,
+	// entirely on the critical path, for answers that were available before the request
+	// asked Emby anything.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ranking = s.rankingInputs(ctx, sess.EmbyUserID)
+	}()
+	if s.store != nil {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			stats, err := s.store.UserRowStats(ctx, sess.EmbyUserID, time.Now().Add(-45*24*time.Hour))
+			if err != nil {
+				s.loggerFor(ctx).Warn("home row preferences unavailable",
+					"user", sess.EmbyUserID, "error", err)
+				return
+			}
+			rowStats, rowStatsOK = stats, true
+		}()
+	}
 	if s.forYou != nil {
 		// The prepared pool is one indexed PostgreSQL read. It runs beside the Emby
 		// calls and deliberately has no live-engine fallback, so Home can never inherit
@@ -226,6 +262,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 
 	wg.Wait()
+	upstream()
 
 	if failures == 4 {
 		writeError(w, http.StatusBadGateway, "could not reach the emby server")
@@ -249,6 +286,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	if recommendations == nil {
 		s.refreshRecommendationsInBackground(sess)
 	}
+	assemble := timing.Start(ctx, timing.StageRows)
 	rows := baseRows(out)
 	nearContinue := make([]recommend.Row, 0, len(forYouRows)+2)
 	if len(forYouRows) > 0 {
@@ -271,34 +309,38 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request, sess store.S
 	}
 	out.Rows = append(rows, recommendations...)
 	out.Rows = s.filterRecommendationPermissions(ctx, sess, out.Rows)
-	if stats, err := s.store.UserRowStats(
-		ctx,
-		sess.EmbyUserID,
-		time.Now().Add(-45*24*time.Hour),
-	); err != nil {
-		s.loggerFor(ctx).Warn("home row preferences unavailable", "user", sess.EmbyUserID, "error", err)
-	} else {
-		out.Rows = personalizeHomeRows(out.Rows, stats)
+	if rowStatsOK {
+		out.Rows = personalizeHomeRows(out.Rows, rowStats)
 	}
-	out.Rows = s.personalizeTitles(ctx, sess, out.Rows)
+	assemble()
+	rank := timing.Start(ctx, timing.StageRank)
+	out.Rows = s.personalizeTitlesWith(out.Rows, ranking)
 	out.Rows = personalizeRowsByTitleScores(selectPersonalizedRows(out.Rows))
 	out.Rows = deduplicateRows(out.Rows)
+	rank()
 	// Ratings ride on the cards themselves. Only what is already stored is attached, so
 	// the launcher pays one indexed read rather than a request per poster, and a card
 	// shows its scores as it is drawn instead of when focus reaches it.
+	decorate := timing.Start(ctx, timing.StageRatings)
 	s.decorateHomeRatings(ctx, &out)
+	decorate()
 	// The hero is composed last, from the finished rows, because that is the only point
 	// at which the ratings it ranks by are already attached. It is prepended rather than
 	// inserted: the television consumes this row instead of drawing it, so its position
 	// among the shelves means nothing, and being first is what lets an older reader that
 	// does draw it put it somewhere sensible.
 	if hero {
-		if row := s.heroRow(ctx, out.Rows, sess.EmbyUserID, time.Now()); row != nil {
+		compose := timing.Start(ctx, timing.StageHero)
+		row := s.heroRow(ctx, out.Rows, sess.EmbyUserID, time.Now())
+		compose()
+		if row != nil {
 			out.Rows = append([]recommend.Row{*row}, out.Rows...)
 		}
 	}
 
+	encode := timing.Start(ctx, timing.StageEncode)
 	body, err := json.Marshal(out)
+	encode()
 	if err != nil {
 		s.loggerFor(ctx).Error("home encode failed", "error", err)
 		writeError(w, http.StatusInternalServerError, "could not build the home payload")