0.2.79 - Slow api fixes

server/internal/store/gateway_settings.go

@@ -46,6 +46,12 @@ type GatewaySettings struct {
 	RadarrAlertMinutes int `json:"radarrAlertMinutes"`
 	// EmbyHealthSeconds is how often the reachability probe asks Emby whether it is there.
 	EmbyHealthSeconds int `json:"embyHealthSeconds"`
+	// SlowRequestMillis is how long a request must take before its log line carries a
+	// stage breakdown — Emby time, database time, cache outcome, gateway work. It is an
+	// override worth having because the useful threshold is a property of the household
+	// rather than of the build: an operator chasing one slow screen wants it at 100ms for
+	// an evening and back at 500 afterwards, and neither is a redeployment.
+	SlowRequestMillis int `json:"slowRequestMillis"`
 	// LibrarySyncMinutes is how often the catalogue sweep asks Emby what changed.
 	//
 	// It is an override worth having because the answer now depends on the household's
@@ -86,6 +92,10 @@ func normalizeGatewaySettings(settings GatewaySettings) GatewaySettings {
 	settings.SonarrAlertMinutes = clampOverride(settings.SonarrAlertMinutes, 1, 24*60, true)
 	settings.RadarrAlertMinutes = clampOverride(settings.RadarrAlertMinutes, 1, 7*24*60, true)
 	settings.EmbyHealthSeconds = clampOverride(settings.EmbyHealthSeconds, 10, 3600, true)
+	// The floor is 1ms rather than 0 because "breakdown on everything" is a real thing to
+	// want for a few minutes, and off is a real thing to want too — this is the one
+	// setting here whose switched-off state costs nothing but a column of text.
+	settings.SlowRequestMillis = clampOverride(settings.SlowRequestMillis, 1, 60000, true)
 	// A day is the ceiling rather than a week: however well the webhooks are working, the
 	// sweep is the only thing that ever notices a file somebody moved by hand.
 	settings.LibrarySyncMinutes = clampOverride(settings.LibrarySyncMinutes, 5, 24*60, true)

server/internal/store/store.go

@@ -294,8 +294,39 @@ type Store struct {
 	pool *pgxpool.Pool
 }
 
+// defaultMaxConns is the floor on the connection pool.
+//
+// pgxpool's own default is max(4, GOMAXPROCS), which on the NAS this runs on is four —
+// and four is fewer than one household's televisions. Every one of them polls
+// /v1/status every ten seconds, and that poll alone reads the feature policy, the
+// notification preferences, the preference revision, the theme and the request
+// allowlist. Queries that each take a millisecond then queue behind each other for
+// hundreds, and the waiting is invisible: the query time is honest and the request is
+// slow anyway. A deployment that has stated pool_max_conns for itself keeps its own
+// answer — this replaces a default nobody chose, not a decision somebody made.
+const defaultMaxConns = 16
+
+// minConns keeps a few connections open. Establishing one costs a TCP connection, TLS
+// where it is configured and Postgres' own backend fork, which is several milliseconds
+// paid by whichever request happened to arrive after an idle period — most often the
+// first thing somebody does after switching a television on.
+const minConns = 4
+
 func Open(ctx context.Context, databaseURL string) (*Store, error) {
-	pool, err := pgxpool.New(ctx, databaseURL)
+	config, err := pgxpool.ParseConfig(databaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("store: parse database url: %w", err)
+	}
+	if !strings.Contains(databaseURL, "pool_max_conns") && config.MaxConns < defaultMaxConns {
+		config.MaxConns = defaultMaxConns
+	}
+	if !strings.Contains(databaseURL, "pool_min_conns") && config.MinConns < minConns {
+		config.MinConns = minConns
+	}
+	// Every query in the gateway passes through here, which is what makes a slow request
+	// able to say how much of itself was Postgres.
+	config.ConnConfig.Tracer = queryTracer{}
+	pool, err := pgxpool.NewWithConfig(ctx, config)
 	if err != nil {
 		return nil, fmt.Errorf("store: connect: %w", err)
 	}

server/internal/store/tracing.go

@@ -0,0 +1,39 @@
+package store
+
+import (
+	"context"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/ponzischeme89/memby/server/internal/timing"
+)
+
+// queryTracer attributes Postgres time to the request that asked for it.
+//
+// pgx's tracer interface is the only place every query in the gateway passes through —
+// there are several hundred call sites across internal/store and instrumenting them by
+// hand would guarantee the one that matters is the one nobody wrapped. The SQL itself
+// is deliberately not recorded: a breakdown is read beside a request line in the admin
+// console, and a query text there would put table and column names in front of anybody
+// who can open the log.
+type queryTracer struct{}
+
+type queryStartKey struct{}
+
+func (queryTracer) TraceQueryStart(
+	ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData,
+) context.Context {
+	if timing.From(ctx) == nil {
+		return ctx
+	}
+	return context.WithValue(ctx, queryStartKey{}, time.Now())
+}
+
+func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryEndData) {
+	began, ok := ctx.Value(queryStartKey{}).(time.Time)
+	if !ok {
+		return
+	}
+	timing.Record(ctx, timing.StageDB, time.Since(began))
+}