0.3.27 - Omni search fixes, Services in genre browser..

This commit is contained in:
ponzischeme89
2026-08-26 08:42:56 +12:00
parent 919a12f8d5
commit 9d11bb7282
26 changed files with 2202 additions and 590 deletions
+6
View File
@@ -286,11 +286,17 @@ func (s *Server) Routes() http.Handler {
v1.Handle("GET /v1/screensaver", s.authed(s.handleScreensaver))
v1.Handle("GET /v1/search", s.authed(s.handleSearch))
v1.Handle("GET /v1/search/stream", s.authed(s.handleSearchStream))
// External discovery is its own route so a keystroke never waits on Sonarr or
// Radarr. See search_discover.go.
v1.Handle("GET /v1/search/discover", s.authed(s.handleSearchDiscover))
// A genre is browsed, not searched: the chip is a filter and this is the route that
// treats it as one. Paged, because a household's Drama shelf is not a screenful.
v1.Handle("GET /v1/genres/{genre}/items", s.authed(s.handleGenreItems))
v1.Handle("GET /v1/library/items", s.authed(s.handleLibraryItems))
v1.Handle("GET /v1/genres/affinity", s.authed(s.handleGenreAffinity))
// A streaming service shortcut on the Genres page is the same shelf, filtered on
// Studios rather than Genres. See handleServiceItems in genres.go.
v1.Handle("GET /v1/services/{service}/items", s.authed(s.handleServiceItems))
v1.Handle("POST /v1/search/history", s.authed(s.handleSearchHistory))
v1.Handle("GET /v1/requests/lookup", s.authed(s.handleRequestLookup))
v1.Handle("GET /v1/requests", s.authed(s.handleMyRequests))
+41 -14
View File
@@ -50,18 +50,45 @@ func (s *Server) handleGenreItems(w http.ResponseWriter, r *http.Request, sess s
writeError(w, http.StatusBadRequest, "a genre is required")
return
}
s.handleBrowseItems(w, r, sess, genre)
s.handleBrowseItems(w, r, sess, "Genres", "genre", genre)
}
func (s *Server) handleLibraryItems(w http.ResponseWriter, r *http.Request, sess store.Session) {
s.handleBrowseItems(w, r, sess, "")
s.handleBrowseItems(w, r, sess, "Genres", "genre", "")
}
// handleServiceItems answers the Genres page's Services shortcuts — Apple TV+, Netflix,
// Disney+ and the rest of the studio/network catalogue in ui/genre/GenreCategories.kt.
//
// It is the same shelf as a genre, filtered on a different Emby field: services are
// filtered on Studios, which is where a metadata agent (TMDb chief among them) records a
// title's production or distribution company — "Netflix", "Apple TV+", "Disney+" are
// ordinary studio names to Emby, so no new metadata or import step is needed. Like
// genres, the catalogue mapping a service to its Emby spellings is a client-side product
// list rather than server data, for the same reason genreCategories() is: the order is a
// design decision that must never jump around while home rows are arriving, and a service
// with no matching titles simply returns an empty page rather than needing to be hidden
// from a central registry.
func (s *Server) handleServiceItems(w http.ResponseWriter, r *http.Request, sess store.Session) {
service := strings.TrimSpace(r.PathValue("service"))
if service == "" {
writeError(w, http.StatusBadRequest, "a service is required")
return
}
s.handleBrowseItems(w, r, sess, "Studios", "service", service)
}
func (s *Server) handleBrowseItems(
w http.ResponseWriter,
r *http.Request,
sess store.Session,
genre string,
// filterField is the Emby query parameter the filter value is written into — "Genres"
// for a genre shelf, "Studios" for a service shelf. filterLabel is only for the cache
// key and the log line, so the two shelves' entries can never collide or be confused
// for one another in a shared keyspace.
filterField string,
filterLabel string,
filterValue string,
) {
ctx := r.Context()
limit := queryInt(r, "limit", genrePageSize, genrePageMax)
@@ -73,8 +100,8 @@ func (s *Server) handleBrowseItems(
}
filterKey := "all"
if genre != "" {
filterKey = "genre:" + genre
if filterValue != "" {
filterKey = filterLabel + ":" + filterValue
}
key := cache.UserKey(viewerKeyOf(ctx, sess), "browse:"+itemType+":"+filterKey+":"+itoa(offset)+":"+itoa(limit))
if raw, err := s.cache.Get(ctx, key); err == nil {
@@ -95,8 +122,8 @@ func (s *Server) handleBrowseItems(
"SortBy": {"PremiereDate,SortName"},
"SortOrder": {"Descending"},
}, fieldsRow)
if genre != "" {
params.Set("Genres", genre)
if filterValue != "" {
params.Set(filterField, filterValue)
}
// rowParams turns this off for the home rows, which never page. Here it is the number
// the scroll stops on.
@@ -115,20 +142,20 @@ func (s *Server) handleBrowseItems(
total := genreTotal(result.TotalRecordCount, offset, len(items), limit)
// The first page is somebody opening a genre, which is a navigation event worth the
// log; the pages after it are one viewer scrolling and would bury it.
if offset == 0 && genre != "" {
s.loggerFor(ctx).Info("genre browsed", "genre", genre, "results", len(items), "total", total)
// The first page is somebody opening a genre or a service, which is a navigation event
// worth the log; the pages after it are one viewer scrolling and would bury it.
if offset == 0 && filterValue != "" {
s.loggerFor(ctx).Info(filterLabel+" browsed", filterLabel, filterValue, "results", len(items), "total", total)
} else if offset == 0 {
s.loggerFor(ctx).Info("library browsed", "type", itemType, "results", len(items), "total", total)
} else if genre != "" {
s.loggerFor(ctx).Debug("genre page", "genre", genre, "offset", offset, "results", len(items))
} else if filterValue != "" {
s.loggerFor(ctx).Debug(filterLabel+" page", filterLabel, filterValue, "offset", offset, "results", len(items))
} else {
s.loggerFor(ctx).Debug("library page", "type", itemType, "offset", offset, "results", len(items))
}
body, err := json.Marshal(genrePage{
Genre: genre,
Genre: filterValue,
Items: items,
Offset: offset,
Limit: limit,
+7 -2
View File
@@ -714,7 +714,9 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, sess store
return
}
limit := queryInt(r, "limit", 40, 100)
key := cache.UserKey(viewerKeyOf(ctx, sess), "search:"+itoa(limit)+":"+term+":d"+sess.DeviceID)
// v2: the row now carries ProviderIds, so an entry written by the previous build
// cannot hide a field the deduplication against external discovery depends on.
key := cache.UserKey(viewerKeyOf(ctx, sess), "search:v2:"+itoa(limit)+":"+term+":d"+sess.DeviceID)
// Every search the tab performs is recorded here, before the cache is consulted, so a
// query answered from Redis counts the same as one that reached Emby. The client also
@@ -736,7 +738,10 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, sess store
"IncludeItemTypes": {"Movie,Series,Episode"},
"Recursive": {"true"},
"Limit": {itoa(limit)},
}, fieldsRow))
// ProviderIds is what lets the television collapse a library result and an *arr
// discovery result for one title into a single card, on a stable id rather than
// on a title string.
}, fieldsRow+",ProviderIds"))
if err != nil {
s.writeUpstreamError(ctx, w, err, "search failed")
return
+281
View File
@@ -0,0 +1,281 @@
package api
import (
"context"
"encoding/json"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/ponzischeme89/memby/server/internal/cache"
"github.com/ponzischeme89/memby/server/internal/store"
)
// External discovery is a separate route from search on purpose.
//
// /v1/search answers out of Emby and Redis and is what a keystroke asks. This asks Sonarr
// and Radarr, which are somebody else's services on somebody else's network — a lookup
// measured in seconds, rate-limited upstream, and worth nothing at all for the letters on
// the way to a word. Splitting them is the whole of the promise that discovery can never
// delay the library: the television makes the two requests independently and renders
// whichever answers, so an *arr that is down or slow costs the second section and never
// the first.
const (
// Three characters, not the search floor of two. Two letters match a large part of any
// catalogue and the answer is noise; the *arrs are also the one backend here that Memby
// does not own, so the floor is set by what is worth asking them rather than by what
// they would tolerate.
minDiscoveryQueryRunes = 3
// How long one query's answer is reused for the whole household. A film's existence in
// TMDb does not change minute to minute, and this is what makes leaving Search and
// coming back, deleting and retyping a title, or a second television asking the same
// thing free. It is deliberately well past the client's own debounce: the debounce
// protects against typing, this protects against everything else.
discoveryCacheTTL = 10 * time.Minute
// A lookup that found nothing is kept for less. Nothing is the answer for a typo, which
// is the query most likely to be corrected and asked again a second later — but it is
// also the answer while an *arr is confused, and that recovers.
discoveryEmptyTTL = 2 * time.Minute
// Bounds the whole route. Longer than a library search, because that is the trade this
// route exists to isolate, and short enough that a television is never left with a
// section claiming it is still finding things for a minute.
discoveryTimeout = 8 * time.Second
)
// discoveryResponse is deliberately not just an item list: a television has to be able to
// tell "there is nothing else to be had" from "we could not ask", because the first is an
// answer and the second is worth a quiet line under the results.
type discoveryResponse struct {
Query string `json:"query"`
Items []json.RawMessage `json:"items"`
// Partial means at least one source failed. The items that did arrive are still good.
Partial bool `json:"partial,omitempty"`
// Skipped names why nothing was asked at all, for the log and for the client's own
// diagnostics. An empty list with no reason means the *arrs simply knew nothing.
Skipped string `json:"skipped,omitempty"`
}
// normaliseDiscoveryQuery is what makes the cache and the client's own deduplication agree
// on when two searches are the same search. "Disclosure", "disclosure" and "disclosure "
// are one question; "disclosure day" is another.
func normaliseDiscoveryQuery(term string) string {
return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(term))), " ")
}
// discoveryQueryEligible is the one rule, counted in runes rather than bytes for the reason
// searchQueryRecordable is: a title in Japanese is rejected at a third of an English one's
// length otherwise.
func discoveryQueryEligible(term string) bool {
return len([]rune(normaliseDiscoveryQuery(term))) >= minDiscoveryQueryRunes
}
func (s *Server) handleSearchDiscover(w http.ResponseWriter, r *http.Request, sess store.Session) {
started := time.Now()
term := normaliseDiscoveryQuery(r.URL.Query().Get("q"))
limit := queryInt(r, "limit", 20, 60)
log := s.loggerFor(r.Context())
if !discoveryQueryEligible(term) {
s.writeDiscovery(w, discoveryResponse{Query: term, Skipped: "query_too_short"})
return
}
sonarrOn := s.sonarrEnabled(r.Context())
radarrOn := s.radarrEnabled(r.Context())
if !sonarrOn && !radarrOn {
s.writeDiscovery(w, discoveryResponse{Query: term, Skipped: "no_sources"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), discoveryTimeout)
defer cancel()
canRequest := s.requestAllowed(r, sess)
type leg struct {
source string
items []json.RawMessage
cached bool
err error
took time.Duration
}
results := make(chan leg, 2)
var wg sync.WaitGroup
// Sonarr and Radarr are independent, and one being unreachable must never cost the
// other's answer — the same reason the two halves of a request lookup run apart.
start := func(source string, load func(context.Context) ([]json.RawMessage, error)) {
wg.Add(1)
go func() {
defer wg.Done()
at := time.Now()
items, cached, err := s.discoverCached(ctx, source, term, limit, load)
results <- leg{source: source, items: items, cached: cached, err: err, took: time.Since(at)}
}()
}
if sonarrOn {
start("sonarr", func(ctx context.Context) ([]json.RawMessage, error) {
return s.discoverSonarr(ctx, term, limit)
})
}
if radarrOn {
start("radarr", func(ctx context.Context) ([]json.RawMessage, error) {
return s.discoverRadarr(ctx, term, limit)
})
}
go func() { wg.Wait(); close(results) }()
merged := make([]json.RawMessage, 0, limit)
response := discoveryResponse{Query: term}
fields := []any{"query", term, "limit", limit}
found := 0
for result := range results {
if result.err != nil {
response.Partial = true
log.Debug("discovery source failed", "query", term, "source", result.source, "error", result.err)
}
found += len(result.items)
for _, item := range result.items {
merged = mergeSearchRaw(merged, item, limit)
}
fields = append(fields,
result.source, len(result.items),
result.source+"_ms", result.took.Milliseconds(),
result.source+"_cache", cacheWord(result.cached))
}
response.Items = applyRequestable(merged, canRequest)
// DEBUG for the reason the search line is: this is the record of what somebody was
// looking for, and it is a per-viewer stream of titles rather than something the
// ordinary log should carry. An operator tuning the debounce turns it on.
log.Debug("search discovery", append(fields,
"results", len(response.Items),
"duplicates", found-len(response.Items),
"partial", response.Partial,
"ms", time.Since(started).Milliseconds())...)
s.writeDiscovery(w, response)
}
func (s *Server) writeDiscovery(w http.ResponseWriter, response discoveryResponse) {
if response.Items == nil {
response.Items = []json.RawMessage{}
}
writeJSON(w, http.StatusOK, response)
}
func cacheWord(hit bool) string {
if hit {
return "hit"
}
return "miss"
}
// discoverCached is where the cooldown lives.
//
// The key is a MetadataKey rather than a UserKey, and that is load-bearing: what Radarr
// knows about "disclosure day" is a fact about the world and the household's catalogue,
// with nothing in it derived from whoever typed it — so one lookup answers for every
// television in the house. Whether *this* viewer may press Request is applied afterwards,
// on the way out, which is the same separation cache.MetadataKey demands everywhere else.
//
// cachedRead brings the other half for free: two televisions missing the same key in the
// same second make one lookup between them rather than two.
func (s *Server) discoverCached(
ctx context.Context,
source, term string,
limit int,
load func(context.Context) ([]json.RawMessage, error),
) ([]json.RawMessage, bool, error) {
key := cache.MetadataKey("discover:v1:" + source + ":" + itoa(limit) + ":" + term)
body, hit, err := s.cachedRead(ctx, key, discoveryCacheTTL, func(ctx context.Context) (json.RawMessage, error) {
items, err := load(ctx)
if err != nil {
return nil, err
}
return json.Marshal(nonNil(items))
})
if err != nil {
return nil, hit, err
}
var items []json.RawMessage
if err := json.Unmarshal(body, &items); err != nil {
return nil, hit, err
}
return items, hit, nil
}
// discoverSonarr and discoverRadarr differ from the streaming search's own legs in one
// way: a title the household already has is dropped rather than listed. The library
// section above it is already showing that title, playable, and one result appearing twice
// under two headings is the single worst thing progressive search can do.
func (s *Server) discoverSonarr(ctx context.Context, term string, limit int) ([]json.RawMessage, error) {
series, err := s.sonarr.Lookup(ctx, term)
if err != nil {
return nil, err
}
ids := make([]int, 0, len(series))
for _, v := range series {
if v.TVDBID > 0 {
ids = append(ids, v.TVDBID)
}
}
inLibrary, _ := s.store.LibraryContainsProviderIDs(ctx, "Tvdb", ids)
items := make([]json.RawMessage, 0, limit)
for _, v := range series {
if v.TVDBID == 0 || inLibrary[v.TVDBID] || len(items) >= limit {
continue
}
state := lookupStatusFor(RequestSubject{
Tracked: v.ID > 0,
Released: seriesReleased(v.Status, v.NextAiring, time.Now()),
}, false)
items = append(items, searchExternalItem(v.Title, v.Year, v.Overview, "Series", "sonarr",
state, strconv.Itoa(v.TVDBID), sonarrCoverURL(v.Images, "poster"), false))
}
return items, nil
}
func (s *Server) discoverRadarr(ctx context.Context, term string, limit int) ([]json.RawMessage, error) {
movies, err := s.radarr.Lookup(ctx, term)
if err != nil {
return nil, err
}
ids := make([]int, 0, len(movies))
for _, v := range movies {
if v.TMDBID > 0 {
ids = append(ids, v.TMDBID)
}
}
inLibrary, _ := s.store.LibraryContainsProviderIDs(ctx, "Tmdb", ids)
items := make([]json.RawMessage, 0, limit)
for _, v := range movies {
if v.TMDBID == 0 || inLibrary[v.TMDBID] || len(items) >= limit {
continue
}
state := lookupStatusFor(RequestSubject{
Tracked: v.ID > 0,
HasFile: v.HasFile,
Released: movieReleased(v.Status),
}, false)
items = append(items, searchExternalItem(v.Title, v.Year, v.Overview, "Movie", "radarr",
state, strconv.Itoa(v.TMDBID), radarrCoverURL(v.Images, "poster"), false))
}
return items, nil
}
// applyRequestable is what keeps the cached half free of the viewer. Whether the button is
// offered is this person's permission and nothing else's, so it is stamped on the way out
// rather than baked into what the household shares.
func applyRequestable(items []json.RawMessage, canRequest bool) []json.RawMessage {
for i, raw := range items {
var item map[string]any
if json.Unmarshal(raw, &item) != nil {
continue
}
state, _ := item["MembySearchState"].(string)
item["MembyRequestable"] = canRequest && state == RequestStatusRequestable
if encoded, err := json.Marshal(item); err == nil {
items[i] = encoded
}
}
return items
}
@@ -0,0 +1,67 @@
package api
import (
"encoding/json"
"testing"
)
func TestDiscoveryQueryNormalisationCollapsesTheSameQuestion(t *testing.T) {
same := []string{"Disclosure Day", "disclosure day", " disclosure day ", "DISCLOSURE Day"}
want := normaliseDiscoveryQuery(same[0])
for _, query := range same[1:] {
if got := normaliseDiscoveryQuery(query); got != want {
t.Fatalf("%q normalised to %q, want %q", query, got, want)
}
}
if normaliseDiscoveryQuery("disclosure") == want {
t.Fatal("a materially different query must not share a cache key")
}
}
func TestDiscoveryFloorIsCountedInRunes(t *testing.T) {
for _, query := range []string{"", " ", "d", "di", " di "} {
if discoveryQueryEligible(query) {
t.Fatalf("%q should not reach Sonarr or Radarr", query)
}
}
for _, query := range []string{"dis", "disclosure day", "君の名は"} {
if !discoveryQueryEligible(query) {
t.Fatalf("%q should be eligible for discovery", query)
}
}
}
func TestRequestableIsStampedOnTheWayOut(t *testing.T) {
items := []json.RawMessage{
searchExternalItem("Disclosure Day", 2027, "", "Movie", "radarr", RequestStatusRequestable, "12", "", false),
searchExternalItem("Something Tracked", 2020, "", "Series", "sonarr", RequestStatusProcessing, "34", "", false),
}
for _, canRequest := range []bool{true, false} {
decorated := applyRequestable(append([]json.RawMessage{}, items...), canRequest)
var requestable, processing map[string]any
if err := json.Unmarshal(decorated[0], &requestable); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(decorated[1], &processing); err != nil {
t.Fatal(err)
}
if requestable["MembyRequestable"] != canRequest {
t.Fatalf("canRequest=%v: requestable candidate got %v", canRequest, requestable["MembyRequestable"])
}
// Only a title nothing has and nobody has asked for offers the button, whatever
// this viewer is permitted to do.
if processing["MembyRequestable"] != false {
t.Fatalf("canRequest=%v: a tracked title must never offer Request", canRequest)
}
}
}
func TestDiscoveryMergeCollapsesOnProviderID(t *testing.T) {
merged := []json.RawMessage{}
one := searchExternalItem("Disclosure Day", 2027, "", "Movie", "radarr", RequestStatusRequestable, "12", "", false)
merged = mergeSearchRaw(merged, one, 20)
merged = mergeSearchRaw(merged, one, 20)
if len(merged) != 1 {
t.Fatalf("one title reported twice produced %d cards", len(merged))
}
}