188 lines
7.3 KiB
Go
188 lines
7.3 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/ponzischeme89/memby/server/internal/library"
|
|
)
|
|
|
|
// News about the service itself, as opposed to news about what is in it.
|
|
//
|
|
// Both producers here answer the same question a viewer would otherwise have to guess
|
|
// at: why the library changed under them, and why playback stopped working. They are
|
|
// the alerts most worth showing *during* a film, which is why they are worded to be read
|
|
// in one glance and never ask for an action.
|
|
const (
|
|
// A refresh is only interesting while it is recent — long enough that a set switched
|
|
// on shortly afterwards still learns why there is something new on the home screen,
|
|
// short enough that it is never yesterday's news.
|
|
librarySyncAlertWindow = 30 * time.Minute
|
|
|
|
// Reachability is a live fact, so its window is short: a TV waking up half an hour
|
|
// after Emby went down should find out by asking, not by being told about the past.
|
|
reachabilityAlertWindow = 10 * time.Minute
|
|
|
|
// embyFailureThreshold is how many probes in a row must fail before it is announced.
|
|
// One timeout is a hiccup — a restart, a slow scan — and telling a room full of
|
|
// people about it is worse than saying nothing.
|
|
embyFailureThreshold = 3
|
|
|
|
// A deployment is announced before it happens, and the window is the whole delivery
|
|
// mechanism: it must stay open across the image build, which is the several minutes
|
|
// during which the gateway is still answering and every open TV will poll at least
|
|
// once. It cannot be made to cover the outage itself — the alert list lives in a
|
|
// Redis with no persistence and no volume, so the swap takes it with everything else.
|
|
// That is why this is published early rather than at the swap.
|
|
deploymentAlertWindow = 20 * time.Minute
|
|
)
|
|
|
|
// Wording is fixed here rather than taken from the caller: the deployment script is not a
|
|
// place for on-screen copy, and a notice that reads differently each release is one
|
|
// viewers have to read twice.
|
|
const (
|
|
deploymentAlertTitle = "Memby server in deployment mode"
|
|
deploymentAlertMessage = "New update coming will be available soon."
|
|
)
|
|
|
|
// AnnounceDeployment tells every open TV that the gateway is about to be replaced.
|
|
//
|
|
// It is published by the deploying operator *before* the old container stops — in fact
|
|
// before the image is even built — because that is the only time there is anything left
|
|
// to say it with. Once the stack is down the status poll fails, and a television shows a
|
|
// connection error with no idea that somebody meant it to happen; publishing at the swap
|
|
// would be too late for the same reason it is needed.
|
|
//
|
|
// Nothing here waits for the deployment to finish. The gateway that publishes this is the
|
|
// one being retired; the one that comes back has no memory of having said it.
|
|
func (s *Server) AnnounceDeployment(ctx context.Context) {
|
|
s.publishAlert(ctx, deploymentAlert(time.Now().UTC()), deploymentAlertWindow)
|
|
}
|
|
|
|
func deploymentAlert(now time.Time) clientAlert {
|
|
return clientAlert{
|
|
// Keyed on the second, so a redeployment ten minutes later is a second notice
|
|
// rather than one the fleet has already dismissed as seen.
|
|
ID: fmt.Sprintf("deploy:%d", now.Unix()),
|
|
Kind: alertKindDeploying,
|
|
Label: "SERVER UPDATING",
|
|
Title: deploymentAlertTitle,
|
|
Message: deploymentAlertMessage,
|
|
AiredAt: now.Format(time.RFC3339),
|
|
}
|
|
}
|
|
|
|
// AnnounceLibrarySync tells every TV that the catalogue moved.
|
|
//
|
|
// Only a run that actually changed something is announced: the import is scheduled, so
|
|
// most passes find nothing, and an hourly "nothing happened" banner would train viewers
|
|
// to ignore the one that matters. Removals alone are deliberately silent — a title
|
|
// disappearing is not something to celebrate mid-film.
|
|
func (s *Server) AnnounceLibrarySync(ctx context.Context, result library.Result) {
|
|
if result.Changed <= 0 {
|
|
return
|
|
}
|
|
now := time.Now().UTC()
|
|
s.publishAlert(ctx, clientAlert{
|
|
// Keyed on the minute the sync finished: two runs are two pieces of news, but a
|
|
// retried publish of the same run is not.
|
|
ID: fmt.Sprintf("library:%d", now.Truncate(time.Minute).Unix()),
|
|
Kind: alertKindLibrarySync,
|
|
Label: "LIBRARY UPDATED",
|
|
Title: librarySyncTitle(result.Changed),
|
|
Message: "Memby has finished refreshing — it is on the home screen now.",
|
|
AiredAt: now.Format(time.RFC3339),
|
|
}, librarySyncAlertWindow)
|
|
}
|
|
|
|
func librarySyncTitle(changed int) string {
|
|
if changed == 1 {
|
|
return "1 title added or updated"
|
|
}
|
|
return fmt.Sprintf("%d titles added or updated", changed)
|
|
}
|
|
|
|
// WatchEmbyReachability announces Emby going away and coming back.
|
|
//
|
|
// This is the one alert that matters more during playback than on the home screen: video
|
|
// direct-plays from Emby, so when Emby stops answering a film stops with no explanation
|
|
// the viewer can act on. The gateway keeps serving /v1/status either way, which is what
|
|
// makes it able to say so.
|
|
//
|
|
// Only *transitions* are announced. A server that is down stays down, and repeating it
|
|
// every minute would bury everything else.
|
|
func (s *Server) WatchEmbyReachability(ctx context.Context, interval time.Duration) {
|
|
if interval <= 0 {
|
|
return
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
// Start out assuming reachable: a gateway booting while Emby is down should not
|
|
// open with a banner about a state nobody has seen change.
|
|
reachable := true
|
|
failures := 0
|
|
// The same probe feeds the live state on /v1/status, so one request to Emby answers
|
|
// both "did this just change" and "is it working right now". Declared before the
|
|
// first tick so a client asking during the opening minute learns the retry interval.
|
|
s.embyHealth.begin(interval, time.Now().UTC())
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if s.quietTimeActive() {
|
|
continue
|
|
}
|
|
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
err := s.emby.Ping(probeCtx)
|
|
cancel()
|
|
s.embyHealth.record(err == nil, time.Now().UTC())
|
|
|
|
if err != nil {
|
|
failures++
|
|
if reachable && failures >= embyFailureThreshold {
|
|
reachable = false
|
|
s.log.Warn("emby unreachable, announcing",
|
|
"component", "emby-health", "failures", failures, "error", err)
|
|
s.publishAlert(ctx, s.reachabilityAlert(false), reachabilityAlertWindow)
|
|
}
|
|
continue
|
|
}
|
|
if !reachable {
|
|
s.log.Info("emby reachable again, announcing", "component", "emby-health")
|
|
s.publishAlert(ctx, s.reachabilityAlert(true), reachabilityAlertWindow)
|
|
}
|
|
reachable = true
|
|
failures = 0
|
|
}
|
|
}
|
|
}
|
|
|
|
// reachabilityAlert is timestamped per transition, so the "back online" banner never
|
|
// collides with the "not responding" one it replaces.
|
|
func (s *Server) reachabilityAlert(up bool) clientAlert {
|
|
now := time.Now().UTC()
|
|
if up {
|
|
return clientAlert{
|
|
ID: fmt.Sprintf("emby:up:%d", now.Unix()),
|
|
Kind: alertKindServerUp,
|
|
Label: "SERVER BACK ONLINE",
|
|
Title: "Emby is responding again",
|
|
Message: "Playback and browsing are working normally.",
|
|
AiredAt: now.Format(time.RFC3339),
|
|
}
|
|
}
|
|
return clientAlert{
|
|
ID: fmt.Sprintf("emby:down:%d", now.Unix()),
|
|
Kind: alertKindServerDown,
|
|
Label: "SERVER NOT RESPONDING",
|
|
Title: "Emby has stopped communicating",
|
|
// Says what the viewer will see rather than what failed: a diagnosis they cannot
|
|
// act on from the sofa is worse than none.
|
|
Message: "Playback may stop until it is back. Memby will say when it returns.",
|
|
AiredAt: now.Format(time.RFC3339),
|
|
}
|
|
}
|