0.2.69 - Homepage loading improvements pass

server/internal/buildinfo/VERSION

@@ -1 +1 @@
-0.1.48
+0.1.49

server/internal/credits/README.md

@@ -79,10 +79,13 @@ viewers is enough only when both rolled into the next episode, which is unambigu
 they left.
 
 **Visual** scanning finds a sustained structural transition — dark, flat, textured with thin
-text, and staying that way for a minute. Darkness is *multiplied* rather than added into the
-frame score, which is the one modelling decision worth defending: under a weighted sum a
-night exterior reaches the credit-like floor on darkness and flatness alone, which is exactly
-how a final scene comes to be reported as a credits roll. A unit test pins that case.
+text, and staying that way for about half a minute. Darkness is *multiplied* rather than added
+into the frame score, which is the one modelling decision worth defending: under a weighted
+sum a night exterior reaches the credit-like floor on darkness and flatness alone, which is
+exactly how a final scene comes to be reported as a credits roll. The candidate frame itself
+must also be genuinely dark, and at least 65% of the remaining frames must stay credit-like;
+those two guards stop a dim final scene or a temporary title card becoming an early marker.
+Unit tests pin both cases.
 
 The two fail in unrelated ways, so agreement between them is worth far more than either
 alone — hence a probabilistic union rather than an average. Disagreement beyond 20 seconds
@@ -163,3 +166,6 @@ in which demand-driven narrowing is doing nothing, and the design would need rev
   on every playback.** Readings wobble by seconds; a ±12s difference is not news.
 - **The detector never learns why an episode was chosen.** That boundary is what stops it
   being tuned to agree with the predictor rather than with the media.
+- **An `ffmpeg` decoder crash gets one conservative retry.** The retry is single-threaded and
+  discards corrupt packets; ordinary network, authentication and timeout failures are not
+  retried by the sampler.

server/internal/credits/detector.go

@@ -20,9 +20,12 @@ import (
 // proportion to a feature whose fallback is simply not showing a button.
 
 const (
-	// sustainSeconds is how long the credit-like state has to persist to count. Shorter than
-	// this and a dark establishing shot at the end of an act qualifies.
-	sustainSeconds = 45
+	// sustainSeconds is how long the credit-like state has to persist to count. Thirty
+	// seconds still rejects an ordinary fade or end-of-act beat, but admits the compact
+	// closing rolls common to half-hour and network television. It also fits inside the
+	// thirty-second forward half of the fine pass; the old forty-five-second requirement
+	// made refinement around a correctly centred transition impossible.
+	sustainSeconds = 30
 
 	// leadSeconds is how much ordinary programme has to precede the transition. Without it
 	// the changepoint can sit at the very first sampled frame, which is not evidence of a
@@ -34,10 +37,23 @@ const (
 	// "slightly more dark" in the middle of a night scene.
 	creditLikeFloor = 0.45
 
+	// onsetMeanCeiling requires the candidate cut itself to be genuinely dark. A dim final
+	// scene can share the roll's edge density and be followed by real credits soon enough to
+	// lift the sustain average; accepting it clips the scene. The detector deliberately does
+	// not claim bright or picture-backed credits without behavioural corroboration.
+	onsetMeanCeiling = 0.10
+
 	// minSeparation is how much better the tail has to score than the head. This is the
-	// primary guard against answering on noise, and it is set high because the cost of a
-	// wrong marker is somebody losing the end of an episode.
-	minSeparation = 0.18
+	// primary guard against answering on noise. A dark programme can make the absolute gap
+	// modest even at its real credits; the stay-credit-like guard below is what makes this
+	// lower bar safe.
+	minSeparation = 0.10
+
+	// minCreditLikeTailFraction says that credits continue to the end of the file. A dark
+	// scene may satisfy the short sustain window, but ordinary programme resumes afterwards;
+	// a real roll leaves most remaining samples credit-like even with black gaps, logos and
+	// production cards mixed through it.
+	minCreditLikeTailFraction = 0.65
 
 	// onsetFraction is how credit-like a frame has to be, relative to the established roll,
 	// to count as already part of it.
@@ -142,13 +158,17 @@ func (d *VisualDetector) Detect(ctx context.Context, media MediaInfo) (Detection
 func creditScore(frame frameStats) float64 {
 	darkness := frame.DarkFraction
 
-	// Text produces a narrow *band* of edge density: a flat black frame has almost none, and
-	// a detailed photograph has far more than titles do. Scoring the band rather than the
-	// magnitude is what stops a bright, busy scene outscoring the credits.
-	const idealEdges = 0.030
+	// Text produces a band of edge density: a flat black frame has almost none, and a
+	// detailed photograph has far more than titles do. The useful band is deliberately
+	// broad. A sparse title card and a dense cast roll are both credits; the old triangle
+	// reached zero at twice one narrow ideal and rejected the latter outright.
+	const (
+		idealEdges = 0.020
+		maxEdges   = 0.120
+	)
 	text := frame.EdgeDensity / idealEdges
 	if text > 1 {
-		text = 2 - text
+		text = 1 - (frame.EdgeDensity-idealEdges)/(maxEdges-idealEdges)
 	}
 	text = clamp01(text)
 
@@ -190,8 +210,13 @@ func findTransition(frames []frameStats, interval time.Duration) (int, float64,
 	// a few hundred frames, but the quadratic version is the kind of thing that stops being
 	// free the moment somebody widens the window.
 	prefix := make([]float64, len(scores)+1)
+	qualifying := make([]int, len(scores)+1)
 	for index, score := range scores {
 		prefix[index+1] = prefix[index] + score
+		qualifying[index+1] = qualifying[index]
+		if score >= creditLikeFloor {
+			qualifying[index+1]++
+		}
 	}
 	segmentMean := func(from, to int) float64 {
 		if to <= from {
@@ -202,6 +227,9 @@ func findTransition(frames []frameStats, interval time.Duration) (int, float64,
 
 	bestIndex, bestSeparation, found := 0, 0.0, false
 	for split := lead; split+sustain <= len(frames); split++ {
+		if frames[split].Mean > onsetMeanCeiling {
+			continue
+		}
 		head := segmentMean(0, split)
 		tail := segmentMean(split, len(frames))
 		// The sustained window immediately after the split has to qualify on its own, not
@@ -210,6 +238,11 @@ func findTransition(frames []frameStats, interval time.Duration) (int, float64,
 		if segmentMean(split, split+sustain) < creditLikeFloor {
 			continue
 		}
+		tailCount := len(frames) - split
+		creditLikeTail := qualifying[len(scores)] - qualifying[split]
+		if float64(creditLikeTail)/float64(tailCount) < minCreditLikeTailFraction {
+			continue
+		}
 		separation := tail - head
 		if separation < minSeparation || separation <= bestSeparation {
 			continue
@@ -222,8 +255,15 @@ func findTransition(frames []frameStats, interval time.Duration) (int, float64,
 	// The separation is reported for the split that earned it, never for the walked-back
 	// frame: confidence is a statement about how clearly the transition was found, and
 	// recomputing it over a fade would report the answer as weaker for having been improved.
-	return backOffToOnset(scores, bestIndex, segmentMean(bestIndex, min(bestIndex+sustain, len(scores))), interval),
-		bestSeparation, true
+	onset := backOffToOnset(
+		scores, bestIndex, segmentMean(bestIndex, min(bestIndex+sustain, len(scores))), interval,
+	)
+	// The score-only walk can step back onto a dim final picture whose texture resembles
+	// text. Preserve the same luminance guard that qualified the split itself.
+	for onset < bestIndex && frames[onset].Mean > onsetMeanCeiling {
+		onset++
+	}
+	return onset, bestSeparation, true
 }
 
 // backOffToOnset walks a split backwards through the credits' fade-in.

server/internal/credits/detector_test.go

@@ -49,6 +49,19 @@ func darkSceneFrame(position time.Duration) frameStats {
 	}
 }
 
+// A dim, textured final scene: close enough to the roll's structural score that what comes
+// after it could lift the sustain average, but visibly still programme.
+func dimTexturedSceneFrame(position time.Duration) frameStats {
+	return frameStats{
+		PositionMs:   position.Milliseconds(),
+		Mean:         0.13,
+		Variance:     0.006,
+		DarkFraction: 0.78,
+		EdgeDensity:  0.015,
+		Diff:         0.08,
+	}
+}
+
 func window(start time.Duration, kinds ...func(time.Duration) frameStats) []frameStats {
 	frames := make([]frameStats, 0, len(kinds))
 	for index, build := range kinds {
@@ -85,6 +98,32 @@ func TestFindsACleanTransition(t *testing.T) {
 	}
 }
 
+// Television rolls are commonly only half a minute long. This is also the amount of media
+// available after a correctly centred transition in the fine pass, so requiring more would
+// make the refinement structurally unable to confirm the coarse answer.
+func TestFindsACompactTelevisionCreditsRoll(t *testing.T) {
+	kinds := append(repeatFrames(20, programmeFrame), repeatFrames(8, creditsFrame)...)
+	frames := window(20*time.Minute, kinds...)
+
+	if index, _, found := findTransition(frames, coarseInterval); !found || index != 20 {
+		t.Fatalf("compact credits transition = %d, found %t; want frame 20", index, found)
+	}
+}
+
+func TestFinePassCanConfirmACentredTransition(t *testing.T) {
+	frames := make([]frameStats, 0, 80)
+	for index := 0; index < 40; index++ {
+		frames = append(frames, programmeFrame(time.Duration(index)*fineInterval))
+	}
+	for index := 40; index < 80; index++ {
+		frames = append(frames, creditsFrame(time.Duration(index)*fineInterval))
+	}
+
+	if index, _, found := findTransition(frames, fineInterval); !found || index != 40 {
+		t.Fatalf("fine transition = %d, found %t; want frame 40", index, found)
+	}
+}
+
 // The whole point of not answering on darkness alone.
 func TestDarkSceneIsNotCredits(t *testing.T) {
 	kinds := append(repeatFrames(30, programmeFrame), repeatFrames(30, darkSceneFrame)...)
@@ -122,6 +161,34 @@ func TestBriefDarkBeatIsIgnored(t *testing.T) {
 	}
 }
 
+// A locally convincing title-like sequence in the middle of the tail is not closing
+// credits when ordinary programme resumes after it. This is the real-media failure shape:
+// looking only at the immediate sustain window marked Westworld and Blue Bloods several
+// minutes before their genuine rolls.
+func TestCreditLikeSequenceThatDoesNotReachTheEndIsIgnored(t *testing.T) {
+	kinds := append(repeatFrames(20, programmeFrame), repeatFrames(8, creditsFrame)...)
+	kinds = append(kinds, repeatFrames(20, programmeFrame)...)
+	frames := window(35*time.Minute, kinds...)
+
+	if _, _, found := findTransition(frames, coarseInterval); found {
+		t.Fatal("a temporary credit-like sequence was reported as closing credits")
+	}
+}
+
+func TestDimFinalSceneBeforeCreditsDoesNotMoveTheMarkerEarly(t *testing.T) {
+	kinds := append(repeatFrames(20, programmeFrame), repeatFrames(8, dimTexturedSceneFrame)...)
+	kinds = append(kinds, repeatFrames(12, creditsFrame)...)
+	frames := window(35*time.Minute, kinds...)
+
+	index, _, found := findTransition(frames, coarseInterval)
+	if !found {
+		t.Fatal("the genuine credits after the dim scene were not found")
+	}
+	if index != 28 {
+		t.Fatalf("transition at frame %d, want the credits at frame 28", index)
+	}
+}
+
 func TestTooFewFramesAnswerNothing(t *testing.T) {
 	frames := window(41*time.Minute, repeatFrames(4, creditsFrame)...)
 	if _, _, found := findTransition(frames, coarseInterval); found {
@@ -151,6 +218,14 @@ func TestCreditScoreSeparatesTheThreeCases(t *testing.T) {
 	}
 }
 
+func TestDenseCreditsStillLookLikeCredits(t *testing.T) {
+	frame := creditsFrame(0)
+	frame.EdgeDensity = 0.075
+	if score := creditScore(frame); score < creditLikeFloor {
+		t.Fatalf("dense credits scored %.2f, below the floor %.2f", score, creditLikeFloor)
+	}
+}
+
 // Confidence from one detector agreeing with itself is not corroboration.
 func TestVisualConfidenceIsCapped(t *testing.T) {
 	if score := visualConfidence(10); score >= 1 {

server/internal/credits/sampler.go

@@ -122,21 +122,51 @@ func (s *Sampler) Sample(
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
+	stats, err := s.samplePass(ctx, url, from, to, interval, false)
+	if err == nil || !decoderCrashed(err) || ctx.Err() != nil {
+		return stats, err
+	}
+	// A decoder crash is local to the ffmpeg process, not evidence that the media is
+	// unreadable. Retry once with conservative decoder settings: single-threaded decoding
+	// avoids the most common native-code race, while corrupt packets are discarded rather
+	// than handed back through the failing path. Ordinary HTTP and authentication failures
+	// are never retried here.
+	stats, retryErr := s.samplePass(ctx, url, from, to, interval, true)
+	if retryErr != nil {
+		return stats, fmt.Errorf("credits: conservative ffmpeg retry: %w", retryErr)
+	}
+	return stats, nil
+}
+
+func (s *Sampler) samplePass(
+	ctx context.Context, url string, from, to time.Duration, interval time.Duration,
+	conservative bool,
+) ([]frameStats, error) {
+
 	// -ss ahead of -i is the whole optimisation: it seeks in the container before opening a
 	// decoder, so the input starts near the credits. Behind -i it would decode from zero and
 	// discard, which is the full read this package exists to avoid.
 	args := []string{
 		"-hide_banner", "-loglevel", "error", "-nostdin",
 		"-ss", formatSeconds(from),
+	}
+	if conservative {
+		args = append(args,
+			"-threads", "1",
+			"-fflags", "+discardcorrupt",
+			"-err_detect", "ignore_err",
+		)
+	}
+	args = append(args,
 		"-i", url,
-		"-t", formatSeconds(to - from),
+		"-t", formatSeconds(to-from),
 		"-an", "-sn", "-dn",
 		"-vf", fmt.Sprintf("fps=%s,scale=%d:%d,format=gray",
 			formatRate(interval), sampleWidth, sampleHeight),
 		"-frames:v", strconv.Itoa(maxFrames),
 		"-f", "rawvideo", "-pix_fmt", "gray",
 		"pipe:1",
-	}
+	)
 
 	cmd := exec.CommandContext(ctx, s.binary(), args...)
 	// Cancel and WaitDelay together are what stop an orphan. CommandContext's default is to
@@ -181,6 +211,24 @@ func (s *Sampler) Sample(
 	return stats, nil
 }
 
+func decoderCrashed(err error) bool {
+	if err == nil {
+		return false
+	}
+	message := strings.ToLower(err.Error())
+	for _, signature := range []string{
+		"segmentation fault",
+		"signal: aborted",
+		"signal: bus error",
+		"access violation",
+	} {
+		if strings.Contains(message, signature) {
+			return true
+		}
+	}
+	return false
+}
+
 // readFrames pulls fixed-size grayscale frames off the pipe and reduces each one as it
 // arrives. The frame buffer is allocated once and reused, so a four-hundred-frame pass
 // allocates fourteen kilobytes rather than five and a half megabytes.

server/internal/credits/sampler_test.go

@@ -0,0 +1,28 @@
+package credits
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestDecoderCrashClassification(t *testing.T) {
+	for _, message := range []string{
+		"credits: ffmpeg: signal: segmentation fault",
+		"credits: ffmpeg: signal: aborted",
+		"credits: ffmpeg: signal: bus error",
+		"credits: ffmpeg: access violation reading location",
+	} {
+		if !decoderCrashed(errors.New(message)) {
+			t.Errorf("%q was not classified as a decoder crash", message)
+		}
+	}
+	for _, message := range []string{
+		"credits: ffmpeg: exit status 1: HTTP error 401 Unauthorized",
+		"context deadline exceeded",
+		"credits: ffmpeg is not available",
+	} {
+		if decoderCrashed(errors.New(message)) {
+			t.Errorf("%q was classified as a decoder crash", message)
+		}
+	}
+}