memby/server/internal/credits/detector.go

package credits

import (
	"context"
	"time"
)

// The visual detector: two sparse passes and a changepoint.
//
// What is being looked for is a *sustained structural transition*, not an understanding of
// the picture. A credits roll is dark, textured with thin text, and — the part that makes it
// findable — it stays that way for a minute or more, where a dark night scene does not. So
// the detector does not try to recognise credits at all. It scores every sampled frame for
// how credit-like it is, then finds the single point in the window where everything after it
// scores markedly higher than everything before it, and refuses to answer when no such point
// is clearly better than its neighbours.
//
// This is deliberately the least clever component in the package. Anything heavier — a model,
// OCR, a full OpenCV pipeline — would be a dependency and a hardware requirement out of all
// proportion to a feature whose fallback is simply not showing a button.

const (
	// sustainSeconds is how long the credit-like state has to persist to count. Shorter than
	// this and a dark establishing shot at the end of an act qualifies.
	sustainSeconds = 45

	// leadSeconds is how much ordinary programme has to precede the transition. Without it
	// the changepoint can sit at the very first sampled frame, which is not evidence of a
	// transition — it is evidence that the window started too late.
	leadSeconds = 12

	// creditLikeFloor is how credit-like the tail has to look in absolute terms. The
	// separation test alone would happily report a transition from "slightly less dark" to
	// "slightly more dark" in the middle of a night scene.
	creditLikeFloor = 0.45

	// minSeparation is how much better the tail has to score than the head. This is the
	// primary guard against answering on noise, and it is set high because the cost of a
	// wrong marker is somebody losing the end of an episode.
	minSeparation = 0.18

	// onsetFraction is how credit-like a frame has to be, relative to the established roll,
	// to count as already part of it.
	//
	// The split itself lands where the credits are *established*, because the sustain window
	// after it has to clear creditLikeFloor on average — and credits usually fade in, so the
	// first second or two of the fade scores below that floor and pushes the split forward.
	// Measured against real media the gap is two to three seconds, which is visible: the roll
	// has plainly begun before the picture moves.
	//
	// So the split is walked backwards through the fade to the first frame that is already
	// mostly credit-like. Two fifths is deliberately generous — this is looking for the start
	// of a ramp, not for more credits — and it is a fraction of the established level rather
	// than an absolute, because a roll over a bright background never reaches the same score
	// as one over black and would otherwise never be walked back at all.
	onsetFraction = 0.4

	// onsetBackoffMs bounds that walk. A fade is a second or two; anything walking further is
	// no longer following one, and the bound is what stops a gradual dimming at the end of a
	// scene dragging the marker back into the programme.
	onsetBackoffMs = 4000
)

// VisualDetector implements Detector over the ffmpeg sampler.
type VisualDetector struct {
	Sampler *Sampler
}

// Detect runs the coarse pass, then a fine pass around whatever it found.
//
// The second pass is what makes the answer usable. A frame every four seconds locates the
// transition to within four seconds, and a Skip Credits button four seconds early clips the
// last line of dialogue. Refining costs a second pass over a single minute of file — about
// eighty more tiny frames — which is a good trade for a marker that will be served for the
// life of the media version.
func (d *VisualDetector) Detect(ctx context.Context, media MediaInfo) (Detection, error) {
	if d == nil || d.Sampler == nil || !media.Window.Valid() {
		return Detection{}, nil
	}
	started := time.Now()

	from := time.Duration(media.Window.StartMs) * time.Millisecond
	to := time.Duration(media.Window.EndMs) * time.Millisecond

	coarse, err := d.Sampler.Sample(ctx, media.URL, from, to, coarseInterval)
	if err != nil {
		return Detection{}, err
	}
	frames := len(coarse)
	index, separation, found := findTransition(coarse, coarseInterval)
	if !found {
		// No transition is a perfectly ordinary answer, and the common one on a film. It is
		// reported as "nothing found" rather than as an error so it can be cached: without
		// that, every playback of a credit-less title would re-scan it.
		return Detection{
			FramesSampled: frames,
			Elapsed:       time.Since(started),
		}, nil
	}
	startMs := coarse[index].PositionMs

	// Refine, but never let the refinement move the answer outside the span the coarse pass
	// actually pointed at — a fine pass that disagreed wildly would be finding a different
	// transition, and the coarse one had the whole window to consider.
	fineFrom := time.Duration(startMs)*time.Millisecond - fineSpan
	if fineFrom < from {
		fineFrom = from
	}
	fineTo := time.Duration(startMs)*time.Millisecond + fineSpan
	if fineTo > to {
		fineTo = to
	}
	if fine, fineErr := d.Sampler.Sample(ctx, media.URL, fineFrom, fineTo, fineInterval); fineErr == nil {
		frames += len(fine)
		if refined, refinedSeparation, ok := findTransition(fine, fineInterval); ok {
			startMs = fine[refined].PositionMs
			if refinedSeparation > separation {
				separation = refinedSeparation
			}
		}
	}

	return Detection{
		Found:         true,
		StartMs:       startMs,
		Confidence:    visualConfidence(separation),
		Method:        MethodVisual,
		FramesSampled: frames,
		Elapsed:       time.Since(started),
	}, nil
}

// creditScore is how credit-like one frame looks, from 0 to 1.
//
// Three properties, and the way they are combined is the substance of the detector. Darkness
// is *multiplied* rather than added, which is what separates credits from the one thing that
// most resembles them: a night exterior is dark and flat and, under a weighted sum, scores
// most of the way to the floor on those two properties alone — which is exactly how a final
// scene comes to be reported as a credits roll. Text is not optional evidence that tops a
// dark frame up; it is the thing being detected, and a dark frame without it must score near
// zero however dark and however flat it is.
func creditScore(frame frameStats) float64 {
	darkness := frame.DarkFraction

	// Text produces a narrow *band* of edge density: a flat black frame has almost none, and
	// a detailed photograph has far more than titles do. Scoring the band rather than the
	// magnitude is what stops a bright, busy scene outscoring the credits.
	const idealEdges = 0.030
	text := frame.EdgeDensity / idealEdges
	if text > 1 {
		text = 2 - text
	}
	text = clamp01(text)

	// Flatness: a credits background is uniform, so nearly all the variance in the frame
	// comes from the text itself and is small. A supporting property, never a deciding one.
	flatness := clamp01(1 - frame.Variance/0.04)

	return clamp01(darkness * (0.75*text + 0.25*flatness))
}

// findTransition locates the point where the window stops looking like programme and starts
// looking like credits.
//
// Pure, and the piece worth testing hardest: it is the whole of the visual decision, and
// everything above it is plumbing. It returns the index of the first credit-like frame, and
// how much better the tail scored than the head — which is what confidence is derived from.
func findTransition(frames []frameStats, interval time.Duration) (int, float64, bool) {
	if interval <= 0 {
		return 0, 0, false
	}
	perSecond := float64(time.Second) / float64(interval)
	lead := int(float64(leadSeconds) * perSecond)
	if lead < 1 {
		lead = 1
	}
	sustain := int(float64(sustainSeconds) * perSecond)
	if sustain < 2 {
		sustain = 2
	}
	if len(frames) < lead+sustain {
		return 0, 0, false
	}

	scores := make([]float64, len(frames))
	for index, frame := range frames {
		scores[index] = creditScore(frame)
	}
	// Prefix sums so every candidate split is evaluated in constant time. The window is only
	// a few hundred frames, but the quadratic version is the kind of thing that stops being
	// free the moment somebody widens the window.
	prefix := make([]float64, len(scores)+1)
	for index, score := range scores {
		prefix[index+1] = prefix[index] + score
	}
	segmentMean := func(from, to int) float64 {
		if to <= from {
			return 0
		}
		return (prefix[to] - prefix[from]) / float64(to-from)
	}

	bestIndex, bestSeparation, found := 0, 0.0, false
	for split := lead; split+sustain <= len(frames); split++ {
		head := segmentMean(0, split)
		tail := segmentMean(split, len(frames))
		// The sustained window immediately after the split has to qualify on its own, not
		// merely drag the average of a long tail up. This is what refuses a dark final shot
		// followed by genuine credits: the split belongs at the credits, not at the shot.
		if segmentMean(split, split+sustain) < creditLikeFloor {
			continue
		}
		separation := tail - head
		if separation < minSeparation || separation <= bestSeparation {
			continue
		}
		bestIndex, bestSeparation, found = split, separation, true
	}
	if !found {
		return 0, 0, false
	}
	// The separation is reported for the split that earned it, never for the walked-back
	// frame: confidence is a statement about how clearly the transition was found, and
	// recomputing it over a fade would report the answer as weaker for having been improved.
	return backOffToOnset(scores, bestIndex, segmentMean(bestIndex, min(bestIndex+sustain, len(scores))), interval),
		bestSeparation, true
}

// backOffToOnset walks a split backwards through the credits' fade-in.
//
// Pure, bounded, and it can only ever move the marker earlier — which is the direction that
// needs the care, so both bounds matter: it stops at the first frame that is not already
// mostly credit-like, and it never travels further than onsetBackoffMs. On a hard cut from
// programme to credits the preceding frame scores near zero and the walk stops immediately,
// which is correct: there is no fade to find and the split was already right.
func backOffToOnset(scores []float64, split int, established float64, interval time.Duration) int {
	if split <= 0 || established <= 0 || interval <= 0 {
		return split
	}
	limit := int(float64(onsetBackoffMs) / float64(interval.Milliseconds()))
	if limit < 1 {
		// A sampling interval coarser than the whole allowance cannot resolve a fade at all.
		// The coarse pass is this case, and it is the one whose answer the fine pass replaces.
		return split
	}
	floor := onsetFraction * established
	earliest := split - limit
	if earliest < 0 {
		earliest = 0
	}
	onset := split
	for index := split - 1; index >= earliest; index-- {
		if scores[index] < floor {
			break
		}
		onset = index
	}
	return onset
}

// visualConfidence maps separation onto a score.
//
// Capped below certainty because a single detector agreeing with itself is not corroboration:
// the ceiling is what forces a genuinely ambiguous file to wait for behavioural agreement
// before a marker is written, rather than being decided by one reading of one window.
func visualConfidence(separation float64) float64 {
	const ceiling = 0.88
	// minSeparation earns the threshold exactly; twice it earns nearly the ceiling. A
	// detector that cleared the bar by a hair should produce a marker that a disagreement
	// can still overturn.
	score := ConfidenceThreshold +
		(ceiling-ConfidenceThreshold)*clamp01((separation-minSeparation)/minSeparation)
	return score
}

func clamp01(value float64) float64 {
	if value < 0 {
		return 0
	}
	if value > 1 {
		return 1
	}
	return value
}

// noopDetector stands in when ffmpeg is absent. It finds nothing, which lets the service run
// on behavioural evidence alone rather than needing a second code path for the case.
type noopDetector struct{}

func (noopDetector) Detect(context.Context, MediaInfo) (Detection, error) {
	return Detection{}, nil
}