memby/server/internal/credits/corpus_test.go

package credits

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

// The corpus harness: the only test in this package that reads a media file.
//
// Every other test here pins a pure function against numbers written by hand, which is the
// right shape for a changepoint rule and no shape at all for the question that actually
// matters — does this find the credits. That question needs media with a known answer, so
// this runs the detector over a corpus whose ground truth was fixed when the clips were
// built (`testdata/gen_corpus.sh`) rather than annotated afterwards by looking at what the
// detector said.
//
// It is skipped unless MEMBY_CREDITS_CORPUS points at such a directory, because the corpus
// is media and media does not belong in the repository. Run it as:
//
//	MEMBY_CREDITS_CORPUS=/path/to/corpus go test ./internal/credits -run Corpus -v
//
// The synthetic corpus is deliberately not a substitute for real episodes. What it is good
// for is the structural cases — credits over footage, a dark scene before the roll, a
// negative with no credits at all — where the failure is a property of the rule rather than
// of any particular show, and where a real library gives you one example a fortnight.

// truthEntry is one clip and the frame its credits genuinely begin at. A negative carries
// -1, which is a claim in its own right: the detector must find nothing.
type truthEntry struct {
	Clip           string `json:"clip"`
	CreditsStartMs int64  `json:"creditsStartMs"`
}

// corpusResult is one clip's outcome, kept apart from the printing so a future run can emit
// something other than a table without disturbing the measurement.
type corpusResult struct {
	Clip     string
	TruthMs  int64
	Detected bool
	StartMs  int64
	// PtsStartMs is the same answer read from the stream's own timestamps rather than from
	// the sampling arithmetic. Where the two disagree the arithmetic is what is wrong.
	Confidence float64
	ErrorMs    int64
	Frames     int
	Elapsed    time.Duration
	Verdict    string
}

func TestCorpusVisualDetector(t *testing.T) {
	dir := strings.TrimSpace(os.Getenv("MEMBY_CREDITS_CORPUS"))
	if dir == "" {
		t.Skip("set MEMBY_CREDITS_CORPUS to a corpus directory to run the media benchmark")
	}
	sampler := &Sampler{Binary: os.Getenv("MEMBY_CREDITS_FFMPEG"), Timeout: 2 * time.Minute}
	if !sampler.Available() {
		t.Skip("ffmpeg is not on the path")
	}

	truth := loadTruth(t, dir)
	detector := &VisualDetector{Sampler: sampler}

	results := make([]corpusResult, 0, len(truth))
	for _, entry := range truth {
		path := filepath.Join(dir, entry.Clip+".mp4")
		runtimeMs, err := probeRuntimeMs(path)
		if err != nil {
			t.Fatalf("probe %s: %v", entry.Clip, err)
		}
		detection, err := detector.Detect(context.Background(), MediaInfo{
			URL:       path,
			RuntimeMs: runtimeMs,
			Window:    GenericTailWindow(runtimeMs),
		})
		if err != nil {
			t.Fatalf("detect %s: %v", entry.Clip, err)
		}
		results = append(results, scoreClip(entry, detection))
	}
	reportCorpus(t, results)
}

// scoreClip turns one detection into a verdict. The four outcomes are kept distinct rather
// than collapsed into pass/fail because they cost quite different things: a miss is a button
// that never appears, a false positive throws somebody past the end of an episode, and those
// are not the same defect however similar the arithmetic looks.
func scoreClip(entry truthEntry, detection Detection) corpusResult {
	result := corpusResult{
		Clip:       entry.Clip,
		TruthMs:    entry.CreditsStartMs,
		Detected:   detection.Found,
		StartMs:    detection.StartMs,
		Confidence: detection.Confidence,
		Frames:     detection.FramesSampled,
		Elapsed:    detection.Elapsed,
	}
	negative := entry.CreditsStartMs < 0
	switch {
	case negative && !detection.Found:
		result.Verdict = "ok (correctly found nothing)"
	case negative && detection.Found:
		result.Verdict = "FALSE POSITIVE"
	case !detection.Found:
		result.Verdict = "MISS"
	default:
		result.ErrorMs = detection.StartMs - entry.CreditsStartMs
		result.Verdict = bandFor(result.ErrorMs)
	}
	return result
}

// bandFor names the accuracy band an error falls in. The bands are the brief's, and the sign
// is kept because early and late are not equally bad: early clips the last line of dialogue,
// late shows the viewer the thing they asked to skip.
func bandFor(errorMs int64) string {
	magnitude := errorMs
	if magnitude < 0 {
		magnitude = -magnitude
	}
	switch {
	case magnitude <= 100:
		return "<=100ms"
	case magnitude <= 500:
		return "<=500ms"
	case magnitude <= 1000:
		return "<=1s"
	case magnitude <= 4000:
		return "<=4s"
	default:
		return "WRONG"
	}
}

func reportCorpus(t *testing.T, results []corpusResult) {
	t.Helper()
	sort.Slice(results, func(a, b int) bool { return results[a].Clip < results[b].Clip })

	var report strings.Builder
	fmt.Fprintf(&report, "\n%-28s %9s %9s %9s %6s %7s %7s  %s\n",
		"CLIP", "TRUTH", "FOUND", "ERROR", "CONF", "FRAMES", "TIME", "VERDICT")
	var (
		within500, positives, falsePositives, misses int
	)
	for _, result := range results {
		found, errorLabel := "-", "-"
		if result.Detected {
			found = formatMs(result.StartMs)
			if result.TruthMs >= 0 {
				errorLabel = fmt.Sprintf("%+.3fs", float64(result.ErrorMs)/1000)
			}
		}
		truthLabel := "none"
		if result.TruthMs >= 0 {
			truthLabel = formatMs(result.TruthMs)
		}
		fmt.Fprintf(&report, "%-28s %9s %9s %9s %6.2f %7d %6.1fs  %s\n",
			result.Clip, truthLabel, found, errorLabel, result.Confidence,
			result.Frames, result.Elapsed.Seconds(), result.Verdict)

		switch {
		case result.Verdict == "FALSE POSITIVE":
			falsePositives++
		case result.Verdict == "MISS":
			misses++
		case result.TruthMs >= 0:
			positives++
			if magnitude := result.ErrorMs; magnitude <= 500 && magnitude >= -500 {
				within500++
			}
		}
	}
	fmt.Fprintf(&report, "\n%d/%d located within 500ms, %d missed, %d false positives\n",
		within500, positives+misses, misses, falsePositives)
	t.Log(report.String())
}

func loadTruth(t *testing.T, dir string) []truthEntry {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join(dir, "truth.json"))
	if err != nil {
		t.Fatalf("read truth: %v", err)
	}
	var truth []truthEntry
	if err := json.Unmarshal(raw, &truth); err != nil {
		t.Fatalf("parse truth: %v", err)
	}
	if len(truth) == 0 {
		t.Fatal("corpus truth is empty")
	}
	return truth
}

func probeRuntimeMs(path string) (int64, error) {
	out, err := exec.Command("ffprobe", "-v", "error",
		"-show_entries", "format=duration", "-of", "csv=p=0", path).Output()
	if err != nil {
		return 0, err
	}
	seconds, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
	if err != nil {
		return 0, err
	}
	return int64(seconds * 1000), nil
}

func formatMs(value int64) string {
	return fmt.Sprintf("%d:%06.3f", value/60000, float64(value%60000)/1000)
}