package credits import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "testing" "time" ) // The corpus harness: the only test in this package that reads a media file. // // Every other test here pins a pure function against numbers written by hand, which is the // right shape for a changepoint rule and no shape at all for the question that actually // matters — does this find the credits. That question needs media with a known answer, so // this runs the detector over a corpus whose ground truth was fixed when the clips were // built (`testdata/gen_corpus.sh`) rather than annotated afterwards by looking at what the // detector said. // // It is skipped unless MEMBY_CREDITS_CORPUS points at such a directory, because the corpus // is media and media does not belong in the repository. Run it as: // // MEMBY_CREDITS_CORPUS=/path/to/corpus go test ./internal/credits -run Corpus -v // // The synthetic corpus is deliberately not a substitute for real episodes. What it is good // for is the structural cases — credits over footage, a dark scene before the roll, a // negative with no credits at all — where the failure is a property of the rule rather than // of any particular show, and where a real library gives you one example a fortnight. // truthEntry is one clip and the frame its credits genuinely begin at. A negative carries // -1, which is a claim in its own right: the detector must find nothing. type truthEntry struct { Clip string `json:"clip"` CreditsStartMs int64 `json:"creditsStartMs"` } // corpusResult is one clip's outcome, kept apart from the printing so a future run can emit // something other than a table without disturbing the measurement. type corpusResult struct { Clip string TruthMs int64 Detected bool StartMs int64 // PtsStartMs is the same answer read from the stream's own timestamps rather than from // the sampling arithmetic. Where the two disagree the arithmetic is what is wrong. Confidence float64 ErrorMs int64 Frames int Elapsed time.Duration Verdict string } func TestCorpusVisualDetector(t *testing.T) { dir := strings.TrimSpace(os.Getenv("MEMBY_CREDITS_CORPUS")) if dir == "" { t.Skip("set MEMBY_CREDITS_CORPUS to a corpus directory to run the media benchmark") } sampler := &Sampler{Binary: os.Getenv("MEMBY_CREDITS_FFMPEG"), Timeout: 2 * time.Minute} if !sampler.Available() { t.Skip("ffmpeg is not on the path") } truth := loadTruth(t, dir) detector := &VisualDetector{Sampler: sampler} results := make([]corpusResult, 0, len(truth)) for _, entry := range truth { path := filepath.Join(dir, entry.Clip+".mp4") runtimeMs, err := probeRuntimeMs(path) if err != nil { t.Fatalf("probe %s: %v", entry.Clip, err) } detection, err := detector.Detect(context.Background(), MediaInfo{ URL: path, RuntimeMs: runtimeMs, Window: GenericTailWindow(runtimeMs), }) if err != nil { t.Fatalf("detect %s: %v", entry.Clip, err) } results = append(results, scoreClip(entry, detection)) } reportCorpus(t, results) } // scoreClip turns one detection into a verdict. The four outcomes are kept distinct rather // than collapsed into pass/fail because they cost quite different things: a miss is a button // that never appears, a false positive throws somebody past the end of an episode, and those // are not the same defect however similar the arithmetic looks. func scoreClip(entry truthEntry, detection Detection) corpusResult { result := corpusResult{ Clip: entry.Clip, TruthMs: entry.CreditsStartMs, Detected: detection.Found, StartMs: detection.StartMs, Confidence: detection.Confidence, Frames: detection.FramesSampled, Elapsed: detection.Elapsed, } negative := entry.CreditsStartMs < 0 switch { case negative && !detection.Found: result.Verdict = "ok (correctly found nothing)" case negative && detection.Found: result.Verdict = "FALSE POSITIVE" case !detection.Found: result.Verdict = "MISS" default: result.ErrorMs = detection.StartMs - entry.CreditsStartMs result.Verdict = bandFor(result.ErrorMs) } return result } // bandFor names the accuracy band an error falls in. The bands are the brief's, and the sign // is kept because early and late are not equally bad: early clips the last line of dialogue, // late shows the viewer the thing they asked to skip. func bandFor(errorMs int64) string { magnitude := errorMs if magnitude < 0 { magnitude = -magnitude } switch { case magnitude <= 100: return "<=100ms" case magnitude <= 500: return "<=500ms" case magnitude <= 1000: return "<=1s" case magnitude <= 4000: return "<=4s" default: return "WRONG" } } func reportCorpus(t *testing.T, results []corpusResult) { t.Helper() sort.Slice(results, func(a, b int) bool { return results[a].Clip < results[b].Clip }) var report strings.Builder fmt.Fprintf(&report, "\n%-28s %9s %9s %9s %6s %7s %7s %s\n", "CLIP", "TRUTH", "FOUND", "ERROR", "CONF", "FRAMES", "TIME", "VERDICT") var ( within500, positives, falsePositives, misses int ) for _, result := range results { found, errorLabel := "-", "-" if result.Detected { found = formatMs(result.StartMs) if result.TruthMs >= 0 { errorLabel = fmt.Sprintf("%+.3fs", float64(result.ErrorMs)/1000) } } truthLabel := "none" if result.TruthMs >= 0 { truthLabel = formatMs(result.TruthMs) } fmt.Fprintf(&report, "%-28s %9s %9s %9s %6.2f %7d %6.1fs %s\n", result.Clip, truthLabel, found, errorLabel, result.Confidence, result.Frames, result.Elapsed.Seconds(), result.Verdict) switch { case result.Verdict == "FALSE POSITIVE": falsePositives++ case result.Verdict == "MISS": misses++ case result.TruthMs >= 0: positives++ if magnitude := result.ErrorMs; magnitude <= 500 && magnitude >= -500 { within500++ } } } fmt.Fprintf(&report, "\n%d/%d located within 500ms, %d missed, %d false positives\n", within500, positives+misses, misses, falsePositives) t.Log(report.String()) } func loadTruth(t *testing.T, dir string) []truthEntry { t.Helper() raw, err := os.ReadFile(filepath.Join(dir, "truth.json")) if err != nil { t.Fatalf("read truth: %v", err) } var truth []truthEntry if err := json.Unmarshal(raw, &truth); err != nil { t.Fatalf("parse truth: %v", err) } if len(truth) == 0 { t.Fatal("corpus truth is empty") } return truth } func probeRuntimeMs(path string) (int64, error) { out, err := exec.Command("ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path).Output() if err != nil { return 0, err } seconds, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) if err != nil { return 0, err } return int64(seconds * 1000), nil } func formatMs(value int64) string { return fmt.Sprintf("%d:%06.3f", value/60000, float64(value%60000)/1000) }