0.2.75
This commit is contained in:
ponzischeme89
@@ -0,0 +1,217 @@
|
||||
package credits
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The corpus harness: the only test in this package that reads a media file.
|
||||
//
|
||||
// Every other test here pins a pure function against numbers written by hand, which is the
|
||||
// right shape for a changepoint rule and no shape at all for the question that actually
|
||||
// matters — does this find the credits. That question needs media with a known answer, so
|
||||
// this runs the detector over a corpus whose ground truth was fixed when the clips were
|
||||
// built (`testdata/gen_corpus.sh`) rather than annotated afterwards by looking at what the
|
||||
// detector said.
|
||||
//
|
||||
// It is skipped unless MEMBY_CREDITS_CORPUS points at such a directory, because the corpus
|
||||
// is media and media does not belong in the repository. Run it as:
|
||||
//
|
||||
// MEMBY_CREDITS_CORPUS=/path/to/corpus go test ./internal/credits -run Corpus -v
|
||||
//
|
||||
// The synthetic corpus is deliberately not a substitute for real episodes. What it is good
|
||||
// for is the structural cases — credits over footage, a dark scene before the roll, a
|
||||
// negative with no credits at all — where the failure is a property of the rule rather than
|
||||
// of any particular show, and where a real library gives you one example a fortnight.
|
||||
|
||||
// truthEntry is one clip and the frame its credits genuinely begin at. A negative carries
|
||||
// -1, which is a claim in its own right: the detector must find nothing.
|
||||
type truthEntry struct {
|
||||
Clip string `json:"clip"`
|
||||
CreditsStartMs int64 `json:"creditsStartMs"`
|
||||
}
|
||||
|
||||
// corpusResult is one clip's outcome, kept apart from the printing so a future run can emit
|
||||
// something other than a table without disturbing the measurement.
|
||||
type corpusResult struct {
|
||||
Clip string
|
||||
TruthMs int64
|
||||
Detected bool
|
||||
StartMs int64
|
||||
// PtsStartMs is the same answer read from the stream's own timestamps rather than from
|
||||
// the sampling arithmetic. Where the two disagree the arithmetic is what is wrong.
|
||||
Confidence float64
|
||||
ErrorMs int64
|
||||
Frames int
|
||||
Elapsed time.Duration
|
||||
Verdict string
|
||||
}
|
||||
|
||||
func TestCorpusVisualDetector(t *testing.T) {
|
||||
dir := strings.TrimSpace(os.Getenv("MEMBY_CREDITS_CORPUS"))
|
||||
if dir == "" {
|
||||
t.Skip("set MEMBY_CREDITS_CORPUS to a corpus directory to run the media benchmark")
|
||||
}
|
||||
sampler := &Sampler{Binary: os.Getenv("MEMBY_CREDITS_FFMPEG"), Timeout: 2 * time.Minute}
|
||||
if !sampler.Available() {
|
||||
t.Skip("ffmpeg is not on the path")
|
||||
}
|
||||
|
||||
truth := loadTruth(t, dir)
|
||||
detector := &VisualDetector{Sampler: sampler}
|
||||
|
||||
results := make([]corpusResult, 0, len(truth))
|
||||
for _, entry := range truth {
|
||||
path := filepath.Join(dir, entry.Clip+".mp4")
|
||||
runtimeMs, err := probeRuntimeMs(path)
|
||||
if err != nil {
|
||||
t.Fatalf("probe %s: %v", entry.Clip, err)
|
||||
}
|
||||
detection, err := detector.Detect(context.Background(), MediaInfo{
|
||||
URL: path,
|
||||
RuntimeMs: runtimeMs,
|
||||
Window: GenericTailWindow(runtimeMs),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("detect %s: %v", entry.Clip, err)
|
||||
}
|
||||
results = append(results, scoreClip(entry, detection))
|
||||
}
|
||||
reportCorpus(t, results)
|
||||
}
|
||||
|
||||
// scoreClip turns one detection into a verdict. The four outcomes are kept distinct rather
|
||||
// than collapsed into pass/fail because they cost quite different things: a miss is a button
|
||||
// that never appears, a false positive throws somebody past the end of an episode, and those
|
||||
// are not the same defect however similar the arithmetic looks.
|
||||
func scoreClip(entry truthEntry, detection Detection) corpusResult {
|
||||
result := corpusResult{
|
||||
Clip: entry.Clip,
|
||||
TruthMs: entry.CreditsStartMs,
|
||||
Detected: detection.Found,
|
||||
StartMs: detection.StartMs,
|
||||
Confidence: detection.Confidence,
|
||||
Frames: detection.FramesSampled,
|
||||
Elapsed: detection.Elapsed,
|
||||
}
|
||||
negative := entry.CreditsStartMs < 0
|
||||
switch {
|
||||
case negative && !detection.Found:
|
||||
result.Verdict = "ok (correctly found nothing)"
|
||||
case negative && detection.Found:
|
||||
result.Verdict = "FALSE POSITIVE"
|
||||
case !detection.Found:
|
||||
result.Verdict = "MISS"
|
||||
default:
|
||||
result.ErrorMs = detection.StartMs - entry.CreditsStartMs
|
||||
result.Verdict = bandFor(result.ErrorMs)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// bandFor names the accuracy band an error falls in. The bands are the brief's, and the sign
|
||||
// is kept because early and late are not equally bad: early clips the last line of dialogue,
|
||||
// late shows the viewer the thing they asked to skip.
|
||||
func bandFor(errorMs int64) string {
|
||||
magnitude := errorMs
|
||||
if magnitude < 0 {
|
||||
magnitude = -magnitude
|
||||
}
|
||||
switch {
|
||||
case magnitude <= 100:
|
||||
return "<=100ms"
|
||||
case magnitude <= 500:
|
||||
return "<=500ms"
|
||||
case magnitude <= 1000:
|
||||
return "<=1s"
|
||||
case magnitude <= 4000:
|
||||
return "<=4s"
|
||||
default:
|
||||
return "WRONG"
|
||||
}
|
||||
}
|
||||
|
||||
func reportCorpus(t *testing.T, results []corpusResult) {
|
||||
t.Helper()
|
||||
sort.Slice(results, func(a, b int) bool { return results[a].Clip < results[b].Clip })
|
||||
|
||||
var report strings.Builder
|
||||
fmt.Fprintf(&report, "\n%-28s %9s %9s %9s %6s %7s %7s %s\n",
|
||||
"CLIP", "TRUTH", "FOUND", "ERROR", "CONF", "FRAMES", "TIME", "VERDICT")
|
||||
var (
|
||||
within500, positives, falsePositives, misses int
|
||||
)
|
||||
for _, result := range results {
|
||||
found, errorLabel := "-", "-"
|
||||
if result.Detected {
|
||||
found = formatMs(result.StartMs)
|
||||
if result.TruthMs >= 0 {
|
||||
errorLabel = fmt.Sprintf("%+.3fs", float64(result.ErrorMs)/1000)
|
||||
}
|
||||
}
|
||||
truthLabel := "none"
|
||||
if result.TruthMs >= 0 {
|
||||
truthLabel = formatMs(result.TruthMs)
|
||||
}
|
||||
fmt.Fprintf(&report, "%-28s %9s %9s %9s %6.2f %7d %6.1fs %s\n",
|
||||
result.Clip, truthLabel, found, errorLabel, result.Confidence,
|
||||
result.Frames, result.Elapsed.Seconds(), result.Verdict)
|
||||
|
||||
switch {
|
||||
case result.Verdict == "FALSE POSITIVE":
|
||||
falsePositives++
|
||||
case result.Verdict == "MISS":
|
||||
misses++
|
||||
case result.TruthMs >= 0:
|
||||
positives++
|
||||
if magnitude := result.ErrorMs; magnitude <= 500 && magnitude >= -500 {
|
||||
within500++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&report, "\n%d/%d located within 500ms, %d missed, %d false positives\n",
|
||||
within500, positives+misses, misses, falsePositives)
|
||||
t.Log(report.String())
|
||||
}
|
||||
|
||||
func loadTruth(t *testing.T, dir string) []truthEntry {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "truth.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read truth: %v", err)
|
||||
}
|
||||
var truth []truthEntry
|
||||
if err := json.Unmarshal(raw, &truth); err != nil {
|
||||
t.Fatalf("parse truth: %v", err)
|
||||
}
|
||||
if len(truth) == 0 {
|
||||
t.Fatal("corpus truth is empty")
|
||||
}
|
||||
return truth
|
||||
}
|
||||
|
||||
func probeRuntimeMs(path string) (int64, error) {
|
||||
out, err := exec.Command("ffprobe", "-v", "error",
|
||||
"-show_entries", "format=duration", "-of", "csv=p=0", path).Output()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
seconds, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(seconds * 1000), nil
|
||||
}
|
||||
|
||||
func formatMs(value int64) string {
|
||||
return fmt.Sprintf("%d:%06.3f", value/60000, float64(value%60000)/1000)
|
||||
}
|
||||
Reference in New Issue
Block a user