0.2.75
This commit is contained in:
ponzischeme89
@@ -0,0 +1,217 @@
|
||||
package credits
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The corpus harness: the only test in this package that reads a media file.
|
||||
//
|
||||
// Every other test here pins a pure function against numbers written by hand, which is the
|
||||
// right shape for a changepoint rule and no shape at all for the question that actually
|
||||
// matters — does this find the credits. That question needs media with a known answer, so
|
||||
// this runs the detector over a corpus whose ground truth was fixed when the clips were
|
||||
// built (`testdata/gen_corpus.sh`) rather than annotated afterwards by looking at what the
|
||||
// detector said.
|
||||
//
|
||||
// It is skipped unless MEMBY_CREDITS_CORPUS points at such a directory, because the corpus
|
||||
// is media and media does not belong in the repository. Run it as:
|
||||
//
|
||||
// MEMBY_CREDITS_CORPUS=/path/to/corpus go test ./internal/credits -run Corpus -v
|
||||
//
|
||||
// The synthetic corpus is deliberately not a substitute for real episodes. What it is good
|
||||
// for is the structural cases — credits over footage, a dark scene before the roll, a
|
||||
// negative with no credits at all — where the failure is a property of the rule rather than
|
||||
// of any particular show, and where a real library gives you one example a fortnight.
|
||||
|
||||
// truthEntry is one clip and the frame its credits genuinely begin at. A negative carries
|
||||
// -1, which is a claim in its own right: the detector must find nothing.
|
||||
type truthEntry struct {
|
||||
Clip string `json:"clip"`
|
||||
CreditsStartMs int64 `json:"creditsStartMs"`
|
||||
}
|
||||
|
||||
// corpusResult is one clip's outcome, kept apart from the printing so a future run can emit
|
||||
// something other than a table without disturbing the measurement.
|
||||
type corpusResult struct {
|
||||
Clip string
|
||||
TruthMs int64
|
||||
Detected bool
|
||||
StartMs int64
|
||||
// PtsStartMs is the same answer read from the stream's own timestamps rather than from
|
||||
// the sampling arithmetic. Where the two disagree the arithmetic is what is wrong.
|
||||
Confidence float64
|
||||
ErrorMs int64
|
||||
Frames int
|
||||
Elapsed time.Duration
|
||||
Verdict string
|
||||
}
|
||||
|
||||
func TestCorpusVisualDetector(t *testing.T) {
|
||||
dir := strings.TrimSpace(os.Getenv("MEMBY_CREDITS_CORPUS"))
|
||||
if dir == "" {
|
||||
t.Skip("set MEMBY_CREDITS_CORPUS to a corpus directory to run the media benchmark")
|
||||
}
|
||||
sampler := &Sampler{Binary: os.Getenv("MEMBY_CREDITS_FFMPEG"), Timeout: 2 * time.Minute}
|
||||
if !sampler.Available() {
|
||||
t.Skip("ffmpeg is not on the path")
|
||||
}
|
||||
|
||||
truth := loadTruth(t, dir)
|
||||
detector := &VisualDetector{Sampler: sampler}
|
||||
|
||||
results := make([]corpusResult, 0, len(truth))
|
||||
for _, entry := range truth {
|
||||
path := filepath.Join(dir, entry.Clip+".mp4")
|
||||
runtimeMs, err := probeRuntimeMs(path)
|
||||
if err != nil {
|
||||
t.Fatalf("probe %s: %v", entry.Clip, err)
|
||||
}
|
||||
detection, err := detector.Detect(context.Background(), MediaInfo{
|
||||
URL: path,
|
||||
RuntimeMs: runtimeMs,
|
||||
Window: GenericTailWindow(runtimeMs),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("detect %s: %v", entry.Clip, err)
|
||||
}
|
||||
results = append(results, scoreClip(entry, detection))
|
||||
}
|
||||
reportCorpus(t, results)
|
||||
}
|
||||
|
||||
// scoreClip turns one detection into a verdict. The four outcomes are kept distinct rather
|
||||
// than collapsed into pass/fail because they cost quite different things: a miss is a button
|
||||
// that never appears, a false positive throws somebody past the end of an episode, and those
|
||||
// are not the same defect however similar the arithmetic looks.
|
||||
func scoreClip(entry truthEntry, detection Detection) corpusResult {
|
||||
result := corpusResult{
|
||||
Clip: entry.Clip,
|
||||
TruthMs: entry.CreditsStartMs,
|
||||
Detected: detection.Found,
|
||||
StartMs: detection.StartMs,
|
||||
Confidence: detection.Confidence,
|
||||
Frames: detection.FramesSampled,
|
||||
Elapsed: detection.Elapsed,
|
||||
}
|
||||
negative := entry.CreditsStartMs < 0
|
||||
switch {
|
||||
case negative && !detection.Found:
|
||||
result.Verdict = "ok (correctly found nothing)"
|
||||
case negative && detection.Found:
|
||||
result.Verdict = "FALSE POSITIVE"
|
||||
case !detection.Found:
|
||||
result.Verdict = "MISS"
|
||||
default:
|
||||
result.ErrorMs = detection.StartMs - entry.CreditsStartMs
|
||||
result.Verdict = bandFor(result.ErrorMs)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// bandFor names the accuracy band an error falls in. The bands are the brief's, and the sign
|
||||
// is kept because early and late are not equally bad: early clips the last line of dialogue,
|
||||
// late shows the viewer the thing they asked to skip.
|
||||
func bandFor(errorMs int64) string {
|
||||
magnitude := errorMs
|
||||
if magnitude < 0 {
|
||||
magnitude = -magnitude
|
||||
}
|
||||
switch {
|
||||
case magnitude <= 100:
|
||||
return "<=100ms"
|
||||
case magnitude <= 500:
|
||||
return "<=500ms"
|
||||
case magnitude <= 1000:
|
||||
return "<=1s"
|
||||
case magnitude <= 4000:
|
||||
return "<=4s"
|
||||
default:
|
||||
return "WRONG"
|
||||
}
|
||||
}
|
||||
|
||||
func reportCorpus(t *testing.T, results []corpusResult) {
|
||||
t.Helper()
|
||||
sort.Slice(results, func(a, b int) bool { return results[a].Clip < results[b].Clip })
|
||||
|
||||
var report strings.Builder
|
||||
fmt.Fprintf(&report, "\n%-28s %9s %9s %9s %6s %7s %7s %s\n",
|
||||
"CLIP", "TRUTH", "FOUND", "ERROR", "CONF", "FRAMES", "TIME", "VERDICT")
|
||||
var (
|
||||
within500, positives, falsePositives, misses int
|
||||
)
|
||||
for _, result := range results {
|
||||
found, errorLabel := "-", "-"
|
||||
if result.Detected {
|
||||
found = formatMs(result.StartMs)
|
||||
if result.TruthMs >= 0 {
|
||||
errorLabel = fmt.Sprintf("%+.3fs", float64(result.ErrorMs)/1000)
|
||||
}
|
||||
}
|
||||
truthLabel := "none"
|
||||
if result.TruthMs >= 0 {
|
||||
truthLabel = formatMs(result.TruthMs)
|
||||
}
|
||||
fmt.Fprintf(&report, "%-28s %9s %9s %9s %6.2f %7d %6.1fs %s\n",
|
||||
result.Clip, truthLabel, found, errorLabel, result.Confidence,
|
||||
result.Frames, result.Elapsed.Seconds(), result.Verdict)
|
||||
|
||||
switch {
|
||||
case result.Verdict == "FALSE POSITIVE":
|
||||
falsePositives++
|
||||
case result.Verdict == "MISS":
|
||||
misses++
|
||||
case result.TruthMs >= 0:
|
||||
positives++
|
||||
if magnitude := result.ErrorMs; magnitude <= 500 && magnitude >= -500 {
|
||||
within500++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&report, "\n%d/%d located within 500ms, %d missed, %d false positives\n",
|
||||
within500, positives+misses, misses, falsePositives)
|
||||
t.Log(report.String())
|
||||
}
|
||||
|
||||
func loadTruth(t *testing.T, dir string) []truthEntry {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "truth.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read truth: %v", err)
|
||||
}
|
||||
var truth []truthEntry
|
||||
if err := json.Unmarshal(raw, &truth); err != nil {
|
||||
t.Fatalf("parse truth: %v", err)
|
||||
}
|
||||
if len(truth) == 0 {
|
||||
t.Fatal("corpus truth is empty")
|
||||
}
|
||||
return truth
|
||||
}
|
||||
|
||||
func probeRuntimeMs(path string) (int64, error) {
|
||||
out, err := exec.Command("ffprobe", "-v", "error",
|
||||
"-show_entries", "format=duration", "-of", "csv=p=0", path).Output()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
seconds, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(seconds * 1000), nil
|
||||
}
|
||||
|
||||
func formatMs(value int64) string {
|
||||
return fmt.Sprintf("%d:%06.3f", value/60000, float64(value%60000)/1000)
|
||||
}
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env bash
|
||||
# Synthetic credits-detection corpus with exact ground truth.
|
||||
#
|
||||
# Every clip is 25fps, 640x360, with audio, and every positive puts credits_start
|
||||
# at exactly frame 1534 = 61.360s. That figure is deliberately not round: at a
|
||||
# 750ms fine-pass interval anchored to the window start, a boundary at 60.000s is
|
||||
# itself a sample point, so a detector sampling on that grid scores a perfect zero
|
||||
# for reasons that have nothing to do with how well it found anything. Off-grid
|
||||
# truth is what makes the reported error the real error.
|
||||
#
|
||||
# Segments are encoded separately and concatenated, so the boundary is a genuine
|
||||
# frame boundary rather than something a filter approximated.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
OUT=corpus
|
||||
rm -rf "$OUT" parts
|
||||
mkdir -p "$OUT" parts
|
||||
FONT="C\\:/Windows/Fonts/arial.ttf"
|
||||
ENC="-c:v libx264 -preset veryfast -pix_fmt yuv420p -g 50 -r 25 -c:a aac -b:a 96k -ar 48000 -ac 2"
|
||||
Q="-hide_banner -loglevel error -y"
|
||||
|
||||
# Frame-exact durations. 1534 frames at 25fps is 61.360s.
|
||||
D_PROG=61.36 # programme before the credits
|
||||
D_PROG_A=46.36 # programme before a dark closing scene
|
||||
D_DARK=15.00 # the dark closing scene
|
||||
D_CRED=30.00
|
||||
D_CRED_SHORT=20.00
|
||||
D_POST=10.00
|
||||
D_DARKTAIL=30.00
|
||||
TRUTH_MS=61360
|
||||
|
||||
cat > parts/credits.txt <<'EOF'
|
||||
DIRECTED BY
|
||||
ALEX MERRIWEATHER
|
||||
WRITTEN BY
|
||||
JORDAN HALE
|
||||
PRODUCED BY
|
||||
SAM OKONKWO
|
||||
CAST
|
||||
RILEY BRENNAN
|
||||
DANA VOSS
|
||||
KIT ARMITAGE
|
||||
MARGO SEELEY
|
||||
DIRECTOR OF PHOTOGRAPHY
|
||||
NOOR HADDAD
|
||||
EDITED BY
|
||||
TOBY LINDQVIST
|
||||
MUSIC BY
|
||||
PRIYA RAGHAVAN
|
||||
PRODUCTION DESIGNER
|
||||
ELLIOT SHAW
|
||||
COSTUME DESIGNER
|
||||
FRANCES ADEYEMI
|
||||
CASTING BY
|
||||
WES TANAKA
|
||||
UNIT PRODUCTION MANAGER
|
||||
HANNAH DELACROIX
|
||||
FIRST ASSISTANT DIRECTOR
|
||||
OSCAR BRENNAN
|
||||
EOF
|
||||
|
||||
# Audio beds. Programme is broadband and non-stationary the way speech is;
|
||||
# credits are a steady two-note pad. The distinction a detector can find here is
|
||||
# stationarity and spectral shape, which is the same distinction real end-credit
|
||||
# music offers against dialogue — synthetic, but not a different mechanism.
|
||||
A_PROG="anoisesrc=color=brown:amplitude=0.35:r=48000,tremolo=f=3.5:d=0.8"
|
||||
A_CRED="sine=frequency=220:r=48000,volume=0.3"
|
||||
|
||||
# $1 out $2 duration $3 video lavfi $4 audio lavfi [$5 extra -vf]
|
||||
seg() {
|
||||
local extra="${5:-null}"
|
||||
ffmpeg $Q -f lavfi -i "$3" -f lavfi -i "$4" -t "$2" -vf "$extra" $ENC "parts/$1.mp4"
|
||||
}
|
||||
|
||||
# scrolling credits over a supplied background. $1 out $2 dur $3 vsrc $4 fontcolour
|
||||
credits_over() {
|
||||
seg "$1" "$2" "$3" "$A_CRED" \
|
||||
"drawtext=fontfile='$FONT':textfile=parts/credits.txt:fontcolor=$4:fontsize=15:line_spacing=10:x=(w-tw)/2:y=h-35*t"
|
||||
}
|
||||
|
||||
join() { # $1 out name, rest: part names
|
||||
local out="$1"; shift
|
||||
: > parts/list.txt
|
||||
# Relative to the list file's own directory: a POSIX $(pwd) from Git Bash is
|
||||
# not a path native ffmpeg can open.
|
||||
for p in "$@"; do echo "file '$p.mp4'" >> parts/list.txt; done
|
||||
ffmpeg $Q -f concat -safe 0 -i parts/list.txt -c copy "$OUT/$out.mp4"
|
||||
}
|
||||
|
||||
PROG="testsrc2=s=640x360:r=25"
|
||||
DARKPROG="color=c=#0d0d10:s=640x360:r=25"
|
||||
BLACK="color=c=black:s=640x360:r=25"
|
||||
WHITE="color=c=white:s=640x360:r=25"
|
||||
|
||||
echo "building parts..."
|
||||
seg prog $D_PROG "$PROG" "$A_PROG"
|
||||
seg progA $D_PROG_A "$PROG" "$A_PROG"
|
||||
seg dark15 $D_DARK "$DARKPROG" "$A_PROG"
|
||||
seg darktail $D_DARKTAIL "$DARKPROG" "$A_PROG"
|
||||
seg post10 $D_POST "$PROG" "$A_PROG"
|
||||
seg progfade $D_PROG "$PROG" "$A_PROG" "fade=t=out:st=59.86:d=1.5"
|
||||
|
||||
credits_over cred_black $D_CRED "$BLACK" white
|
||||
credits_over cred_short $D_CRED_SHORT "$BLACK" white
|
||||
credits_over cred_over $D_CRED "$PROG" white
|
||||
credits_over cred_white $D_CRED "$WHITE" black
|
||||
|
||||
# static centred card credits (no scroll)
|
||||
seg cred_static $D_CRED "$BLACK" "$A_CRED" \
|
||||
"drawtext=fontfile='$FONT':textfile=parts/credits.txt:fontcolor=white:fontsize=13:line_spacing=6:x=(w-tw)/2:y=(h-th)/2"
|
||||
|
||||
echo "assembling clips..."
|
||||
join hard-cut-black prog cred_black
|
||||
join fade-to-black progfade cred_black
|
||||
join credits-over-footage prog cred_over
|
||||
join bright-credits prog cred_white
|
||||
join static-card-credits prog cred_static
|
||||
join dark-scene-then-credits progA dark15 cred_black
|
||||
join short-credits-postcred prog cred_short post10
|
||||
join negative-dark-ending progA dark15 darktail
|
||||
|
||||
# Ground truth travels with the corpus rather than being written into the
|
||||
# harness: a truth table kept apart from the media it describes is one that
|
||||
# silently stops matching when a clip is regenerated.
|
||||
{
|
||||
echo '['
|
||||
first=1
|
||||
for f in "$OUT"/*.mp4; do
|
||||
n=$(basename "$f" .mp4)
|
||||
[ $first -eq 1 ] || echo ','
|
||||
first=0
|
||||
if [ "$n" = "negative-dark-ending" ]; then
|
||||
printf ' {"clip":"%s","creditsStartMs":-1}' "$n"
|
||||
else
|
||||
printf ' {"clip":"%s","creditsStartMs":%s}' "$n" "$TRUTH_MS"
|
||||
fi
|
||||
done
|
||||
echo; echo ']'
|
||||
} > "$OUT/truth.json"
|
||||
|
||||
echo
|
||||
printf '%-28s %-9s %s\n' CLIP DURATION TRUTH
|
||||
for f in "$OUT"/*.mp4; do
|
||||
n=$(basename "$f" .mp4)
|
||||
d=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$f")
|
||||
if [ "$n" = "negative-dark-ending" ]; then t="none"; else t="61.360"; fi
|
||||
printf '%-28s %-9.2f %s\n' "$n" "$d" "$t"
|
||||
done
|
||||
@@ -0,0 +1,54 @@
|
||||
package credits
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDebugDump(t *testing.T) {
|
||||
path := os.Getenv("MEMBY_DEBUG_CLIP")
|
||||
if path == "" {
|
||||
t.Skip("no clip")
|
||||
}
|
||||
s := &Sampler{Timeout: 2 * time.Minute}
|
||||
coarse, err := s.Sample(context.Background(), path, 0, 91380*time.Millisecond, coarseInterval)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("coarse frames=%d", len(coarse))
|
||||
for i, f := range coarse {
|
||||
t.Logf(" [%2d] pos=%6d mean=%.3f dark=%.3f edge=%.4f var=%.4f diff=%.4f score=%.3f",
|
||||
i, f.PositionMs, f.Mean, f.DarkFraction, f.EdgeDensity, f.Variance, f.Diff, creditScore(f))
|
||||
}
|
||||
idx, sep, ok := findTransition(coarse, coarseInterval)
|
||||
t.Logf("coarse transition idx=%d pos=%v sep=%.3f ok=%v", idx, func() int64 {
|
||||
if ok {
|
||||
return coarse[idx].PositionMs
|
||||
}
|
||||
return -1
|
||||
}(), sep, ok)
|
||||
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
start := time.Duration(coarse[idx].PositionMs) * time.Millisecond
|
||||
from := start - fineSpan
|
||||
if from < 0 {
|
||||
from = 0
|
||||
}
|
||||
to := start + fineSpan
|
||||
fine, err := s.Sample(context.Background(), path, from, to, fineInterval)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fmt.Println("fine from", from, "to", to, "frames", len(fine))
|
||||
fidx, fsep, fok := findTransition(fine, fineInterval)
|
||||
if fok {
|
||||
t.Logf("fine transition idx=%d pos=%d sep=%.3f", fidx, fine[fidx].PositionMs, fsep)
|
||||
} else {
|
||||
t.Logf("fine transition: NONE (fine pass did not refine)")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user