0.1.38 gateway

2026-08-14 09:40:03 +12:00
parent abc392d30b
commit 5e2ed3d12e
2847 changed files with 1072928 additions and 3783 deletions
@@ -0,0 +1,491 @@
+// Package scheduler is the gateway's reusable background-job service.
+//
+// Before it, every recurring job in the gateway was its own goroutine with its own
+// time.Ticker, its own idea of what to do when it failed and no way for an operator to
+// see whether it had run. The point of a registry is not that a ticker is hard to write —
+// it is that "when did the housekeeping last run, how long did it take, and did it work"
+// is a question no scattered ticker can answer, and that a job an operator cannot start
+// by hand is one they can only restart the container to retry.
+package scheduler
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/ponzischeme89/memby/server/internal/adminevents"
+	"github.com/ponzischeme89/memby/server/internal/store"
+)
+
+// tick is how often the scheduler looks for work. Deliberately coarse: nothing registered
+// here is time-critical, and a task declaring a five-minute interval genuinely does not
+// care whether it starts on the minute.
+const tick = 30 * time.Second
+
+// TaskFunc does the work. The detail it returns is the one line the console prints beside
+// the run — "412 events removed" — so it should describe what happened, and be empty when
+// nothing did. That emptiness is load-bearing: it is what stops a task running every ten
+// minutes announcing itself into the notification bell every ten minutes.
+type TaskFunc func(ctx context.Context) (detail string, err error)
+
+// Task is a declaration. Everything about it except the operator's overrides is code, so
+// the registry is readable as a list of what the gateway does in the background.
+type Task struct {
+	ID          string
+	Name        string
+	Description string
+	Group       string
+	Interval    time.Duration
+	// Timeout bounds one run. A housekeeping job that hangs must not hold its slot for
+	// ever, and without this the only recovery is a restart.
+	Timeout time.Duration
+	// RunOnStart runs the task once shortly after boot regardless of when it last ran.
+	// For jobs whose cost is trivial and whose value is highest immediately.
+	RunOnStart bool
+	Run        TaskFunc
+}
+
+// Status is one task as the console reads it: the declaration, the operator's overrides,
+// the last run and when the next one is due.
+type Status struct {
+	ID          string         `json:"id"`
+	Name        string         `json:"name"`
+	Description string         `json:"description"`
+	Group       string         `json:"group"`
+	Interval    int64          `json:"intervalSeconds"`
+	Enabled     bool           `json:"enabled"`
+	Running     bool           `json:"running"`
+	NextRun     *time.Time     `json:"nextRun,omitempty"`
+	LastRun     *store.TaskRun `json:"lastRun,omitempty"`
+}
+
+type registered struct {
+	task Task
+
+	mu       sync.Mutex
+	enabled  bool
+	interval time.Duration
+	running  bool
+	nextRun  time.Time
+	lastRun  *store.TaskRun
+}
+
+// effectiveInterval is the operator's override where they set one, and the declared
+// interval otherwise. Zero means "never on a schedule" — a task that only ever runs when
+// somebody presses the button, which is a legitimate declaration rather than a mistake.
+func (r *registered) effectiveInterval() time.Duration {
+	if r.interval > 0 {
+		return r.interval
+	}
+	return r.task.Interval
+}
+
+type Scheduler struct {
+	store  *store.Store
+	log    *slog.Logger
+	events *adminevents.Bus
+
+	mu    sync.RWMutex
+	tasks map[string]*registered
+	order []string
+
+	started bool
+}
+
+func New(st *store.Store, log *slog.Logger, events *adminevents.Bus) *Scheduler {
+	return &Scheduler{
+		store: st, log: log.With("component", "scheduler"), events: events,
+		tasks: map[string]*registered{},
+	}
+}
+
+// Register adds a task. Registration happens during start-up wiring, before Start, and a
+// duplicate id panics rather than silently replacing: two tasks sharing an id would share
+// a history and an enabled switch, and the operator would have no way to tell which of
+// them the console was describing.
+func (s *Scheduler) Register(task Task) {
+	if s == nil {
+		return
+	}
+	if task.ID == "" || task.Run == nil {
+		panic("scheduler: a task needs an id and a function")
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if _, exists := s.tasks[task.ID]; exists {
+		panic(fmt.Sprintf("scheduler: duplicate task id %q", task.ID))
+	}
+	if task.Timeout <= 0 {
+		task.Timeout = 10 * time.Minute
+	}
+	s.tasks[task.ID] = &registered{task: task, enabled: true}
+	s.order = append(s.order, task.ID)
+}
+
+// Start restores state and begins the loop. It returns immediately; the loop stops with
+// the context.
+//
+// The restore is what makes a restart not a re-run. Without reading the last successful
+// start time, a container replaced at three in the morning would run every overnight job
+// the moment it came up, and a deployment during the day would do it again.
+func (s *Scheduler) Start(ctx context.Context) {
+	if s == nil {
+		return
+	}
+	s.mu.Lock()
+	if s.started {
+		s.mu.Unlock()
+		return
+	}
+	s.started = true
+	s.mu.Unlock()
+
+	s.restore(ctx)
+	go s.loop(ctx)
+}
+
+func (s *Scheduler) restore(ctx context.Context) {
+	if s.store == nil {
+		return
+	}
+	if abandoned, err := s.store.AbandonRunningTasks(ctx); err != nil {
+		s.log.Warn("could not close interrupted task runs", "error", err)
+	} else if abandoned > 0 {
+		s.log.Info("closed interrupted task runs", "runs", abandoned)
+	}
+
+	settings, err := s.store.TaskSettingsAll(ctx)
+	if err != nil {
+		s.log.Warn("could not read task settings", "error", err)
+	}
+	lastSuccess, err := s.store.TaskLastSuccess(ctx)
+	if err != nil {
+		s.log.Warn("could not read task history", "error", err)
+	}
+	latest, err := s.store.LatestTaskRuns(ctx)
+	if err != nil {
+		s.log.Warn("could not read latest task runs", "error", err)
+	}
+
+	now := time.Now()
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	for id, entry := range s.tasks {
+		entry.mu.Lock()
+		if override, ok := settings[id]; ok {
+			entry.enabled = override.Enabled
+			entry.interval = time.Duration(override.IntervalSeconds) * time.Second
+		}
+		if run, ok := latest[id]; ok {
+			copied := run
+			entry.lastRun = &copied
+		}
+		interval := entry.effectiveInterval()
+		switch {
+		case entry.task.RunOnStart:
+			// Shortly, not immediately: boot is already the busiest the gateway gets.
+			entry.nextRun = now.Add(45 * time.Second)
+		case interval <= 0:
+			entry.nextRun = time.Time{}
+		case !lastSuccess[id].IsZero():
+			entry.nextRun = lastSuccess[id].Add(interval)
+		default:
+			entry.nextRun = now.Add(interval)
+		}
+		entry.mu.Unlock()
+	}
+}
+
+func (s *Scheduler) loop(ctx context.Context) {
+	ticker := time.NewTicker(tick)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			s.runDue(ctx)
+		}
+	}
+}
+
+func (s *Scheduler) runDue(ctx context.Context) {
+	now := time.Now()
+	s.mu.RLock()
+	entries := make([]*registered, 0, len(s.tasks))
+	for _, entry := range s.tasks {
+		entries = append(entries, entry)
+	}
+	s.mu.RUnlock()
+
+	for _, entry := range entries {
+		entry.mu.Lock()
+		due := entry.enabled && !entry.running &&
+			!entry.nextRun.IsZero() && !now.Before(entry.nextRun)
+		entry.mu.Unlock()
+		if due {
+			go s.execute(ctx, entry, store.TriggerSchedule)
+		}
+	}
+}
+
+// RunNow starts a task by hand. It reports whether the task started, not whether it
+// succeeded: the console watches the run history for the outcome, and a button that
+// blocked until an overnight job finished would appear to hang.
+func (s *Scheduler) RunNow(ctx context.Context, id string) error {
+	s.mu.RLock()
+	entry, ok := s.tasks[id]
+	s.mu.RUnlock()
+	if !ok {
+		return fmt.Errorf("scheduler: no task %q", id)
+	}
+	entry.mu.Lock()
+	if entry.running {
+		entry.mu.Unlock()
+		return fmt.Errorf("scheduler: %s is already running", id)
+	}
+	entry.mu.Unlock()
+	// Detached: the run outlives the admin request that asked for it, which has already
+	// been answered.
+	go s.execute(context.WithoutCancel(ctx), entry, store.TriggerManual)
+	return nil
+}
+
+func (s *Scheduler) execute(ctx context.Context, entry *registered, trigger string) {
+	entry.mu.Lock()
+	if entry.running {
+		entry.mu.Unlock()
+		return
+	}
+	entry.running = true
+	task := entry.task
+	interval := entry.effectiveInterval()
+	entry.mu.Unlock()
+
+	// The next run is scheduled from the start of this one rather than from its end, so a
+	// task that takes four minutes on a five-minute interval keeps its cadence instead of
+	// drifting a run later every time.
+	started := time.Now()
+	defer func() {
+		entry.mu.Lock()
+		entry.running = false
+		if interval > 0 {
+			next := started.Add(interval)
+			// A run that overran its own interval goes again on the next tick rather than
+			// immediately: back-to-back execution is how a slow task starves everything
+			// else sharing the pool.
+			if next.Before(time.Now()) {
+				next = time.Now().Add(tick)
+			}
+			entry.nextRun = next
+		} else {
+			entry.nextRun = time.Time{}
+		}
+		entry.mu.Unlock()
+	}()
+
+	var runID int64
+	if s.store != nil {
+		id, err := s.store.BeginTaskRun(ctx, task.ID, trigger)
+		if err != nil {
+			s.log.Warn("could not open task run", "task", task.ID, "error", err)
+		} else {
+			runID = id
+		}
+	}
+
+	runCtx, cancel := context.WithTimeout(ctx, task.Timeout)
+	detail, err := s.safeRun(runCtx, task)
+	cancel()
+	elapsed := time.Since(started)
+
+	status := store.TaskSuccess
+	failure := ""
+	if err != nil {
+		status, failure = store.TaskFailed, err.Error()
+	}
+	if s.store != nil && runID != 0 {
+		if closeErr := s.store.FinishTaskRun(
+			context.WithoutCancel(ctx), runID, status, detail, failure,
+		); closeErr != nil {
+			s.log.Warn("could not close task run", "task", task.ID, "error", closeErr)
+		}
+	}
+
+	finished := time.Now()
+	entry.mu.Lock()
+	entry.lastRun = &store.TaskRun{
+		ID: runID, TaskID: task.ID, Trigger: trigger, Status: status,
+		StartedAt: started, FinishedAt: &finished,
+		DurationMS: elapsed.Milliseconds(), Detail: detail, Error: failure,
+	}
+	entry.mu.Unlock()
+
+	s.announce(ctx, task, trigger, status, detail, failure, elapsed)
+}
+
+// safeRun turns a panicking task into a failed run. A background job is the one place a
+// panic takes the whole process down for a reason nobody is watching for, and one
+// housekeeping job with a nil map must not be able to stop the gateway serving
+// television.
+func (s *Scheduler) safeRun(ctx context.Context, task Task) (detail string, err error) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			err = fmt.Errorf("task panicked: %v", recovered)
+		}
+	}()
+	return task.Run(ctx)
+}
+
+// announce writes the log line and, when it is worth an operator's attention, publishes
+// an administrative event.
+//
+// A *failure* is always news. A success is news only when the run reports having done
+// something — the AnnounceLibrarySync rule: most passes of a housekeeping job find
+// nothing, and an hourly "nothing to do" in the notification bell is what trains an
+// operator to stop reading it.
+func (s *Scheduler) announce(
+	ctx context.Context, task Task, trigger, status, detail, failure string,
+	elapsed time.Duration,
+) {
+	log := s.log.With("task", task.ID, "trigger", trigger,
+		"duration_ms", elapsed.Milliseconds())
+	if status == store.TaskFailed {
+		log.Error("scheduled task failed", "error", failure)
+		s.events.Publish(ctx, adminevents.Event{
+			Type:     adminevents.TypeTaskFailed,
+			Severity: adminevents.SeverityError,
+			Title:    task.Name + " failed",
+			Summary:  failure,
+			Actor:    trigger,
+			Target:   task.ID,
+			Link:     "/admin/tasks",
+			Metadata: adminevents.Meta(map[string]any{
+				"taskId": task.ID, "durationMs": elapsed.Milliseconds(),
+			}),
+		})
+		return
+	}
+	if detail == "" {
+		log.Debug("scheduled task finished")
+		return
+	}
+	log.Info("scheduled task finished", "detail", detail)
+	s.events.Publish(ctx, adminevents.Event{
+		Type:     adminevents.TypeTaskCompleted,
+		Severity: adminevents.SeverityInfo,
+		Title:    task.Name,
+		Summary:  detail,
+		Actor:    trigger,
+		Target:   task.ID,
+		Link:     "/admin/tasks",
+		Metadata: adminevents.Meta(map[string]any{
+			"taskId": task.ID, "durationMs": elapsed.Milliseconds(),
+		}),
+	})
+}
+
+// SetEnabled turns a task on or off and persists the choice.
+func (s *Scheduler) SetEnabled(ctx context.Context, id string, enabled bool) error {
+	return s.override(ctx, id, func(entry *registered) {
+		entry.enabled = enabled
+		if enabled && entry.nextRun.IsZero() && entry.effectiveInterval() > 0 {
+			entry.nextRun = time.Now().Add(entry.effectiveInterval())
+		}
+	})
+}
+
+// SetInterval overrides a task's cadence. Zero restores the declared interval.
+func (s *Scheduler) SetInterval(ctx context.Context, id string, interval time.Duration) error {
+	if interval < 0 {
+		interval = 0
+	}
+	if interval > 0 && interval < time.Minute {
+		interval = time.Minute
+	}
+	return s.override(ctx, id, func(entry *registered) {
+		entry.interval = interval
+		if next := entry.effectiveInterval(); next > 0 {
+			entry.nextRun = time.Now().Add(next)
+		} else {
+			entry.nextRun = time.Time{}
+		}
+	})
+}
+
+func (s *Scheduler) override(ctx context.Context, id string, apply func(*registered)) error {
+	s.mu.RLock()
+	entry, ok := s.tasks[id]
+	s.mu.RUnlock()
+	if !ok {
+		return fmt.Errorf("scheduler: no task %q", id)
+	}
+	entry.mu.Lock()
+	apply(entry)
+	settings := store.TaskSettings{
+		TaskID: id, Enabled: entry.enabled,
+		IntervalSeconds: int(entry.interval / time.Second),
+	}
+	entry.mu.Unlock()
+	if s.store == nil {
+		return nil
+	}
+	return s.store.SetTaskSettings(ctx, settings)
+}
+
+// Snapshot is every task as the console reads it, in registration order — which is the
+// order the registry is written in, and therefore groups related jobs together without
+// anything having to sort them.
+func (s *Scheduler) Snapshot() []Status {
+	if s == nil {
+		return []Status{}
+	}
+	s.mu.RLock()
+	order := append([]string(nil), s.order...)
+	tasks := make(map[string]*registered, len(s.tasks))
+	for id, entry := range s.tasks {
+		tasks[id] = entry
+	}
+	s.mu.RUnlock()
+
+	statuses := make([]Status, 0, len(order))
+	for _, id := range order {
+		entry := tasks[id]
+		entry.mu.Lock()
+		status := Status{
+			ID: entry.task.ID, Name: entry.task.Name,
+			Description: entry.task.Description, Group: entry.task.Group,
+			Interval: int64(entry.effectiveInterval() / time.Second),
+			Enabled:  entry.enabled, Running: entry.running,
+		}
+		if !entry.nextRun.IsZero() && entry.enabled {
+			next := entry.nextRun
+			status.NextRun = &next
+		}
+		if entry.lastRun != nil {
+			last := *entry.lastRun
+			status.LastRun = &last
+		}
+		entry.mu.Unlock()
+		statuses = append(statuses, status)
+	}
+	return statuses
+}
+
+// Groups lists the distinct task groups, so the console can lay the registry out in
+// sections without a hard-coded list that drifts from the tasks themselves.
+func (s *Scheduler) Groups() []string {
+	seen := map[string]bool{}
+	groups := []string{}
+	for _, status := range s.Snapshot() {
+		if status.Group != "" && !seen[status.Group] {
+			seen[status.Group] = true
+			groups = append(groups, status.Group)
+		}
+	}
+	sort.Strings(groups)
+	return groups
+}
@@ -0,0 +1,221 @@
+package scheduler
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"sync"
+	"testing"
+	"time"
+)
+
+func quietScheduler() *Scheduler {
+	return New(nil, slog.New(slog.NewTextHandler(io.Discard, nil)), nil)
+}
+
+// The registry's guarantees are the ones worth pinning: a task cannot be registered
+// twice, a run cannot overlap itself, and a panicking job cannot take the gateway down.
+
+func TestDuplicateTaskIDsPanic(t *testing.T) {
+	// Two tasks sharing an id would share a history, an enabled switch and a Run Now
+	// button, and the console would have no way to say which it was describing.
+	sched := quietScheduler()
+	sched.Register(Task{ID: "one", Name: "One", Run: noop})
+	defer func() {
+		if recover() == nil {
+			t.Fatal("registering a duplicate id should panic at start-up, not be tolerated")
+		}
+	}()
+	sched.Register(Task{ID: "one", Name: "One again", Run: noop})
+}
+
+func TestATaskNeedsAnIDAndAFunction(t *testing.T) {
+	sched := quietScheduler()
+	defer func() {
+		if recover() == nil {
+			t.Fatal("a task with no function should be refused at registration")
+		}
+	}()
+	sched.Register(Task{ID: "broken"})
+}
+
+func TestAPanickingTaskBecomesAFailedRun(t *testing.T) {
+	// A background job is the one place a panic takes the whole process down for a reason
+	// nobody is watching for. One housekeeping job with a nil map must not be able to stop
+	// the gateway serving television.
+	sched := quietScheduler()
+	sched.Register(Task{ID: "panics", Name: "Panics", Run: func(context.Context) (string, error) {
+		panic("nil map")
+	}})
+	if err := sched.RunNow(context.Background(), "panics"); err != nil {
+		t.Fatalf("RunNow: %v", err)
+	}
+	status := waitForRun(t, sched, "panics")
+	if status.LastRun.Status != "failed" {
+		t.Fatalf("expected a failed run, got %q", status.LastRun.Status)
+	}
+	if status.LastRun.Error == "" {
+		t.Fatal("the recovered panic should be recorded as the failure")
+	}
+}
+
+func TestRunNowRefusesToStackASecondRun(t *testing.T) {
+	// The button is pressable while a job is running, and an overnight job started twice
+	// would double whatever it does.
+	release := make(chan struct{})
+	sched := quietScheduler()
+	sched.Register(Task{ID: "slow", Name: "Slow", Run: func(context.Context) (string, error) {
+		<-release
+		return "", nil
+	}})
+	if err := sched.RunNow(context.Background(), "slow"); err != nil {
+		t.Fatalf("first run: %v", err)
+	}
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) && !running(sched, "slow") {
+		time.Sleep(5 * time.Millisecond)
+	}
+	if err := sched.RunNow(context.Background(), "slow"); err == nil {
+		t.Fatal("a second RunNow while the first is in flight should be refused")
+	}
+	close(release)
+}
+
+func TestRunNowRejectsAnUnknownTask(t *testing.T) {
+	if err := quietScheduler().RunNow(context.Background(), "nothing"); err == nil {
+		t.Fatal("an unknown task id should be an error, not a silent no-op")
+	}
+}
+
+func TestAFailingTaskRecordsItsError(t *testing.T) {
+	sched := quietScheduler()
+	sched.Register(Task{ID: "fails", Name: "Fails", Run: func(context.Context) (string, error) {
+		return "", errors.New("postgres did not answer")
+	}})
+	_ = sched.RunNow(context.Background(), "fails")
+	status := waitForRun(t, sched, "fails")
+	if status.LastRun.Error != "postgres did not answer" {
+		t.Fatalf("unexpected error %q", status.LastRun.Error)
+	}
+}
+
+func TestSetIntervalZeroRestoresTheDeclaredCadence(t *testing.T) {
+	// Zero is "as the code declares", so a task whose schedule changes in a later release
+	// takes effect for every operator who never overrode it.
+	sched := quietScheduler()
+	sched.Register(Task{ID: "daily", Name: "Daily", Interval: 24 * time.Hour, Run: noop})
+	if err := sched.SetInterval(context.Background(), "daily", 2*time.Hour); err != nil {
+		t.Fatalf("SetInterval: %v", err)
+	}
+	if got := statusOf(t, sched, "daily").Interval; got != int64((2 * time.Hour).Seconds()) {
+		t.Fatalf("override not applied, got %d", got)
+	}
+	if err := sched.SetInterval(context.Background(), "daily", 0); err != nil {
+		t.Fatalf("SetInterval: %v", err)
+	}
+	if got := statusOf(t, sched, "daily").Interval; got != int64((24 * time.Hour).Seconds()) {
+		t.Fatalf("expected the declared interval back, got %d", got)
+	}
+}
+
+func TestIntervalsAreFloored(t *testing.T) {
+	// The scheduler only wakes every thirty seconds, so a one-second interval is a promise
+	// it cannot keep — and a job set to run every second by a slipped decimal point is one
+	// that would sit permanently due.
+	sched := quietScheduler()
+	sched.Register(Task{ID: "daily", Name: "Daily", Interval: 24 * time.Hour, Run: noop})
+	_ = sched.SetInterval(context.Background(), "daily", time.Second)
+	if got := statusOf(t, sched, "daily").Interval; got != 60 {
+		t.Fatalf("expected a one-minute floor, got %d", got)
+	}
+}
+
+func TestADisabledTaskAdvertisesNoNextRun(t *testing.T) {
+	// The console prints "next run" from this field; a disabled task showing a time would
+	// read as one that is going to run anyway.
+	sched := quietScheduler()
+	sched.Register(Task{ID: "daily", Name: "Daily", Interval: time.Hour, Run: noop})
+	_ = sched.SetEnabled(context.Background(), "daily", false)
+	if status := statusOf(t, sched, "daily"); status.NextRun != nil {
+		t.Fatalf("a disabled task should advertise no next run, got %v", status.NextRun)
+	}
+}
+
+func TestSnapshotKeepsRegistrationOrder(t *testing.T) {
+	// The registry is written as a readable list, and the console lays it out in that
+	// order; a map iteration would reshuffle the page on every poll.
+	sched := quietScheduler()
+	for _, id := range []string{"a", "b", "c", "d"} {
+		sched.Register(Task{ID: id, Name: id, Run: noop})
+	}
+	for attempt := 0; attempt < 5; attempt++ {
+		snapshot := sched.Snapshot()
+		for i, id := range []string{"a", "b", "c", "d"} {
+			if snapshot[i].ID != id {
+				t.Fatalf("order changed: %v", snapshot)
+			}
+		}
+	}
+}
+
+func TestGroupsAreDistinctAndSorted(t *testing.T) {
+	sched := quietScheduler()
+	sched.Register(Task{ID: "a", Group: "System", Run: noop})
+	sched.Register(Task{ID: "b", Group: "Housekeeping", Run: noop})
+	sched.Register(Task{ID: "c", Group: "System", Run: noop})
+	sched.Register(Task{ID: "d", Run: noop}) // ungrouped
+	groups := sched.Groups()
+	if len(groups) != 2 || groups[0] != "Housekeeping" || groups[1] != "System" {
+		t.Fatalf("unexpected groups %v", groups)
+	}
+}
+
+func TestNilSchedulerIsSafe(t *testing.T) {
+	// Every unit test in the api package builds a Server with no scheduler, and the admin
+	// handler calls straight through to it.
+	var sched *Scheduler
+	if len(sched.Snapshot()) != 0 {
+		t.Fatal("a nil scheduler should answer with an empty registry")
+	}
+	sched.Register(Task{ID: "a", Run: noop})
+	sched.Start(context.Background())
+}
+
+func noop(context.Context) (string, error) { return "", nil }
+
+func running(sched *Scheduler, id string) bool {
+	for _, status := range sched.Snapshot() {
+		if status.ID == id {
+			return status.Running
+		}
+	}
+	return false
+}
+
+func statusOf(t *testing.T, sched *Scheduler, id string) Status {
+	t.Helper()
+	for _, status := range sched.Snapshot() {
+		if status.ID == id {
+			return status
+		}
+	}
+	t.Fatalf("no task %q", id)
+	return Status{}
+}
+
+func waitForRun(t *testing.T, sched *Scheduler, id string) Status {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		status := statusOf(t, sched, id)
+		if status.LastRun != nil && !status.Running {
+			return status
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("task %q did not finish", id)
+	return Status{}
+}
+
+var _ sync.Locker = (*sync.Mutex)(nil)