Homelabtoolkit v1
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
"""Service layer for the User Favourites feature.
|
||||
|
||||
Every module here depends only on an injected ``client`` object exposing four
|
||||
async methods (``get``, ``get_all``, ``post``, ``delete``). Production code passes
|
||||
an adapter around the existing Emby helpers in ``app.py``; tests pass a fake. This
|
||||
keeps the services free of FastAPI/Emby/Pillow import weight and fully unit
|
||||
testable.
|
||||
"""
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""SQLite database layer for HomelabToolkit.
|
||||
|
||||
Boring and database-first: a single local SQLite file holds the music-library
|
||||
scan results, MusicBrainz cache, and collection-completeness data. The schema is
|
||||
created idempotently at startup (``init_db``), which doubles as the migration —
|
||||
every statement uses ``IF NOT EXISTS``.
|
||||
|
||||
Connections are opened per call (cheap for SQLite) so each thread/request gets
|
||||
its own handle. WAL mode lets the UI keep reading while a scan job writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(os.environ.get("DB_PATH", "cache/homelab.db"))
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connect():
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(DB_PATH), timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.execute("PRAGMA busy_timeout=8000")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS library_scan_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kind TEXT NOT NULL, -- 'scan' | 'metadata'
|
||||
status TEXT NOT NULL, -- 'running' | 'completed' | 'failed'
|
||||
started_at TEXT,
|
||||
completed_at TEXT,
|
||||
error_message TEXT,
|
||||
files_scanned INTEGER DEFAULT 0,
|
||||
albums_found INTEGER DEFAULT 0,
|
||||
artists_found INTEGER DEFAULT 0,
|
||||
progress TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS library_artists (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
name_normalized TEXT NOT NULL UNIQUE,
|
||||
mbid TEXT,
|
||||
is_various INTEGER DEFAULT 0,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS library_albums (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artist_id INTEGER NOT NULL REFERENCES library_artists(id) ON DELETE CASCADE,
|
||||
title TEXT NOT NULL,
|
||||
title_normalized TEXT NOT NULL,
|
||||
year INTEGER DEFAULT 0,
|
||||
mbid TEXT,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TEXT,
|
||||
updated_at TEXT,
|
||||
UNIQUE(artist_id, title_normalized, year)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS library_tracks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
album_id INTEGER NOT NULL REFERENCES library_albums(id) ON DELETE CASCADE,
|
||||
title TEXT,
|
||||
track_number INTEGER,
|
||||
disc_number INTEGER,
|
||||
file_path TEXT NOT NULL UNIQUE,
|
||||
file_mtime REAL,
|
||||
file_size INTEGER,
|
||||
mbid TEXT,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
last_seen_scan_id INTEGER,
|
||||
created_at TEXT,
|
||||
updated_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS external_artist_matches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artist_id INTEGER NOT NULL UNIQUE REFERENCES library_artists(id) ON DELETE CASCADE,
|
||||
mb_artist_mbid TEXT,
|
||||
mb_artist_name TEXT,
|
||||
confidence REAL DEFAULT 0,
|
||||
status TEXT, -- 'matched' | 'not_found' | 'error' | 'skipped'
|
||||
checked_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS external_releases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artist_id INTEGER NOT NULL REFERENCES library_artists(id) ON DELETE CASCADE,
|
||||
mb_release_group_mbid TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
title_normalized TEXT,
|
||||
first_release_year INTEGER DEFAULT 0,
|
||||
primary_type TEXT,
|
||||
secondary_types TEXT, -- JSON array
|
||||
fetched_at TEXT,
|
||||
UNIQUE(artist_id, mb_release_group_mbid)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS collection_completeness (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artist_id INTEGER NOT NULL REFERENCES library_artists(id) ON DELETE CASCADE,
|
||||
release_group_mbid TEXT NOT NULL,
|
||||
local_album_id INTEGER REFERENCES library_albums(id) ON DELETE SET NULL,
|
||||
title TEXT,
|
||||
year INTEGER DEFAULT 0,
|
||||
status TEXT NOT NULL, -- owned|probably_owned|missing|ignored|uncertain
|
||||
confidence REAL DEFAULT 0,
|
||||
reason TEXT,
|
||||
source TEXT DEFAULT 'musicbrainz',
|
||||
manual_override INTEGER DEFAULT 0,
|
||||
updated_at TEXT,
|
||||
UNIQUE(artist_id, release_group_mbid)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mb_cache (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
payload TEXT,
|
||||
fetched_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_albums_artist ON library_albums(artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tracks_album ON library_tracks(album_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tracks_path ON library_tracks(file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_releases_artist ON external_releases(artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_completeness_artist ON collection_completeness(artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_completeness_status ON collection_completeness(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_runs_kind ON library_scan_runs(kind, id);
|
||||
"""
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
with connect() as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Favourites collection detection and item operations.
|
||||
|
||||
A "favourites collection" is any Emby collection (a ``BoxSet``) named
|
||||
``"{UserName} Favorites"``. Unlike playlists, collection membership is keyed by
|
||||
the item's own id (there is no per-entry id), so additions and removals operate
|
||||
on item ids via ``/Collections/{id}/Items``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
FAVORITES_SUFFIX = "Favorites"
|
||||
|
||||
# Fields requested for every collection/candidate item so the recommendation
|
||||
# engine and the UI have what they need in one round trip.
|
||||
ITEM_FIELDS = (
|
||||
"Genres,Studios,Tags,People,ProductionYear,RunTimeTicks,"
|
||||
"SeriesName,CommunityRating,MediaType,Overview"
|
||||
)
|
||||
|
||||
# Emby stores runtime as 100ns ticks. 1 minute = 60 * 1e7 ticks.
|
||||
_TICKS_PER_MINUTE = 600_000_000
|
||||
|
||||
|
||||
def parse_favorites_owner(collection_name: str | None) -> str | None:
|
||||
"""Return the owner name from ``"{Name} Favorites"`` or ``None``.
|
||||
|
||||
Matching is case-insensitive on the suffix but preserves the owner's casing.
|
||||
``"Favorites"`` on its own (no owner) is not a per-user favourites collection.
|
||||
"""
|
||||
if not collection_name:
|
||||
return None
|
||||
name = collection_name.strip()
|
||||
suffix = " " + FAVORITES_SUFFIX
|
||||
if len(name) <= len(suffix):
|
||||
return None
|
||||
if name[-len(suffix):].casefold() != suffix.casefold():
|
||||
return None
|
||||
owner = name[: -len(suffix)].strip()
|
||||
return owner or None
|
||||
|
||||
|
||||
def normalize_item(raw: dict) -> dict:
|
||||
"""Flatten an Emby item into the shape the feature uses everywhere."""
|
||||
user_data = raw.get("UserData") or {}
|
||||
ticks = raw.get("RunTimeTicks")
|
||||
runtime_minutes = round(ticks / _TICKS_PER_MINUTE) if ticks else None
|
||||
people = raw.get("People") or []
|
||||
return {
|
||||
"id": raw.get("Id", ""),
|
||||
"title": raw.get("Name", ""),
|
||||
"type": raw.get("Type", ""),
|
||||
"media_type": raw.get("MediaType", "") or raw.get("Type", ""),
|
||||
"year": raw.get("ProductionYear"),
|
||||
"runtime_minutes": runtime_minutes,
|
||||
"watched": bool(user_data.get("Played", False)),
|
||||
"community_rating": raw.get("CommunityRating"),
|
||||
"genres": [g for g in (raw.get("Genres") or []) if g],
|
||||
"studios": [s.get("Name") for s in (raw.get("Studios") or []) if s.get("Name")],
|
||||
"tags": [t for t in (raw.get("Tags") or []) if t],
|
||||
"series_name": raw.get("SeriesName"),
|
||||
"directors": [p.get("Name") for p in people if p.get("Type") == "Director" and p.get("Name")],
|
||||
"actors": [p.get("Name") for p in people if p.get("Type") == "Actor" and p.get("Name")],
|
||||
}
|
||||
|
||||
|
||||
async def find_all_collections(client) -> list[dict]:
|
||||
"""Return every Emby collection (BoxSet), sorted by name.
|
||||
|
||||
``owner_name`` is the parsed ``"{Name} Favorites"`` owner (or ``None``), and
|
||||
``is_favorites`` flags whether the collection follows that convention.
|
||||
``[{"collection_id", "collection_name", "owner_name", "is_favorites", "item_count"}]``
|
||||
"""
|
||||
data = await client.get_all(
|
||||
"/Items",
|
||||
{
|
||||
"IncludeItemTypes": "BoxSet",
|
||||
"Recursive": "true",
|
||||
"Fields": "ChildCount",
|
||||
},
|
||||
)
|
||||
collections = []
|
||||
for raw in data:
|
||||
owner = parse_favorites_owner(raw.get("Name"))
|
||||
collections.append(
|
||||
{
|
||||
"collection_id": raw.get("Id", ""),
|
||||
"collection_name": raw.get("Name", ""),
|
||||
"owner_name": owner,
|
||||
"is_favorites": owner is not None,
|
||||
"item_count": raw.get("ChildCount"),
|
||||
}
|
||||
)
|
||||
collections.sort(key=lambda c: (c["collection_name"] or "").casefold())
|
||||
return collections
|
||||
|
||||
|
||||
async def find_favorites_collections(client) -> list[dict]:
|
||||
"""Return only the detected ``"{Name} Favorites"`` collections."""
|
||||
return [c for c in await find_all_collections(client) if c["is_favorites"]]
|
||||
|
||||
|
||||
async def find_user_favorites_collection(client, user_name: str) -> dict | None:
|
||||
"""Find the ``"{user_name} Favorites"`` collection, or ``None``."""
|
||||
if not user_name:
|
||||
return None
|
||||
target = user_name.strip().casefold()
|
||||
for collection in await find_favorites_collections(client):
|
||||
if collection["owner_name"].casefold() == target:
|
||||
return collection
|
||||
return None
|
||||
|
||||
|
||||
async def list_collection_items(client, collection_id: str, user_id: str) -> list[dict]:
|
||||
"""List a collection's items with watched status resolved for ``user_id``.
|
||||
|
||||
Collection children are retrieved via ``ParentId``. Passing the user scope
|
||||
makes Emby populate ``UserData.Played`` for that specific user, which is what
|
||||
makes the watched flag user-specific.
|
||||
"""
|
||||
data = await client.get(
|
||||
f"/Users/{user_id}/Items",
|
||||
{
|
||||
"ParentId": collection_id,
|
||||
"Fields": ITEM_FIELDS,
|
||||
"EnableUserData": "true",
|
||||
},
|
||||
)
|
||||
items = data.get("Items", []) if isinstance(data, dict) else (data or [])
|
||||
return [normalize_item(raw) for raw in items]
|
||||
|
||||
|
||||
async def add_collection_items(client, collection_id: str, item_ids: list[str]):
|
||||
"""Add items to a collection by id. No-op for an empty list."""
|
||||
if not item_ids:
|
||||
return None
|
||||
return await client.post(
|
||||
f"/Collections/{collection_id}/Items",
|
||||
{"Ids": ",".join(item_ids)},
|
||||
)
|
||||
|
||||
|
||||
async def remove_collection_items(client, collection_id: str, item_ids: list[str]):
|
||||
"""Remove items from a collection by id. No-op when empty."""
|
||||
if not item_ids:
|
||||
return None
|
||||
return await client.delete(
|
||||
f"/Collections/{collection_id}/Items",
|
||||
{"Ids": ",".join(item_ids)},
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Emby user lookups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def _normalize_user(raw: dict) -> dict:
|
||||
return {"id": raw.get("Id", ""), "name": raw.get("Name", "")}
|
||||
|
||||
|
||||
async def fetch_users(client) -> list[dict]:
|
||||
"""Return all Emby users as ``[{"id", "name"}]``.
|
||||
|
||||
``GET /Users`` returns a bare JSON array on most Emby builds, but some return
|
||||
a ``{"Items": [...]}`` envelope. Handle both.
|
||||
"""
|
||||
data = await client.get("/Users")
|
||||
raw_users = data.get("Items", []) if isinstance(data, dict) else (data or [])
|
||||
return [_normalize_user(u) for u in raw_users if u.get("Id")]
|
||||
|
||||
|
||||
async def resolve_user_id_by_name(client, name: str) -> str | None:
|
||||
"""Case-insensitive lookup of a user id by display name."""
|
||||
if not name:
|
||||
return None
|
||||
target = name.strip().casefold()
|
||||
for user in await fetch_users(client):
|
||||
if user["name"].casefold() == target:
|
||||
return user["id"]
|
||||
return None
|
||||
|
||||
|
||||
async def get_user(client, user_id: str) -> dict | None:
|
||||
"""Return ``{"id", "name"}`` for a user id, or ``None`` if not found."""
|
||||
if not user_id:
|
||||
return None
|
||||
for user in await fetch_users(client):
|
||||
if user["id"] == user_id:
|
||||
return user
|
||||
return None
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Per-user watch history.
|
||||
|
||||
Every query here is scoped to a single ``user_id`` via the ``/Users/{id}/Items``
|
||||
endpoint, so one user's history is never mixed with another's.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .emby_collections import ITEM_FIELDS, normalize_item
|
||||
|
||||
# Movies and series are what we recommend; episodes are folded into their series.
|
||||
WATCHED_ITEM_TYPES = "Movie,Series"
|
||||
|
||||
|
||||
async def get_watched_items(client, user_id: str, item_types: str = WATCHED_ITEM_TYPES) -> list[dict]:
|
||||
"""Return the user's played items (normalized) for the given types."""
|
||||
if not user_id:
|
||||
return []
|
||||
raw_items = await client.get_all(
|
||||
f"/Users/{user_id}/Items",
|
||||
{
|
||||
"Recursive": "true",
|
||||
"IsPlayed": "true",
|
||||
"Filters": "IsPlayed",
|
||||
"IncludeItemTypes": item_types,
|
||||
"Fields": ITEM_FIELDS,
|
||||
"EnableUserData": "true",
|
||||
},
|
||||
)
|
||||
return [normalize_item(raw) for raw in raw_items]
|
||||
|
||||
|
||||
async def get_watched_item_ids(client, user_id: str, item_types: str = WATCHED_ITEM_TYPES) -> set[str]:
|
||||
"""Return the set of item ids the user has watched."""
|
||||
return {item["id"] for item in await get_watched_items(client, user_id, item_types) if item["id"]}
|
||||
|
||||
|
||||
async def is_item_watched(client, user_id: str, item_id: str) -> bool:
|
||||
"""Whether ``user_id`` has played ``item_id`` (user-specific)."""
|
||||
if not (user_id and item_id):
|
||||
return False
|
||||
data = await client.get(
|
||||
f"/Users/{user_id}/Items",
|
||||
{"Ids": item_id, "EnableUserData": "true"},
|
||||
)
|
||||
items = data.get("Items", []) if isinstance(data, dict) else (data or [])
|
||||
if not items:
|
||||
return False
|
||||
user_data = items[0].get("UserData") or {}
|
||||
return bool(user_data.get("Played", False))
|
||||
@@ -0,0 +1,311 @@
|
||||
"""Orchestration for the User Favourites feature.
|
||||
|
||||
Combines the user / collection / watch-history / recommendation services into the
|
||||
operations the API exposes: list users, browse collections, view a collection's
|
||||
items, clean up watched items, and regenerate recommendations. Dry-run is the
|
||||
default for both destructive (cleanup) and bulk (regenerate) actions, and the
|
||||
destructive actions only run on a user's own ``"{Name} Favorites"`` collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from . import emby_collections, emby_users, emby_watch_history, recommendations
|
||||
from .recommendations import DEFAULT_TARGET_SIZE
|
||||
|
||||
LOG_DIR = Path("logs")
|
||||
_logger: logging.Logger | None = None
|
||||
|
||||
|
||||
class FavoritesError(Exception):
|
||||
"""Raised for expected, user-facing problems (missing user/collection, etc.).
|
||||
|
||||
Carries an HTTP-ish ``status`` so the route layer can map it cleanly.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, status: int = 400):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status = status
|
||||
|
||||
|
||||
def _get_logger() -> logging.Logger:
|
||||
"""Lazily configure a dedicated favourites logger with a file handler.
|
||||
|
||||
Guards against duplicate handlers across uvicorn reloads.
|
||||
"""
|
||||
global _logger
|
||||
if _logger is not None:
|
||||
return _logger
|
||||
log = logging.getLogger("homelabtoolkit.favorites")
|
||||
log.setLevel(logging.INFO)
|
||||
if not any(isinstance(h, logging.FileHandler) for h in log.handlers):
|
||||
try:
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
handler = logging.FileHandler(LOG_DIR / "favorites.log", encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
log.addHandler(handler)
|
||||
except OSError:
|
||||
# Filesystem unavailable (read-only container): fall back to console.
|
||||
pass
|
||||
_logger = log
|
||||
return log
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _log_action(user_name, collection_name, item, reason, action):
|
||||
record = {
|
||||
"timestamp": _now_iso(),
|
||||
"action": action,
|
||||
"user": user_name,
|
||||
"collection": collection_name,
|
||||
"title": item.get("title", ""),
|
||||
"item_id": item.get("id", ""),
|
||||
"reason": reason,
|
||||
}
|
||||
_get_logger().info(
|
||||
"%s | user=%s | collection=%s | item=%s (%s) | reason=%s",
|
||||
action, user_name, collection_name, record["title"], record["item_id"], reason,
|
||||
)
|
||||
return record
|
||||
|
||||
|
||||
async def _resolve_collection(client, collection_id: str, user_id: str) -> tuple[dict, dict]:
|
||||
"""Resolve ``(user, collection)`` by id or raise :class:`FavoritesError`."""
|
||||
user = await emby_users.get_user(client, user_id)
|
||||
if not user:
|
||||
raise FavoritesError(f"No Emby user found for id {user_id!r}.", status=404)
|
||||
collection = next(
|
||||
(c for c in await emby_collections.find_all_collections(client) if c["collection_id"] == collection_id),
|
||||
None,
|
||||
)
|
||||
if not collection:
|
||||
raise FavoritesError(f"No collection found for id {collection_id!r}.", status=404)
|
||||
return user, collection
|
||||
|
||||
|
||||
async def list_favorites_users(client) -> list[dict]:
|
||||
"""Users that have a detected ``"{Name} Favorites"`` collection."""
|
||||
users = await emby_users.fetch_users(client)
|
||||
by_name = {u["name"].casefold(): u for u in users}
|
||||
result = []
|
||||
for collection in await emby_collections.find_favorites_collections(client):
|
||||
user = by_name.get(collection["owner_name"].casefold())
|
||||
if not user:
|
||||
continue # orphan collection with no matching user
|
||||
result.append(
|
||||
{
|
||||
"user_id": user["id"],
|
||||
"user_name": user["name"],
|
||||
"collection_id": collection["collection_id"],
|
||||
"collection_name": collection["collection_name"],
|
||||
"item_count": collection.get("item_count"),
|
||||
}
|
||||
)
|
||||
result.sort(key=lambda r: r["user_name"].casefold())
|
||||
return result
|
||||
|
||||
|
||||
async def list_collections_overview(client) -> dict:
|
||||
"""All collections plus all users, for the browse pickers.
|
||||
|
||||
Each collection is annotated with ``owner_user_id`` when its ``"{Name}
|
||||
Favorites"`` owner resolves to a real Emby user.
|
||||
"""
|
||||
users = await emby_users.fetch_users(client)
|
||||
by_name = {u["name"].casefold(): u for u in users}
|
||||
collections = []
|
||||
for collection in await emby_collections.find_all_collections(client):
|
||||
owner = by_name.get(collection["owner_name"].casefold()) if collection["owner_name"] else None
|
||||
collections.append({**collection, "owner_user_id": owner["id"] if owner else None})
|
||||
return {"collections": collections, "users": users}
|
||||
|
||||
|
||||
async def get_collection_items_view(client, collection_id: str, user_id: str) -> dict:
|
||||
"""View any collection's items with watched status resolved for ``user_id``.
|
||||
|
||||
Watched status is user-specific. ``actions_enabled`` is true only when the
|
||||
collection is a ``"{Name} Favorites"`` collection and the selected user is its
|
||||
owner, so cleanup/regenerate never touch shared or themed collections.
|
||||
"""
|
||||
user = await emby_users.get_user(client, user_id)
|
||||
if not user:
|
||||
raise FavoritesError(f"No Emby user found for id {user_id!r}.", status=404)
|
||||
|
||||
collection = next(
|
||||
(c for c in await emby_collections.find_all_collections(client) if c["collection_id"] == collection_id),
|
||||
None,
|
||||
)
|
||||
if not collection:
|
||||
raise FavoritesError(f"No collection found for id {collection_id!r}.", status=404)
|
||||
|
||||
items = await emby_collections.list_collection_items(client, collection_id, user_id)
|
||||
watched_count = sum(1 for i in items if i["watched"])
|
||||
|
||||
# Cleanup/regenerate are available for any collection. They act on the
|
||||
# selected user's watch data; removal affects the shared collection itself.
|
||||
actions_enabled = True
|
||||
actions_reason = ""
|
||||
|
||||
return {
|
||||
"collection_id": collection_id,
|
||||
"collection_name": collection["collection_name"],
|
||||
"is_favorites": collection["is_favorites"],
|
||||
"owner_name": collection["owner_name"],
|
||||
"user_id": user["id"],
|
||||
"user_name": user["name"],
|
||||
"actions_enabled": actions_enabled,
|
||||
"actions_reason": actions_reason,
|
||||
"items": items,
|
||||
"summary": {
|
||||
"current_count": len(items),
|
||||
"watched_count": watched_count,
|
||||
"unwatched_count": len(items) - watched_count,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def cleanup_watched(client, collection_id: str, user_id: str, dry_run: bool = True) -> dict:
|
||||
"""Preview or remove items the selected user has already watched.
|
||||
|
||||
Which items are "watched" is resolved per user (via ``UserData.Played`` for
|
||||
this user only). The removal itself operates on the collection, which is
|
||||
shared, so removed items leave the collection for everyone.
|
||||
"""
|
||||
user, collection = await _resolve_collection(client, collection_id, user_id)
|
||||
items = await emby_collections.list_collection_items(client, collection_id, user_id)
|
||||
|
||||
watched = [i for i in items if i["watched"]]
|
||||
|
||||
records = []
|
||||
applied = False
|
||||
if not dry_run and watched:
|
||||
await emby_collections.remove_collection_items(
|
||||
client, collection["collection_id"], [i["id"] for i in watched]
|
||||
)
|
||||
applied = True
|
||||
records = [
|
||||
_log_action(user["name"], collection["collection_name"], i, "watched-by-user", "removed")
|
||||
for i in watched
|
||||
]
|
||||
|
||||
final_count = len(items) - (len(watched) if applied else 0)
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"applied": applied,
|
||||
"user_id": user["id"],
|
||||
"user_name": user["name"],
|
||||
"collection_name": collection["collection_name"],
|
||||
"watched_found": len(watched),
|
||||
"removed": [
|
||||
{"title": i["title"], "item_id": i["id"], "reason": "watched-by-user"}
|
||||
for i in watched
|
||||
],
|
||||
"log": records,
|
||||
"summary": {
|
||||
"current_count": len(items),
|
||||
"watched_count": len(watched),
|
||||
"removed_count": len(watched) if applied else 0,
|
||||
"final_count": final_count,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def regenerate(
|
||||
client,
|
||||
collection_id: str,
|
||||
user_id: str,
|
||||
dry_run: bool = True,
|
||||
target_size: int = DEFAULT_TARGET_SIZE,
|
||||
) -> dict:
|
||||
"""Preview or add recommendations derived from the selected user's history.
|
||||
|
||||
Candidates exclude items the user has watched and items already in the
|
||||
collection, and are never watched items. New items are added until the
|
||||
collection reaches ``target_size``.
|
||||
"""
|
||||
if target_size < 0:
|
||||
raise FavoritesError("targetSize must be zero or greater.", status=400)
|
||||
|
||||
user, collection = await _resolve_collection(client, collection_id, user_id)
|
||||
items = await emby_collections.list_collection_items(client, collection_id, user_id)
|
||||
current_ids = {i["id"] for i in items if i["id"]}
|
||||
|
||||
watched_items = await emby_watch_history.get_watched_items(client, user_id)
|
||||
watched_ids = {i["id"] for i in watched_items if i["id"]}
|
||||
profile = recommendations.build_profile(watched_items)
|
||||
|
||||
summary_base = {
|
||||
"current_count": len(items),
|
||||
"watched_history_count": len(watched_items),
|
||||
"target_size": target_size,
|
||||
}
|
||||
|
||||
if profile.is_empty:
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"applied": False,
|
||||
"user_id": user["id"],
|
||||
"user_name": user["name"],
|
||||
"collection_name": collection["collection_name"],
|
||||
"message": "No watch history for this user yet, so no recommendations can be made.",
|
||||
"recommended": [],
|
||||
"log": [],
|
||||
"summary": {**summary_base, "recommended_count": 0, "added_count": 0, "final_count": len(items)},
|
||||
}
|
||||
|
||||
exclude_ids = current_ids | watched_ids
|
||||
candidates = await recommendations.build_candidates(client, user_id, profile, exclude_ids)
|
||||
ranked = recommendations.rank_candidates(candidates, profile)
|
||||
|
||||
need = max(0, target_size - len(items))
|
||||
chosen = ranked[:need]
|
||||
|
||||
records = []
|
||||
applied = False
|
||||
if not dry_run and chosen:
|
||||
await emby_collections.add_collection_items(
|
||||
client, collection["collection_id"], [c["id"] for c in chosen]
|
||||
)
|
||||
applied = True
|
||||
records = [
|
||||
_log_action(
|
||||
user["name"], collection["collection_name"], c,
|
||||
f"recommended (score={c['score']})", "added",
|
||||
)
|
||||
for c in chosen
|
||||
]
|
||||
|
||||
final_count = len(items) + (len(chosen) if applied else 0)
|
||||
return {
|
||||
"dry_run": dry_run,
|
||||
"applied": applied,
|
||||
"user_id": user["id"],
|
||||
"user_name": user["name"],
|
||||
"collection_name": collection["collection_name"],
|
||||
"recommended": [
|
||||
{
|
||||
"title": c["title"],
|
||||
"item_id": c["id"],
|
||||
"type": c["type"],
|
||||
"year": c["year"],
|
||||
"runtime_minutes": c["runtime_minutes"],
|
||||
"community_rating": c["community_rating"],
|
||||
"score": c["score"],
|
||||
}
|
||||
for c in chosen
|
||||
],
|
||||
"log": records,
|
||||
"summary": {
|
||||
**summary_base,
|
||||
"recommended_count": len(chosen),
|
||||
"added_count": len(chosen) if applied else 0,
|
||||
"final_count": final_count,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
"""Music library maintenance, refactored from the original ``music-covers.py`` CLI.
|
||||
|
||||
The interactive terminal UI is gone; what remains is pure, importable logic the
|
||||
web app drives. Two entry points matter:
|
||||
|
||||
* :func:`scan_library` — read-only analysis. Walks ``MUSIC_ROOT`` and reports, per
|
||||
album, what each maintenance mode *would* do. Safe to call any time.
|
||||
* :func:`process_library` — performs the work. Honours ``dry_run`` (the web UI
|
||||
default) so nothing is renamed, deleted, or downloaded unless the caller opts in.
|
||||
|
||||
Both are synchronous (filesystem + blocking HTTP); call them from FastAPI via
|
||||
``asyncio.to_thread``. Progress is reported through an optional ``log`` callback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import requests
|
||||
|
||||
try: # Optional: only needed for the "find missing year/cover" online lookups.
|
||||
import musicbrainzngs
|
||||
|
||||
musicbrainzngs.set_useragent("HomelabToolkit", "1.0", "homelab-toolkit@example.com")
|
||||
_HAS_MUSICBRAINZ = True
|
||||
except Exception: # pragma: no cover - optional dependency
|
||||
_HAS_MUSICBRAINZ = False
|
||||
|
||||
from mutagen import File as MutagenFile, MutagenError
|
||||
|
||||
MUSIC_ROOT = Path(os.environ.get("MUSIC_ROOT", r"\\Matt-htpc\d\Music"))
|
||||
|
||||
COVER_NAME_PRIORITY = ("cover.jpg", "folder.jpg", "front.jpg")
|
||||
COVER_NAMES = set(COVER_NAME_PRIORITY)
|
||||
COVER_MISSING_MARKER = ".cover-not-found"
|
||||
LYRICS_MISSING_MARKER = ".lyrics-not-found"
|
||||
LYRICS_SIDECAR_EXTENSIONS = {".lrc", ".txt"}
|
||||
AUDIO_EXTENSIONS = {".mp3", ".flac", ".m4a"}
|
||||
YEAR_ALBUM_FOLDER_RE = re.compile(r"^\s*(\d{4})\s*-\s*(.+?)\s*$")
|
||||
|
||||
LogCallback = Callable[[dict], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessOptions:
|
||||
folder_cleanup: bool = False
|
||||
rename: bool = False
|
||||
file_cleanup: bool = False
|
||||
lyrics: bool = False
|
||||
covers: bool = True
|
||||
dry_run: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ProcessOptions":
|
||||
return cls(
|
||||
folder_cleanup=bool(data.get("folder_cleanup", False)),
|
||||
rename=bool(data.get("rename", False)),
|
||||
file_cleanup=bool(data.get("file_cleanup", False)),
|
||||
lyrics=bool(data.get("lyrics", False)),
|
||||
covers=bool(data.get("covers", True)),
|
||||
dry_run=bool(data.get("dry_run", True)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Recorder:
|
||||
"""Collects structured action records and forwards them to an optional sink."""
|
||||
|
||||
sink: LogCallback | None = None
|
||||
actions: list[dict] = field(default_factory=list)
|
||||
counts: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def emit(self, level: str, action: str, message: str, **extra) -> None:
|
||||
record = {"level": level, "action": action, "message": message, **extra}
|
||||
self.actions.append(record)
|
||||
self.counts[action] = self.counts.get(action, 0) + 1
|
||||
if self.sink:
|
||||
self.sink(record)
|
||||
|
||||
|
||||
# ── pure string helpers ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def clean_name(text: str) -> str:
|
||||
text = re.sub(r"\[(.*?)\]|\((.*?)\)", "", text)
|
||||
text = text.replace("_", " ").replace("-", " ")
|
||||
return " ".join(text.split()).strip()
|
||||
|
||||
|
||||
def clean_album_folder_name(text: str) -> str:
|
||||
match = YEAR_ALBUM_FOLDER_RE.match(text)
|
||||
if match:
|
||||
text = match.group(2)
|
||||
return clean_name(text)
|
||||
|
||||
|
||||
def get_year_from_album_folder_name(text: str) -> str | None:
|
||||
match = YEAR_ALBUM_FOLDER_RE.match(text)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def safe_filename(text: str) -> str:
|
||||
text = re.sub(r'[<>:"/\\|?*]', "", text)
|
||||
text = text.strip().rstrip(".")
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def clean_track_number(value) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
text = str(value[0] if isinstance(value, list) else value).strip()
|
||||
text = text.split("/")[0].strip()
|
||||
return text.zfill(2) if text.isdigit() else None
|
||||
|
||||
|
||||
def extract_year(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"\b(19\d{2}|20\d{2})\b", str(value))
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
# ── metadata reading ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _load_audio(path: Path):
|
||||
try:
|
||||
return MutagenFile(path, easy=True)
|
||||
except (MutagenError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _first_tag(audio, names):
|
||||
for name in names:
|
||||
value = audio.get(name)
|
||||
if value:
|
||||
return str(value[0]).strip()
|
||||
return None
|
||||
|
||||
|
||||
def get_album_metadata_from_files(album_folder: Path):
|
||||
for file in album_folder.iterdir():
|
||||
if not file.is_file() or file.suffix.lower() not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
audio = _load_audio(file)
|
||||
if audio is None:
|
||||
continue
|
||||
artist = _first_tag(audio, ["albumartist", "artist"])
|
||||
album = _first_tag(audio, ["album"])
|
||||
year = extract_year(_first_tag(audio, ["date", "originaldate", "year"]))
|
||||
if artist or album or year:
|
||||
return artist, album, year
|
||||
return None, None, None
|
||||
|
||||
|
||||
def get_track_metadata(path: Path):
|
||||
audio = _load_audio(path)
|
||||
if audio is None:
|
||||
return None, None, None, None
|
||||
artist = _first_tag(audio, ["artist", "albumartist"])
|
||||
album = _first_tag(audio, ["album"])
|
||||
title = _first_tag(audio, ["title"])
|
||||
duration = None
|
||||
info = getattr(audio, "info", None)
|
||||
if info and getattr(info, "length", None):
|
||||
duration = round(info.length)
|
||||
return artist, album, title, duration
|
||||
|
||||
|
||||
# ── album inspection ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cover_to_keep(album_folder: Path) -> str | None:
|
||||
existing = {p.name.lower() for p in album_folder.iterdir() if p.is_file()}
|
||||
for name in COVER_NAME_PRIORITY:
|
||||
if name in existing:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _has_cover(album_folder: Path) -> bool:
|
||||
existing = {p.name.lower() for p in album_folder.iterdir() if p.is_file()}
|
||||
return any(name in existing for name in COVER_NAMES)
|
||||
|
||||
|
||||
def _should_keep_file(path: Path, cover_to_keep: str | None) -> bool:
|
||||
name = path.name.lower()
|
||||
suffix = path.suffix.lower()
|
||||
return (
|
||||
suffix in AUDIO_EXTENSIONS
|
||||
or suffix in LYRICS_SIDECAR_EXTENSIONS
|
||||
or name == cover_to_keep
|
||||
)
|
||||
|
||||
|
||||
def _has_lyrics(audio_file: Path) -> bool:
|
||||
return any(
|
||||
audio_file.with_suffix(extension).exists()
|
||||
for extension in LYRICS_SIDECAR_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
def analyze_album(album_folder: Path) -> dict:
|
||||
"""Read-only summary of an album folder and the pending maintenance work."""
|
||||
tag_artist, tag_album, tag_year = get_album_metadata_from_files(album_folder)
|
||||
artist = tag_artist or clean_name(album_folder.parent.name)
|
||||
album = tag_album or clean_album_folder_name(album_folder.name)
|
||||
year = tag_year or get_year_from_album_folder_name(album_folder.name)
|
||||
|
||||
audio_files = [
|
||||
f for f in album_folder.iterdir()
|
||||
if f.is_file() and f.suffix.lower() in AUDIO_EXTENSIONS
|
||||
]
|
||||
cover_to_keep = _cover_to_keep(album_folder)
|
||||
extra_files = [
|
||||
f.name for f in album_folder.iterdir()
|
||||
if f.is_file() and not _should_keep_file(f, cover_to_keep)
|
||||
]
|
||||
|
||||
suggested_folder = None
|
||||
if album and year:
|
||||
candidate = f"{year} - {safe_filename(album)}"
|
||||
if candidate != album_folder.name:
|
||||
suggested_folder = candidate
|
||||
|
||||
missing_lyrics = sum(1 for f in audio_files if not _has_lyrics(f))
|
||||
|
||||
return {
|
||||
"path": str(album_folder),
|
||||
"folder_name": album_folder.name,
|
||||
"artist": artist or "",
|
||||
"album": album or "",
|
||||
"year": year,
|
||||
"track_count": len(audio_files),
|
||||
"has_cover": _has_cover(album_folder),
|
||||
"suggested_folder": suggested_folder,
|
||||
"needs_folder_rename": suggested_folder is not None,
|
||||
"extra_files": extra_files,
|
||||
"extra_file_count": len(extra_files),
|
||||
"missing_lyrics_count": missing_lyrics,
|
||||
}
|
||||
|
||||
|
||||
def _iter_album_folders(root: Path):
|
||||
for first_level in root.iterdir():
|
||||
if not first_level.is_dir():
|
||||
continue
|
||||
try:
|
||||
has_audio = any(
|
||||
p.is_file() and p.suffix.lower() in AUDIO_EXTENSIONS
|
||||
for p in first_level.iterdir()
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
if has_audio:
|
||||
# Top-level folder holding audio directly is not an Artist/Album tree.
|
||||
continue
|
||||
for album_folder in first_level.iterdir():
|
||||
if album_folder.is_dir():
|
||||
yield album_folder
|
||||
|
||||
|
||||
def scan_library(root: Path | None = None) -> dict:
|
||||
root = root or MUSIC_ROOT
|
||||
if not root.exists():
|
||||
return {"root": str(root), "exists": False, "albums": []}
|
||||
|
||||
albums = []
|
||||
for album_folder in _iter_album_folders(root):
|
||||
try:
|
||||
albums.append(analyze_album(album_folder))
|
||||
except OSError:
|
||||
continue
|
||||
albums.sort(key=lambda a: (a["artist"].lower(), a["year"] or "", a["album"].lower()))
|
||||
|
||||
return {
|
||||
"root": str(root),
|
||||
"exists": True,
|
||||
"album_count": len(albums),
|
||||
"missing_cover_count": sum(1 for a in albums if not a["has_cover"]),
|
||||
"needs_rename_count": sum(1 for a in albums if a["needs_folder_rename"]),
|
||||
"extra_file_count": sum(a["extra_file_count"] for a in albums),
|
||||
"albums": albums,
|
||||
}
|
||||
|
||||
|
||||
# ── online lookups (year / cover / lyrics) ───────────────────────────────────
|
||||
|
||||
|
||||
def find_album_year(artist: str, album: str) -> str | None:
|
||||
if not _HAS_MUSICBRAINZ:
|
||||
return None
|
||||
try:
|
||||
result = musicbrainzngs.search_releases(artist=artist, release=album, limit=5)
|
||||
for release in result.get("release-list", []):
|
||||
year = extract_year(release.get("date"))
|
||||
if year:
|
||||
return year
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def find_album_cover(artist: str, album: str) -> bytes | None:
|
||||
if not _HAS_MUSICBRAINZ:
|
||||
return None
|
||||
try:
|
||||
result = musicbrainzngs.search_releases(artist=artist, release=album, limit=3)
|
||||
releases = result.get("release-list", [])
|
||||
if not releases:
|
||||
return None
|
||||
mbid = releases[0]["id"]
|
||||
url = f"https://coverartarchive.org/release/{mbid}/front-500"
|
||||
response = requests.get(url, timeout=20, allow_redirects=True)
|
||||
if response.status_code == 200 and response.headers.get("content-type", "").startswith("image"):
|
||||
return response.content
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def find_track_lyrics(artist: str, album: str, title: str, duration: int | None):
|
||||
if not duration:
|
||||
return None
|
||||
try:
|
||||
response = requests.get(
|
||||
"https://lrclib.net/api/get",
|
||||
params={
|
||||
"artist_name": artist,
|
||||
"track_name": title,
|
||||
"album_name": album,
|
||||
"duration": duration,
|
||||
},
|
||||
headers={"User-Agent": "HomelabToolkit/1.0 (local music library tool)"},
|
||||
timeout=20,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
data = response.json()
|
||||
if data.get("syncedLyrics"):
|
||||
return ".lrc", data["syncedLyrics"].strip() + "\n"
|
||||
if data.get("plainLyrics"):
|
||||
return ".txt", data["plainLyrics"].strip() + "\n"
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
# ── mutating operations (respect dry_run) ────────────────────────────────────
|
||||
|
||||
|
||||
def _is_top_level(folder: Path, root: Path) -> bool:
|
||||
try:
|
||||
return folder.resolve().parent == root.resolve()
|
||||
except OSError:
|
||||
return folder.parent == root
|
||||
|
||||
|
||||
def _rename_album_folder(album_folder, artist, album, year, root, opts, rec) -> Path:
|
||||
if not album or not year or _is_top_level(album_folder, root):
|
||||
return album_folder
|
||||
new_name = f"{year} - {safe_filename(album)}"
|
||||
if album_folder.name == new_name:
|
||||
return album_folder
|
||||
new_folder = album_folder.with_name(new_name)
|
||||
if new_folder.exists():
|
||||
rec.emit("skip", "folder", f"Rename target already exists: {new_name}")
|
||||
return album_folder
|
||||
rec.emit(
|
||||
"dry" if opts.dry_run else "ok",
|
||||
"folder",
|
||||
f"{album_folder.name} → {new_name}",
|
||||
path=str(album_folder),
|
||||
)
|
||||
if opts.dry_run:
|
||||
return album_folder
|
||||
album_folder.rename(new_folder)
|
||||
return new_folder
|
||||
|
||||
|
||||
def _rename_track(path: Path, opts, rec) -> Path:
|
||||
audio = _load_audio(path)
|
||||
if audio is None:
|
||||
return path
|
||||
artist = _first_tag(audio, ["artist", "albumartist"])
|
||||
album = _first_tag(audio, ["album"])
|
||||
title = _first_tag(audio, ["title"])
|
||||
track = clean_track_number(audio.get("tracknumber"))
|
||||
if not (artist and album and title and track):
|
||||
return path
|
||||
new_name = f"{track} - {safe_filename(title)}{path.suffix.lower()}"
|
||||
if path.name == new_name:
|
||||
return path
|
||||
new_path = path.with_name(new_name)
|
||||
if new_path.exists():
|
||||
rec.emit("skip", "rename", f"Target exists: {new_name}")
|
||||
return path
|
||||
rec.emit("dry" if opts.dry_run else "ok", "rename", f"{path.name} → {new_name}")
|
||||
if opts.dry_run:
|
||||
return path
|
||||
path.rename(new_path)
|
||||
return new_path
|
||||
|
||||
|
||||
def _clean_files(album_folder: Path, opts, rec) -> None:
|
||||
cover_to_keep = _cover_to_keep(album_folder)
|
||||
for file in album_folder.iterdir():
|
||||
if not file.is_file() or _should_keep_file(file, cover_to_keep):
|
||||
continue
|
||||
rec.emit("dry" if opts.dry_run else "ok", "remove", f"Remove {file.name}", path=str(file))
|
||||
if opts.dry_run:
|
||||
continue
|
||||
try:
|
||||
file.unlink()
|
||||
except OSError as exc:
|
||||
rec.emit("warn", "remove", f"Could not remove {file.name}: {exc}")
|
||||
|
||||
|
||||
def _fetch_cover(album_folder, artist, album, opts, rec) -> None:
|
||||
if _has_cover(album_folder):
|
||||
return
|
||||
if not (artist and album):
|
||||
return
|
||||
rec.emit("info", "cover", f"Looking up cover: {artist} - {album}")
|
||||
image = find_album_cover(artist, album)
|
||||
if not image:
|
||||
rec.emit("skip", "cover", f"No cover found: {artist} - {album}")
|
||||
return
|
||||
output = album_folder / "cover.jpg"
|
||||
rec.emit("dry" if opts.dry_run else "ok", "cover", f"Save cover.jpg for {album}")
|
||||
if not opts.dry_run:
|
||||
output.write_bytes(image)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _fetch_lyrics(audio_file, album_artist, album_name, opts, rec) -> None:
|
||||
if _has_lyrics(audio_file):
|
||||
return
|
||||
t_artist, t_album, title, duration = get_track_metadata(audio_file)
|
||||
artist = t_artist or album_artist
|
||||
album = t_album or album_name
|
||||
if not (artist and album and title):
|
||||
return
|
||||
lyrics = find_track_lyrics(artist, album, title, duration)
|
||||
if not lyrics:
|
||||
rec.emit("skip", "lyrics", f"No lyrics: {artist} - {title}")
|
||||
return
|
||||
extension, text = lyrics
|
||||
output = audio_file.with_suffix(extension)
|
||||
rec.emit("dry" if opts.dry_run else "ok", "lyrics", f"Save lyrics for {title}")
|
||||
if not opts.dry_run:
|
||||
output.write_text(text, encoding="utf-8")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _process_album(album_folder: Path, root: Path, opts: ProcessOptions, rec: _Recorder) -> None:
|
||||
tag_artist, tag_album, tag_year = get_album_metadata_from_files(album_folder)
|
||||
artist = tag_artist or clean_name(album_folder.parent.name)
|
||||
album = tag_album or clean_album_folder_name(album_folder.name)
|
||||
year = tag_year or get_year_from_album_folder_name(album_folder.name)
|
||||
|
||||
if opts.folder_cleanup and not year and artist and album:
|
||||
year = find_album_year(artist, album)
|
||||
if opts.folder_cleanup:
|
||||
album_folder = _rename_album_folder(album_folder, artist, album, year, root, opts, rec)
|
||||
|
||||
audio_files = [
|
||||
f for f in album_folder.iterdir()
|
||||
if f.is_file() and f.suffix.lower() in AUDIO_EXTENSIONS
|
||||
]
|
||||
if opts.rename:
|
||||
audio_files = [_rename_track(f, opts, rec) for f in audio_files]
|
||||
|
||||
if opts.lyrics:
|
||||
for file in audio_files:
|
||||
_fetch_lyrics(file, artist, album, opts, rec)
|
||||
|
||||
if opts.covers:
|
||||
_fetch_cover(album_folder, artist, album, opts, rec)
|
||||
|
||||
if opts.file_cleanup:
|
||||
_clean_files(album_folder, opts, rec)
|
||||
|
||||
|
||||
def process_library(
|
||||
options: ProcessOptions,
|
||||
*,
|
||||
root: Path | None = None,
|
||||
log: LogCallback | None = None,
|
||||
album_paths: list[str] | None = None,
|
||||
) -> dict:
|
||||
"""Run the selected maintenance modes. Honours ``options.dry_run``.
|
||||
|
||||
``album_paths`` optionally limits the run to specific album folders.
|
||||
"""
|
||||
root = root or MUSIC_ROOT
|
||||
rec = _Recorder(sink=log)
|
||||
|
||||
if not root.exists():
|
||||
rec.emit("warn", "root", f"Music root does not exist: {root}")
|
||||
return {"root": str(root), "dry_run": options.dry_run, "actions": rec.actions, "counts": rec.counts}
|
||||
|
||||
if album_paths:
|
||||
wanted = {str(Path(p)) for p in album_paths}
|
||||
folders = [Path(p) for p in album_paths] if all(Path(p).exists() for p in album_paths) else [
|
||||
f for f in _iter_album_folders(root) if str(f) in wanted
|
||||
]
|
||||
else:
|
||||
folders = list(_iter_album_folders(root))
|
||||
|
||||
rec.emit(
|
||||
"info",
|
||||
"start",
|
||||
f"{'Dry run' if options.dry_run else 'Applying'} across {len(folders)} album(s)",
|
||||
)
|
||||
for album_folder in folders:
|
||||
try:
|
||||
_process_album(album_folder, root, options, rec)
|
||||
except OSError as exc:
|
||||
rec.emit("warn", "album", f"Error processing {album_folder.name}: {exc}")
|
||||
|
||||
rec.emit("info", "done", "Finished")
|
||||
return {
|
||||
"root": str(root),
|
||||
"dry_run": options.dry_run,
|
||||
"actions": rec.actions,
|
||||
"counts": rec.counts,
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
"""Music-library scanning, MusicBrainz enrichment, and completeness logic.
|
||||
|
||||
Design rules (per the feature spec):
|
||||
* Disk is only walked by an explicit background job, never on page render.
|
||||
* The UI reads exclusively from the database.
|
||||
* Jobs write progress to ``library_scan_runs`` so the UI can poll.
|
||||
* Manual user decisions (ignore / mark owned / mark missing) are never
|
||||
overwritten by a metadata refresh.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from . import db, musicbrainz
|
||||
from .music_covers import AUDIO_EXTENSIONS, MUSIC_ROOT, _first_tag, _load_audio, clean_name, clean_track_number, extract_year
|
||||
from .text_normalize import (
|
||||
MISSING,
|
||||
OWNED,
|
||||
classify_release,
|
||||
is_various_artists,
|
||||
normalize_artist,
|
||||
normalize_title,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("homelabtoolkit.music_library")
|
||||
|
||||
_OWNED_STATUSES = (OWNED, "probably_owned")
|
||||
_BATCH_SIZE = 200
|
||||
|
||||
# Only one job of each kind runs at a time (single process).
|
||||
_running: set[str] = set()
|
||||
_running_lock = threading.Lock()
|
||||
|
||||
|
||||
# ── job guards ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _try_acquire(kind: str) -> bool:
|
||||
with _running_lock:
|
||||
if kind in _running:
|
||||
return False
|
||||
_running.add(kind)
|
||||
return True
|
||||
|
||||
|
||||
def _release(kind: str) -> None:
|
||||
with _running_lock:
|
||||
_running.discard(kind)
|
||||
|
||||
|
||||
def _is_running(kind: str) -> bool:
|
||||
with _running_lock:
|
||||
return kind in _running
|
||||
|
||||
|
||||
# ── scan run bookkeeping ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _create_run(kind: str) -> int:
|
||||
with db.connect() as conn:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO library_scan_runs(kind, status, started_at) VALUES(?, 'running', ?)",
|
||||
(kind, db.now_iso()),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def _update_run(run_id: int, **fields) -> None:
|
||||
if not fields:
|
||||
return
|
||||
cols = ", ".join(f"{k}=?" for k in fields)
|
||||
with db.connect() as conn:
|
||||
conn.execute(f"UPDATE library_scan_runs SET {cols} WHERE id=?", (*fields.values(), run_id))
|
||||
|
||||
|
||||
def _finish_run(run_id: int, status: str, **fields) -> None:
|
||||
_update_run(run_id, status=status, completed_at=db.now_iso(), **fields)
|
||||
|
||||
|
||||
# ── tag reading ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_tags(path: Path) -> dict | None:
|
||||
audio = _load_audio(path)
|
||||
if audio is None:
|
||||
return None
|
||||
album_artist = _first_tag(audio, ["albumartist", "album artist"])
|
||||
track_artist = _first_tag(audio, ["artist", "albumartist"])
|
||||
artist = album_artist or track_artist or clean_name(path.parent.parent.name)
|
||||
album = _first_tag(audio, ["album"]) or clean_name(path.parent.name)
|
||||
title = _first_tag(audio, ["title"]) or path.stem
|
||||
track_no = clean_track_number(audio.get("tracknumber"))
|
||||
disc_no = clean_track_number(audio.get("discnumber"))
|
||||
return {
|
||||
"artist": artist,
|
||||
"album": album,
|
||||
"title": title,
|
||||
"track_number": int(track_no) if track_no else None,
|
||||
"disc_number": int(disc_no) if disc_no else None,
|
||||
"year": int(extract_year(_first_tag(audio, ["date", "originaldate", "year"])) or 0),
|
||||
"artist_mbid": _first_tag(audio, ["musicbrainz_artistid"]),
|
||||
"album_mbid": _first_tag(audio, ["musicbrainz_releasegroupid", "musicbrainz_albumid"]),
|
||||
"track_mbid": _first_tag(audio, ["musicbrainz_trackid"]),
|
||||
}
|
||||
|
||||
|
||||
def _iter_audio_files(root: Path):
|
||||
"""Yield audio file paths lazily so we never hold the library in memory."""
|
||||
for dirpath, _dirnames, filenames in os.walk(root):
|
||||
for name in filenames:
|
||||
if Path(name).suffix.lower() in AUDIO_EXTENSIONS:
|
||||
yield Path(dirpath) / name
|
||||
|
||||
|
||||
# ── upserts (in-run caches keep artist/album lookups cheap) ───────────────────
|
||||
|
||||
|
||||
def _get_artist_id(conn, cache: dict, meta: dict) -> int:
|
||||
name = meta["artist"]
|
||||
norm = normalize_artist(name)
|
||||
if norm in cache:
|
||||
return cache[norm]
|
||||
now = db.now_iso()
|
||||
various = 1 if is_various_artists(name) else 0
|
||||
row = conn.execute("SELECT id FROM library_artists WHERE name_normalized=?", (norm,)).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE library_artists SET is_active=1, is_various=?, updated_at=?, mbid=COALESCE(mbid, ?) WHERE id=?",
|
||||
(various, now, meta.get("artist_mbid"), row["id"]),
|
||||
)
|
||||
artist_id = row["id"]
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO library_artists(name, name_normalized, mbid, is_various, is_active, created_at, updated_at) "
|
||||
"VALUES(?,?,?,?,1,?,?)",
|
||||
(name, norm, meta.get("artist_mbid"), various, now, now),
|
||||
)
|
||||
artist_id = cur.lastrowid
|
||||
cache[norm] = artist_id
|
||||
return artist_id
|
||||
|
||||
|
||||
def _get_album_id(conn, cache: dict, artist_id: int, meta: dict) -> int:
|
||||
title = meta["album"]
|
||||
norm = normalize_title(title)
|
||||
year = meta.get("year") or 0
|
||||
key = (artist_id, norm, year)
|
||||
if key in cache:
|
||||
return cache[key]
|
||||
now = db.now_iso()
|
||||
row = conn.execute(
|
||||
"SELECT id FROM library_albums WHERE artist_id=? AND title_normalized=? AND year=?",
|
||||
(artist_id, norm, year),
|
||||
).fetchone()
|
||||
if row:
|
||||
conn.execute(
|
||||
"UPDATE library_albums SET is_active=1, updated_at=?, mbid=COALESCE(mbid, ?) WHERE id=?",
|
||||
(now, meta.get("album_mbid"), row["id"]),
|
||||
)
|
||||
album_id = row["id"]
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO library_albums(artist_id, title, title_normalized, year, mbid, is_active, created_at, updated_at) "
|
||||
"VALUES(?,?,?,?,?,1,?,?)",
|
||||
(artist_id, title, norm, year, meta.get("album_mbid"), now, now),
|
||||
)
|
||||
album_id = cur.lastrowid
|
||||
cache[key] = album_id
|
||||
return album_id
|
||||
|
||||
|
||||
def _upsert_track(conn, album_id: int, meta: dict, path: Path, size: int, mtime: float, run_id: int) -> None:
|
||||
now = db.now_iso()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO library_tracks(album_id, title, track_number, disc_number, file_path, file_mtime,
|
||||
file_size, mbid, is_active, last_seen_scan_id, created_at, updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,1,?,?,?)
|
||||
ON CONFLICT(file_path) DO UPDATE SET
|
||||
album_id=excluded.album_id, title=excluded.title, track_number=excluded.track_number,
|
||||
disc_number=excluded.disc_number, file_mtime=excluded.file_mtime, file_size=excluded.file_size,
|
||||
mbid=excluded.mbid, is_active=1, last_seen_scan_id=excluded.last_seen_scan_id, updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
album_id, meta["title"], meta["track_number"], meta["disc_number"], str(path), mtime,
|
||||
size, meta.get("track_mbid"), run_id, now, now,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ── scanner ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run_scan(root: Path | None = None) -> dict:
|
||||
"""Walk the library and upsert artists/albums/tracks. Unchanged files
|
||||
(same path + size + mtime) are skipped without reading tags. Files that
|
||||
vanished are marked inactive, never hard-deleted."""
|
||||
root = Path(root) if root else MUSIC_ROOT
|
||||
run_id = _create_run("scan")
|
||||
logger.info("Scan %d started: %s", run_id, root)
|
||||
|
||||
if not root.exists():
|
||||
_finish_run(run_id, "failed", error_message=f"Music root not found: {root}")
|
||||
logger.warning("Scan %d aborted: root missing", run_id)
|
||||
return {"run_id": run_id, "status": "failed"}
|
||||
|
||||
files_scanned = 0
|
||||
try:
|
||||
with db.connect() as conn:
|
||||
artist_cache: dict = {}
|
||||
album_cache: dict = {}
|
||||
batch = 0
|
||||
for path in _iter_audio_files(root):
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
size, mtime = stat.st_size, stat.st_mtime
|
||||
existing = conn.execute(
|
||||
"SELECT id, file_size, file_mtime FROM library_tracks WHERE file_path=?",
|
||||
(str(path),),
|
||||
).fetchone()
|
||||
if existing and existing["file_size"] == size and abs((existing["file_mtime"] or 0) - mtime) < 1:
|
||||
conn.execute(
|
||||
"UPDATE library_tracks SET is_active=1, last_seen_scan_id=? WHERE id=?",
|
||||
(run_id, existing["id"]),
|
||||
)
|
||||
else:
|
||||
meta = _read_tags(path)
|
||||
if meta is not None:
|
||||
artist_id = _get_artist_id(conn, artist_cache, meta)
|
||||
album_id = _get_album_id(conn, album_cache, artist_id, meta)
|
||||
_upsert_track(conn, album_id, meta, path, size, mtime, run_id)
|
||||
files_scanned += 1
|
||||
batch += 1
|
||||
if batch >= _BATCH_SIZE:
|
||||
conn.commit()
|
||||
batch = 0
|
||||
conn.execute(
|
||||
"UPDATE library_scan_runs SET files_scanned=?, progress=? WHERE id=?",
|
||||
(files_scanned, f"Scanned {files_scanned} files", run_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.commit()
|
||||
|
||||
# Mark vanished files inactive, then cascade activity up.
|
||||
conn.execute(
|
||||
"UPDATE library_tracks SET is_active=0 WHERE COALESCE(last_seen_scan_id, -1) != ? AND is_active=1",
|
||||
(run_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE library_albums SET is_active = "
|
||||
"CASE WHEN EXISTS(SELECT 1 FROM library_tracks t WHERE t.album_id=library_albums.id AND t.is_active=1) "
|
||||
"THEN 1 ELSE 0 END"
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE library_artists SET is_active = "
|
||||
"CASE WHEN EXISTS(SELECT 1 FROM library_albums al WHERE al.artist_id=library_artists.id AND al.is_active=1) "
|
||||
"THEN 1 ELSE 0 END"
|
||||
)
|
||||
artists_found = conn.execute("SELECT COUNT(*) c FROM library_artists WHERE is_active=1").fetchone()["c"]
|
||||
albums_found = conn.execute("SELECT COUNT(*) c FROM library_albums WHERE is_active=1").fetchone()["c"]
|
||||
conn.commit()
|
||||
|
||||
_finish_run(
|
||||
run_id, "completed",
|
||||
files_scanned=files_scanned, albums_found=albums_found, artists_found=artists_found,
|
||||
progress="Done",
|
||||
)
|
||||
logger.info("Scan %d completed: %d files, %d artists, %d albums", run_id, files_scanned, artists_found, albums_found)
|
||||
return {"run_id": run_id, "status": "completed", "files_scanned": files_scanned}
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.exception("Scan %d failed", run_id)
|
||||
_finish_run(run_id, "failed", files_scanned=files_scanned, error_message=str(exc))
|
||||
return {"run_id": run_id, "status": "failed", "error": str(exc)}
|
||||
|
||||
|
||||
# ── completeness ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _local_albums_for_artist(conn, artist_id: int) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"SELECT id, title, title_normalized, year, mbid FROM library_albums WHERE artist_id=? AND is_active=1",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def recompute_completeness(conn, artist_id: int) -> None:
|
||||
"""Rebuild completeness rows for one artist from stored external releases and
|
||||
local albums. Manual decisions (manual_override=1) are preserved."""
|
||||
local_albums = _local_albums_for_artist(conn, artist_id)
|
||||
releases = conn.execute(
|
||||
"SELECT mb_release_group_mbid, title, first_release_year, primary_type, secondary_types "
|
||||
"FROM external_releases WHERE artist_id=?",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
|
||||
existing = {
|
||||
r["release_group_mbid"]: dict(r)
|
||||
for r in conn.execute(
|
||||
"SELECT release_group_mbid, manual_override FROM collection_completeness WHERE artist_id=?",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
}
|
||||
|
||||
qualifying_mbids: list[str] = []
|
||||
now = db.now_iso()
|
||||
for rel in releases:
|
||||
secondary = json.loads(rel["secondary_types"] or "[]")
|
||||
if not musicbrainz.is_official_album(rel["primary_type"], secondary):
|
||||
continue
|
||||
mbid = rel["mb_release_group_mbid"]
|
||||
qualifying_mbids.append(mbid)
|
||||
|
||||
prior = existing.get(mbid)
|
||||
if prior and prior["manual_override"]:
|
||||
# Keep the user's decision; only refresh descriptive fields.
|
||||
conn.execute(
|
||||
"UPDATE collection_completeness SET title=?, year=?, source='musicbrainz', updated_at=? "
|
||||
"WHERE artist_id=? AND release_group_mbid=?",
|
||||
(rel["title"], rel["first_release_year"] or 0, now, artist_id, mbid),
|
||||
)
|
||||
continue
|
||||
|
||||
status, confidence, reason, local_id = classify_release(
|
||||
rel["title"], rel["first_release_year"], mbid, local_albums
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO collection_completeness(artist_id, release_group_mbid, local_album_id, title, year,
|
||||
status, confidence, reason, source, manual_override, updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?, 'musicbrainz', 0, ?)
|
||||
ON CONFLICT(artist_id, release_group_mbid) DO UPDATE SET
|
||||
local_album_id=excluded.local_album_id, title=excluded.title, year=excluded.year,
|
||||
status=excluded.status, confidence=excluded.confidence, reason=excluded.reason,
|
||||
source='musicbrainz', updated_at=excluded.updated_at
|
||||
WHERE collection_completeness.manual_override=0
|
||||
""",
|
||||
(artist_id, mbid, local_id, rel["title"], rel["first_release_year"] or 0,
|
||||
status, confidence, reason, now),
|
||||
)
|
||||
|
||||
# Drop non-manual rows that are no longer qualifying (e.g. filter changes).
|
||||
placeholders = ",".join("?" for _ in qualifying_mbids) or "''"
|
||||
conn.execute(
|
||||
f"DELETE FROM collection_completeness WHERE artist_id=? AND manual_override=0 "
|
||||
f"AND release_group_mbid NOT IN ({placeholders})",
|
||||
(artist_id, *qualifying_mbids),
|
||||
)
|
||||
|
||||
|
||||
# ── metadata refresh job (per-artist MusicBrainz lookups) ─────────────────────
|
||||
|
||||
|
||||
def run_metadata_refresh() -> dict:
|
||||
run_id = _create_run("metadata")
|
||||
logger.info("Metadata refresh %d started", run_id)
|
||||
processed = 0
|
||||
try:
|
||||
with db.connect() as conn:
|
||||
artists = conn.execute(
|
||||
"SELECT id, name, mbid, is_various FROM library_artists WHERE is_active=1 ORDER BY name"
|
||||
).fetchall()
|
||||
total = len(artists)
|
||||
|
||||
for artist in artists:
|
||||
artist_id = artist["id"]
|
||||
if artist["is_various"]:
|
||||
with db.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO external_artist_matches(artist_id, status, checked_at) VALUES(?, 'skipped', ?) "
|
||||
"ON CONFLICT(artist_id) DO UPDATE SET status='skipped', checked_at=excluded.checked_at",
|
||||
(artist_id, db.now_iso()),
|
||||
)
|
||||
processed += 1
|
||||
continue
|
||||
|
||||
match = musicbrainz.search_artist(artist["name"])
|
||||
with db.connect() as conn:
|
||||
if not match or not match.get("mbid"):
|
||||
conn.execute(
|
||||
"INSERT INTO external_artist_matches(artist_id, status, checked_at) VALUES(?, 'not_found', ?) "
|
||||
"ON CONFLICT(artist_id) DO UPDATE SET status='not_found', checked_at=excluded.checked_at",
|
||||
(artist_id, db.now_iso()),
|
||||
)
|
||||
processed += 1
|
||||
_bump_metadata_progress(run_id, processed, total)
|
||||
continue
|
||||
conn.execute(
|
||||
"INSERT INTO external_artist_matches(artist_id, mb_artist_mbid, mb_artist_name, confidence, status, checked_at) "
|
||||
"VALUES(?,?,?,?, 'matched', ?) "
|
||||
"ON CONFLICT(artist_id) DO UPDATE SET mb_artist_mbid=excluded.mb_artist_mbid, "
|
||||
"mb_artist_name=excluded.mb_artist_name, confidence=excluded.confidence, status='matched', checked_at=excluded.checked_at",
|
||||
(artist_id, match["mbid"], match["name"], match["confidence"], db.now_iso()),
|
||||
)
|
||||
|
||||
release_groups = musicbrainz.fetch_release_groups(match["mbid"])
|
||||
with db.connect() as conn:
|
||||
for rg in release_groups:
|
||||
if not rg.get("mbid"):
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO external_releases(artist_id, mb_release_group_mbid, title, title_normalized,
|
||||
first_release_year, primary_type, secondary_types, fetched_at)
|
||||
VALUES(?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(artist_id, mb_release_group_mbid) DO UPDATE SET
|
||||
title=excluded.title, title_normalized=excluded.title_normalized,
|
||||
first_release_year=excluded.first_release_year, primary_type=excluded.primary_type,
|
||||
secondary_types=excluded.secondary_types, fetched_at=excluded.fetched_at
|
||||
""",
|
||||
(
|
||||
artist_id, rg["mbid"], rg["title"], normalize_title(rg["title"]),
|
||||
rg["first_release_year"], rg["primary_type"], json.dumps(rg["secondary_types"]),
|
||||
db.now_iso(),
|
||||
),
|
||||
)
|
||||
recompute_completeness(conn, artist_id)
|
||||
|
||||
processed += 1
|
||||
_bump_metadata_progress(run_id, processed, total)
|
||||
|
||||
_finish_run(run_id, "completed", artists_found=processed, progress=f"Checked {processed} artists")
|
||||
logger.info("Metadata refresh %d completed: %d artists", run_id, processed)
|
||||
return {"run_id": run_id, "status": "completed", "artists": processed}
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.exception("Metadata refresh %d failed", run_id)
|
||||
_finish_run(run_id, "failed", error_message=str(exc))
|
||||
return {"run_id": run_id, "status": "failed", "error": str(exc)}
|
||||
|
||||
|
||||
def _bump_metadata_progress(run_id: int, processed: int, total: int) -> None:
|
||||
_update_run(run_id, artists_found=processed, progress=f"Checked {processed}/{total} artists")
|
||||
|
||||
|
||||
# ── job launchers ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def start_scan_job(root: Path | None = None) -> dict:
|
||||
if not _try_acquire("scan"):
|
||||
return {"started": False, "reason": "A scan is already running."}
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
run_scan(root)
|
||||
finally:
|
||||
_release("scan")
|
||||
|
||||
threading.Thread(target=_worker, name="scan_music_collection", daemon=True).start()
|
||||
return {"started": True}
|
||||
|
||||
|
||||
def start_metadata_job() -> dict:
|
||||
if not _try_acquire("metadata"):
|
||||
return {"started": False, "reason": "A metadata refresh is already running."}
|
||||
|
||||
def _worker():
|
||||
try:
|
||||
run_metadata_refresh()
|
||||
finally:
|
||||
_release("metadata")
|
||||
|
||||
threading.Thread(target=_worker, name="refresh_music_metadata", daemon=True).start()
|
||||
return {"started": True}
|
||||
|
||||
|
||||
# ── read-side queries (UI; database only) ─────────────────────────────────────
|
||||
|
||||
|
||||
def _latest_run(conn, kind: str) -> dict | None:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM library_scan_runs WHERE kind=? ORDER BY id DESC LIMIT 1", (kind,)
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def get_status() -> dict:
|
||||
with db.connect() as conn:
|
||||
scan = _latest_run(conn, "scan")
|
||||
metadata = _latest_run(conn, "metadata")
|
||||
return {
|
||||
"scan": scan,
|
||||
"metadata": metadata,
|
||||
"scan_running": _is_running("scan"),
|
||||
"metadata_running": _is_running("metadata"),
|
||||
}
|
||||
|
||||
|
||||
def get_overview() -> dict:
|
||||
with db.connect() as conn:
|
||||
scan = _latest_run(conn, "scan")
|
||||
metadata = _latest_run(conn, "metadata")
|
||||
totals = conn.execute(
|
||||
"""
|
||||
SELECT
|
||||
SUM(CASE WHEN status IN ('owned','probably_owned') THEN 1 ELSE 0 END) AS owned,
|
||||
SUM(CASE WHEN status='missing' THEN 1 ELSE 0 END) AS missing,
|
||||
SUM(CASE WHEN status='uncertain' THEN 1 ELSE 0 END) AS uncertain,
|
||||
SUM(CASE WHEN status='ignored' THEN 1 ELSE 0 END) AS ignored
|
||||
FROM collection_completeness c
|
||||
JOIN library_artists a ON a.id=c.artist_id AND a.is_active=1
|
||||
"""
|
||||
).fetchone()
|
||||
artist_count = conn.execute("SELECT COUNT(*) c FROM library_artists WHERE is_active=1").fetchone()["c"]
|
||||
album_count = conn.execute("SELECT COUNT(*) c FROM library_albums WHERE is_active=1").fetchone()["c"]
|
||||
|
||||
owned = totals["owned"] or 0
|
||||
missing = totals["missing"] or 0
|
||||
uncertain = totals["uncertain"] or 0
|
||||
ignored = totals["ignored"] or 0
|
||||
denom = owned + missing + uncertain
|
||||
completeness = round(owned / denom * 100, 1) if denom else 0.0
|
||||
return {
|
||||
"last_scan": scan,
|
||||
"last_metadata": metadata,
|
||||
"scan_running": _is_running("scan"),
|
||||
"metadata_running": _is_running("metadata"),
|
||||
"owned": owned,
|
||||
"missing": missing,
|
||||
"uncertain": uncertain,
|
||||
"ignored": ignored,
|
||||
"completeness": completeness,
|
||||
"library_artists": artist_count,
|
||||
"library_albums": album_count,
|
||||
}
|
||||
|
||||
|
||||
def get_artists_completeness(search: str = "") -> list[dict]:
|
||||
where = "WHERE a.is_active=1"
|
||||
params: list = []
|
||||
if search.strip():
|
||||
where += " AND a.name LIKE ?"
|
||||
params.append(f"%{search.strip()}%")
|
||||
with db.connect() as conn:
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT a.id, a.name,
|
||||
SUM(CASE WHEN c.status IN ('owned','probably_owned') THEN 1 ELSE 0 END) AS owned,
|
||||
SUM(CASE WHEN c.status='missing' THEN 1 ELSE 0 END) AS missing,
|
||||
SUM(CASE WHEN c.status='uncertain' THEN 1 ELSE 0 END) AS uncertain,
|
||||
SUM(CASE WHEN c.status='ignored' THEN 1 ELSE 0 END) AS ignored,
|
||||
COUNT(c.id) AS total
|
||||
FROM library_artists a
|
||||
JOIN collection_completeness c ON c.artist_id=a.id
|
||||
{where}
|
||||
GROUP BY a.id
|
||||
HAVING total > 0
|
||||
ORDER BY missing DESC, a.name COLLATE NOCASE
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
result = []
|
||||
for r in rows:
|
||||
owned, missing, uncertain = r["owned"] or 0, r["missing"] or 0, r["uncertain"] or 0
|
||||
denom = owned + missing + uncertain
|
||||
result.append(
|
||||
{
|
||||
"id": r["id"],
|
||||
"name": r["name"],
|
||||
"owned": owned,
|
||||
"missing": missing,
|
||||
"uncertain": uncertain,
|
||||
"ignored": r["ignored"] or 0,
|
||||
"completeness": round(owned / denom * 100, 1) if denom else 0.0,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def get_artist_albums(artist_id: int) -> dict:
|
||||
with db.connect() as conn:
|
||||
artist = conn.execute("SELECT id, name FROM library_artists WHERE id=?", (artist_id,)).fetchone()
|
||||
rows = conn.execute(
|
||||
"SELECT id, release_group_mbid, title, year, status, confidence, reason, source, manual_override "
|
||||
"FROM collection_completeness WHERE artist_id=? ORDER BY year, title COLLATE NOCASE",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
return {
|
||||
"artist": dict(artist) if artist else None,
|
||||
"albums": [dict(r) for r in rows],
|
||||
}
|
||||
|
||||
|
||||
def set_album_decision(completeness_id: int, action: str) -> dict:
|
||||
action = (action or "").lower()
|
||||
valid = {"ignore", "owned", "missing", "reset"}
|
||||
if action not in valid:
|
||||
raise ValueError(f"Unknown action: {action}")
|
||||
|
||||
now = db.now_iso()
|
||||
with db.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, artist_id FROM collection_completeness WHERE id=?", (completeness_id,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise LookupError("Completeness row not found.")
|
||||
|
||||
if action == "ignore":
|
||||
conn.execute(
|
||||
"UPDATE collection_completeness SET status='ignored', manual_override=1, reason='Manually ignored', updated_at=? WHERE id=?",
|
||||
(now, completeness_id),
|
||||
)
|
||||
elif action == "owned":
|
||||
conn.execute(
|
||||
"UPDATE collection_completeness SET status='owned', confidence=1.0, manual_override=1, reason='Manually marked owned', updated_at=? WHERE id=?",
|
||||
(now, completeness_id),
|
||||
)
|
||||
elif action == "missing":
|
||||
conn.execute(
|
||||
"UPDATE collection_completeness SET status='missing', confidence=0, manual_override=1, reason='Manually marked missing', updated_at=? WHERE id=?",
|
||||
(now, completeness_id),
|
||||
)
|
||||
elif action == "reset":
|
||||
conn.execute(
|
||||
"UPDATE collection_completeness SET manual_override=0, updated_at=? WHERE id=?",
|
||||
(now, completeness_id),
|
||||
)
|
||||
recompute_completeness(conn, row["artist_id"])
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,187 @@
|
||||
"""MusicBrainz client: the default free metadata source.
|
||||
|
||||
Deliberately small and polite:
|
||||
* one global throttle so we never exceed ~1 request/second (MB's published limit);
|
||||
* a descriptive User-Agent (MB rejects anonymous clients);
|
||||
* retry/backoff on 503 and network errors;
|
||||
* responses cached in the ``mb_cache`` table so completeness never hits the API
|
||||
during a normal page render.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
from . import db
|
||||
|
||||
logger = logging.getLogger("homelabtoolkit.musicbrainz")
|
||||
|
||||
MB_BASE = "https://musicbrainz.org/ws/2"
|
||||
USER_AGENT = os.environ.get(
|
||||
"MUSICBRAINZ_USER_AGENT",
|
||||
"HomelabToolkit/1.0 ( https://github.com/homelabtoolkit )",
|
||||
)
|
||||
MIN_INTERVAL_SECONDS = 1.1
|
||||
CACHE_MAX_AGE_DAYS = 30
|
||||
|
||||
# Release-group filtering. Primary type must be Album; any of these secondary
|
||||
# types excludes it (live albums, compilations, soundtracks, remixes, etc.).
|
||||
# Kept as module constants so the filter can be relaxed later in one place.
|
||||
INCLUDED_PRIMARY_TYPES = {"album"}
|
||||
EXCLUDED_SECONDARY_TYPES = {
|
||||
"live",
|
||||
"compilation",
|
||||
"soundtrack",
|
||||
"remix",
|
||||
"dj-mix",
|
||||
"mixtape/street",
|
||||
"demo",
|
||||
"interview",
|
||||
"audiobook",
|
||||
"audio drama",
|
||||
"spokenword",
|
||||
}
|
||||
|
||||
_throttle_lock = threading.Lock()
|
||||
_last_request_at = 0.0
|
||||
|
||||
|
||||
def _throttle() -> None:
|
||||
global _last_request_at
|
||||
with _throttle_lock:
|
||||
wait = MIN_INTERVAL_SECONDS - (time.monotonic() - _last_request_at)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
_last_request_at = time.monotonic()
|
||||
|
||||
|
||||
def _cache_get(cache_key: str, max_age_days: int = CACHE_MAX_AGE_DAYS):
|
||||
cutoff = time.time() - max_age_days * 86400
|
||||
with db.connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT payload, fetched_at FROM mb_cache WHERE cache_key=?", (cache_key,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
# fetched_at is ISO; treat anything older than the cutoff as a miss.
|
||||
try:
|
||||
import datetime as _dt
|
||||
|
||||
fetched = _dt.datetime.fromisoformat((row["fetched_at"] or "").replace("Z", "+00:00"))
|
||||
if fetched.timestamp() < cutoff:
|
||||
return None
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return json.loads(row["payload"])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _cache_put(cache_key: str, payload) -> None:
|
||||
with db.connect() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO mb_cache(cache_key, payload, fetched_at) VALUES(?,?,?) "
|
||||
"ON CONFLICT(cache_key) DO UPDATE SET payload=excluded.payload, fetched_at=excluded.fetched_at",
|
||||
(cache_key, json.dumps(payload), db.now_iso()),
|
||||
)
|
||||
|
||||
|
||||
def _request(path: str, params: dict, cache_key: str, *, use_cache: bool = True):
|
||||
if use_cache:
|
||||
cached = _cache_get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
query = {**params, "fmt": "json"}
|
||||
for attempt in range(4):
|
||||
_throttle()
|
||||
try:
|
||||
resp = requests.get(f"{MB_BASE}/{path}", params=query, headers=headers, timeout=25)
|
||||
if resp.status_code == 503:
|
||||
time.sleep(2.0 * (attempt + 1))
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
_cache_put(cache_key, data)
|
||||
return data
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("MusicBrainz request failed (%s attempt %d): %s", path, attempt + 1, exc)
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
return None
|
||||
|
||||
|
||||
def search_artist(name: str) -> dict | None:
|
||||
"""Best artist match for a name, with a 0–1 confidence score."""
|
||||
cache_key = f"artist_search::{name.lower()}"
|
||||
data = _request("artist", {"query": name, "limit": 5}, cache_key)
|
||||
if not data:
|
||||
return None
|
||||
artists = data.get("artists") or []
|
||||
if not artists:
|
||||
return None
|
||||
best = artists[0]
|
||||
return {
|
||||
"mbid": best.get("id"),
|
||||
"name": best.get("name") or "",
|
||||
"confidence": round(int(best.get("score", 0)) / 100.0, 3),
|
||||
}
|
||||
|
||||
|
||||
def fetch_release_groups(artist_mbid: str) -> list[dict]:
|
||||
"""All release groups for an artist (paged). Filtering happens later so the
|
||||
completeness filters can change without re-fetching."""
|
||||
results: list[dict] = []
|
||||
offset = 0
|
||||
limit = 100
|
||||
while True:
|
||||
cache_key = f"release_groups::{artist_mbid}::{offset}"
|
||||
data = _request(
|
||||
"release-group",
|
||||
{"artist": artist_mbid, "type": "album", "limit": limit, "offset": offset},
|
||||
cache_key,
|
||||
)
|
||||
if not data:
|
||||
break
|
||||
batch = data.get("release-groups") or []
|
||||
for rg in batch:
|
||||
results.append(
|
||||
{
|
||||
"mbid": rg.get("id"),
|
||||
"title": rg.get("title") or "",
|
||||
"first_release_year": _year(rg.get("first-release-date")),
|
||||
"primary_type": (rg.get("primary-type") or "").strip(),
|
||||
"secondary_types": [s.strip() for s in (rg.get("secondary-types") or [])],
|
||||
}
|
||||
)
|
||||
total = int(data.get("release-group-count", len(results)))
|
||||
offset += limit
|
||||
if offset >= total or not batch:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def is_official_album(primary_type: str | None, secondary_types: list[str] | None) -> bool:
|
||||
"""Apply the default album filter (excludes EP/single/live/comp/etc.)."""
|
||||
if (primary_type or "").strip().lower() not in INCLUDED_PRIMARY_TYPES:
|
||||
return False
|
||||
for secondary in secondary_types or []:
|
||||
if secondary.strip().lower() in EXCLUDED_SECONDARY_TYPES:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _year(date_str: str | None) -> int:
|
||||
if not date_str:
|
||||
return 0
|
||||
try:
|
||||
return int(str(date_str)[:4])
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
@@ -0,0 +1,308 @@
|
||||
"""Navidrome integration via the Subsonic API.
|
||||
|
||||
Navidrome speaks the Subsonic/OpenSubsonic REST API. Authentication uses the
|
||||
salted-token scheme (``t = md5(password + salt)``) so the password never travels
|
||||
in the clear. All functions take an ``httpx.AsyncClient`` so they share the
|
||||
app-wide client and stay easy to test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import secrets
|
||||
|
||||
import httpx
|
||||
|
||||
NAVIDROME_URL = os.environ.get("NAVIDROME_URL", "http://10.0.0.2:4533")
|
||||
NAVIDROME_USER = os.environ.get("NAVIDROME_USER", "")
|
||||
NAVIDROME_PASSWORD = os.environ.get("NAVIDROME_PASSWORD", "")
|
||||
NAVIDROME_CLIENT = "HomelabToolkit"
|
||||
SUBSONIC_API_VERSION = "1.16.1"
|
||||
|
||||
|
||||
class NavidromeError(Exception):
|
||||
def __init__(self, message: str, status: int = 502):
|
||||
self.message = message
|
||||
self.status = status
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
return bool(NAVIDROME_URL and NAVIDROME_USER and NAVIDROME_PASSWORD)
|
||||
|
||||
|
||||
def _require_configured() -> None:
|
||||
if not is_configured():
|
||||
raise NavidromeError(
|
||||
"Navidrome is not configured. Set NAVIDROME_URL, NAVIDROME_USER and "
|
||||
"NAVIDROME_PASSWORD.",
|
||||
status=503,
|
||||
)
|
||||
|
||||
|
||||
def _auth_params() -> dict:
|
||||
salt = secrets.token_hex(8)
|
||||
token = hashlib.md5((NAVIDROME_PASSWORD + salt).encode("utf-8")).hexdigest()
|
||||
return {
|
||||
"u": NAVIDROME_USER,
|
||||
"t": token,
|
||||
"s": salt,
|
||||
"v": SUBSONIC_API_VERSION,
|
||||
"c": NAVIDROME_CLIENT,
|
||||
"f": "json",
|
||||
}
|
||||
|
||||
|
||||
def _base_url() -> str:
|
||||
return NAVIDROME_URL.rstrip("/")
|
||||
|
||||
|
||||
async def _call(client: httpx.AsyncClient, method: str, params: dict | None = None) -> dict:
|
||||
_require_configured()
|
||||
url = f"{_base_url()}/rest/{method}.view"
|
||||
request_params = {**_auth_params(), **(params or {})}
|
||||
try:
|
||||
response = await client.get(url, params=request_params)
|
||||
except httpx.RequestError as exc:
|
||||
raise NavidromeError(f"Could not reach Navidrome at {NAVIDROME_URL}: {exc}") from exc
|
||||
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise NavidromeError(
|
||||
f"Navidrome returned HTTP {response.status_code} for {method}.", status=502
|
||||
) from exc
|
||||
|
||||
try:
|
||||
payload = response.json().get("subsonic-response", {})
|
||||
except ValueError as exc:
|
||||
raise NavidromeError("Navidrome returned an invalid response.", status=502) from exc
|
||||
|
||||
if payload.get("status") != "ok":
|
||||
error = payload.get("error") or {}
|
||||
message = error.get("message") or "Navidrome request failed."
|
||||
code = error.get("code")
|
||||
# Subsonic auth failures (codes 40/41) are the user's credentials, not a
|
||||
# server error — surface as 401 so the UI can prompt for setup.
|
||||
status = 401 if code in (40, 41, 44) else 502
|
||||
raise NavidromeError(message, status=status)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _cover_url(cover_art: str | None) -> str | None:
|
||||
if not cover_art:
|
||||
return None
|
||||
return f"/api/navidrome/cover/{cover_art}"
|
||||
|
||||
|
||||
def _map_album(raw: dict) -> dict:
|
||||
return {
|
||||
"id": raw.get("id"),
|
||||
"name": raw.get("name") or raw.get("album") or "",
|
||||
"artist": raw.get("artist") or "",
|
||||
"artist_id": raw.get("artistId"),
|
||||
"year": raw.get("year"),
|
||||
"genre": raw.get("genre"),
|
||||
"song_count": raw.get("songCount") or 0,
|
||||
"duration": raw.get("duration") or 0,
|
||||
"cover_art": raw.get("coverArt"),
|
||||
"cover_url": _cover_url(raw.get("coverArt")),
|
||||
"created": raw.get("created"),
|
||||
"starred": bool(raw.get("starred")),
|
||||
}
|
||||
|
||||
|
||||
def _map_song(raw: dict) -> dict:
|
||||
return {
|
||||
"id": raw.get("id"),
|
||||
"title": raw.get("title") or "",
|
||||
"track": raw.get("track"),
|
||||
"disc": raw.get("discNumber"),
|
||||
"artist": raw.get("artist") or "",
|
||||
"album": raw.get("album") or "",
|
||||
"year": raw.get("year"),
|
||||
"duration": raw.get("duration") or 0,
|
||||
"bitrate": raw.get("bitRate"),
|
||||
"suffix": raw.get("suffix"),
|
||||
"size": raw.get("size"),
|
||||
"path": raw.get("path"),
|
||||
}
|
||||
|
||||
|
||||
async def ping(client: httpx.AsyncClient) -> dict:
|
||||
if not is_configured():
|
||||
return {"connected": False, "configured": False, "url": NAVIDROME_URL}
|
||||
try:
|
||||
payload = await _call(client, "ping")
|
||||
return {
|
||||
"connected": True,
|
||||
"configured": True,
|
||||
"url": NAVIDROME_URL,
|
||||
"version": payload.get("version"),
|
||||
"server": payload.get("type") or payload.get("serverVersion"),
|
||||
}
|
||||
except NavidromeError as exc:
|
||||
return {
|
||||
"connected": False,
|
||||
"configured": True,
|
||||
"url": NAVIDROME_URL,
|
||||
"error": exc.message,
|
||||
}
|
||||
|
||||
|
||||
async def get_artists(client: httpx.AsyncClient) -> list[dict]:
|
||||
payload = await _call(client, "getArtists")
|
||||
indexes = ((payload.get("artists") or {}).get("index")) or []
|
||||
artists: list[dict] = []
|
||||
for index in indexes:
|
||||
for artist in index.get("artist") or []:
|
||||
artists.append(
|
||||
{
|
||||
"id": artist.get("id"),
|
||||
"name": artist.get("name") or "",
|
||||
"album_count": artist.get("albumCount") or 0,
|
||||
"cover_art": artist.get("coverArt"),
|
||||
"cover_url": _cover_url(artist.get("coverArt")),
|
||||
}
|
||||
)
|
||||
artists.sort(key=lambda entry: entry["name"].lower())
|
||||
return artists
|
||||
|
||||
|
||||
async def get_albums(
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
list_type: str = "alphabeticalByName",
|
||||
size: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
payload = await _call(
|
||||
client,
|
||||
"getAlbumList2",
|
||||
{"type": list_type, "size": size, "offset": offset},
|
||||
)
|
||||
raw_albums = ((payload.get("albumList2") or {}).get("album")) or []
|
||||
return [_map_album(album) for album in raw_albums]
|
||||
|
||||
|
||||
async def search_albums(client: httpx.AsyncClient, query: str, *, count: int = 60) -> list[dict]:
|
||||
payload = await _call(
|
||||
client,
|
||||
"search3",
|
||||
{"query": query, "albumCount": count, "artistCount": 0, "songCount": 0},
|
||||
)
|
||||
raw_albums = ((payload.get("searchResult3") or {}).get("album")) or []
|
||||
return [_map_album(album) for album in raw_albums]
|
||||
|
||||
|
||||
async def get_album(client: httpx.AsyncClient, album_id: str) -> dict:
|
||||
payload = await _call(client, "getAlbum", {"id": album_id})
|
||||
raw = payload.get("album") or {}
|
||||
album = _map_album(raw)
|
||||
album["songs"] = [_map_song(song) for song in (raw.get("song") or [])]
|
||||
return album
|
||||
|
||||
|
||||
async def get_cover_art(
|
||||
client: httpx.AsyncClient, cover_id: str, size: int | None = None
|
||||
) -> tuple[bytes, str]:
|
||||
_require_configured()
|
||||
url = f"{_base_url()}/rest/getCoverArt.view"
|
||||
params = {**_auth_params(), "id": cover_id}
|
||||
if size:
|
||||
params["size"] = size
|
||||
try:
|
||||
response = await client.get(url, params=params)
|
||||
except httpx.RequestError as exc:
|
||||
raise NavidromeError(f"Could not reach Navidrome at {NAVIDROME_URL}: {exc}") from exc
|
||||
if response.status_code != 200:
|
||||
raise NavidromeError("Cover art not found.", status=404)
|
||||
content_type = (response.headers.get("content-type") or "image/jpeg").split(";")[0]
|
||||
if not content_type.startswith("image/"):
|
||||
# Subsonic returns a JSON error document on failure.
|
||||
raise NavidromeError("Cover art not found.", status=404)
|
||||
return response.content, content_type
|
||||
|
||||
|
||||
async def start_scan(client: httpx.AsyncClient, *, full: bool = False) -> dict:
|
||||
"""Trigger a Navidrome library scan (Subsonic ``startScan`` extension)."""
|
||||
payload = await _call(client, "startScan", {"fullScan": "true" if full else "false"})
|
||||
status = payload.get("scanStatus") or {}
|
||||
return {"scanning": bool(status.get("scanning")), "count": status.get("count")}
|
||||
|
||||
|
||||
async def get_genres(client: httpx.AsyncClient) -> list[dict]:
|
||||
payload = await _call(client, "getGenres")
|
||||
raw = ((payload.get("genres") or {}).get("genre")) or []
|
||||
genres = [
|
||||
{
|
||||
"name": g.get("value") or g.get("name") or "Unknown",
|
||||
"song_count": g.get("songCount") or 0,
|
||||
"album_count": g.get("albumCount") or 0,
|
||||
}
|
||||
for g in raw
|
||||
]
|
||||
genres.sort(key=lambda g: g["song_count"], reverse=True)
|
||||
return genres
|
||||
|
||||
|
||||
async def get_format_breakdown(
|
||||
client: httpx.AsyncClient, *, page_size: int = 500, max_pages: int = 400
|
||||
) -> dict:
|
||||
"""Count tracks by file format (flac, mp3, m4a, …) by paging all songs.
|
||||
|
||||
Subsonic has no aggregate format endpoint, so we walk ``search3`` with an
|
||||
empty query (Navidrome returns the whole library) and tally each song's
|
||||
``suffix``. Cap the page count so a runaway library can't loop forever.
|
||||
"""
|
||||
counts: dict[str, int] = {}
|
||||
offset = 0
|
||||
for _ in range(max_pages):
|
||||
payload = await _call(
|
||||
client,
|
||||
"search3",
|
||||
{
|
||||
"query": "",
|
||||
"artistCount": 0,
|
||||
"albumCount": 0,
|
||||
"songCount": page_size,
|
||||
"songOffset": offset,
|
||||
},
|
||||
)
|
||||
songs = ((payload.get("searchResult3") or {}).get("song")) or []
|
||||
if not songs:
|
||||
break
|
||||
for song in songs:
|
||||
suffix = (song.get("suffix") or "").lower() or "other"
|
||||
counts[suffix] = counts.get(suffix, 0) + 1
|
||||
if len(songs) < page_size:
|
||||
break
|
||||
offset += page_size
|
||||
|
||||
total = sum(counts.values())
|
||||
formats = sorted(
|
||||
({"format": fmt, "count": count} for fmt, count in counts.items()),
|
||||
key=lambda entry: entry["count"],
|
||||
reverse=True,
|
||||
)
|
||||
return {"total": total, "formats": formats}
|
||||
|
||||
|
||||
async def get_stats(client: httpx.AsyncClient) -> dict:
|
||||
"""Library stats for the dashboard: artists, albums, tracks and genres."""
|
||||
artists = await get_artists(client)
|
||||
album_count = sum(artist["album_count"] for artist in artists)
|
||||
try:
|
||||
genres = await get_genres(client)
|
||||
except NavidromeError:
|
||||
genres = []
|
||||
song_count = sum(g["song_count"] for g in genres)
|
||||
return {
|
||||
"artist_count": len(artists),
|
||||
"album_count": album_count,
|
||||
"song_count": song_count,
|
||||
"genre_count": len(genres),
|
||||
"top_genres": genres[:8],
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Deterministic, watch-history-based recommendation engine.
|
||||
|
||||
The scoring model is intentionally simple and explainable (section 8 of the
|
||||
spec). It is pure: ``build_profile`` and ``score_candidate`` touch no I/O and are
|
||||
the unit under test. ``build_candidates`` is the only function that calls Emby.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .emby_collections import ITEM_FIELDS, normalize_item
|
||||
|
||||
# Points awarded per matching facet. Genre is the strongest signal.
|
||||
SCORE_GENRE = 5 # per shared genre
|
||||
SCORE_SERIES = 4 # same series / franchise
|
||||
SCORE_DIRECTOR = 3 # per shared director
|
||||
SCORE_ACTOR = 2 # per shared actor
|
||||
SCORE_STUDIO = 2 # per shared studio
|
||||
SCORE_DECADE = 1 # release decade seen in history
|
||||
SCORE_MEDIA_TYPE = 1 # media type seen in history
|
||||
|
||||
DEFAULT_TARGET_SIZE = 25
|
||||
_CANDIDATE_FETCH_LIMIT = 300
|
||||
_MAX_GENRE_QUERY = 8
|
||||
|
||||
|
||||
def _decade(year) -> int | None:
|
||||
try:
|
||||
return (int(year) // 10) * 10
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TasteProfile:
|
||||
"""Aggregated facets of a user's watch history."""
|
||||
|
||||
genres: set[str] = field(default_factory=set)
|
||||
series: set[str] = field(default_factory=set)
|
||||
directors: set[str] = field(default_factory=set)
|
||||
actors: set[str] = field(default_factory=set)
|
||||
studios: set[str] = field(default_factory=set)
|
||||
decades: set[int] = field(default_factory=set)
|
||||
media_types: set[str] = field(default_factory=set)
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
return not (
|
||||
self.genres or self.series or self.directors or self.actors
|
||||
or self.studios or self.decades or self.media_types
|
||||
)
|
||||
|
||||
|
||||
def build_profile(watched_items: list[dict]) -> TasteProfile:
|
||||
"""Aggregate normalized watched items into a :class:`TasteProfile`."""
|
||||
profile = TasteProfile()
|
||||
for item in watched_items:
|
||||
profile.genres.update(item.get("genres") or [])
|
||||
profile.directors.update(item.get("directors") or [])
|
||||
profile.actors.update(item.get("actors") or [])
|
||||
profile.studios.update(item.get("studios") or [])
|
||||
profile.media_types.add(item.get("media_type") or item.get("type") or "")
|
||||
series = item.get("series_name")
|
||||
if series:
|
||||
profile.series.add(series)
|
||||
# A watched series is itself a "franchise" anchor for similar items.
|
||||
if (item.get("type") == "Series") and item.get("title"):
|
||||
profile.series.add(item["title"])
|
||||
decade = _decade(item.get("year"))
|
||||
if decade is not None:
|
||||
profile.decades.add(decade)
|
||||
profile.media_types.discard("")
|
||||
return profile
|
||||
|
||||
|
||||
def score_candidate(candidate: dict, profile: TasteProfile) -> int:
|
||||
"""Score a candidate against the profile. Higher is more similar."""
|
||||
score = 0
|
||||
score += SCORE_GENRE * len(set(candidate.get("genres") or []) & profile.genres)
|
||||
score += SCORE_DIRECTOR * len(set(candidate.get("directors") or []) & profile.directors)
|
||||
score += SCORE_ACTOR * len(set(candidate.get("actors") or []) & profile.actors)
|
||||
score += SCORE_STUDIO * len(set(candidate.get("studios") or []) & profile.studios)
|
||||
|
||||
series = candidate.get("series_name") or (candidate.get("title") if candidate.get("type") == "Series" else None)
|
||||
if series and series in profile.series:
|
||||
score += SCORE_SERIES
|
||||
|
||||
decade = _decade(candidate.get("year"))
|
||||
if decade is not None and decade in profile.decades:
|
||||
score += SCORE_DECADE
|
||||
|
||||
media_type = candidate.get("media_type") or candidate.get("type")
|
||||
if media_type and media_type in profile.media_types:
|
||||
score += SCORE_MEDIA_TYPE
|
||||
|
||||
return score
|
||||
|
||||
|
||||
def rank_candidates(candidates: list[dict], profile: TasteProfile) -> list[dict]:
|
||||
"""Score, filter to score > 0, and sort candidates.
|
||||
|
||||
Order: score desc, then community rating desc, then title asc. Each returned
|
||||
item carries an added ``score`` key for transparency in the UI/logs.
|
||||
"""
|
||||
scored = []
|
||||
for candidate in candidates:
|
||||
score = score_candidate(candidate, profile)
|
||||
if score <= 0:
|
||||
continue
|
||||
scored.append({**candidate, "score": score})
|
||||
scored.sort(
|
||||
key=lambda c: (-c["score"], -(c.get("community_rating") or 0.0), (c.get("title") or "").casefold())
|
||||
)
|
||||
return scored
|
||||
|
||||
|
||||
async def build_candidates(client, user_id: str, profile: TasteProfile, exclude_ids: set[str]) -> list[dict]:
|
||||
"""Fetch unplayed library items similar to the profile, minus exclusions.
|
||||
|
||||
Uses the user-scoped ``IsUnplayed`` filter so already-watched items never
|
||||
enter the pool. Anything in ``exclude_ids`` (playlist members, defensively
|
||||
re-checked watched ids) is dropped.
|
||||
"""
|
||||
if profile.is_empty:
|
||||
return []
|
||||
|
||||
genres = sorted(profile.genres)[:_MAX_GENRE_QUERY]
|
||||
media_types = ",".join(sorted(t for t in profile.media_types if t)) or "Movie,Series"
|
||||
params = {
|
||||
"Recursive": "true",
|
||||
"Filters": "IsUnplayed",
|
||||
"IsPlayed": "false",
|
||||
"IncludeItemTypes": media_types if media_types in ("Movie", "Series", "Movie,Series") else "Movie,Series",
|
||||
"Fields": ITEM_FIELDS,
|
||||
"EnableUserData": "true",
|
||||
"SortBy": "CommunityRating",
|
||||
"SortOrder": "Descending",
|
||||
"Limit": str(_CANDIDATE_FETCH_LIMIT),
|
||||
}
|
||||
if genres:
|
||||
# Emby treats "|" as OR across genre values.
|
||||
params["Genres"] = "|".join(genres)
|
||||
|
||||
data = await client.get(f"/Users/{user_id}/Items", params)
|
||||
raw_items = data.get("Items", []) if isinstance(data, dict) else (data or [])
|
||||
|
||||
seen: set[str] = set()
|
||||
candidates: list[dict] = []
|
||||
for raw in raw_items:
|
||||
item = normalize_item(raw)
|
||||
item_id = item["id"]
|
||||
if not item_id or item_id in exclude_ids or item_id in seen:
|
||||
continue
|
||||
if item.get("watched"): # defensive: never recommend a watched item
|
||||
continue
|
||||
seen.add(item_id)
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Runtime settings store.
|
||||
|
||||
Configuration can come from two places: environment variables (the deploy-time
|
||||
defaults) and a JSON file written by the in-app Settings page. The file, when
|
||||
present, wins. ``load`` returns the effective settings; ``save`` persists the
|
||||
editable subset and returns the new effective settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
SETTINGS_FILE = Path(os.environ.get("SETTINGS_FILE", "cache/settings.json"))
|
||||
|
||||
FIELDS = (
|
||||
"emby_url",
|
||||
"emby_api_key",
|
||||
"navidrome_url",
|
||||
"navidrome_user",
|
||||
"navidrome_password",
|
||||
"music_root",
|
||||
)
|
||||
|
||||
|
||||
def env_defaults() -> dict:
|
||||
return {
|
||||
"emby_url": os.environ.get("EMBY_URL", "http://10.0.0.2:8096"),
|
||||
"emby_api_key": os.environ.get("EMBY_API_KEY", ""),
|
||||
"navidrome_url": os.environ.get("NAVIDROME_URL", "http://10.0.0.2:4533"),
|
||||
"navidrome_user": os.environ.get("NAVIDROME_USER", ""),
|
||||
"navidrome_password": os.environ.get("NAVIDROME_PASSWORD", ""),
|
||||
"music_root": os.environ.get("MUSIC_ROOT", r"\\Matt-htpc\d\Music"),
|
||||
}
|
||||
|
||||
|
||||
def _read_file() -> dict:
|
||||
if not SETTINGS_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return {k: str(v) for k, v in data.items() if k in FIELDS and v is not None}
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
"""Effective settings: env defaults overlaid with the saved file."""
|
||||
values = env_defaults()
|
||||
values.update(_read_file())
|
||||
return values
|
||||
|
||||
|
||||
def save(updates: dict) -> dict:
|
||||
"""Persist the editable subset of ``updates`` and return effective settings."""
|
||||
current = _read_file()
|
||||
for key in FIELDS:
|
||||
if key in updates and updates[key] is not None:
|
||||
current[key] = str(updates[key]).strip()
|
||||
SETTINGS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
SETTINGS_FILE.write_text(json.dumps(current, indent=2), encoding="utf-8")
|
||||
return load()
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Name normalisation and fuzzy matching for collection completeness.
|
||||
|
||||
Normalisation never mutates the stored original — callers keep both the original
|
||||
and normalised values. The point is to make "Album (Deluxe Edition)" and
|
||||
"Album - 2009 Remaster" collapse to the same comparable key without losing the
|
||||
display name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
_BRACKET_RE = re.compile(r"[\(\[\{].*?[\)\]\}]")
|
||||
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
|
||||
_WS_RE = re.compile(r"\s+")
|
||||
|
||||
# Edition / remaster qualifiers stripped from album titles before comparison.
|
||||
_EDITION_PATTERNS = [
|
||||
r"\bsuper deluxe( edition)?\b",
|
||||
r"\bdeluxe( edition| version)?\b",
|
||||
r"\bexpanded( edition| version)?\b",
|
||||
r"\bspecial edition\b",
|
||||
r"\bcollector'?s edition\b",
|
||||
r"\blegacy edition\b",
|
||||
r"\banniversary( edition)?\b",
|
||||
r"\b\d{1,3}(st|nd|rd|th) anniversary\b",
|
||||
r"\bremaster(ed)?\b",
|
||||
r"\bre-?master(ed)?\b",
|
||||
r"\breissue\b",
|
||||
r"\bbonus track(s)?( version)?\b",
|
||||
r"\bbonus edition\b",
|
||||
r"\b\d{4} remaster\b",
|
||||
r"\bexplicit( version)?\b",
|
||||
r"\bclean( version)?\b",
|
||||
r"\bmono\b",
|
||||
r"\bstereo\b",
|
||||
r"\bdisc \d+\b",
|
||||
r"\bcd\d+\b",
|
||||
]
|
||||
_EDITION_RE = re.compile("|".join(_EDITION_PATTERNS), re.IGNORECASE)
|
||||
|
||||
VARIOUS_ARTISTS = {
|
||||
"various artists",
|
||||
"various",
|
||||
"va",
|
||||
"v a",
|
||||
"soundtrack",
|
||||
"original soundtrack",
|
||||
}
|
||||
|
||||
|
||||
def _base_clean(text: str) -> str:
|
||||
text = text.lower()
|
||||
text = _BRACKET_RE.sub(" ", text) # drop (...) [...] {...}
|
||||
text = _EDITION_RE.sub(" ", text) # drop edition/remaster words
|
||||
text = text.replace("&", " and ")
|
||||
text = _PUNCT_RE.sub(" ", text) # drop remaining punctuation
|
||||
return _WS_RE.sub(" ", text).strip()
|
||||
|
||||
|
||||
def normalize_title(text: str | None) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
return _base_clean(text)
|
||||
|
||||
|
||||
def normalize_artist(text: str | None) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
cleaned = _base_clean(text)
|
||||
if cleaned.startswith("the "):
|
||||
cleaned = cleaned[4:]
|
||||
return cleaned
|
||||
|
||||
|
||||
def is_various_artists(name: str | None) -> bool:
|
||||
if not name:
|
||||
return False
|
||||
return normalize_artist(name) in VARIOUS_ARTISTS or _base_clean(name) in VARIOUS_ARTISTS
|
||||
|
||||
|
||||
def similarity(a: str | None, b: str | None) -> float:
|
||||
na, nb = normalize_title(a), normalize_title(b)
|
||||
if not na or not nb:
|
||||
return 0.0
|
||||
if na == nb:
|
||||
return 1.0
|
||||
return SequenceMatcher(None, na, nb).ratio()
|
||||
|
||||
|
||||
# ── Completeness classification ───────────────────────────────────────────────
|
||||
# Statuses: owned | probably_owned | missing | uncertain (ignored is manual).
|
||||
|
||||
OWNED = "owned"
|
||||
PROBABLY_OWNED = "probably_owned"
|
||||
UNCERTAIN = "uncertain"
|
||||
MISSING = "missing"
|
||||
|
||||
FUZZY_PROBABLE = 0.88
|
||||
FUZZY_UNCERTAIN = 0.60
|
||||
|
||||
|
||||
def classify_release(
|
||||
ext_title: str,
|
||||
ext_year: int | None,
|
||||
ext_mbid: str | None,
|
||||
local_albums: list[dict],
|
||||
) -> tuple[str, float, str, int | None]:
|
||||
"""Decide a status for one external release group against local albums.
|
||||
|
||||
``local_albums`` items: ``{id, title, title_normalized, year, mbid}``.
|
||||
Match order: MusicBrainz id → normalised title (+year) → fuzzy title.
|
||||
Returns ``(status, confidence, reason, local_album_id|None)``.
|
||||
"""
|
||||
ext_norm = normalize_title(ext_title)
|
||||
|
||||
# 1) Exact MusicBrainz id match.
|
||||
if ext_mbid:
|
||||
for la in local_albums:
|
||||
if la.get("mbid") and la["mbid"] == ext_mbid:
|
||||
return (OWNED, 1.0, "MusicBrainz ID match", la["id"])
|
||||
|
||||
# 2) Normalised title (with year corroboration).
|
||||
if ext_norm:
|
||||
for la in local_albums:
|
||||
if la.get("title_normalized") == ext_norm:
|
||||
ly, ey = la.get("year") or 0, ext_year or 0
|
||||
if ly and ey and abs(ly - ey) <= 1:
|
||||
return (OWNED, 0.95, "Title and year match", la["id"])
|
||||
return (PROBABLY_OWNED, 0.85, "Normalised title match", la["id"])
|
||||
|
||||
# 3) Fuzzy title.
|
||||
best_ratio, best = 0.0, None
|
||||
for la in local_albums:
|
||||
r = similarity(ext_title, la.get("title"))
|
||||
if r > best_ratio:
|
||||
best_ratio, best = r, la
|
||||
if best is not None:
|
||||
if best_ratio >= FUZZY_PROBABLE:
|
||||
return (PROBABLY_OWNED, round(best_ratio, 3), f"Fuzzy title match ({best_ratio:.2f})", best["id"])
|
||||
if best_ratio >= FUZZY_UNCERTAIN:
|
||||
return (UNCERTAIN, round(best_ratio, 3), f"Weak title match ({best_ratio:.2f})", best["id"])
|
||||
|
||||
return (MISSING, 0.0, "No matching local album", None)
|
||||
Reference in New Issue
Block a user