From 131b6f8d024b0e2315c3dc7b240a9b43797d2a17 Mon Sep 17 00:00:00 2001
From: ponzischeme89 <matt@mattcohen.net>
Date: Sun, 18 Jan 2026 23:18:38 +1300
Subject: [PATCH] 1.0.9 improves library loading performance

---
 frontend/src/App.svelte                       |  4 +-
 frontend/src/components/AppSidebar.svelte     |  2 +-
 .../{ => library}/LibraryPanel.svelte         | 92 +++++++++++++------
 .../components/{ => scan}/ResultsList.svelte  | 10 +-
 .../components/{ => scan}/ScanPanel.svelte    | 18 ++--
 frontend/src/lib/api.js                       | 10 +-
 server/app.py                                 | 28 +++++-
 server/core/database.py                       | 51 +++++++---
 server/core/file_scanner.py                   |  6 +-
 9 files changed, 154 insertions(+), 67 deletions(-)
 rename frontend/src/components/{ => library}/LibraryPanel.svelte (67%)
 rename frontend/src/components/{ => scan}/ResultsList.svelte (99%)
 rename frontend/src/components/{ => scan}/ScanPanel.svelte (98%)

diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte
index cd6aee5..3f36295 100644
--- a/frontend/src/App.svelte
+++ b/frontend/src/App.svelte
@@ -5,9 +5,9 @@
   import * as Sidebar from "./lib/components/ui/sidebar";
   import { Button } from "./lib/components/ui/button";
   import SettingsPanel from "./components/SettingsPanel.svelte";
-  import ScanPanel from "./components/ScanPanel.svelte";
+  import ScanPanel from "./components/scan/ScanPanel.svelte";
   import HistoryPanel from "./components/HistoryPanel.svelte";
-  import LibraryPanel from "./components/LibraryPanel.svelte";
+  import LibraryPanel from "./components/library/LibraryPanel.svelte";
   import { Menu } from "lucide-svelte";
   import ToastHost from "./components/ToastHost.svelte";
   import { healthCheck } from "./lib/api.js";
diff --git a/frontend/src/components/AppSidebar.svelte b/frontend/src/components/AppSidebar.svelte
index 0822ab8..c048eef 100644
--- a/frontend/src/components/AppSidebar.svelte
+++ b/frontend/src/components/AppSidebar.svelte
@@ -154,7 +154,7 @@
       >
         {#if !collapsed}
           <Badge className="bg-white/10 text-text-secondary"
-            >v1.0.8 Release Candiate</Badge
+            >v1.0.9 Release Candiate</Badge
           >
         {:else}
           <Badge className="bg-white/10 text-text-secondary">v</Badge>
diff --git a/frontend/src/components/LibraryPanel.svelte b/frontend/src/components/library/LibraryPanel.svelte
similarity index 67%
rename from frontend/src/components/LibraryPanel.svelte
rename to frontend/src/components/library/LibraryPanel.svelte
index 41d7cda..22f3c4b 100644
--- a/frontend/src/components/LibraryPanel.svelte
+++ b/frontend/src/components/library/LibraryPanel.svelte
@@ -1,30 +1,31 @@
 <script>
   import { onMount } from "svelte";
-  import { getLibraryReport } from "../lib/api.js";
-  import { Button } from "../lib/components/ui/button";
+  import { getLibraryReport } from "../../lib/api.js";
+  import { Button } from "../../lib/components/ui/button";
+  import { Skeleton } from "../../lib/components/ui/skeleton";
   import { ChevronDown, ChevronUp, RefreshCcw, FileText } from "lucide-svelte";
 
   let items = [];
   let loading = false;
   let error = null;
   let expanded = {};
-  let page = 0;
+  let page = 1;
   const pageSize = 200;
+  let totalItems = 0;
+  let totalPages = 1;
+  let hasMore = false;
 
-  async function loadLibrary(reset = true) {
+  async function loadLibrary(nextPage = 1) {
     loading = true;
     error = null;
     try {
-      if (reset) {
-        page = 0;
-        items = [];
-      }
-      const response = await getLibraryReport(pageSize, page * pageSize);
-      const nextItems = response.items || [];
-      items = reset ? nextItems : [...items, ...nextItems];
-      if (nextItems.length > 0) {
-        page += 1;
-      }
+      page = nextPage;
+      expanded = {};
+      const response = await getLibraryReport(page, pageSize);
+      items = response.items || [];
+      totalItems = response.total_items ?? items.length;
+      totalPages = Math.max(1, Math.ceil(totalItems / pageSize));
+      hasMore = response.has_more ?? page < totalPages;
     } catch (err) {
       error = `Failed to load library report: ${err.message}`;
     } finally {
@@ -36,7 +37,7 @@
     expanded = { ...expanded, [key]: !expanded[key] };
   }
 
-  onMount(() => loadLibrary(true));
+  onMount(() => loadLibrary(1));
 </script>
 
 <div class="space-y-6">
@@ -51,7 +52,7 @@
       variant="outline"
       size="sm"
       className="border-white/15 text-text-secondary hover:bg-white/10"
-      on:click={() => loadLibrary(true)}
+      on:click={() => loadLibrary(1)}
       disabled={loading}
     >
       <RefreshCcw class="h-4 w-4" />
@@ -66,7 +67,24 @@
   {/if}
 
   {#if loading}
-    <div class="text-[13px] text-text-secondary">Loading library report...</div>
+    <div class="space-y-4">
+      {#each Array(4) as _, index}
+        <div class="rounded-2xl border border-border bg-card/60 overflow-hidden">
+          <div class="px-6 py-4 flex flex-col gap-3 md:flex-row md:items-center md:justify-between">
+            <div class="space-y-2">
+              <Skeleton className="h-4 w-40" />
+              <Skeleton className="h-3 w-24" />
+            </div>
+            <div class="flex items-center gap-3">
+              <Skeleton className="h-6 w-20 rounded-full" />
+              <Skeleton className="h-6 w-20 rounded-full" />
+              <Skeleton className="h-6 w-20 rounded-full" />
+              <Skeleton className="h-6 w-6 rounded-full" />
+            </div>
+          </div>
+        </div>
+      {/each}
+    </div>
   {:else if items.length === 0}
     <div class="border border-border rounded-2xl p-12 text-center">
       <div class="flex flex-col items-center gap-4">
@@ -102,10 +120,10 @@
               </span>
               <button
                 class="ml-2 text-text-secondary hover:text-white transition-colors"
-                on:click={() => toggleScan(item.title)}
+                on:click={() => toggleScan(item.year ? `${item.title} (${item.year})` : item.title)}
                 aria-label="Toggle scan details"
               >
-                {#if expanded[item.title]}
+                {#if expanded[item.year ? `${item.title} (${item.year})` : item.title]}
                   <ChevronUp class="h-4 w-4" />
                 {:else}
                   <ChevronDown class="h-4 w-4" />
@@ -114,7 +132,7 @@
             </div>
           </div>
 
-          {#if expanded[item.title]}
+          {#if expanded[item.year ? `${item.title} (${item.year})` : item.title]}
             <div class="border-t border-border bg-bg-secondary/40">
               <div class="px-6 py-4 overflow-x-auto">
                 <table class="min-w-full text-[12px] text-text-secondary">
@@ -158,16 +176,30 @@
           {/if}
         </div>
       {/each}
-      <div class="flex justify-center">
-        <Button
-          variant="outline"
-          size="sm"
-          className="border-white/15 text-text-secondary hover:bg-white/10"
-          on:click={() => loadLibrary(false)}
-          disabled={loading}
-        >
-          Load more
-        </Button>
+      <div class="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 pt-2">
+        <div class="text-[11px] text-text-tertiary">
+          Page {page} of {totalPages}
+        </div>
+        <div class="flex items-center gap-2">
+          <Button
+            variant="outline"
+            size="sm"
+            className="border-white/15 text-text-secondary hover:bg-white/10"
+            on:click={() => loadLibrary(page - 1)}
+            disabled={loading || page <= 1}
+          >
+            Previous
+          </Button>
+          <Button
+            variant="outline"
+            size="sm"
+            className="border-white/15 text-text-secondary hover:bg-white/10"
+            on:click={() => loadLibrary(page + 1)}
+            disabled={loading || !hasMore}
+          >
+            Next
+          </Button>
+        </div>
       </div>
     </div>
   {/if}
diff --git a/frontend/src/components/ResultsList.svelte b/frontend/src/components/scan/ResultsList.svelte
similarity index 99%
rename from frontend/src/components/ResultsList.svelte
rename to frontend/src/components/scan/ResultsList.svelte
index c902900..59ff50d 100644
--- a/frontend/src/components/ResultsList.svelte
+++ b/frontend/src/components/scan/ResultsList.svelte
@@ -1,8 +1,8 @@
 <script>
   import { createEventDispatcher } from "svelte";
-  import StatusBadge from "./StatusBadge.svelte";
-  import { Button } from "../lib/components/ui/button";
-  import { Skeleton } from "../lib/components/ui/skeleton";
+  import StatusBadge from "../StatusBadge.svelte";
+  import { Button } from "../../lib/components/ui/button";
+  import { Skeleton } from "../../lib/components/ui/skeleton";
   import {
     Table,
     TableBody,
@@ -10,12 +10,12 @@
     TableHead,
     TableHeader,
     TableRow,
-  } from "../lib/components/ui/table";
+  } from "../../lib/components/ui/table";
   import {
     searchTitle,
     saveSuggestedMatches,
     processBatch,
-  } from "../lib/api.js";
+  } from "../../lib/api.js";
 
   const dispatch = createEventDispatcher();
 
diff --git a/frontend/src/components/ScanPanel.svelte b/frontend/src/components/scan/ScanPanel.svelte
similarity index 98%
rename from frontend/src/components/ScanPanel.svelte
rename to frontend/src/components/scan/ScanPanel.svelte
index 165dc5e..b0a61bd 100644
--- a/frontend/src/components/ScanPanel.svelte
+++ b/frontend/src/components/scan/ScanPanel.svelte
@@ -8,22 +8,22 @@
     clearAllSuggestedMatches,
     getFolderRules,
     getScanHistory,
-  } from "../lib/api.js";
+  } from "../../lib/api.js";
   import ResultsList from "./ResultsList.svelte";
-  import TypewriterQuote from "./TypewriterQuote.svelte";
-  import { scanResults } from "../lib/scanStore.js";
-  import { Button } from "../lib/components/ui/button";
-  import { addToast } from "../lib/toastStore.js";
+  import TypewriterQuote from "../TypewriterQuote.svelte";
+  import { scanResults } from "../../lib/scanStore.js";
+  import { Button } from "../../lib/components/ui/button";
+  import { addToast } from "../../lib/toastStore.js";
   import {
     Card,
     CardContent,
     CardDescription,
     CardHeader,
     CardTitle,
-  } from "../lib/components/ui/card";
-  import { Input } from "../lib/components/ui/input";
-  import { Skeleton } from "../lib/components/ui/skeleton";
-  import { Combobox } from "../lib/components/ui/combobox";
+  } from "../../lib/components/ui/card";
+  import { Input } from "../../lib/components/ui/input";
+  import { Skeleton } from "../../lib/components/ui/skeleton";
+  import { Combobox } from "../../lib/components/ui/combobox";
   import { FileText, Folder, Info, Plug, Scan } from "lucide-svelte";
 
   // ------------------------------------------------------------
diff --git a/frontend/src/lib/api.js b/frontend/src/lib/api.js
index 05513aa..d29b7f2 100644
--- a/frontend/src/lib/api.js
+++ b/frontend/src/lib/api.js
@@ -336,10 +336,14 @@ export async function getStatistics() {
 
 /**
  * GET /api/library - Get library health report
- * Returns: { success, scans: [...] }
+ * Returns: { success, items: [...], total_items, page, page_size, has_more }
  */
-export async function getLibraryReport(limit = 200, offset = 0) {
-  return apiFetch(`/library?limit=${limit}&offset=${offset}`)
+export async function getLibraryReport(page = 1, pageSize = 200) {
+  const params = new URLSearchParams({
+    page: String(page),
+    page_size: String(pageSize)
+  })
+  return apiFetch(`/library?${params.toString()}`)
 }
 
 // ============ SCHEDULED SCANS API ============
diff --git a/server/app.py b/server/app.py
index de7f7af..858b118 100644
--- a/server/app.py
+++ b/server/app.py
@@ -287,7 +287,7 @@ def _group_key(title: str, year: str | None) -> str:
     return f"{base} ({year})" if year else base
 
 
-def _build_library_items(files: list[dict], latest_results: dict, limit: int) -> list[dict]:
+def _build_library_items(files: list[dict], latest_results: dict, limit: int | None) -> list[dict]:
     """Aggregate scan files into library items."""
     grouped = {}
     for file_info in files:
@@ -354,6 +354,8 @@ def _build_library_items(files: list[dict], latest_results: dict, limit: int) ->
         ),
         reverse=True
     )
+    if limit is None:
+        return items
     return items[:limit]
 
 def get_format_options_from_settings() -> SubtitleFormatOptions:
@@ -1644,14 +1646,30 @@ def get_scan_history():
 def get_library_report():
     """Get library health report with scan files and issue summaries"""
     try:
-        limit = request.args.get('limit', 200, type=int)
-        offset = request.args.get('offset', 0, type=int)
-        latest_files = db.get_latest_scan_files(limit=limit, offset=offset)
+        page_size = request.args.get('page_size', type=int)
+        page = request.args.get('page', type=int)
+        if page_size is None:
+            page_size = request.args.get('limit', 200, type=int)
+        if page is None:
+            offset = request.args.get('offset', 0, type=int)
+            page = (offset // page_size) + 1 if page_size else 1
+
+        latest_files = db.get_latest_scan_files(limit=None, offset=0)
         latest_results = db.get_latest_file_results()
+        items = _build_library_items(latest_files, latest_results, None)
+
+        total_items = len(items)
+        start = max(0, (page - 1) * page_size)
+        end = start + page_size
+        page_items = items[start:end]
 
         return jsonify({
             "success": True,
-            "items": _build_library_items(latest_files, latest_results, limit)
+            "items": page_items,
+            "total_items": total_items,
+            "page": page,
+            "page_size": page_size,
+            "has_more": end < total_items
         })
     except Exception as e:
         logger.error(f"Error fetching library report: {e}")
diff --git a/server/core/database.py b/server/core/database.py
index 7d4a040..3f667db 100644
--- a/server/core/database.py
+++ b/server/core/database.py
@@ -4,7 +4,7 @@ Handles persistent storage for settings, runs, and history
 """
 from datetime import datetime
 from pathlib import Path
-from sqlalchemy import create_engine, Column, Integer, String, DateTime, Boolean, Float, Text, ForeignKey, text
+from sqlalchemy import create_engine, Column, Integer, String, DateTime, Boolean, Float, Text, ForeignKey, text, func
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker, relationship, scoped_session
 import json
@@ -506,17 +506,28 @@ class DatabaseManager:
             session.close()
 
     def get_latest_scan_files(self, limit=500, offset=0):
-        """Get latest scan entry per file path, paged by scan_files.created_at"""
+        """Get latest scan entry per file path, optionally paged by created_at."""
         session = self.get_session()
         try:
-            files = session.query(ScanFile).order_by(
-                ScanFile.created_at.desc()
-            ).offset(offset).limit(limit).all()
-            latest = {}
-            for file_entry in files:
-                if file_entry.file_path in latest:
-                    continue
-                latest[file_entry.file_path] = {
+            latest_subquery = session.query(
+                ScanFile.file_path,
+                func.max(ScanFile.created_at).label("max_created_at")
+            ).group_by(ScanFile.file_path).subquery()
+
+            query = session.query(ScanFile).join(
+                latest_subquery,
+                (ScanFile.file_path == latest_subquery.c.file_path)
+                & (ScanFile.created_at == latest_subquery.c.max_created_at)
+            ).order_by(ScanFile.created_at.desc())
+
+            if offset:
+                query = query.offset(offset)
+            if limit is not None:
+                query = query.limit(limit)
+
+            files = query.all()
+            return [
+                {
                     "path": file_entry.file_path,
                     "name": file_entry.file_name,
                     "title": file_entry.title,
@@ -526,7 +537,25 @@ class DatabaseManager:
                     "status": file_entry.status,
                     "summary": file_entry.summary
                 }
-            return list(latest.values())
+                for file_entry in files
+            ]
+        finally:
+            session.close()
+
+    def get_latest_scan_files_count(self):
+        """Count distinct latest scan entries per file path."""
+        session = self.get_session()
+        try:
+            latest_subquery = session.query(
+                ScanFile.file_path,
+                func.max(ScanFile.created_at).label("max_created_at")
+            ).group_by(ScanFile.file_path).subquery()
+
+            return session.query(ScanFile).join(
+                latest_subquery,
+                (ScanFile.file_path == latest_subquery.c.file_path)
+                & (ScanFile.created_at == latest_subquery.c.max_created_at)
+            ).count()
         finally:
             session.close()
 
diff --git a/server/core/file_scanner.py b/server/core/file_scanner.py
index 6f5a7f4..369a7df 100644
--- a/server/core/file_scanner.py
+++ b/server/core/file_scanner.py
@@ -138,13 +138,17 @@ class FileScanner:
                         file_path.name, e
                     )
 
+            status = "Has Plot" if has_plot else "Not Loaded"
+            if plot_marker_count > 1:
+                status = "Duplicate Plot"
+
             batch.append({
                 "path": str(file_path),
                 "name": file_path.name,
                 "has_plot": has_plot,
                 "plot_marker_count": plot_marker_count,
                 "duplicate_plot": plot_marker_count > 1,
-                "status": "Has Plot" if has_plot else "Not Loaded",
+                "status": status,
                 "summary": metadata.get("summary", ""),
                 "plot": metadata.get("summary", ""),
                 "title": metadata.get("title"),