from __future__ import annotations import asyncio import ipaddress import json import logging import os import posixpath import time from datetime import datetime from pathlib import Path from stat import S_ISDIR from typing import Any try: import paramiko except ImportError: # pragma: no cover - exercised indirectly via status checks paramiko = None from services import settings as settings_service DEFAULT_REMOTE_APP_DIR = "/share/Docker/homelabtoolkit" DEPLOY_TIMEOUT_SECONDS = int(os.environ.get("DEPLOY_TIMEOUT_SECONDS", "1800")) ROOT_DIR = Path(__file__).resolve().parent.parent REMOTE_CONTAINER_NAME = os.environ.get("DEPLOY_CONTAINER_NAME", "homelabtoolkit").strip() or "homelabtoolkit" TOP_LEVEL_FILES = ("app.py", "rotate_preroll.py", "Dockerfile", "docker-compose.yml", "requirements.txt") FRONTEND_ROOT_FILES = ("package.json", "package-lock.json", "vite.config.ts", "tsconfig.json", "index.html") LOGO_EXTENSIONS = {".png", ".jpg", ".jpeg"} runtime: dict[str, Any] = { "running": False, "last_started_at": None, "last_finished_at": None, "last_status": "idle", "last_message": None, "last_output_tail": [], } _lock = asyncio.Lock() _task: asyncio.Task | None = None logging.getLogger("paramiko").setLevel(logging.WARNING) logger = logging.getLogger("homelabtoolkit.update") def _now() -> str: return datetime.now().astimezone().isoformat(timespec="seconds") def normalize_client_host(value: str | None) -> str: text = (value or "").strip() if not text: return "" if "," in text: text = text.split(",", 1)[0].strip() if text.startswith("::ffff:"): text = text[len("::ffff:") :] return text def is_local_client(host: str | None) -> bool: text = normalize_client_host(host) if not text: return False if text.lower() == "localhost": return True try: ip = ipaddress.ip_address(text) except ValueError: return False return ip.is_loopback or ip.is_private def is_paramiko_available() -> bool: return paramiko is not None def is_configured(values: dict[str, Any]) -> bool: return bool(str(values.get("deploy_nas_host") or "").strip() and str(values.get("deploy_nas_user") or "").strip()) def remote_app_dir(values: dict[str, Any]) -> str: return str(values.get("deploy_remote_app_dir") or DEFAULT_REMOTE_APP_DIR).strip() or DEFAULT_REMOTE_APP_DIR def deploy_password(values: dict[str, Any]) -> str: return str(values.get("deploy_nas_password") or "") def using_saved_password(values: dict[str, Any]) -> bool: return bool(deploy_password(values)) def deploy_music_host_path(values: dict[str, Any]) -> str: return str(values.get("deploy_music_host_path") or "/share/Music").strip() or "/share/Music" def _remote(values: dict[str, Any]) -> str: return f'{str(values.get("deploy_nas_user") or "").strip()}@{str(values.get("deploy_nas_host") or "").strip()}' def _tail(lines: list[str], max_lines: int = 40) -> list[str]: return lines[-max_lines:] def _push_runtime_output(log_lines: list[str]) -> None: runtime["last_output_tail"] = list(_tail(log_lines)) def _log(log_lines: list[str], message: str) -> None: text = message.rstrip() if not text: return log_lines.append(text) _push_runtime_output(log_lines) logger.info(text) def _ensure_remote_dir(sftp, remote_dir: str) -> None: normalized = posixpath.normpath(remote_dir) parts = [part for part in normalized.split("/") if part] current = "/" if normalized.startswith("/") else "" for part in parts: current = posixpath.join(current, part) if current not in ("", "/") else f"{current}{part}" if current == "/" else part try: attrs = sftp.stat(current) if not S_ISDIR(attrs.st_mode): raise RuntimeError(f"Remote path exists but is not a directory: {current}") except OSError: sftp.mkdir(current) def _upload_file(sftp, local_path: Path, remote_path: str, log_lines: list[str]) -> None: _ensure_remote_dir(sftp, posixpath.dirname(remote_path)) sftp.put(str(local_path), remote_path) _log(log_lines, f"Uploaded {local_path.relative_to(ROOT_DIR).as_posix()} -> {remote_path}") def _upload_text(sftp, content: str, remote_path: str, label: str, log_lines: list[str]) -> None: _ensure_remote_dir(sftp, posixpath.dirname(remote_path)) with sftp.file(remote_path, "w") as handle: handle.write(content) _log(log_lines, f"Rendered {label} -> {remote_path}") def _upload_tree(sftp, local_dir: Path, remote_dir: str, log_lines: list[str]) -> None: if not local_dir.exists(): return _ensure_remote_dir(sftp, remote_dir) for path in sorted(local_dir.rglob("*")): if path.is_dir(): _ensure_remote_dir(sftp, posixpath.join(remote_dir, path.relative_to(local_dir).as_posix())) continue if "__pycache__" in path.parts: continue remote_path = posixpath.join(remote_dir, path.relative_to(local_dir).as_posix()) _upload_file(sftp, path, remote_path, log_lines) def _stream_output(channel, log_lines: list[str]) -> None: stdout_buffer = "" stderr_buffer = "" while True: had_output = False if channel.recv_ready(): stdout_buffer += channel.recv(4096).decode("utf-8", errors="replace") had_output = True while "\n" in stdout_buffer: line, stdout_buffer = stdout_buffer.split("\n", 1) _log(log_lines, line) if channel.recv_stderr_ready(): stderr_buffer += channel.recv_stderr(4096).decode("utf-8", errors="replace") had_output = True while "\n" in stderr_buffer: line, stderr_buffer = stderr_buffer.split("\n", 1) _log(log_lines, line) if channel.exit_status_ready() and not channel.recv_ready() and not channel.recv_stderr_ready(): break if not had_output: time.sleep(0.1) if stdout_buffer.strip(): _log(log_lines, stdout_buffer) if stderr_buffer.strip(): _log(log_lines, stderr_buffer) def _run_remote_command(client, command: str, log_lines: list[str], allow_failure: bool = False) -> tuple[int, list[str]]: stdin, stdout, stderr = client.exec_command(command, timeout=DEPLOY_TIMEOUT_SECONDS) if stdin: stdin.close() channel = stdout.channel command_lines: list[str] = [] _stream_output(channel, command_lines) for line in command_lines: _log(log_lines, line) exit_code = stdout.channel.recv_exit_status() if exit_code != 0 and not allow_failure: raise RuntimeError(f"Remote command failed with exit code {exit_code}") return exit_code, command_lines def _connect(values: dict[str, Any]): if paramiko is None: raise RuntimeError("Paramiko is not installed.") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) connect_kwargs: dict[str, Any] = { "hostname": str(values.get("deploy_nas_host") or "").strip(), "username": str(values.get("deploy_nas_user") or "").strip(), "timeout": 20, "banner_timeout": 20, "auth_timeout": 20, } password = deploy_password(values) if password: connect_kwargs["password"] = password connect_kwargs["look_for_keys"] = False connect_kwargs["allow_agent"] = False client.connect(**connect_kwargs) return client def render_remote_compose(values: dict[str, Any]) -> str: source = (ROOT_DIR / "docker-compose.yml").read_text(encoding="utf-8") remote_dir = remote_app_dir(values).rstrip("/") music_host = deploy_music_host_path(values) rendered = source rendered = rendered.replace("/share/Docker/homelabtoolkit/output:/app/output", f"{remote_dir}/output:/app/output") rendered = rendered.replace("/share/Docker/homelabtoolkit/cache:/app/cache", f"{remote_dir}/cache:/app/cache") rendered = rendered.replace("/share/Music:/music", f"{music_host}:/music") return rendered def render_remote_settings(values: dict[str, Any]) -> str: payload = {key: values.get(key) for key in settings_service.FIELD_SPECS if key in values} return json.dumps(payload, indent=2) def _check_container_health(client, base_dir: str, log_lines: list[str]) -> tuple[bool, str]: inspect_cmd = ( f"docker inspect -f '{{{{.State.Status}}}}|{{{{.State.Running}}}}|{{{{.State.ExitCode}}}}|{{{{.State.Error}}}}' " f"{REMOTE_CONTAINER_NAME} 2>/dev/null || true" ) _, lines = _run_remote_command(client, inspect_cmd, log_lines, allow_failure=True) raw = (lines[-1] if lines else "").strip() if not raw: _log(log_lines, f"Container {REMOTE_CONTAINER_NAME} was not found after deploy.") return False, "container missing" parts = raw.split("|", 3) state = parts[0] if len(parts) > 0 else "unknown" running = parts[1].lower() == "true" if len(parts) > 1 else False exit_code = parts[2] if len(parts) > 2 else "" error = parts[3].strip() if len(parts) > 3 else "" _log(log_lines, f"Container status: state={state} running={running} exit_code={exit_code or 'n/a'}") if running and state == "running": return True, state _log(log_lines, "Container did not reach running state. Fetching recent logs…") _run_remote_command(client, f"docker logs --tail 80 {REMOTE_CONTAINER_NAME} 2>&1 || true", log_lines, allow_failure=True) if error: _log(log_lines, f"Container error: {error}") return False, state or "unknown" def status_payload(values: dict[str, Any], client_host: str | None) -> dict[str, Any]: allowed = is_local_client(client_host) configured = is_configured(values) transport_ready = is_paramiko_available() available = allowed and configured and transport_ready reasons: list[str] = [] if not allowed: reasons.append("Updates can only be triggered from a local/private network client.") if not configured: reasons.append("Set a NAS host and NAS user first.") if not transport_ready: reasons.append("Python SSH support is not installed on this host yet.") return { "available": available, "allowed": allowed, "configured": configured, "transport": "paramiko" if transport_ready else None, "transport_ready": transport_ready, "password_configured": using_saved_password(values), "client_host": normalize_client_host(client_host), "nas_host": str(values.get("deploy_nas_host") or "").strip(), "nas_user": str(values.get("deploy_nas_user") or "").strip(), "remote_app_dir": remote_app_dir(values), "runtime": dict(runtime), "reason": " ".join(reasons).strip() or None, } def _run_sync(values: dict[str, Any]) -> dict[str, Any]: log_lines: list[str] = [] client = None sftp = None try: _log(log_lines, f"Deploying HomelabToolkit to {_remote(values)}:{remote_app_dir(values)}") _log( log_lines, "Auth mode: saved password" if using_saved_password(values) else "Auth mode: SSH keys / agent" ) _log(log_lines, "Connecting to remote host…") client = _connect(values) _log(log_lines, "SSH connection established.") sftp = client.open_sftp() _log(log_lines, "SFTP channel opened.") base_dir = remote_app_dir(values).rstrip("/") for directory in ( base_dir, f"{base_dir}/output", f"{base_dir}/cache", f"{base_dir}/static", f"{base_dir}/static/studios", f"{base_dir}/services", f"{base_dir}/frontend", ): _ensure_remote_dir(sftp, directory) _log(log_lines, "Remote directory structure ensured.") for name in TOP_LEVEL_FILES: source = ROOT_DIR / name if source.exists(): if name == "docker-compose.yml": _upload_text(sftp, render_remote_compose(values), f"{base_dir}/{name}", name, log_lines) else: _upload_file(sftp, source, f"{base_dir}/{name}", log_lines) _upload_text(sftp, render_remote_settings(values), f"{base_dir}/cache/settings.json", "settings.json", log_lines) _upload_tree(sftp, ROOT_DIR / "static", f"{base_dir}/static", log_lines) _upload_tree(sftp, ROOT_DIR / "services", f"{base_dir}/services", log_lines) _log(log_lines, "Cleaning remote frontend build directories…") _run_remote_command(client, f"rm -rf {base_dir}/frontend/node_modules {base_dir}/frontend/dist", log_lines) for name in FRONTEND_ROOT_FILES: source = ROOT_DIR / "frontend" / name if source.exists(): _upload_file(sftp, source, f"{base_dir}/frontend/{name}", log_lines) _upload_tree(sftp, ROOT_DIR / "frontend" / "src", f"{base_dir}/frontend/src", log_lines) for path in sorted(ROOT_DIR.iterdir()): if path.is_file() and path.suffix.lower() in LOGO_EXTENSIONS: _upload_file(sftp, path, f"{base_dir}/{path.name}", log_lines) _log(log_lines, "Starting remote docker compose build and update…") _run_remote_command( client, ( f"cd {base_dir} && " "if command -v docker-compose >/dev/null 2>&1; then " "docker-compose build && docker-compose up -d; " "else " "docker compose build && docker compose up -d; " "fi" ), log_lines, ) healthy, state = _check_container_health(client, base_dir, log_lines) if not healthy: return { "ok": False, "message": f"Deployment finished but container state is {state}.", "output_tail": _tail(log_lines), } _log(log_lines, "Deployment complete.") _log(log_lines, f"To view logs: ssh {_remote(values)} 'cd {base_dir} && docker compose logs -f'") return { "ok": True, "message": "Deployment completed.", "output_tail": _tail(log_lines), } except Exception as exc: return { "ok": False, "message": f"Deployment failed: {exc}", "output_tail": _tail(log_lines), } finally: if sftp is not None: sftp.close() if client is not None: client.close() async def run_update(values: dict[str, Any], client_host: str | None) -> dict[str, Any]: async with _lock: status = status_payload(values, client_host) if not status["allowed"]: return {"ok": False, "message": status["reason"] or "Update not allowed."} if not status["configured"]: return {"ok": False, "message": status["reason"] or "Update target is not configured."} if not status["transport_ready"]: return {"ok": False, "message": status["reason"] or "Python SSH support is not available."} runtime["running"] = True runtime["last_started_at"] = _now() runtime["last_finished_at"] = None runtime["last_status"] = "running" runtime["last_message"] = None runtime["last_output_tail"] = [] try: result = await asyncio.wait_for(asyncio.to_thread(_run_sync, values), timeout=DEPLOY_TIMEOUT_SECONDS) except asyncio.TimeoutError: result = { "ok": False, "message": f"Deployment timed out after {DEPLOY_TIMEOUT_SECONDS} seconds.", "output_tail": [], } runtime["running"] = False runtime["last_finished_at"] = _now() runtime["last_status"] = "ok" if result.get("ok") else "error" runtime["last_message"] = result.get("message") runtime["last_output_tail"] = list(result.get("output_tail") or []) return result async def start_update(values: dict[str, Any], client_host: str | None) -> dict[str, Any]: global _task status = status_payload(values, client_host) if not status["allowed"]: return {"ok": False, "message": status["reason"] or "Update not allowed."} if not status["configured"]: return {"ok": False, "message": status["reason"] or "Update target is not configured."} if not status["transport_ready"]: return {"ok": False, "message": status["reason"] or "Python SSH support is not available."} if runtime["running"]: return {"ok": False, "message": "Deployment is already running."} _task = asyncio.create_task(run_update(values, client_host)) return {"ok": True, "message": "Deployment started."}