441 lines
16 KiB
Python
441 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import os
|
|
import posixpath
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from stat import S_ISDIR
|
|
from typing import Any
|
|
|
|
try:
|
|
import paramiko
|
|
except ImportError: # pragma: no cover - exercised indirectly via status checks
|
|
paramiko = None
|
|
|
|
from services import settings as settings_service
|
|
|
|
DEFAULT_REMOTE_APP_DIR = "/share/Docker/homelabtoolkit"
|
|
DEPLOY_TIMEOUT_SECONDS = int(os.environ.get("DEPLOY_TIMEOUT_SECONDS", "1800"))
|
|
ROOT_DIR = Path(__file__).resolve().parent.parent
|
|
REMOTE_CONTAINER_NAME = os.environ.get("DEPLOY_CONTAINER_NAME", "homelabtoolkit").strip() or "homelabtoolkit"
|
|
|
|
TOP_LEVEL_FILES = ("app.py", "rotate_preroll.py", "Dockerfile", "docker-compose.yml", "requirements.txt")
|
|
FRONTEND_ROOT_FILES = ("package.json", "package-lock.json", "vite.config.ts", "tsconfig.json", "index.html")
|
|
LOGO_EXTENSIONS = {".png", ".jpg", ".jpeg"}
|
|
|
|
runtime: dict[str, Any] = {
|
|
"running": False,
|
|
"last_started_at": None,
|
|
"last_finished_at": None,
|
|
"last_status": "idle",
|
|
"last_message": None,
|
|
"last_output_tail": [],
|
|
}
|
|
_lock = asyncio.Lock()
|
|
_task: asyncio.Task | None = None
|
|
|
|
logging.getLogger("paramiko").setLevel(logging.WARNING)
|
|
logger = logging.getLogger("homelabtoolkit.update")
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now().astimezone().isoformat(timespec="seconds")
|
|
|
|
|
|
def normalize_client_host(value: str | None) -> str:
|
|
text = (value or "").strip()
|
|
if not text:
|
|
return ""
|
|
if "," in text:
|
|
text = text.split(",", 1)[0].strip()
|
|
if text.startswith("::ffff:"):
|
|
text = text[len("::ffff:") :]
|
|
return text
|
|
|
|
|
|
def is_local_client(host: str | None) -> bool:
|
|
text = normalize_client_host(host)
|
|
if not text:
|
|
return False
|
|
if text.lower() == "localhost":
|
|
return True
|
|
try:
|
|
ip = ipaddress.ip_address(text)
|
|
except ValueError:
|
|
return False
|
|
return ip.is_loopback or ip.is_private
|
|
|
|
|
|
def is_paramiko_available() -> bool:
|
|
return paramiko is not None
|
|
|
|
|
|
def is_configured(values: dict[str, Any]) -> bool:
|
|
return bool(str(values.get("deploy_nas_host") or "").strip() and str(values.get("deploy_nas_user") or "").strip())
|
|
|
|
|
|
def remote_app_dir(values: dict[str, Any]) -> str:
|
|
return str(values.get("deploy_remote_app_dir") or DEFAULT_REMOTE_APP_DIR).strip() or DEFAULT_REMOTE_APP_DIR
|
|
|
|
|
|
def deploy_password(values: dict[str, Any]) -> str:
|
|
return str(values.get("deploy_nas_password") or "")
|
|
|
|
|
|
def using_saved_password(values: dict[str, Any]) -> bool:
|
|
return bool(deploy_password(values))
|
|
|
|
|
|
def deploy_music_host_path(values: dict[str, Any]) -> str:
|
|
return str(values.get("deploy_music_host_path") or "/share/Music").strip() or "/share/Music"
|
|
|
|
|
|
def _remote(values: dict[str, Any]) -> str:
|
|
return f'{str(values.get("deploy_nas_user") or "").strip()}@{str(values.get("deploy_nas_host") or "").strip()}'
|
|
|
|
|
|
def _tail(lines: list[str], max_lines: int = 40) -> list[str]:
|
|
return lines[-max_lines:]
|
|
|
|
|
|
def _push_runtime_output(log_lines: list[str]) -> None:
|
|
runtime["last_output_tail"] = list(_tail(log_lines))
|
|
|
|
|
|
def _log(log_lines: list[str], message: str) -> None:
|
|
text = message.rstrip()
|
|
if not text:
|
|
return
|
|
log_lines.append(text)
|
|
_push_runtime_output(log_lines)
|
|
logger.info(text)
|
|
|
|
|
|
def _ensure_remote_dir(sftp, remote_dir: str) -> None:
|
|
normalized = posixpath.normpath(remote_dir)
|
|
parts = [part for part in normalized.split("/") if part]
|
|
current = "/" if normalized.startswith("/") else ""
|
|
for part in parts:
|
|
current = posixpath.join(current, part) if current not in ("", "/") else f"{current}{part}" if current == "/" else part
|
|
try:
|
|
attrs = sftp.stat(current)
|
|
if not S_ISDIR(attrs.st_mode):
|
|
raise RuntimeError(f"Remote path exists but is not a directory: {current}")
|
|
except OSError:
|
|
sftp.mkdir(current)
|
|
|
|
|
|
def _upload_file(sftp, local_path: Path, remote_path: str, log_lines: list[str]) -> None:
|
|
_ensure_remote_dir(sftp, posixpath.dirname(remote_path))
|
|
sftp.put(str(local_path), remote_path)
|
|
_log(log_lines, f"Uploaded {local_path.relative_to(ROOT_DIR).as_posix()} -> {remote_path}")
|
|
|
|
|
|
def _upload_text(sftp, content: str, remote_path: str, label: str, log_lines: list[str]) -> None:
|
|
_ensure_remote_dir(sftp, posixpath.dirname(remote_path))
|
|
with sftp.file(remote_path, "w") as handle:
|
|
handle.write(content)
|
|
_log(log_lines, f"Rendered {label} -> {remote_path}")
|
|
|
|
|
|
def _upload_tree(sftp, local_dir: Path, remote_dir: str, log_lines: list[str]) -> None:
|
|
if not local_dir.exists():
|
|
return
|
|
_ensure_remote_dir(sftp, remote_dir)
|
|
for path in sorted(local_dir.rglob("*")):
|
|
if path.is_dir():
|
|
_ensure_remote_dir(sftp, posixpath.join(remote_dir, path.relative_to(local_dir).as_posix()))
|
|
continue
|
|
if "__pycache__" in path.parts:
|
|
continue
|
|
remote_path = posixpath.join(remote_dir, path.relative_to(local_dir).as_posix())
|
|
_upload_file(sftp, path, remote_path, log_lines)
|
|
|
|
|
|
def _stream_output(channel, log_lines: list[str]) -> None:
|
|
stdout_buffer = ""
|
|
stderr_buffer = ""
|
|
while True:
|
|
had_output = False
|
|
if channel.recv_ready():
|
|
stdout_buffer += channel.recv(4096).decode("utf-8", errors="replace")
|
|
had_output = True
|
|
while "\n" in stdout_buffer:
|
|
line, stdout_buffer = stdout_buffer.split("\n", 1)
|
|
_log(log_lines, line)
|
|
if channel.recv_stderr_ready():
|
|
stderr_buffer += channel.recv_stderr(4096).decode("utf-8", errors="replace")
|
|
had_output = True
|
|
while "\n" in stderr_buffer:
|
|
line, stderr_buffer = stderr_buffer.split("\n", 1)
|
|
_log(log_lines, line)
|
|
if channel.exit_status_ready() and not channel.recv_ready() and not channel.recv_stderr_ready():
|
|
break
|
|
if not had_output:
|
|
time.sleep(0.1)
|
|
|
|
if stdout_buffer.strip():
|
|
_log(log_lines, stdout_buffer)
|
|
if stderr_buffer.strip():
|
|
_log(log_lines, stderr_buffer)
|
|
|
|
|
|
def _run_remote_command(client, command: str, log_lines: list[str], allow_failure: bool = False) -> tuple[int, list[str]]:
|
|
stdin, stdout, stderr = client.exec_command(command, timeout=DEPLOY_TIMEOUT_SECONDS)
|
|
if stdin:
|
|
stdin.close()
|
|
channel = stdout.channel
|
|
command_lines: list[str] = []
|
|
_stream_output(channel, command_lines)
|
|
for line in command_lines:
|
|
_log(log_lines, line)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
if exit_code != 0 and not allow_failure:
|
|
raise RuntimeError(f"Remote command failed with exit code {exit_code}")
|
|
return exit_code, command_lines
|
|
|
|
|
|
def _connect(values: dict[str, Any]):
|
|
if paramiko is None:
|
|
raise RuntimeError("Paramiko is not installed.")
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
connect_kwargs: dict[str, Any] = {
|
|
"hostname": str(values.get("deploy_nas_host") or "").strip(),
|
|
"username": str(values.get("deploy_nas_user") or "").strip(),
|
|
"timeout": 20,
|
|
"banner_timeout": 20,
|
|
"auth_timeout": 20,
|
|
}
|
|
password = deploy_password(values)
|
|
if password:
|
|
connect_kwargs["password"] = password
|
|
connect_kwargs["look_for_keys"] = False
|
|
connect_kwargs["allow_agent"] = False
|
|
client.connect(**connect_kwargs)
|
|
return client
|
|
|
|
|
|
def render_remote_compose(values: dict[str, Any]) -> str:
|
|
source = (ROOT_DIR / "docker-compose.yml").read_text(encoding="utf-8")
|
|
remote_dir = remote_app_dir(values).rstrip("/")
|
|
music_host = deploy_music_host_path(values)
|
|
rendered = source
|
|
rendered = rendered.replace("/share/Docker/homelabtoolkit/output:/app/output", f"{remote_dir}/output:/app/output")
|
|
rendered = rendered.replace("/share/Docker/homelabtoolkit/cache:/app/cache", f"{remote_dir}/cache:/app/cache")
|
|
rendered = rendered.replace("/share/Music:/music", f"{music_host}:/music")
|
|
return rendered
|
|
|
|
|
|
def render_remote_settings(values: dict[str, Any]) -> str:
|
|
payload = {key: values.get(key) for key in settings_service.FIELD_SPECS if key in values}
|
|
return json.dumps(payload, indent=2)
|
|
|
|
|
|
def _check_container_health(client, base_dir: str, log_lines: list[str]) -> tuple[bool, str]:
|
|
inspect_cmd = (
|
|
f"docker inspect -f '{{{{.State.Status}}}}|{{{{.State.Running}}}}|{{{{.State.ExitCode}}}}|{{{{.State.Error}}}}' "
|
|
f"{REMOTE_CONTAINER_NAME} 2>/dev/null || true"
|
|
)
|
|
_, lines = _run_remote_command(client, inspect_cmd, log_lines, allow_failure=True)
|
|
raw = (lines[-1] if lines else "").strip()
|
|
if not raw:
|
|
_log(log_lines, f"Container {REMOTE_CONTAINER_NAME} was not found after deploy.")
|
|
return False, "container missing"
|
|
|
|
parts = raw.split("|", 3)
|
|
state = parts[0] if len(parts) > 0 else "unknown"
|
|
running = parts[1].lower() == "true" if len(parts) > 1 else False
|
|
exit_code = parts[2] if len(parts) > 2 else ""
|
|
error = parts[3].strip() if len(parts) > 3 else ""
|
|
_log(log_lines, f"Container status: state={state} running={running} exit_code={exit_code or 'n/a'}")
|
|
if running and state == "running":
|
|
return True, state
|
|
|
|
_log(log_lines, "Container did not reach running state. Fetching recent logs…")
|
|
_run_remote_command(client, f"docker logs --tail 80 {REMOTE_CONTAINER_NAME} 2>&1 || true", log_lines, allow_failure=True)
|
|
if error:
|
|
_log(log_lines, f"Container error: {error}")
|
|
return False, state or "unknown"
|
|
|
|
|
|
def status_payload(values: dict[str, Any], client_host: str | None) -> dict[str, Any]:
|
|
allowed = is_local_client(client_host)
|
|
configured = is_configured(values)
|
|
transport_ready = is_paramiko_available()
|
|
available = allowed and configured and transport_ready
|
|
|
|
reasons: list[str] = []
|
|
if not allowed:
|
|
reasons.append("Updates can only be triggered from a local/private network client.")
|
|
if not configured:
|
|
reasons.append("Set a NAS host and NAS user first.")
|
|
if not transport_ready:
|
|
reasons.append("Python SSH support is not installed on this host yet.")
|
|
|
|
return {
|
|
"available": available,
|
|
"allowed": allowed,
|
|
"configured": configured,
|
|
"transport": "paramiko" if transport_ready else None,
|
|
"transport_ready": transport_ready,
|
|
"password_configured": using_saved_password(values),
|
|
"client_host": normalize_client_host(client_host),
|
|
"nas_host": str(values.get("deploy_nas_host") or "").strip(),
|
|
"nas_user": str(values.get("deploy_nas_user") or "").strip(),
|
|
"remote_app_dir": remote_app_dir(values),
|
|
"runtime": dict(runtime),
|
|
"reason": " ".join(reasons).strip() or None,
|
|
}
|
|
|
|
|
|
def _run_sync(values: dict[str, Any]) -> dict[str, Any]:
|
|
log_lines: list[str] = []
|
|
client = None
|
|
sftp = None
|
|
try:
|
|
_log(log_lines, f"Deploying HomelabToolkit to {_remote(values)}:{remote_app_dir(values)}")
|
|
_log(
|
|
log_lines,
|
|
"Auth mode: saved password" if using_saved_password(values) else "Auth mode: SSH keys / agent"
|
|
)
|
|
_log(log_lines, "Connecting to remote host…")
|
|
|
|
client = _connect(values)
|
|
_log(log_lines, "SSH connection established.")
|
|
sftp = client.open_sftp()
|
|
_log(log_lines, "SFTP channel opened.")
|
|
|
|
base_dir = remote_app_dir(values).rstrip("/")
|
|
for directory in (
|
|
base_dir,
|
|
f"{base_dir}/output",
|
|
f"{base_dir}/cache",
|
|
f"{base_dir}/static",
|
|
f"{base_dir}/static/studios",
|
|
f"{base_dir}/services",
|
|
f"{base_dir}/frontend",
|
|
):
|
|
_ensure_remote_dir(sftp, directory)
|
|
_log(log_lines, "Remote directory structure ensured.")
|
|
|
|
for name in TOP_LEVEL_FILES:
|
|
source = ROOT_DIR / name
|
|
if source.exists():
|
|
if name == "docker-compose.yml":
|
|
_upload_text(sftp, render_remote_compose(values), f"{base_dir}/{name}", name, log_lines)
|
|
else:
|
|
_upload_file(sftp, source, f"{base_dir}/{name}", log_lines)
|
|
|
|
_upload_text(sftp, render_remote_settings(values), f"{base_dir}/cache/settings.json", "settings.json", log_lines)
|
|
|
|
_upload_tree(sftp, ROOT_DIR / "static", f"{base_dir}/static", log_lines)
|
|
_upload_tree(sftp, ROOT_DIR / "services", f"{base_dir}/services", log_lines)
|
|
|
|
_log(log_lines, "Cleaning remote frontend build directories…")
|
|
_run_remote_command(client, f"rm -rf {base_dir}/frontend/node_modules {base_dir}/frontend/dist", log_lines)
|
|
for name in FRONTEND_ROOT_FILES:
|
|
source = ROOT_DIR / "frontend" / name
|
|
if source.exists():
|
|
_upload_file(sftp, source, f"{base_dir}/frontend/{name}", log_lines)
|
|
_upload_tree(sftp, ROOT_DIR / "frontend" / "src", f"{base_dir}/frontend/src", log_lines)
|
|
|
|
for path in sorted(ROOT_DIR.iterdir()):
|
|
if path.is_file() and path.suffix.lower() in LOGO_EXTENSIONS:
|
|
_upload_file(sftp, path, f"{base_dir}/{path.name}", log_lines)
|
|
|
|
_log(log_lines, "Starting remote docker compose build and update…")
|
|
_run_remote_command(
|
|
client,
|
|
(
|
|
f"cd {base_dir} && "
|
|
"if command -v docker-compose >/dev/null 2>&1; then "
|
|
"docker-compose build && docker-compose up -d; "
|
|
"else "
|
|
"docker compose build && docker compose up -d; "
|
|
"fi"
|
|
),
|
|
log_lines,
|
|
)
|
|
|
|
healthy, state = _check_container_health(client, base_dir, log_lines)
|
|
if not healthy:
|
|
return {
|
|
"ok": False,
|
|
"message": f"Deployment finished but container state is {state}.",
|
|
"output_tail": _tail(log_lines),
|
|
}
|
|
|
|
_log(log_lines, "Deployment complete.")
|
|
_log(log_lines, f"To view logs: ssh {_remote(values)} 'cd {base_dir} && docker compose logs -f'")
|
|
return {
|
|
"ok": True,
|
|
"message": "Deployment completed.",
|
|
"output_tail": _tail(log_lines),
|
|
}
|
|
except Exception as exc:
|
|
return {
|
|
"ok": False,
|
|
"message": f"Deployment failed: {exc}",
|
|
"output_tail": _tail(log_lines),
|
|
}
|
|
finally:
|
|
if sftp is not None:
|
|
sftp.close()
|
|
if client is not None:
|
|
client.close()
|
|
|
|
|
|
async def run_update(values: dict[str, Any], client_host: str | None) -> dict[str, Any]:
|
|
async with _lock:
|
|
status = status_payload(values, client_host)
|
|
if not status["allowed"]:
|
|
return {"ok": False, "message": status["reason"] or "Update not allowed."}
|
|
if not status["configured"]:
|
|
return {"ok": False, "message": status["reason"] or "Update target is not configured."}
|
|
if not status["transport_ready"]:
|
|
return {"ok": False, "message": status["reason"] or "Python SSH support is not available."}
|
|
|
|
runtime["running"] = True
|
|
runtime["last_started_at"] = _now()
|
|
runtime["last_finished_at"] = None
|
|
runtime["last_status"] = "running"
|
|
runtime["last_message"] = None
|
|
runtime["last_output_tail"] = []
|
|
|
|
try:
|
|
result = await asyncio.wait_for(asyncio.to_thread(_run_sync, values), timeout=DEPLOY_TIMEOUT_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
result = {
|
|
"ok": False,
|
|
"message": f"Deployment timed out after {DEPLOY_TIMEOUT_SECONDS} seconds.",
|
|
"output_tail": [],
|
|
}
|
|
|
|
runtime["running"] = False
|
|
runtime["last_finished_at"] = _now()
|
|
runtime["last_status"] = "ok" if result.get("ok") else "error"
|
|
runtime["last_message"] = result.get("message")
|
|
runtime["last_output_tail"] = list(result.get("output_tail") or [])
|
|
return result
|
|
|
|
|
|
async def start_update(values: dict[str, Any], client_host: str | None) -> dict[str, Any]:
|
|
global _task
|
|
status = status_payload(values, client_host)
|
|
if not status["allowed"]:
|
|
return {"ok": False, "message": status["reason"] or "Update not allowed."}
|
|
if not status["configured"]:
|
|
return {"ok": False, "message": status["reason"] or "Update target is not configured."}
|
|
if not status["transport_ready"]:
|
|
return {"ok": False, "message": status["reason"] or "Python SSH support is not available."}
|
|
if runtime["running"]:
|
|
return {"ok": False, "message": "Deployment is already running."}
|
|
_task = asyncio.create_task(run_update(values, client_host))
|
|
return {"ok": True, "message": "Deployment started."}
|