307 lines
13 KiB
Bash
307 lines
13 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# predeployment-check.sh
|
||
|
|
#
|
||
|
|
# Run on the production server to capture full environment state before making
|
||
|
|
# changes. Safe — read-only, no writes, no restarts.
|
||
|
|
#
|
||
|
|
# Usage (from local machine):
|
||
|
|
# ssh root@<host> 'bash -s' < deploy/predeployment-check.sh
|
||
|
|
#
|
||
|
|
# Usage (on the server directly):
|
||
|
|
# bash /srv/lean101-clients/deploy/predeployment-check.sh
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
REMOTE_PATH="${REMOTE_PATH:-/srv/lean101-clients}"
|
||
|
|
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}"
|
||
|
|
ENV_FILE="${ENV_FILE:-.env.production}"
|
||
|
|
BACKEND_CONTAINER="lean101-clients-backend"
|
||
|
|
FRONTEND_CONTAINER="lean101-clients-frontend"
|
||
|
|
NGINX_CONTAINER="lean101-clients"
|
||
|
|
DB_CONTAINER="lean101-clients-db"
|
||
|
|
|
||
|
|
# Colours
|
||
|
|
RED='\033[0;31m'; YELLOW='\033[1;33m'; GREEN='\033[0;32m'
|
||
|
|
CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m'
|
||
|
|
|
||
|
|
sep() { echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}"; }
|
||
|
|
h1() { sep; echo -e "${BOLD}${CYAN} $1${RESET}"; sep; }
|
||
|
|
ok() { echo -e " ${GREEN}✔${RESET} $1"; }
|
||
|
|
warn() { echo -e " ${YELLOW}⚠${RESET} $1"; }
|
||
|
|
fail() { echo -e " ${RED}✘${RESET} $1"; }
|
||
|
|
kv() { printf " %-30s %s\n" "$1" "$2"; }
|
||
|
|
|
||
|
|
# Helper: run a command inside a container, return empty string on failure
|
||
|
|
cexec() {
|
||
|
|
local container="$1"; shift
|
||
|
|
docker exec "$container" "$@" 2>/dev/null || true
|
||
|
|
}
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD} LEAN 101 CLIENTS — Pre-Deployment Check${RESET}"
|
||
|
|
echo -e " Generated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
||
|
|
echo -e " Host: $(hostname -f 2>/dev/null || hostname)"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "1. HOST SYSTEM"
|
||
|
|
# =============================================================================
|
||
|
|
kv "OS:" "$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || uname -s)"
|
||
|
|
kv "Kernel:" "$(uname -r)"
|
||
|
|
kv "Uptime:" "$(uptime -p 2>/dev/null || uptime)"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Disk usage (df -h):"
|
||
|
|
df -h --output=source,size,used,avail,pcent,target 2>/dev/null | grep -v tmpfs | grep -v udev | sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Memory (free -h):"
|
||
|
|
free -h 2>/dev/null | sed 's/^/ /' || vm_stat 2>/dev/null | sed 's/^/ /'
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "2. DOCKER & COMPOSE"
|
||
|
|
# =============================================================================
|
||
|
|
if command -v docker &>/dev/null; then
|
||
|
|
ok "Docker installed"
|
||
|
|
kv "Docker version:" "$(docker --version)"
|
||
|
|
kv "Compose version:" "$(docker compose version 2>/dev/null || docker-compose --version 2>/dev/null || echo 'not found')"
|
||
|
|
else
|
||
|
|
fail "Docker not found on PATH"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "3. REPOSITORY STATE"
|
||
|
|
# =============================================================================
|
||
|
|
if [ -d "$REMOTE_PATH/.git" ]; then
|
||
|
|
cd "$REMOTE_PATH"
|
||
|
|
ok "Repo found at $REMOTE_PATH"
|
||
|
|
kv "Branch:" "$(git rev-parse --abbrev-ref HEAD 2>/dev/null)"
|
||
|
|
kv "Latest commit:" "$(git log -1 --format='%h %s (%cr)' 2>/dev/null)"
|
||
|
|
kv "Commit author:" "$(git log -1 --format='%an <%ae>' 2>/dev/null)"
|
||
|
|
kv "Remote origin:" "$(git remote get-url origin 2>/dev/null)"
|
||
|
|
|
||
|
|
DIRTY=$(git status --porcelain 2>/dev/null)
|
||
|
|
if [ -n "$DIRTY" ]; then
|
||
|
|
warn "Working tree has uncommitted changes:"
|
||
|
|
git status --short | sed 's/^/ /'
|
||
|
|
else
|
||
|
|
ok "Working tree clean"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Recent commits (last 5):"
|
||
|
|
git log -5 --format=' %h %s (%cr)' 2>/dev/null
|
||
|
|
else
|
||
|
|
fail "No git repo found at $REMOTE_PATH"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "4. ENVIRONMENT FILE"
|
||
|
|
# =============================================================================
|
||
|
|
ENV_PATH="$REMOTE_PATH/$ENV_FILE"
|
||
|
|
if [ -f "$ENV_PATH" ]; then
|
||
|
|
ok "Env file: $ENV_PATH"
|
||
|
|
kv "Modified:" "$(stat -c '%y' "$ENV_PATH" 2>/dev/null | cut -d'.' -f1 || stat -f '%Sm' "$ENV_PATH" 2>/dev/null)"
|
||
|
|
kv "Permissions:" "$(stat -c '%a %U:%G' "$ENV_PATH" 2>/dev/null || stat -f '%Sp %Su:%Sg' "$ENV_PATH" 2>/dev/null)"
|
||
|
|
echo ""
|
||
|
|
echo " Env keys present (values redacted):"
|
||
|
|
grep -v '^#' "$ENV_PATH" | grep '=' | cut -d'=' -f1 | sort | sed 's/^/ /'
|
||
|
|
else
|
||
|
|
fail "Env file NOT found at $ENV_PATH"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "5. DOCKER STACK — CONTAINER STATUS"
|
||
|
|
# =============================================================================
|
||
|
|
cd "$REMOTE_PATH" 2>/dev/null || true
|
||
|
|
|
||
|
|
echo " All containers on this host:"
|
||
|
|
docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}\t{{.Ports}}' | sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Compose stack status ($COMPOSE_FILE):"
|
||
|
|
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" ps 2>/dev/null | sed 's/^/ /' || \
|
||
|
|
warn "Could not run docker compose ps (compose file or env file missing?)"
|
||
|
|
|
||
|
|
# Per-container inspection
|
||
|
|
for CNAME in "$BACKEND_CONTAINER" "$FRONTEND_CONTAINER" "$NGINX_CONTAINER" "$DB_CONTAINER"; do
|
||
|
|
STATUS=$(docker inspect --format='{{.State.Status}}' "$CNAME" 2>/dev/null || echo "missing")
|
||
|
|
HEALTH=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}}' "$CNAME" 2>/dev/null || echo "missing")
|
||
|
|
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$CNAME" 2>/dev/null || echo "unknown")
|
||
|
|
STARTED=$(docker inspect --format='{{.State.StartedAt}}' "$CNAME" 2>/dev/null | cut -d'.' -f1 || echo "unknown")
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e " ${BOLD}$CNAME${RESET}"
|
||
|
|
kv " Status:" "$STATUS"
|
||
|
|
kv " Health:" "$HEALTH"
|
||
|
|
kv " Image:" "$IMAGE"
|
||
|
|
kv " Started:" "$STARTED"
|
||
|
|
|
||
|
|
if [ "$STATUS" = "running" ]; then
|
||
|
|
MEM=$(docker stats "$CNAME" --no-stream --format '{{.MemUsage}}' 2>/dev/null || echo "n/a")
|
||
|
|
CPU=$(docker stats "$CNAME" --no-stream --format '{{.CPUPerc}}' 2>/dev/null || echo "n/a")
|
||
|
|
kv " CPU / Mem:" "$CPU / $MEM"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "6. DOCKER VOLUMES"
|
||
|
|
# =============================================================================
|
||
|
|
echo " Named volumes:"
|
||
|
|
docker volume ls --format 'table {{.Name}}\t{{.Driver}}\t{{.Mountpoint}}' | grep -i lean | sed 's/^/ /' || echo " (none matching lean)"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " All volumes:"
|
||
|
|
docker volume ls --format ' {{.Name}}' | head -30
|
||
|
|
|
||
|
|
# Postgres data volume size
|
||
|
|
PG_MOUNT=$(docker volume inspect lean101-clients_clients_db_data --format '{{.Mountpoint}}' 2>/dev/null || \
|
||
|
|
docker volume inspect clients_db_data --format '{{.Mountpoint}}' 2>/dev/null || true)
|
||
|
|
if [ -n "$PG_MOUNT" ]; then
|
||
|
|
PG_SIZE=$(du -sh "$PG_MOUNT" 2>/dev/null | cut -f1 || echo "n/a")
|
||
|
|
kv " Postgres volume size:" "$PG_SIZE ($PG_MOUNT)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "7. POSTGRESQL — DATABASE STATE"
|
||
|
|
# =============================================================================
|
||
|
|
DB_STATUS=$(docker inspect --format='{{.State.Status}}' "$DB_CONTAINER" 2>/dev/null || echo "missing")
|
||
|
|
|
||
|
|
if [ "$DB_STATUS" = "running" ]; then
|
||
|
|
ok "Database container is running"
|
||
|
|
|
||
|
|
PG_USER=$(docker exec "$DB_CONTAINER" printenv POSTGRES_USER 2>/dev/null || echo "lean101")
|
||
|
|
PG_DB=$(docker exec "$DB_CONTAINER" printenv POSTGRES_DB 2>/dev/null || echo "lean101")
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " PostgreSQL version:"
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c "SELECT version();" | sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Database size:"
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" \
|
||
|
|
-c "SELECT pg_database.datname, pg_size_pretty(pg_database_size(pg_database.datname)) AS size FROM pg_database ORDER BY pg_database_size(pg_database.datname) DESC;" \
|
||
|
|
| sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Tables and row counts:"
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c "
|
||
|
|
SELECT
|
||
|
|
schemaname,
|
||
|
|
relname AS table_name,
|
||
|
|
n_live_tup AS row_count,
|
||
|
|
pg_size_pretty(pg_total_relation_size(quote_ident(schemaname)||'.'||quote_ident(relname))) AS total_size
|
||
|
|
FROM pg_stat_user_tables
|
||
|
|
ORDER BY n_live_tup DESC;
|
||
|
|
" | sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Alembic migration state:"
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c \
|
||
|
|
"SELECT version_num, is_current FROM alembic_version LEFT JOIN (SELECT true AS is_current) t ON true;" \
|
||
|
|
2>/dev/null | sed 's/^/ /' || \
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c \
|
||
|
|
"SELECT version_num FROM alembic_version;" 2>/dev/null | sed 's/^/ /' || \
|
||
|
|
warn "alembic_version table not found or inaccessible"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Active connections:"
|
||
|
|
cexec "$DB_CONTAINER" psql -U "$PG_USER" -d "$PG_DB" -c "
|
||
|
|
SELECT count(*) AS total_connections,
|
||
|
|
sum(CASE WHEN state = 'active' THEN 1 ELSE 0 END) AS active
|
||
|
|
FROM pg_stat_activity
|
||
|
|
WHERE datname = '$PG_DB';
|
||
|
|
" | sed 's/^/ /'
|
||
|
|
else
|
||
|
|
fail "Database container status: $DB_STATUS — skipping DB checks"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "8. BACKEND — HEALTH & API"
|
||
|
|
# =============================================================================
|
||
|
|
if [ "$(docker inspect --format='{{.State.Status}}' "$BACKEND_CONTAINER" 2>/dev/null)" = "running" ]; then
|
||
|
|
echo " Health endpoint (internal):"
|
||
|
|
HEALTH_RESP=$(cexec "$BACKEND_CONTAINER" python -c \
|
||
|
|
"import urllib.request, json; r=urllib.request.urlopen('http://127.0.0.1:8000/health'); print(r.read().decode())" 2>/dev/null || echo "failed")
|
||
|
|
echo " $HEALTH_RESP"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Python / package versions inside backend container:"
|
||
|
|
cexec "$BACKEND_CONTAINER" python --version 2>&1 | sed 's/^/ /'
|
||
|
|
cexec "$BACKEND_CONTAINER" pip show fastapi sqlalchemy alembic psycopg 2>/dev/null | \
|
||
|
|
grep -E '^(Name|Version):' | sed 's/^/ /'
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " DATABASE_URL in backend (secret masked):"
|
||
|
|
cexec "$BACKEND_CONTAINER" printenv DATABASE_URL | \
|
||
|
|
sed 's|://[^:]*:[^@]*@|://***:***@|g' | sed 's/^/ /'
|
||
|
|
else
|
||
|
|
fail "Backend container not running — skipping API checks"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "9. NGINX — CONFIGURATION"
|
||
|
|
# =============================================================================
|
||
|
|
if [ "$(docker inspect --format='{{.State.Status}}' "$NGINX_CONTAINER" 2>/dev/null)" = "running" ]; then
|
||
|
|
ok "Nginx container is running"
|
||
|
|
echo ""
|
||
|
|
echo " nginx -t output:"
|
||
|
|
cexec "$NGINX_CONTAINER" nginx -t 2>&1 | sed 's/^/ /'
|
||
|
|
echo ""
|
||
|
|
echo " Active listening ports (nginx container):"
|
||
|
|
docker exec "$NGINX_CONTAINER" sh -c 'cat /etc/nginx/conf.d/default.conf 2>/dev/null | grep -E "listen|server_name|proxy_pass"' | sed 's/^/ /' || true
|
||
|
|
echo ""
|
||
|
|
echo " Host port binding:"
|
||
|
|
docker port "$NGINX_CONTAINER" | sed 's/^/ /'
|
||
|
|
else
|
||
|
|
warn "Nginx container not running"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "10. RECENT CONTAINER LOGS (last 30 lines each)"
|
||
|
|
# =============================================================================
|
||
|
|
for CNAME in "$BACKEND_CONTAINER" "$FRONTEND_CONTAINER" "$NGINX_CONTAINER" "$DB_CONTAINER"; do
|
||
|
|
STATUS=$(docker inspect --format='{{.State.Status}}' "$CNAME" 2>/dev/null || echo "missing")
|
||
|
|
echo ""
|
||
|
|
echo -e " ${BOLD}$CNAME${RESET} [$STATUS]"
|
||
|
|
if [ "$STATUS" != "missing" ]; then
|
||
|
|
docker logs "$CNAME" --tail=30 2>&1 | sed 's/^/ /'
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "11. HOST NETWORK & FIREWALL"
|
||
|
|
# =============================================================================
|
||
|
|
echo " Listening ports on host (ss -tlnp):"
|
||
|
|
ss -tlnp 2>/dev/null | sed 's/^/ /' || \
|
||
|
|
netstat -tlnp 2>/dev/null | sed 's/^/ /' || \
|
||
|
|
warn "Neither ss nor netstat available"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " UFW status:"
|
||
|
|
ufw status 2>/dev/null | sed 's/^/ /' || warn "UFW not available"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo " Docker network list:"
|
||
|
|
docker network ls | sed 's/^/ /'
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
h1 "12. COMPOSE FILE DIFF (local vs what would deploy)"
|
||
|
|
# =============================================================================
|
||
|
|
echo " Compose file on server ($COMPOSE_FILE):"
|
||
|
|
if [ -f "$REMOTE_PATH/$COMPOSE_FILE" ]; then
|
||
|
|
kv " Modified:" "$(stat -c '%y' "$REMOTE_PATH/$COMPOSE_FILE" 2>/dev/null | cut -d'.' -f1)"
|
||
|
|
kv " Size:" "$(wc -l < "$REMOTE_PATH/$COMPOSE_FILE") lines"
|
||
|
|
ok "File exists"
|
||
|
|
else
|
||
|
|
fail "$COMPOSE_FILE not found at $REMOTE_PATH"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
sep
|
||
|
|
echo -e "${BOLD} Check complete — paste this output into Claude for analysis.${RESET}"
|
||
|
|
sep
|
||
|
|
echo ""
|