Claude/update prod runner script lq2g s

deploy.sh

@@ -11,7 +11,7 @@
 #     vps    — triggered by a manual SSH/shell call on the VPS
 #
 # What it does:
-#   0. Skip deploy if VPS already has the latest commit
+#   0. Skip deploy if VPS already has the latest commit (and service is active)
 #   1. Stop the systemd service (prevents file locks)
 #   2. git fetch --all + git reset --hard origin/main + git clean
 #   3. npm ci --omit=dev  (install/update production deps)
@@ -38,14 +38,16 @@ say "Date:    $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
 cd "$PROJECT_DIR"
 
 # ---------------------------------------------------------
-# Load .env so BOT_TOKEN and ADMIN_IDS are available for
-# Telegram notifications throughout the script.
+# Load only the variables needed for Telegram notifications.
+# Using set -o allexport would export every line in .env
+# (including unrelated or potentially sensitive variables).
+# Instead, extract only BOT_TOKEN and ADMIN_IDS explicitly.
 # ---------------------------------------------------------
 if [[ -f "$PROJECT_DIR/.env" ]]; then
-    set -o allexport
-    # shellcheck source=/dev/null
-    source "$PROJECT_DIR/.env"
-    set +o allexport
+    _env_file="$PROJECT_DIR/.env"
+    BOT_TOKEN="$(grep -E '^BOT_TOKEN=' "$_env_file" | head -1 | cut -d= -f2- | tr -d '"'"'"' | tr -d $'\r')" || true
+    ADMIN_IDS="$(grep -E '^ADMIN_IDS=' "$_env_file" | head -1 | cut -d= -f2- | tr -d '"'"'"' | tr -d $'\r')" || true
+    export BOT_TOKEN ADMIN_IDS
 fi
 
 # ---------------------------------------------------------
@@ -72,14 +74,31 @@ send_admin() {
 }
 
 # ---------------------------------------------------------
-# Error trap — always notify admins if the script fails
+# Track whether the service was stopped so the ERR trap can
+# attempt a recovery restart if the deploy fails mid-way.
+# ---------------------------------------------------------
+_SERVICE_STOPPED=false
+
+# ---------------------------------------------------------
+# Error trap — notify admins and attempt service recovery.
+# ${BASH_LINENO[0]} gives the caller's line number when
+# evaluated inside an ERR trap (more reliable than $LINENO).
 # ---------------------------------------------------------
 _on_error() {
-    local line="$1"
+    local line="${BASH_LINENO[0]:-?}"
     warn "Script failed at line $line"
-    send_admin "❌ Deploy failed at line $line — check VPS logs. (source: $DEPLOY_SOURCE)"
+    send_admin "❌ Deploy FAILED at line $line — check VPS logs. (source: $DEPLOY_SOURCE)"
+    # If the service was already stopped before the failure, try to bring it
+    # back up on whatever code is currently on disk so the bot isn't left down.
+    if [[ "$_SERVICE_STOPPED" == "true" ]] && command -v systemctl >/dev/null 2>&1; then
+        warn "Attempting service recovery restart on existing code…"
+        systemctl start "${APP_NAME}.service" || true
+        local _recovery_status
+        _recovery_status="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo unknown)"
+        send_admin "🔁 Recovery restart attempted after deploy failure — service is now: $_recovery_status"
+    fi
 }
-trap '_on_error "$LINENO"' ERR
+trap '_on_error' ERR
 
 # ---------------------------------------------------------
 # 0) Skip deploy if VPS already has the latest commit
@@ -119,6 +138,7 @@ if command -v systemctl >/dev/null 2>&1; then
     say "Step 1/4 — Stopping ${APP_NAME} service…"
     send_admin "🛑 Stopping bot service…"
     systemctl stop "${APP_NAME}.service" || true
+    _SERVICE_STOPPED=true
 fi
 
 # ---------------------------------------------------------
@@ -147,21 +167,37 @@ else
     # Restart the service so the bot comes back up on the old code
     if command -v systemctl >/dev/null 2>&1; then
         systemctl start "${APP_NAME}.service" || true
+        _SERVICE_STOPPED=false
     fi
     exit 1
 fi
 
 # ---------------------------------------------------------
 # 3) Install / update production dependencies
+#    Wrapped in a conditional identical to the git guard:
+#    if npm fails the service is left down without this guard.
+#    On failure we restart on the existing node_modules so
+#    the bot comes back up on whatever was last working.
 # ---------------------------------------------------------
 say "Step 3/4 — Installing production dependencies…"
 send_admin "📦 Installing dependencies…"
-if [[ -f package-lock.json ]]; then
-    npm ci --omit=dev
+
+NPM_CMD="npm install --omit=dev"
+[[ -f package-lock.json ]] && NPM_CMD="npm ci --omit=dev"
+
+NPM_OUT=""
+if NPM_OUT=$(${NPM_CMD} 2>&1); then
+    say "Dependencies installed."
 else
-    npm install --omit=dev
+    warn "npm install failed — restarting bot on existing node_modules"
+    warn "npm output: $NPM_OUT"
+    send_admin "❌ npm install failed — restarting bot on existing deps. Error: ${NPM_OUT:0:200} (source: $DEPLOY_SOURCE)"
+    if command -v systemctl >/dev/null 2>&1; then
+        systemctl start "${APP_NAME}.service" || true
+        _SERVICE_STOPPED=false
+    fi
+    exit 1
 fi
-say "Dependencies installed."
 
 # ---------------------------------------------------------
 # 4) Start bot via systemctl
@@ -170,6 +206,7 @@ say "Step 4/4 — Starting bot via systemctl…"
 send_admin "🚀 Starting bot service…"
 if command -v systemctl >/dev/null 2>&1; then
     systemctl start "${APP_NAME}.service"
+    _SERVICE_STOPPED=false
     say "Bot started."
 else
     warn "systemctl not found. Start manually: node $PROJECT_DIR/index.js"

scripts/disk-protect.sh

@@ -26,9 +26,28 @@ JOURNAL_VACUUM_MB=300
 
 # ── Logging ────────────────────────────────────────────────────────────────────
 
-log() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] $*"; }
+log()  { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] $*"; }
 warn() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] WARN: $*" >&2; }
 
+# ── Helper: delete files found by find and log each one reliably ───────────────
+# Uses process substitution + null-delimited names to handle filenames with
+# spaces or special characters, and separates print/delete so logging is never
+# suppressed by find implementation differences.
+delete_found() {
+    # Usage: delete_found <find_args...>
+    # Example: delete_found "$LOG_DIR" -type f -mtime +7
+    local _count=0
+    while IFS= read -r -d '' f; do
+        if rm -f "$f"; then
+            log "  Deleted: $f"
+            _count=$((_count + 1))
+        else
+            warn "  Failed to delete: $f"
+        fi
+    done < <(find "$@" -print0 2>/dev/null)
+    log "  Total deleted: $_count file(s)"
+}
+
 # ── 1. Log rotation (via logrotate if available) ───────────────────────────────
 log "Step 1: Log rotation"
 if command -v logrotate >/dev/null 2>&1; then
@@ -45,25 +64,27 @@ fi
 # ── 2. Compress logs older than 1 day ─────────────────────────────────────────
 log "Step 2: Compress logs older than 1 day"
 if [[ -d "$LOG_DIR" ]]; then
-    find "$LOG_DIR" -type f \( -name "*.log" -o -name "*.out" \) -mtime +1 ! -name "*.gz" | while read -r f; do
-        gzip -f "$f" && log "  Compressed: $f"
-    done
+    while IFS= read -r -d '' f; do
+        if gzip -f "$f" 2>/dev/null; then
+            log "  Compressed: $f"
+        else
+            warn "  Failed to compress: $f"
+        fi
+    done < <(find "$LOG_DIR" -type f \( -name "*.log" -o -name "*.out" \) -mtime +1 ! -name "*.gz" -print0 2>/dev/null)
 else
     log "  Log directory not found: $LOG_DIR — skipping"
 fi
 
 # ── 3. Delete logs older than LOG_MAX_DAYS days ────────────────────────────────
 log "Step 3: Delete logs older than ${LOG_MAX_DAYS} days"
 if [[ -d "$LOG_DIR" ]]; then
-    find "$LOG_DIR" -type f -mtime "+${LOG_MAX_DAYS}" -delete -print | while read -r f; do
-        log "  Deleted old log: $f"
-    done
+    delete_found "$LOG_DIR" -type f -mtime "+${LOG_MAX_DAYS}"
 fi
 
 # ── 4. journalctl vacuum ───────────────────────────────────────────────────────
 log "Step 4: journalctl vacuum (keep up to ${JOURNAL_VACUUM_MB}MB)"
 if command -v journalctl >/dev/null 2>&1; then
-    journalctl --vacuum-size="${JOURNAL_VACUUM_MB}M" 2>&1 | grep -v "^$" | while read -r line; do
+    journalctl --vacuum-size="${JOURNAL_VACUUM_MB}M" 2>&1 | grep -v "^$" | while IFS= read -r line; do
         log "  [journal] $line"
     done
 else
@@ -73,55 +94,55 @@ fi
 # ── 5. Clear npm cache ─────────────────────────────────────────────────────────
 log "Step 5: Clear npm cache"
 if command -v npm >/dev/null 2>&1; then
-    npm cache clean --force 2>&1 | tail -1 | while read -r line; do log "  [npm] $line"; done
+    npm cache clean --force 2>&1 | tail -1 | while IFS= read -r line; do log "  [npm] $line"; done
     log "  npm cache cleared"
 else
     log "  npm not found — skipping"
 fi
 
 # ── 6. Clear temp files ────────────────────────────────────────────────────────
 log "Step 6: Clear temp files older than ${TEMP_MAX_DAYS} days"
-# Clear any .tmp files left by atomic writes inside the app
 if [[ -d "$DATA_DIR" ]]; then
-    find "$DATA_DIR" -type f -name "*.tmp.*" -mtime "+${TEMP_MAX_DAYS}" -delete -print | while read -r f; do
-        log "  Deleted temp file: $f"
-    done
+    delete_found "$DATA_DIR" -type f -name "*.tmp.*" -mtime "+${TEMP_MAX_DAYS}"
 fi
-# Clear /tmp files with our app name
-find /tmp -type f -name "runewager-*" -mtime "+${TEMP_MAX_DAYS}" -delete 2>/dev/null | true
+# Clear /tmp files with our app name (best-effort; ignore errors)
+while IFS= read -r -d '' f; do
+    rm -f "$f" 2>/dev/null && log "  Deleted /tmp file: $f" || true
+done < <(find /tmp -type f -name "runewager-*" -mtime "+${TEMP_MAX_DAYS}" -print0 2>/dev/null)
 
 # ── 7. Delete old runtime state snapshots older than SNAPSHOT_MAX_DAYS ─────────
 log "Step 7: Delete runtime state snapshots older than ${SNAPSHOT_MAX_DAYS} days"
 if [[ -d "$BACKUP_DIR" ]]; then
-    find "$BACKUP_DIR" -type f -name "runtime-state-*.json" -mtime "+${SNAPSHOT_MAX_DAYS}" -delete -print | while read -r f; do
-        log "  Deleted old snapshot: $f"
-    done
+    delete_found "$BACKUP_DIR" -type f -name "runtime-state-*.json" -mtime "+${SNAPSHOT_MAX_DAYS}"
 else
     log "  Backup dir not found: $BACKUP_DIR — skipping"
 fi
 
 # ── 8. NEVER delete critical files — safety check ─────────────────────────────
+# Verifies that the cleanup did not accidentally remove key runtime files.
+# Sets CRITICAL_OK=false (and emits a warning) for every file that is missing.
 log "Step 8: Safety check — verifying critical files are intact"
 CRITICAL_OK=true
 for f in "${APP_DIR}/.env" "${DATA_DIR}/runtime-state.json"; do
     if [[ -f "$f" ]]; then
         log "  OK: $f"
     else
-        log "  (not present, not required): $f"
+        warn "  MISSING: $f — expected file not found after cleanup"
+        CRITICAL_OK=false
     fi
 done
 if [[ "$CRITICAL_OK" != "true" ]]; then
-    warn "  ⚠️ Critical file check failed — review above output"
+    warn "⚠️ One or more critical files are missing — review the output above"
 fi
 
 # ── 9. Disk usage summary ──────────────────────────────────────────────────────
 log "Step 9: Disk usage summary"
-df -h / 2>/dev/null | tail -1 | while read -r line; do log "  / : $line"; done
+df -h / 2>/dev/null | tail -1 | while IFS= read -r line; do log "  / : $line"; done
 if [[ -d "$APP_DIR" ]]; then
-    du -sh "$APP_DIR" 2>/dev/null | while read -r line; do log "  App dir: $line"; done
+    du -sh "$APP_DIR" 2>/dev/null | while IFS= read -r line; do log "  App dir: $line"; done
 fi
 if [[ -d "$LOG_DIR" ]]; then
-    du -sh "$LOG_DIR" 2>/dev/null | while read -r line; do log "  Logs: $line"; done
+    du -sh "$LOG_DIR" 2>/dev/null | while IFS= read -r line; do log "  Logs: $line"; done
 fi
 
 log "Disk protection complete."