Improve Backup and Add CV Scores to Model

.env

@@ -11,12 +11,12 @@ LOCAL_PORT=5433
 DB_NAME=postgres
 DB_USER=postgres
 DB_PASSWORD=SolarFalconBreeze
-BACKUP_FILE=/home/boundys/data4/django/agile_predict/.local/backup.sql
+BACKUP_DIR=/home/boundys/data4/django/agile_predict/.local
+LOG_DIR=/home/boundys/data4/django/agile_predict/logs
 APP_NAME=prices-db
 
 # Local PostgreSQL restore
 PGUSER=postgres
 PGPASSWORD=RainbowHenryGibbon
 DBNAME=agile_predict
-WIN_BACKUP_FILE=.local\backup.sql
-
+WIN_BACKUP_FILE=.local\backup.sql

README.md

@@ -1,4 +1,4 @@
-# Agile Predict v2.1.0
+# Agile Predict v2.1.1
 
 This model forecasts Octopus Agile electricity prices up to 14 days in advance using a Machine Learning model trained
 on data from the Balancing Mechanism Reporting System (<a href="https://bmrs.elexon.co.uk/">BRMS</a>), National Grid

bin/backup.sh

@@ -1,21 +1,41 @@
 #!/bin/bash
 
+# Timestamp for logs and backup file naming
+NOW=$(date +"%Y-%m-%d_%H-%M-%S")
+
 # Load environment
-if [ -f .env ]; then
+if [ -f /home/boundys/data4/django/agile_predict/.env ]; then
     set -a
-    . .env
+    . /home/boundys/data4/django/agile_predict/.env
     set +a
 else
     echo "Missing .env file"
     exit 1
 fi
 
+# Ensure LOG_DIR, BACKUP_DIR, and BACKUPS_TO_KEEP are set
+: "${LOG_DIR:?LOG_DIR not set in .env}"
+: "${BACKUP_DIR:?BACKUP_DIR not set in .env}"
+BACKUPS_TO_KEEP="${BACKUPS_TO_KEEP:-3}"
+
+mkdir -p "$LOG_DIR"
+LOG_FILE="$LOG_DIR/backup_$NOW.log"
+
+# Redirect all output to log file
+exec > >(tee -a "$LOG_FILE") 2>&1
+
+echo "================ $(date) ================"
+echo "Starting backup script..."
+
+# Ensure PATH includes location for fly and pg_dump
+export PATH="/home/boundys/.fly/bin:/usr/bin:/bin:$PATH"
+
 echo "APP_NAME: $APP_NAME"
 echo "LOCAL_PORT: $LOCAL_PORT"
 echo "Starting Fly proxy to database..."
 
 # Start Fly proxy
-fly proxy ${LOCAL_PORT}:5432 -a $APP_NAME &
+/home/boundys/.fly/bin/fly proxy ${LOCAL_PORT}:5432 -a $APP_NAME &
 PROXY_PID=$!
 
 # Wait for proxy to be ready
@@ -37,18 +57,38 @@ if ! nc -z localhost "$LOCAL_PORT"; then
     exit 1
 fi
 
-
 echo "Starting backup..."
 export PGPASSWORD=$DB_PASSWORD
 
-pg_dump -h localhost -p $LOCAL_PORT -U $DB_USER -d $DB_NAME > "$BACKUP_FILE"
+mkdir -p "$BACKUP_DIR"
+BACKUP_FILE="$BACKUP_DIR/${DB_NAME}_backup_$NOW.sql"
+LATEST_BACKUP_FILE="$BACKUP_DIR/backup.sql"
+
+/usr/bin/pg_dump -h localhost -p $LOCAL_PORT -U $DB_USER -d $DB_NAME > "$BACKUP_FILE"
 
 if [ $? -eq 0 ]; then
     echo "Backup completed successfully: $BACKUP_FILE"
+    cp "$BACKUP_FILE" "$LATEST_BACKUP_FILE"
+    echo "Updated latest backup symlink: $LATEST_BACKUP_FILE"
 else
     echo "Backup FAILED"
 fi
 
+# Cleanup old backups and their logs beyond $BACKUPS_TO_KEEP
+BACKUP_PATTERN="$BACKUP_DIR/${DB_NAME}_backup_*.sql"
+LOG_PATTERN="$LOG_DIR/backup_*.log"
+
+BACKUP_LIST=$(ls -1t $BACKUP_PATTERN 2>/dev/null)
+LOG_LIST=$(ls -1t $LOG_PATTERN 2>/dev/null)
+
+DELETE_BACKUPS=$(echo "$BACKUP_LIST" | tail -n +$((BACKUPS_TO_KEEP + 1)))
+DELETE_LOGS=$(echo "$LOG_LIST" | tail -n +$((BACKUPS_TO_KEEP + 1)))
+
+echo "$DELETE_BACKUPS" | xargs -r rm --
+echo "$DELETE_LOGS" | xargs -r rm --
+
+echo "Old backups and logs cleaned. Keeping last $BACKUPS_TO_KEEP."
+
 # Kill proxy if still running
 if kill -0 $PROXY_PID 2>/dev/null; then
     kill $PROXY_PID

bin/update.sh

@@ -1,3 +1,3 @@
-/home/boundys/.fly/bin/fly ssh console -C "python manage.py update3 --debug" -a prices 
+/home/boundys/.fly/bin/fly ssh console -C "python manage.py update --debug" -a prices 
 /home/boundys/.fly/bin/fly app restart prices
 

logs/backup_2025-04-16_14-40-43.log

@@ -0,0 +1,19 @@
+================ Wed Apr 16 14:40:43 BST 2025 ================
+Starting backup script...
+APP_NAME: prices-db
+LOCAL_PORT: 5433
+Starting Fly proxy to database...
+Waiting for proxy to be ready...
+Waiting for proxy to be ready...
+Proxying local port 5433 to remote [prices-db.internal]:5432
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Proxy is ready!
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Starting backup...
+pg_dump: warning: there are circular foreign-key constraints on this table:
+pg_dump: detail: nodes
+pg_dump: hint: You might not be able to restore the dump without using --disable-triggers or temporarily dropping the constraints.
+pg_dump: hint: Consider using a full dump instead of a --data-only dump to avoid this problem.
+Backup completed successfully: /home/boundys/data4/django/agile_predict/.local/postgres_backup_2025-04-16_14-40-43.sql
+Updated latest backup symlink: /home/boundys/data4/django/agile_predict/.local/backup.sql
+Proxy closed.

logs/backup_2025-04-16_14-43-01.log

@@ -0,0 +1,20 @@
+================ Wed Apr 16 14:43:01 BST 2025 ================
+Starting backup script...
+APP_NAME: prices-db
+LOCAL_PORT: 5433
+Starting Fly proxy to database...
+Waiting for proxy to be ready...
+Waiting for proxy to be ready...
+Proxying local port 5433 to remote [prices-db.internal]:5432
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Proxy is ready!
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Starting backup...
+pg_dump: warning: there are circular foreign-key constraints on this table:
+pg_dump: detail: nodes
+pg_dump: hint: You might not be able to restore the dump without using --disable-triggers or temporarily dropping the constraints.
+pg_dump: hint: Consider using a full dump instead of a --data-only dump to avoid this problem.
+Backup completed successfully: /home/boundys/data4/django/agile_predict/.local/postgres_backup_2025-04-16_14-43-01.sql
+Updated latest backup symlink: /home/boundys/data4/django/agile_predict/.local/backup.sql
+Old backups and logs cleaned. Keeping last 3.
+Proxy closed.

logs/backup_2025-04-16_14-46-01.log

@@ -0,0 +1,21 @@
+================ Wed Apr 16 14:46:01 BST 2025 ================
+Starting backup script...
+APP_NAME: prices-db
+LOCAL_PORT: 5433
+Starting Fly proxy to database...
+Waiting for proxy to be ready...
+Waiting for proxy to be ready...
+Waiting for proxy to be ready...
+Proxying local port 5433 to remote [prices-db.internal]:5432
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Proxy is ready!
+Connection to localhost (127.0.0.1) 5433 port [tcp/*] succeeded!
+Starting backup...
+pg_dump: warning: there are circular foreign-key constraints on this table:
+pg_dump: detail: nodes
+pg_dump: hint: You might not be able to restore the dump without using --disable-triggers or temporarily dropping the constraints.
+pg_dump: hint: Consider using a full dump instead of a --data-only dump to avoid this problem.
+Backup completed successfully: /home/boundys/data4/django/agile_predict/.local/postgres_backup_2025-04-16_14-46-01.sql
+Updated latest backup symlink: /home/boundys/data4/django/agile_predict/.local/backup.sql
+Old backups and logs cleaned. Keeping last 3.
+Proxy closed.