Skip to content

Commit

Permalink
Simplyfi logic around old backup deletion
Browse files Browse the repository at this point in the history 
Limit number of settings backup files to 45 and delete the oldest.
Previously FPP was only checking if they were >90 days old before deleting which causes too many files being kept if lots of changes are made

Include a shell script to clear out old backups just in cases (to avoid issues with 1000's of files)
  • Loading branch information
@jaredb7 @dkulp
jaredb7 authored and dkulp committed Dec 9, 2023
1 parent fbe5436 commit 64a5229
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 52 deletions.
12 changes: 12 additions & 0 deletions upgrade/81/upgrade.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
#####################################

BINDIR=$(cd $(dirname $0) && pwd)
. ${BINDIR}/../../scripts/common

#Purge any old backups, leaving 45 of the most recent files
#This is a initial bypass of the backup script cleanup which may run out of memory if there are thousand of backups to delete
if [ -d ${MEDIADIR}/config/backups ]; then
cd ${MEDIADIR}/config/backups
ls -tr | head -n -45 | xargs --no-run-if-empty rm
fi
108 changes: 57 additions & 51 deletions www/backup.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2349,63 +2349,69 @@ function pruneOrRemoveAgedBackupFiles()
$config_dir_files = file_get_contents('http://localhost/api/backups/configuration/list');
$config_dir_files = json_decode($config_dir_files, true);

//If the number of backup files that exist IS LESS than what the minimum we want to keep, return and stop processing
//If the number of backup files that exist IS LESS than what the minimum we want to keep, return and stop processing
if (count($config_dir_files) < $fpp_backup_min_number_kept) {
$aged_backup_removal_message = "SETTINGS BACKUP: Not removing JSON Settings backup files older than $fpp_backup_max_age days. Since there are (" . count($config_dir_files) . ") backups available and this is less than the minimum backups we want to keep ($fpp_backup_min_number_kept)";
$aged_backup_removal_message = "SETTINGS BACKUP: NOT removing JSON Settings backup files older than $fpp_backup_max_age days. Since there are (" . count($config_dir_files) . ") backups available and this is less than the minimum backups we want to keep ($fpp_backup_min_number_kept) \r\n";
error_log($aged_backup_removal_message);
return;
}

//Reverse the results so we get oldest to newest, the API by default returns results with newest first
$config_dir_files = array_reverse($config_dir_files);

//loop over the backup files we've found
foreach ($config_dir_files as $backup_file_index => $backup_file_meta_data) {
$backup_filename = $backup_file_meta_data['backup_filename'];

if ((stripos(strtolower($backup_filename), "-backup_") !== false) && (stripos(strtolower($backup_filename), ".json") !== false)) {
//File is one of our backup files, this check is just in case there is other json files in this directory placed there by the user which we want to avoid touching.

//reconstruct the path
$backup_file_location = $backup_file_meta_data['backup_filedirectory'];
$backup_file_path = $backup_file_location . $backup_filename;
//Backup Directory
$backup_directory = !$backup_file_meta_data['backup_alternative_location'] ? 'JsonBackups' : 'JsonBackupsAlternate';
//Collect the backup file
$backup_time = $backup_file_meta_data['backup_time_unix'];
//backup max_age time in seconds, since we are dealing with UNIX epoc time
$backup_max_age_seconds = ($fpp_backup_max_age * 86400);

//Check the age of the file by checking the file modification time compared to now
if ((time() - ($backup_time)) > ($backup_max_age_seconds)) {
//Remove the file via API instead
//Build the URL
$url = 'http://localhost/api/backups/configuration/' . $backup_directory . '/' . $backup_filename;
//options for the request
$options = array(
'http' => array(
'header' => "Content-type: text/plain",
'method' => 'DELETE',
),
);
$context = stream_context_create($options);

if (file_get_contents($url, false, $context) !== false) {
//Note what happened in the logs also
$aged_backup_removal_message = "SETTINGS BACKUP: Removed old JSON settings backup file ($backup_file_path) since it was " . ceil((time() - ($backup_time)) / 86400) . " days old. Max age is $fpp_backup_max_age days.";
error_log($aged_backup_removal_message);
} else {
$aged_backup_removal_message = "SETTINGS BACKUP: Something went wrong pruning old JSON settings backup files.";
error_log($aged_backup_removal_message);
}
} else {
if ($backups_verbose_logging == true) {
$aged_backup_removal_message = "SETTINGS BACKUP: NOT REMOVING backup file ($backup_file_path) since it is only " . ceil((time() - ($backup_time)) / 86400) . " days old.";
error_log($aged_backup_removal_message);
}
}
}
}
// $config_dir_files = array_reverse($config_dir_files);

//Select the backups that lie outside what we want to keep, because the array is a list of backups with the newest first then we'll be selecting older/dated backups
$backups_to_delete = array_slice($config_dir_files, $fpp_backup_min_number_kept);
$num_backups_deleted = 0;

//loop over the backup files we've found
foreach ($backups_to_delete as $backup_file_index => $backup_file_meta_data) {
$backup_filename = $backup_file_meta_data['backup_filename'];

if ((stripos(strtolower($backup_filename), "-backup_") !== false) && (stripos(strtolower($backup_filename), ".json") !== false)) {
//File is one of our backup files, this check is just in case there is other json files in this directory placed there by the user which we want to avoid touching.

//reconstruct the path
$backup_file_location = $backup_file_meta_data['backup_filedirectory'];
$backup_file_path = $backup_file_location . $backup_filename;
//Backup Directory
$backup_directory = !$backup_file_meta_data['backup_alternative_location'] ? 'JsonBackups' : 'JsonBackupsAlternate';
//Collect the backup file
$backup_time = $backup_file_meta_data['backup_time_unix'];
//backup max_age time in seconds, since we are dealing with UNIX epoc time
$backup_max_age_seconds = ($fpp_backup_max_age * 86400);

//Remove the file via API instead
//Build the URL
$url = 'http://localhost/api/backups/configuration/' . $backup_directory . '/' . $backup_filename;
//options for the request
$options = array(
'http' => array(
'header' => "Content-type: text/plain",
'method' => 'DELETE',
),
);
$context = stream_context_create($options);

//Make the call to the API to delete the backup file
$backup_deleted_call = file_get_contents($url, false, $context);
$backup_deleted_call_decoded = json_decode($backup_deleted_call, true);

if ($backup_deleted_call !== false && $backup_deleted_call_decoded['Status'] == 'OK') {
//Track how many were deleted
$num_backups_deleted++;
} else {
$aged_backup_removal_message = "SETTINGS BACKUP: Something went wrong pruning old JSON settings backup file ($backup_file_path). Error: " . $backup_deleted_call;
error_log($aged_backup_removal_message . PHP_EOL);
}
}
}

//Note how many were deleted
if ($num_backups_deleted > 0) {
//Note what happened in the logs also
$aged_backup_removal_message = "SETTINGS BACKUP: Removed ($num_backups_deleted) old JSON settings backup files, because we only want to keep the $fpp_backup_min_number_kept most recent backups";
error_log($aged_backup_removal_message . PHP_EOL);
}
}

/**
Expand Down
5 changes: 4 additions & 1 deletion www/common.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2460,7 +2460,7 @@ function read_directory_files($directory, $return_data = true, $sort_by_date = f
$file_list = array();
$file_data = false;

if ($handle = opendir($directory)) {
if ($handle = @opendir($directory)) {
while (false !== ($file = readdir($handle))) {
// do something with the file
// note that '.' and '..' is returned even
Expand All @@ -2476,6 +2476,9 @@ function read_directory_files($directory, $return_data = true, $sort_by_date = f
}
}
closedir($handle);
unset($handle);
unset($file);
unset($file_data);
}

//Sort the results if sort flag is true
Expand Down

0 comments on commit 64a5229

Please sign in to comment.