From e00a241d94012df5ad0dab01f0e3cd5c53598666 Mon Sep 17 00:00:00 2001 From: vdbsh <> Date: Mon, 16 Aug 2021 12:34:13 +0300 Subject: [PATCH] General refactoring, Always remove failed archives --- README.md | 1 - backy.go | 205 +++++++++++++++++++++----------------- scripts/example_task.json | 1 - 3 files changed, 114 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 7f180cb..99bb9a2 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ { "destination": "~/Backup", "archiving_cycle": "monthly", - "remove_failed_archives": true, "multiprocessing": true, "verbose_log": false, diff --git a/backy.go b/backy.go index 178b50f..74f9f7f 100644 --- a/backy.go +++ b/backy.go @@ -13,12 +13,11 @@ import ( "time" ) -const version string = "1.1" +const version string = "1.2" type Task struct { VerboseLog bool `json:"verbose_log"` Multiprocessing bool `json:"multiprocessing"` - RemoveFailedArchives bool `json:"remove_failed_archives"` Destination string `json:"destination"` ArchivingCycle string `json:"archiving_cycle"` Exclude []string `json:"exclude"` @@ -42,18 +41,27 @@ func removeDuplicates(elements []string) (unique_elements []string) { func normilizePaths(paths []string) (normalized_paths []string) { homedir, _ := os.UserHomeDir() - for i, path := range paths { - if string(path[0]) == "~" { - path = strings.Replace(path, "~", homedir, 1) + for i, p := range paths { + if string(p[0]) == "~" { + p = strings.Replace(p, "~", homedir, 1) } - paths[i] = strings.TrimRight(string(path), "/") + paths[i] = strings.TrimRight(string(p), "/") } return paths } +func getBasePath(path string) (base_path string) { + hostname, err := os.Hostname() + + if err != nil { + hostname = "local" + } + return strings.Replace(hostname+path, "/", "_", -1) +} + func formatExcludeArgs(exclude_args_list []string) (exclude_args []string) { - for _, el := range exclude_args_list { - exclude_args = append(exclude_args, "--exclude="+el) + for _, i := range exclude_args_list { + exclude_args = append(exclude_args, "--exclude="+i) } return exclude_args } @@ -63,13 +71,7 @@ func generateArchiveFilePath(path string, archiving_cycle string) (file_path str dt := time.Now() _, iso_week := dt.ISOWeek() - - hostname, err := os.Hostname() - if err != nil { - hostname = "local" - } - - base_path := strings.Replace(hostname+path, "/", "_", -1) + base_path := getBasePath(path) switch archiving_cycle { case "hourly": @@ -113,57 +115,75 @@ func getTaskFromJson(file_path string) (config_err error, task_config Task) { return config_err, task_config } -func executeProcesses(c string, args [][]string, multiprocessing bool, progress_indicator string, print_outtput bool) (statuses []error) { - var processes []*exec.Cmd - progress_indicator_step := " " + progress_indicator - - for _, arg_set := range args { - cmd := exec.Command(c, arg_set...) - if print_outtput == true { - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr +func filterArchiveDirs(archive_dirs []string, archiving_cycle string, dest string) (filtered_archive_dirs []string) { + for _, i := range archive_dirs { + _, e := os.Stat(path.Join(dest, generateArchiveFilePath(i, archiving_cycle))) + if os.IsNotExist(e) { + filtered_archive_dirs = append(filtered_archive_dirs, i) } + } + return filtered_archive_dirs +} - if multiprocessing == true { - processes = append(processes, cmd) - cmd.Start() - } else { - log.Println(progress_indicator) - statuses = append(statuses, cmd.Run()) - progress_indicator += progress_indicator_step - } +func logProgress(amount int, indicator string) { + progress_indicator_step := " " + indicator + for i := 1; i < amount; i++ { + indicator += progress_indicator_step } + log.Println(indicator) +} - if multiprocessing == true { - for _, process := range processes { - log.Println(progress_indicator) - statuses = append(statuses, process.Wait()) - if print_outtput == true { - log.Println() - } - progress_indicator += progress_indicator_step - } +func checkStatus(status error, process string, element string) (status_err error) { + if status != nil { + log.Println("๐Ÿ’ข", process, "failed for:", element) + status_err = errors.New("process failed") } - return statuses + return status_err } -func rsync(src_dirs []string, dest string, args []string, multiprocessing bool, print_outtput bool) (rsync_err error) { - if len(src_dirs) > 0 { - var task_args [][]string +func setProcessOutput(process *exec.Cmd, print_outtput bool) { + if print_outtput == true { + process.Stdout = os.Stdout + process.Stderr = os.Stderr + } +} + +func runProcess(c string, args []string, print_outtput bool) (status error) { + cmd := exec.Command(c, args...) + setProcessOutput(cmd, print_outtput) + return cmd.Run() +} + +func startProcess(c string, args []string, print_outtput bool) (handle *exec.Cmd) { + cmd := exec.Command(c, args...) + setProcessOutput(cmd, print_outtput) + cmd.Start() + return cmd +} + +func startRsync(src_dirs []string, dest string, args []string, multiprocessing bool, print_outtput bool) (rsync_err error) { + var processes []*exec.Cmd + if len(src_dirs) > 0 { log.Println("๐Ÿ“ค", "Syncing from:", src_dirs) log.Println("๐Ÿ“ฅ", "To:", dest, "...") - for _, dir := range src_dirs { - task_args = append(task_args, append(args, []string{dir, dest}...)) + for p, i := range src_dirs { + if multiprocessing == true { + processes = append(processes, startProcess("rsync", append(args, []string{i, dest}...), print_outtput)) + } else { + logProgress(p+1, "๐Ÿ—‚") + if checkStatus(runProcess("rsync", append(args, []string{i, dest}...), print_outtput), "Syncronization", i) != nil { + rsync_err = errors.New("rsync failed") + } + } } - - results := executeProcesses("rsync", task_args, multiprocessing, "๐Ÿ—‚", print_outtput) - - for i, status := range results { - if status != nil { - log.Println("๐Ÿ’ข", "Syncronisation failed for:", src_dirs[i]) - rsync_err = errors.New("rsync failed") + if multiprocessing == true { + for i, p := range processes { + logProgress(i+1, "๐Ÿ—‚") + if checkStatus(p.Wait(), "Syncronization", src_dirs[i]) != nil { + rsync_err = errors.New("rsync failed") + } } } } else { @@ -172,50 +192,53 @@ func rsync(src_dirs []string, dest string, args []string, multiprocessing bool, return rsync_err } -func tar(archive_dirs []string, dest string, mode string, args []string, archiving_cycle string, multiprocessing bool, print_outtput bool, remove_failed_archives bool) (tar_err error) { - var archives []string - var dirs_to_archive []string - var task_args [][]string - - for _, dir := range archive_dirs { - file_path := generateArchiveFilePath(dir, archiving_cycle) - archives = append(archives, path.Join(dest, file_path)) - } - - for i, archive := range archives { - in_progress_archive := archive + ".part" - - if _, err := os.Stat(archive); err != nil { - if _, err := os.Stat(in_progress_archive); err == nil { - os.Remove(in_progress_archive) - } - task_args = append(task_args, append([]string{mode, in_progress_archive}, append(args, archive_dirs[i])...)) - dirs_to_archive = append(dirs_to_archive, archive_dirs[i]) - } - } +func startTar(archive_dirs []string, dest string, mode string, args []string, archiving_cycle string, multiprocessing bool, print_outtput bool) (tar_err error) { + var processes []*exec.Cmd + dirs_to_archive := filterArchiveDirs(archive_dirs, archiving_cycle, dest) if len(dirs_to_archive) > 0 { log.Println("๐Ÿ“ค", "Archiving:", dirs_to_archive) log.Println("๐Ÿ“ฅ", "To:", dest, "...") - } else { - log.Println("๐Ÿ’ค", "Nothing to archive") - } - results := executeProcesses("tar", task_args, multiprocessing, "๐Ÿ“ฆ", print_outtput) + for p, i := range dirs_to_archive { + completed_archive := path.Join(dest, generateArchiveFilePath(i, archiving_cycle)) + in_progress_archive := completed_archive + ".part" + task_args := append([]string{mode, in_progress_archive}, append(args, i)...) - for i, status := range results { - completed_archive := path.Join(dest, generateArchiveFilePath(dirs_to_archive[i], archiving_cycle)) + _, e := os.Stat(in_progress_archive) + if !os.IsNotExist(e) { + os.Remove(in_progress_archive) + } - if status != nil { - log.Println("๐Ÿ’ข", "Archiving failed for:", dirs_to_archive[i]) - if remove_failed_archives == true { - log.Println("๐Ÿงน", "Removing failed archive") - os.Remove(completed_archive + ".part") + if multiprocessing == true { + processes = append(processes, startProcess("tar", task_args, print_outtput)) + } else { + logProgress(p+1, "๐Ÿ“ฆ") + if checkStatus(runProcess("tar", task_args, print_outtput), "Archiving", i) != nil { + os.Remove(in_progress_archive) + tar_err = errors.New("tar failed") + } else { + os.Rename(in_progress_archive, completed_archive) + } + } + } + + if multiprocessing == true { + for i, p := range processes { + completed_archive := path.Join(dest, generateArchiveFilePath(dirs_to_archive[i], archiving_cycle)) + in_progress_archive := completed_archive + ".part" + logProgress(i+1, "๐Ÿ“ฆ") + if checkStatus(p.Wait(), "Archiving", dirs_to_archive[i]) != nil { + os.Remove(in_progress_archive) + tar_err = errors.New("tar failed") + } else { + os.Rename(in_progress_archive, completed_archive) + } } - tar_err = errors.New("tar failed") - } else { - os.Rename(completed_archive+".part", completed_archive) } + + } else { + log.Println("๐Ÿ’ค", "Nothing to archive") } return tar_err } @@ -237,12 +260,12 @@ func main() { status = 2 } else { - if rsync(task.DirectoriesToSync, task.Destination, append([]string{"-avW", "--delete", "--delete-excluded"}, task.Exclude...), task.Multiprocessing, task.VerboseLog) != nil { + if startRsync(task.DirectoriesToSync, task.Destination, append([]string{"-avW", "--delete", "--delete-excluded"}, task.Exclude...), task.Multiprocessing, task.VerboseLog) != nil { status = 3 log.Println("โ—๏ธ", "Synchronization completed with errors") } - if tar(task.DirectoriesToArchive, task.Destination, "-jcvf", task.Exclude, task.ArchivingCycle, task.Multiprocessing, task.VerboseLog, task.RemoveFailedArchives) != nil { + if startTar(task.DirectoriesToArchive, task.Destination, "-jcvf", task.Exclude, task.ArchivingCycle, task.Multiprocessing, task.VerboseLog) != nil { if status != 3 { status = 4 } else { diff --git a/scripts/example_task.json b/scripts/example_task.json index 00283d9..5d68e34 100644 --- a/scripts/example_task.json +++ b/scripts/example_task.json @@ -1,7 +1,6 @@ { "destination": "~/Backup", "archiving_cycle": "monthly", - "remove_failed_archives": true, "multiprocessing": true, "verbose_log": false,