From 4bfc696618177c6cb6c2b599cfd0ec3945112960 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Thu, 4 Apr 2024 18:41:30 +0530 Subject: [PATCH 01/30] chage for admin in separate folder --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 8 ++++++-- .../Backup/Mongodump/MongodumpThread.py | 2 +- mongodb_consistent_backup/Common/LocalCommand.py | 11 +++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index edf2c314..ff367d82 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -30,18 +30,22 @@ def run(self): try: backup_base_dir = os.path.dirname(self.backup_dir) backup_base_name = os.path.basename(self.backup_dir) + admin_backup_file = "_".join("admin",self.output_file) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] + cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] + admin_cmd_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "admin", "--remove-files"] + if self.do_gzip(): log_msg = "Archiving and compressing directory: %s" % self.backup_dir cmd_flags.append("-z") + admin_cmd_flags.append("-z") cmd_flags.append(backup_base_name) logging.info(log_msg) self.running = True - self._command = LocalCommand(self.binary, cmd_flags, self.verbose) + self._command = LocalCommand(self.binary, cmd_flags, admin_cmd_flags, self.verbose) self.exit_code = self._command.run() except Exception, e: return self.result(False, "Failed archiving file: %s!" % self.output_file, e) diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index addd2844..5275e368 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -173,7 +173,7 @@ def mongodump_cmd(self): logging.info("MongoDump Version higher that 4.2.0 found extending mongodump with snppy compressor flag") ## https://www.mongodb.com/docs/drivers/node/v4.4/fundamentals/connection/network-compression/ mongodump_flags.extend([ - "--compressors=%s" % "snappy,zlib,zstd" + "--compressors=%s" % "zstd,snappy,zlib" ]) # --numParallelCollections diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 5c99bd31..c9ccf6d7 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -7,11 +7,14 @@ class LocalCommand: - def __init__(self, command, command_flags=None, verbose=False): + def __init__(self, command, command_flags=None, admin_cmd_flags=None, verbose=False): if command_flags is None: command_flags = [] + if admin_command_flags is None: + admin_command_flags = [] self.command = command self.command_flags = command_flags + self.admin_command_flags = command_flags self.verbose = verbose self.output = [] @@ -21,6 +24,10 @@ def __init__(self, command, command_flags=None, verbose=False): if len(self.command_flags): self.command_line.extend(self.command_flags) + self.admin_command_line = [self.command] + if len(self.admin_command_flags): + self.admin_command_line.extend(self.command_flags) + def parse_output(self): if self._process: try: @@ -36,7 +43,7 @@ def parse_output(self): def run(self): try: - self._process = Popen(self.command_line, stdout=PIPE, stderr=PIPE) + self._process = Popen(self.command_line+" && "+ self.admin_command_line, stdout=PIPE, stderr=PIPE) while self._process.poll() is None: self.parse_output() sleep(0.1) From 598851635fbc6973b0532837a75937bd2ec1bdbd Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Thu, 4 Apr 2024 20:45:26 +0530 Subject: [PATCH 02/30] first --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9df886c4..428b770e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.2 +1.4.3 From 91ebeae71bf821f3f35ddfc8b3a35cbb94c05361 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Tue, 16 Apr 2024 19:31:52 +0530 Subject: [PATCH 03/30] somelog --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index ff367d82..e29d1974 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -28,6 +28,7 @@ def run(self): if os.path.isdir(self.backup_dir): if not os.path.isfile(self.output_file): try: + print("outputfile ",self.output_file) backup_base_dir = os.path.dirname(self.backup_dir) backup_base_name = os.path.basename(self.backup_dir) admin_backup_file = "_".join("admin",self.output_file) From efdd6b90bbdcb8456dc37d692635e78f80d6d32f Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:11:46 +0530 Subject: [PATCH 04/30] admincmd --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 6 +++--- mongodb_consistent_backup/Common/LocalCommand.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index e29d1974..c1d0cee7 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -35,18 +35,18 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] - admin_cmd_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "admin", "--remove-files"] + admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "admin", "--remove-files"] if self.do_gzip(): log_msg = "Archiving and compressing directory: %s" % self.backup_dir cmd_flags.append("-z") - admin_cmd_flags.append("-z") + admin_command_flags.append("-z") cmd_flags.append(backup_base_name) logging.info(log_msg) self.running = True - self._command = LocalCommand(self.binary, cmd_flags, admin_cmd_flags, self.verbose) + self._command = LocalCommand(self.binary, cmd_flags, admin_command_flags, self.verbose) self.exit_code = self._command.run() except Exception, e: return self.result(False, "Failed archiving file: %s!" % self.output_file, e) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index c9ccf6d7..b55b9f13 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -7,7 +7,7 @@ class LocalCommand: - def __init__(self, command, command_flags=None, admin_cmd_flags=None, verbose=False): + def __init__(self, command, command_flags=None, admin_command_flags=None, verbose=False): if command_flags is None: command_flags = [] if admin_command_flags is None: From a5546890d493951a0e057e6f1dd5fbd840f9bcbb Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:21:45 +0530 Subject: [PATCH 05/30] admincmd --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index c1d0cee7..c30e3315 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -1,6 +1,7 @@ import os import logging + from mongodb_consistent_backup.Common import LocalCommand from mongodb_consistent_backup.Pipeline import PoolThread @@ -31,7 +32,12 @@ def run(self): print("outputfile ",self.output_file) backup_base_dir = os.path.dirname(self.backup_dir) backup_base_name = os.path.basename(self.backup_dir) - admin_backup_file = "_".join("admin",self.output_file) + output_file_dir = os.path.dirname(self.output_file) + output_file_basename= os.path.basename(self.output_file) + admin_backup_file = output_file_dir +"_".join("admin",output_file_basename) + print(output_file_dir) + print(output_file_basename) + print(admin_backup_file) log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] From 3252c573f3bb5b63d61e18261e78ec307d8b38d9 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:26:25 +0530 Subject: [PATCH 06/30] admincmd --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index c30e3315..412d39fb 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -35,7 +35,7 @@ def run(self): output_file_dir = os.path.dirname(self.output_file) output_file_basename= os.path.basename(self.output_file) admin_backup_file = output_file_dir +"_".join("admin",output_file_basename) - print(output_file_dir) + print("output_file_dir",output_file_dir) print(output_file_basename) print(admin_backup_file) From ea4f1897c0c415c892d2bdbf6da0365e0781193a Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:31:29 +0530 Subject: [PATCH 07/30] admincmd --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 412d39fb..1b4cad44 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -34,10 +34,11 @@ def run(self): backup_base_name = os.path.basename(self.backup_dir) output_file_dir = os.path.dirname(self.output_file) output_file_basename= os.path.basename(self.output_file) - admin_backup_file = output_file_dir +"_".join("admin",output_file_basename) print("output_file_dir",output_file_dir) print(output_file_basename) print(admin_backup_file) + admin_backup_file = output_file_dir +"_".join("admin",output_file_basename) + log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] From 3e834dbd36cf3c90c919f340454722aa71ef7325 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:44:56 +0530 Subject: [PATCH 08/30] admincmd --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 1b4cad44..4418e6ce 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -29,15 +29,12 @@ def run(self): if os.path.isdir(self.backup_dir): if not os.path.isfile(self.output_file): try: - print("outputfile ",self.output_file) backup_base_dir = os.path.dirname(self.backup_dir) backup_base_name = os.path.basename(self.backup_dir) output_file_dir = os.path.dirname(self.output_file) output_file_basename= os.path.basename(self.output_file) - print("output_file_dir",output_file_dir) - print(output_file_basename) + admin_backup_file = output_file_dir +"_".join(["admin",output_file_basename]) print(admin_backup_file) - admin_backup_file = output_file_dir +"_".join("admin",output_file_basename) log_msg = "Archiving directory: %s" % self.backup_dir From 13dcf092210c94871bbea0f5370b9206f98ffb7a Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 14:53:25 +0530 Subject: [PATCH 09/30] fix / --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 4418e6ce..9a70acfb 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -33,7 +33,7 @@ def run(self): backup_base_name = os.path.basename(self.backup_dir) output_file_dir = os.path.dirname(self.output_file) output_file_basename= os.path.basename(self.output_file) - admin_backup_file = output_file_dir +"_".join(["admin",output_file_basename]) + admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) print(admin_backup_file) From e85399184909b98d650f9a77ff9933d53cfd187c Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 15:01:02 +0530 Subject: [PATCH 10/30] admincommand --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 -- mongodb_consistent_backup/Common/LocalCommand.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 9a70acfb..46a279fe 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -34,8 +34,6 @@ def run(self): output_file_dir = os.path.dirname(self.output_file) output_file_basename= os.path.basename(self.output_file) admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) - print(admin_backup_file) - log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index b55b9f13..1e2f430f 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -14,7 +14,7 @@ def __init__(self, command, command_flags=None, admin_command_flags=None, verbos admin_command_flags = [] self.command = command self.command_flags = command_flags - self.admin_command_flags = command_flags + self.admin_command_flags = admin_command_flags self.verbose = verbose self.output = [] @@ -43,6 +43,8 @@ def parse_output(self): def run(self): try: + print(self.command_line) + print(self.admin_command_line) self._process = Popen(self.command_line+" && "+ self.admin_command_line, stdout=PIPE, stderr=PIPE) while self._process.poll() is None: self.parse_output() From 7da3128c35e74f0696a18921287cf05203bbf793 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 15:14:01 +0530 Subject: [PATCH 11/30] list --- mongodb_consistent_backup/Common/LocalCommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 1e2f430f..11fe434c 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -45,7 +45,7 @@ def run(self): try: print(self.command_line) print(self.admin_command_line) - self._process = Popen(self.command_line+" && "+ self.admin_command_line, stdout=PIPE, stderr=PIPE) + self._process = Popen(self.command_line+["&&"]+ self.admin_command_line, stdout=PIPE, stderr=PIPE) while self._process.poll() is None: self.parse_output() sleep(0.1) From 190c12e9542dd8eb12b03c0e97b9da17cb5b2725 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 15:24:49 +0530 Subject: [PATCH 12/30] list --- mongodb_consistent_backup/Common/LocalCommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 11fe434c..179628ef 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -26,7 +26,7 @@ def __init__(self, command, command_flags=None, admin_command_flags=None, verbos self.admin_command_line = [self.command] if len(self.admin_command_flags): - self.admin_command_line.extend(self.command_flags) + self.admin_command_line.extend(self.admin_command_flags) def parse_output(self): if self._process: From d7e392e2754610c8ddf5ac408780cf738c4df444 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 15:34:18 +0530 Subject: [PATCH 13/30] list --- mongodb_consistent_backup/Common/LocalCommand.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 179628ef..4c86fbb8 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,7 @@ def parse_output(self): def run(self): try: - print(self.command_line) - print(self.admin_command_line) + print(self.command_line+["&&"]+ self.admin_command_line) self._process = Popen(self.command_line+["&&"]+ self.admin_command_line, stdout=PIPE, stderr=PIPE) while self._process.poll() is None: self.parse_output() From 3a25bc93d28adf7c024e41f3da912e72d1e2bc8a Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 16:34:11 +0530 Subject: [PATCH 14/30] shell --- mongodb_consistent_backup/Common/LocalCommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 4c86fbb8..03b6114b 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -44,7 +44,7 @@ def parse_output(self): def run(self): try: print(self.command_line+["&&"]+ self.admin_command_line) - self._process = Popen(self.command_line+["&&"]+ self.admin_command_line, stdout=PIPE, stderr=PIPE) + self._process = Popen(self.command_line+["&&"]+ self.admin_command_line, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: self.parse_output() sleep(0.1) From 37c6daef00f06f7a46f8e4461273ceafdf2e2086 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 16:59:42 +0530 Subject: [PATCH 15/30] shell --- mongodb_consistent_backup/Common/LocalCommand.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 03b6114b..b11759eb 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,9 @@ def parse_output(self): def run(self): try: - print(self.command_line+["&&"]+ self.admin_command_line) - self._process = Popen(self.command_line+["&&"]+ self.admin_command_line, stdout=PIPE, stderr=PIPE,shell= True) + print("".join(self.admin_command_line+ ["&&"]+ self.command_line)) + cmd = "".join(self.admin_command_line+ ["&&"]+ self.command_line) + self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: self.parse_output() sleep(0.1) From 9293a6059c3f92ebc6f157a87c1b017c5d1a3deb Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 17:02:19 +0530 Subject: [PATCH 16/30] shell --- mongodb_consistent_backup/Common/LocalCommand.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index b11759eb..9e64e462 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,8 @@ def parse_output(self): def run(self): try: - print("".join(self.admin_command_line+ ["&&"]+ self.command_line)) - cmd = "".join(self.admin_command_line+ ["&&"]+ self.command_line) + print(" ".join(self.admin_command_line+ ["&&"]+ self.command_line)) + cmd = " ".join(self.admin_command_line+ ["&&"]+ self.command_line) self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: self.parse_output() From 8b6aa479d94e18c5afd2bca2445ca81eb6c17594 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 17:26:11 +0530 Subject: [PATCH 17/30] shell --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 46a279fe..ee7b25e1 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -36,8 +36,8 @@ def run(self): admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--exclude", "admin" ,"--remove-files"] - admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "admin", "--remove-files"] + cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, ] + admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file,] if self.do_gzip(): @@ -45,7 +45,8 @@ def run(self): cmd_flags.append("-z") admin_command_flags.append("-z") - cmd_flags.append(backup_base_name) + cmd_flags.append(backup_base_name+"/.") + admin_command_flags.append(backup_base_name+"/.") logging.info(log_msg) self.running = True self._command = LocalCommand(self.binary, cmd_flags, admin_command_flags, self.verbose) From f28823c1b88150027168b6fab1b6dcde6218e02d Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 17:27:01 +0530 Subject: [PATCH 18/30] --remove-files --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index ee7b25e1..9c82a0ab 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -36,8 +36,8 @@ def run(self): admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, ] - admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file,] + cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] + admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): From 539cb89bb6251ecbb1ed8c47b20b29d2057e8c01 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 17:31:46 +0530 Subject: [PATCH 19/30] --remove-files --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 9c82a0ab..9c693497 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -36,7 +36,7 @@ def run(self): admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] + cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file] admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "--remove-files"] @@ -45,8 +45,8 @@ def run(self): cmd_flags.append("-z") admin_command_flags.append("-z") - cmd_flags.append(backup_base_name+"/.") - admin_command_flags.append(backup_base_name+"/.") + cmd_flags.append(backup_base_name) + admin_command_flags.append(backup_base_name) logging.info(log_msg) self.running = True self._command = LocalCommand(self.binary, cmd_flags, admin_command_flags, self.verbose) From fc6edd6413f147e219b29e86ce1ac88c52c62b26 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Wed, 17 Apr 2024 17:39:16 +0530 Subject: [PATCH 20/30] --remove-files --- mongodb_consistent_backup/Common/LocalCommand.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 9e64e462..ee79746c 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,8 @@ def parse_output(self): def run(self): try: - print(" ".join(self.admin_command_line+ ["&&"]+ self.command_line)) - cmd = " ".join(self.admin_command_line+ ["&&"]+ self.command_line) + print(" ".join(self.command_line+ ["&&"]+ self.admin_command_line)) + cmd = " ".join(self.command_line+ ["&&"]+ self.admin_command_line) self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: self.parse_output() From 1f79c5ead9ddb8a064c735d9c1e915adce0f1599 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 13:12:09 +0530 Subject: [PATCH 21/30] finall check --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 9c693497..56cf2389 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file] - admin_command_flags = ["-C", backup_base_dir, "-c", "-f", admin_backup_file, "--remove-files"] + admin_command_flags = ["-C", backup_base_dir+"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): From 71445f64aa3104802e4e52b7fe1d62bbc19d90bb Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 13:18:34 +0530 Subject: [PATCH 22/30] finall check --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 56cf2389..ac2a0dd6 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file] - admin_command_flags = ["-C", backup_base_dir+"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] + admin_command_flags = ["-C", backup_base_dir+backup_base_name+"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): From a52cae2927538fbe8b6c505e53a0523876823e7a Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 13:22:02 +0530 Subject: [PATCH 23/30] self.backup_dir --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index ac2a0dd6..a7d7356f 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file] - admin_command_flags = ["-C", backup_base_dir+backup_base_name+"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] + admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): From 1980f1decb4d52e6cc98b506408ce5aa92447473 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:14:13 +0530 Subject: [PATCH 24/30] admin --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 4 ++-- mongodb_consistent_backup/Common/LocalCommand.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index a7d7356f..05e860b0 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -36,7 +36,7 @@ def run(self): admin_backup_file = output_file_dir+"/" +"_".join(["admin",output_file_basename]) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file] + cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] @@ -46,7 +46,7 @@ def run(self): admin_command_flags.append("-z") cmd_flags.append(backup_base_name) - admin_command_flags.append(backup_base_name) + admin_command_flags.append(".") logging.info(log_msg) self.running = True self._command = LocalCommand(self.binary, cmd_flags, admin_command_flags, self.verbose) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index ee79746c..9e64e462 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,8 @@ def parse_output(self): def run(self): try: - print(" ".join(self.command_line+ ["&&"]+ self.admin_command_line)) - cmd = " ".join(self.command_line+ ["&&"]+ self.admin_command_line) + print(" ".join(self.admin_command_line+ ["&&"]+ self.command_line)) + cmd = " ".join(self.admin_command_line+ ["&&"]+ self.command_line) self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: self.parse_output() From 7c22550c793fc19d090407fa3e7c604e2d4e9d95 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:21:00 +0530 Subject: [PATCH 25/30] admin1 --- mongodb_consistent_backup/Common/LocalCommand.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index 9e64e462..d0ee38d9 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,7 +43,8 @@ def parse_output(self): def run(self): try: - print(" ".join(self.admin_command_line+ ["&&"]+ self.command_line)) + print("cmd : ") + print(" ".join( self.admin_command_line+ ["&&"]+ self.command_line)) cmd = " ".join(self.admin_command_line+ ["&&"]+ self.command_line) self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: From 88ede1713112dd4d013525a01b7651425a28af4f Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:24:32 +0530 Subject: [PATCH 26/30] admin1 --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 05e860b0..96c16954 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] - admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] + admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file] if self.do_gzip(): From 063762519e72c9f07df6f4abbe297689e5f16407 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:28:21 +0530 Subject: [PATCH 27/30] admin1 --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 96c16954..9ec46352 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] - admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file] + admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): @@ -46,7 +46,7 @@ def run(self): admin_command_flags.append("-z") cmd_flags.append(backup_base_name) - admin_command_flags.append(".") + admin_command_flags.append("admin") logging.info(log_msg) self.running = True self._command = LocalCommand(self.binary, cmd_flags, admin_command_flags, self.verbose) From 67ff8c9e4d09935d3c489abf899a9a09eab18449 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:32:24 +0530 Subject: [PATCH 28/30] work --- mongodb_consistent_backup/Archive/Tar/TarThread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index 9ec46352..e51fc832 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -37,7 +37,7 @@ def run(self): log_msg = "Archiving directory: %s" % self.backup_dir cmd_flags = ["--exclude", "admin" ,"-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] - admin_command_flags = ["-C", self.backup_dir +"/dump/admin", "-c", "-f", admin_backup_file, "--remove-files"] + admin_command_flags = ["-C", self.backup_dir +"/dump/", "-c", "-f", admin_backup_file, "--remove-files"] if self.do_gzip(): From c272472b34d86b618491c8176021f25967007b24 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 14:38:53 +0530 Subject: [PATCH 29/30] readme update --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 5e7ea970..0c6770fa 100644 --- a/README.rst +++ b/README.rst @@ -256,3 +256,6 @@ Contact ~~~~~~~ `Contact Percona `__ + +Take Bacup OR Instances: +rocket_consistent_backup_wrapper `instaneid` \ No newline at end of file From d4f87b76f04f32a9916c1fe3c61dc84c1ce01408 Mon Sep 17 00:00:00 2001 From: Biswajit Mohanty Date: Fri, 19 Apr 2024 18:56:24 +0530 Subject: [PATCH 30/30] logging --- mongodb_consistent_backup/Common/LocalCommand.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mongodb_consistent_backup/Common/LocalCommand.py b/mongodb_consistent_backup/Common/LocalCommand.py index d0ee38d9..7d4e7879 100644 --- a/mongodb_consistent_backup/Common/LocalCommand.py +++ b/mongodb_consistent_backup/Common/LocalCommand.py @@ -43,8 +43,6 @@ def parse_output(self): def run(self): try: - print("cmd : ") - print(" ".join( self.admin_command_line+ ["&&"]+ self.command_line)) cmd = " ".join(self.admin_command_line+ ["&&"]+ self.command_line) self._process = Popen(cmd, stdout=PIPE, stderr=PIPE,shell= True) while self._process.poll() is None: