From ca6dab6df399f334d60af75a4594ddca42b38211 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Tue, 5 Jan 2016 22:08:02 -0800 Subject: [PATCH 01/39] Add backup.full_backup config property --- swb/config/example.ini | 2 ++ swb/local.py | 10 ++++++---- swb/remote.py | 35 ++++++++++++++++++++++++++++------- tests/files/config.ini | 1 + tests/test_local.py | 4 ++-- tests/test_remote.py | 3 ++- 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/swb/config/example.ini b/swb/config/example.ini index 87f1c0b..ece1074 100644 --- a/swb/config/example.ini +++ b/swb/config/example.ini @@ -21,3 +21,5 @@ compressor = bzip2 -v decompressor = bzip2 -d # The maximum number of local backups to keep max_local_backups = 3 +# Whether to backup the entire site directory in addition to the database +full_backup = hey diff --git a/swb/local.py b/swb/local.py index 02a43f7..6997d30 100755 --- a/swb/local.py +++ b/swb/local.py @@ -77,10 +77,10 @@ def quote_arg(arg): if hasattr(shlex, 'quote'): # shlex.quote was introduced in v3.3 - quoted_arg = shlex.quote(arg) + quoted_arg = shlex.quote(str(arg)) else: # pipes.quote is deprecated, but use it if shlex.quote is unavailable - quoted_arg = pipes.quote(arg) + quoted_arg = pipes.quote(str(arg)) quoted_arg = unquote_home_dir(quoted_arg) return quoted_arg @@ -141,12 +141,13 @@ def transfer_file(ssh_user, ssh_hostname, ssh_port, # Execute remote backup script to create remote backup def create_remote_backup(ssh_user, ssh_hostname, ssh_port, wordpress_path, remote_backup_path, - backup_compressor, *, stdout, stderr): + backup_compressor, full_backup, *, stdout, stderr): exec_on_remote(ssh_user, ssh_hostname, ssh_port, 'back-up', [ wordpress_path, backup_compressor, - remote_backup_path + remote_backup_path, + full_backup ], stdout=stdout, stderr=stderr) @@ -230,6 +231,7 @@ def back_up(config, *, stdout=None, stderr=None): config.get('paths', 'wordpress'), expanded_remote_backup_path, config.get('backup', 'compressor'), + config.getboolean('backup', 'full_backup'), stdout=stdout, stderr=stderr) create_dir_structure(expanded_local_backup_path) diff --git a/swb/remote.py b/swb/remote.py index 7c9bf70..236d5b0 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -4,6 +4,7 @@ import os.path import re import shlex +import shutil import subprocess import sys @@ -81,17 +82,37 @@ def purge_downloaded_backup(backup_path): os.remove(backup_path) -def back_up(wordpress_path, backup_compressor, backup_path): +def back_up(wordpress_path, backup_compressor, backup_path, full_backup): backup_path = os.path.expanduser(backup_path) create_dir_structure(backup_path) - db_info = get_db_info(wordpress_path) - dump_db( - db_info['name'], db_info['host'], - db_info['user'], db_info['password'], - backup_compressor, backup_path) - verify_backup_integrity(backup_path) + if full_backup == 'True': + + # backup_path is assumed to refer to entire site directory backup + wordpress_site_name = os.path.basename(wordpress_path) + backup_pwd_path = os.path.dirname(backup_path) + shutil.copy2(wordpress_path, backup_pwd_path) + + db_backup_name = '{}.sql'.format() + db_backup_path = os.path.join( + backup_pwd_path, wordpress_site_name, db_backup_name) + db_info = get_db_info(wordpress_path) + dump_db( + db_info['name'], db_info['host'], + db_info['user'], db_info['password'], + backup_compressor, db_backup_path) + verify_backup_integrity(backup_path) + + else: + + # backup_path is assumed to refer to SQL database file backup + db_info = get_db_info(wordpress_path) + dump_db( + db_info['name'], db_info['host'], + db_info['user'], db_info['password'], + backup_compressor, backup_path) + verify_backup_integrity(backup_path) # Decompress the given backup file to a database file in the same directory diff --git a/tests/files/config.ini b/tests/files/config.ini index 6f78478..f88ab76 100644 --- a/tests/files/config.ini +++ b/tests/files/config.ini @@ -12,3 +12,4 @@ local_backup = ~/Backups/mysite.sql.bz2 compressor = bzip2 decompressor = bzip2 -d max_local_backups = 3 +full_backup = no diff --git a/tests/test_local.py b/tests/test_local.py index 8e33932..1c1a39b 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -32,8 +32,8 @@ def test_create_remote_backup(): swb.back_up(config) swb.subprocess.Popen.assert_any_call([ 'ssh', '-p 2222', 'myname@mysite.com', 'python3', '-', 'back-up', - '~/\'public_html/mysite\'', 'bzip2', '~/\'backups/mysite.sql.bz2\''], - stdin=ANY, stdout=None, stderr=None) + '~/\'public_html/mysite\'', 'bzip2', '~/\'backups/mysite.sql.bz2\'', + 'False'], stdin=ANY, stdout=None, stderr=None) @nose.with_setup(set_up, tear_down) diff --git a/tests/test_remote.py b/tests/test_remote.py index 681ccfa..d212bed 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -12,11 +12,12 @@ BACKUP_COMPRESSOR = 'bzip2 -v' BACKUP_DECOMPRESSOR = 'bzip2 -d' BACKUP_PATH = '~/backups/mysite.sql.bz2' +FULL_BACKUP = False DB_PATH = BACKUP_PATH.replace('.bz2', '') def run_back_up(): - swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH) + swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) def run_restore(): From 26fce38733bb7f696b461de9861594ec9d88ced7 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 15:07:18 -0800 Subject: [PATCH 02/39] Add remote logic for creating tar file --- swb/config/example.ini | 2 +- swb/remote.py | 75 +++++++++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/swb/config/example.ini b/swb/config/example.ini index ece1074..c4f7782 100644 --- a/swb/config/example.ini +++ b/swb/config/example.ini @@ -22,4 +22,4 @@ decompressor = bzip2 -d # The maximum number of local backups to keep max_local_backups = 3 # Whether to backup the entire site directory in addition to the database -full_backup = hey +full_backup = no diff --git a/swb/remote.py b/swb/remote.py index 236d5b0..45e34d1 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 +import io import os import os.path import re import shlex -import shutil import subprocess import sys +import tarfile # Read contents of wp-config.php for a WordPress installation @@ -44,9 +45,8 @@ def get_db_info(wordpress_path): return db_info -# Dump MySQL database to compressed file on remote -def dump_db(db_name, db_host, db_user, db_password, - backup_compressor, backup_path): +# Dump MySQL database to file and return subprocess +def get_mysqldump(db_name, db_host, db_user, db_password): mysqldump = subprocess.Popen([ 'mysqldump', @@ -57,6 +57,15 @@ def dump_db(db_name, db_host, db_user, db_password, '--add-drop-table' ], stdout=subprocess.PIPE) + return mysqldump + + +# Dump MySQL database to compressed file +def dump_compressed_db(db_name, db_host, db_user, db_password, + backup_compressor, backup_path): + + mysqldump = get_mysqldump(db_name, db_host, db_user, db_password) + # Create remote backup so as to write output of dump/compress to file with open(backup_path, 'w') as backup_file: @@ -69,6 +78,16 @@ def dump_db(db_name, db_host, db_user, db_password, compressor.wait() +# Dump MySQL database to uncompressed file at the given path +def dump_uncompressed_db(db_name, db_host, db_user, db_password): + + mysqldump = get_mysqldump(db_name, db_host, db_user, db_password) + db_contents = mysqldump.communicate()[0] + mysqldump.wait() + + return db_contents + + # Verify integrity of remote backup by checking its size def verify_backup_integrity(backup_path): @@ -82,6 +101,37 @@ def purge_downloaded_backup(backup_path): os.remove(backup_path) +# Add a database to the given tar file under the given name +def add_db_to_tar(tar_file, db_file_name, db_contents): + + db_file_obj = io.BytesIO() + db_file_obj.write(db_contents) + db_tar_info = tarfile.TarInfo(db_file_name) + db_tar_info.size = len(db_file_obj.getvalue()) + db_file_obj.seek(0) + tar_file.addfile(db_tar_info, db_file_obj) + + +# Create the full backup file by tar'ing both the wordpress site directory and +# the the dumped database contents +def create_full_backup(backup_path, wordpress_path, db_contents): + + backup_name = os.path.basename(backup_path) + tar_name = os.path.splitext(backup_name)[0] + + wordpress_site_name = os.path.basename(wordpress_path) + db_file_name = '{}.sql'.format(wordpress_site_name) + + backup_pwd_path = os.path.dirname(backup_path) + tar_path = os.path.join(backup_pwd_path, tar_name) + + tar_file = tarfile.open(tar_path, 'w') + tar_file.add(wordpress_path, arcname=wordpress_site_name) + add_db_to_tar(tar_file, db_file_name, db_contents) + tar_file.close() + + +# Back up WordPress database or installation def back_up(wordpress_path, backup_compressor, backup_path, full_backup): backup_path = os.path.expanduser(backup_path) @@ -90,25 +140,18 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): if full_backup == 'True': # backup_path is assumed to refer to entire site directory backup - wordpress_site_name = os.path.basename(wordpress_path) - backup_pwd_path = os.path.dirname(backup_path) - shutil.copy2(wordpress_path, backup_pwd_path) - - db_backup_name = '{}.sql'.format() - db_backup_path = os.path.join( - backup_pwd_path, wordpress_site_name, db_backup_name) db_info = get_db_info(wordpress_path) - dump_db( + db_contents = dump_uncompressed_db( db_info['name'], db_info['host'], - db_info['user'], db_info['password'], - backup_compressor, db_backup_path) - verify_backup_integrity(backup_path) + db_info['user'], db_info['password']) + create_full_backup(backup_path, wordpress_path, db_contents) + # verify_backup_integrity(backup_path) else: # backup_path is assumed to refer to SQL database file backup db_info = get_db_info(wordpress_path) - dump_db( + dump_compressed_db( db_info['name'], db_info['host'], db_info['user'], db_info['password'], backup_compressor, backup_path) From 134858086d10fa59efda197f4520b8d79e3e7e48 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 15:09:07 -0800 Subject: [PATCH 03/39] Use os.path.splitext instead of re.sub --- swb/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swb/remote.py b/swb/remote.py index 45e34d1..12b5d76 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -169,7 +169,7 @@ def decompress_backup(backup_path, backup_decompressor): # Construct path to decompressed database file from given backup file def get_db_path(backup_path): - return re.sub('\.([A-Za-z0-9]+)$', '', backup_path) + return os.path.splitext(backup_path)[0] # Replace a WordPress database with the database at the given path From 887a71bf544cd09b9c2d77cb67e3554026958615 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 15:22:15 -0800 Subject: [PATCH 04/39] Add logic to compress created tar backup file --- swb/remote.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index 12b5d76..3b9b905 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -101,6 +101,14 @@ def purge_downloaded_backup(backup_path): os.remove(backup_path) +# Compressed an existing tar backup file using the chosen compressor +def compress_tar(tar_path, backup_path, backup_compressor): + + compressor = subprocess.Popen( + shlex.split(backup_compressor) + [backup_path, tar_path]) + compressor.wait() + + # Add a database to the given tar file under the given name def add_db_to_tar(tar_file, db_file_name, db_contents): @@ -114,7 +122,8 @@ def add_db_to_tar(tar_file, db_file_name, db_contents): # Create the full backup file by tar'ing both the wordpress site directory and # the the dumped database contents -def create_full_backup(backup_path, wordpress_path, db_contents): +def create_full_backup(wordpress_path, db_contents, + backup_path, backup_compressor): backup_name = os.path.basename(backup_path) tar_name = os.path.splitext(backup_name)[0] @@ -130,6 +139,8 @@ def create_full_backup(backup_path, wordpress_path, db_contents): add_db_to_tar(tar_file, db_file_name, db_contents) tar_file.close() + compress_tar(tar_path, backup_path, backup_compressor) + # Back up WordPress database or installation def back_up(wordpress_path, backup_compressor, backup_path, full_backup): @@ -144,7 +155,9 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): db_contents = dump_uncompressed_db( db_info['name'], db_info['host'], db_info['user'], db_info['password']) - create_full_backup(backup_path, wordpress_path, db_contents) + create_full_backup( + wordpress_path, db_contents, + backup_path, backup_compressor) # verify_backup_integrity(backup_path) else: From d0d5f98abe8a4432b9250265d38e83a04ae48ce1 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 15:27:00 -0800 Subject: [PATCH 05/39] Parameterize remote test helper functions --- tests/test_remote.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index d212bed..63d9461 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -16,12 +16,14 @@ DB_PATH = BACKUP_PATH.replace('.bz2', '') -def run_back_up(): - swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) +def run_back_up(wp_path=WP_PATH, backup_compressor=BACKUP_COMPRESSOR, + backup_path=BACKUP_PATH, full_backup=FULL_BACKUP): + swb.back_up(wp_path, backup_compressor, backup_path, full_backup) -def run_restore(): - swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) +def run_restore(wp_path=WP_PATH, backup_path=BACKUP_PATH, + backup_decompressor=BACKUP_DECOMPRESSOR): + swb.restore(wp_path, backup_path, backup_decompressor) @nose.with_setup(set_up, tear_down) From ff496273baf18907bf4fdcc4e184154680e1fcac Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 15:40:46 -0800 Subject: [PATCH 06/39] Revert "Parameterize remote test helper functions" This reverts commit d0d5f98abe8a4432b9250265d38e83a04ae48ce1. --- tests/test_remote.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 63d9461..d212bed 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -16,14 +16,12 @@ DB_PATH = BACKUP_PATH.replace('.bz2', '') -def run_back_up(wp_path=WP_PATH, backup_compressor=BACKUP_COMPRESSOR, - backup_path=BACKUP_PATH, full_backup=FULL_BACKUP): - swb.back_up(wp_path, backup_compressor, backup_path, full_backup) +def run_back_up(): + swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) -def run_restore(wp_path=WP_PATH, backup_path=BACKUP_PATH, - backup_decompressor=BACKUP_DECOMPRESSOR): - swb.restore(wp_path, backup_path, backup_decompressor) +def run_restore(): + swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) @nose.with_setup(set_up, tear_down) From aa939493d368c27ffa7a2478543ab297e6153322 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 16:47:08 -0800 Subject: [PATCH 07/39] Add unit tests for all full backup functions --- swb/remote.py | 2 +- tests/files/{ => mysite}/wp-config.php | 0 tests/files/mysite/wp-content/foo.txt | 1 + tests/fixtures/remote.py | 8 +- tests/test_remote.py | 119 +--------------------- tests/test_remote_db.py | 123 ++++++++++++++++++++++ tests/test_remote_full.py | 135 +++++++++++++++++++++++++ 7 files changed, 268 insertions(+), 120 deletions(-) rename tests/files/{ => mysite}/wp-config.php (100%) create mode 100644 tests/files/mysite/wp-content/foo.txt create mode 100644 tests/test_remote_db.py create mode 100644 tests/test_remote_full.py diff --git a/swb/remote.py b/swb/remote.py index 3b9b905..d39dd8a 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -158,7 +158,7 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): create_full_backup( wordpress_path, db_contents, backup_path, backup_compressor) - # verify_backup_integrity(backup_path) + verify_backup_integrity(backup_path) else: diff --git a/tests/files/wp-config.php b/tests/files/mysite/wp-config.php similarity index 100% rename from tests/files/wp-config.php rename to tests/files/mysite/wp-config.php diff --git a/tests/files/mysite/wp-content/foo.txt b/tests/files/mysite/wp-content/foo.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/tests/files/mysite/wp-content/foo.txt @@ -0,0 +1 @@ +hello world diff --git a/tests/fixtures/remote.py b/tests/fixtures/remote.py index 8f77b80..02f99fa 100644 --- a/tests/fixtures/remote.py +++ b/tests/fixtures/remote.py @@ -4,8 +4,8 @@ from mock import Mock, mock_open, patch -WP_CONFIG_PATH = 'tests/files/wp-config.php' -with open(WP_CONFIG_PATH) as wp_config: +WP_CONFIG_PATH = 'tests/files/mysite/wp-config.php' +with open(WP_CONFIG_PATH, 'r') as wp_config: WP_CONFIG_CONTENTS = wp_config.read() @@ -13,8 +13,10 @@ patch_remove = patch('os.remove') patch_rmdir = patch('os.rmdir') patch_getsize = patch('os.path.getsize', return_value=54321) +mock_communicate = Mock(return_value=[b'db contents', b'no errors']) patch_popen = patch('subprocess.Popen', - return_value=Mock(returncode=0)) + return_value=Mock( + returncode=0, communicate=mock_communicate)) patch_open = None diff --git a/tests/test_remote.py b/tests/test_remote.py index d212bed..4ec1d29 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,29 +1,11 @@ #!/usr/bin/env python3 -import os -import subprocess import nose.tools as nose import swb.remote as swb -from mock import ANY, patch +from mock import patch from tests.fixtures.remote import set_up, tear_down -WP_PATH = '~/mysite' -BACKUP_COMPRESSOR = 'bzip2 -v' -BACKUP_DECOMPRESSOR = 'bzip2 -d' -BACKUP_PATH = '~/backups/mysite.sql.bz2' -FULL_BACKUP = False -DB_PATH = BACKUP_PATH.replace('.bz2', '') - - -def run_back_up(): - swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) - - -def run_restore(): - swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) - - @nose.with_setup(set_up, tear_down) @patch('swb.remote.back_up') @patch('sys.argv', [swb.__file__, 'back-up', 'a', 'b', 'c']) @@ -51,103 +33,8 @@ def test_route_purge_backup(purge): purge.assert_called_once_with('a', 'b', 'c') -@nose.with_setup(set_up, tear_down) -def test_create_dir_structure(): - """should create intermediate directories""" - run_back_up() - swb.os.makedirs.assert_called_with(os.path.expanduser('~/backups')) - - -@nose.with_setup(set_up, tear_down) -@patch('os.makedirs', side_effect=OSError) -def test_create_dir_structure_silent_fail(makedirs): - """should fail silently if intermediate directories already exist""" - run_back_up() - makedirs.assert_called_with(os.path.expanduser('~/backups')) - - -@nose.with_setup(set_up, tear_down) -def test_dump_db(): - """should dump database to stdout""" - run_back_up() - swb.subprocess.Popen.assert_any_call([ - 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) - - -@nose.with_setup(set_up, tear_down) -def test_compress_db(): - """should compress dumped database""" - run_back_up() - swb.subprocess.Popen.assert_any_call(['bzip2', '-v'], - stdin=ANY, stdout=ANY) - - -@nose.with_setup(set_up, tear_down) -@patch('os.path.getsize', return_value=20) -def test_corrupted_backup(getsize): - """should raise OSError if backup is corrupted""" - with nose.assert_raises(OSError): - run_back_up() - - @nose.with_setup(set_up, tear_down) def test_purge_downloaded_backup(): """should purge remote backup after download""" - swb.purge_downloaded_backup(BACKUP_PATH) - swb.os.remove.assert_called_once_with(BACKUP_PATH) - - -@nose.with_setup(set_up, tear_down) -def test_restore_verify(): - """should verify backup on restore""" - run_restore() - swb.os.path.getsize.assert_called_once_with( - os.path.expanduser(BACKUP_PATH)) - - -@nose.with_setup(set_up, tear_down) -def test_decompress_backup(): - """should decompress backup on restore""" - run_restore() - swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( - BACKUP_PATH)]) - - -@nose.with_setup(set_up, tear_down) -def test_replace_db(): - """should replace database with decompressed revision""" - run_restore() - swb.subprocess.Popen.assert_any_call([ - 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword'], stdin=swb.open()) - - -@nose.with_setup(set_up, tear_down) -def test_purge_restored_backup(): - """should purge remote backup/database after restore""" - run_restore() - swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) - swb.os.remove.assert_any_call(os.path.expanduser(DB_PATH)) - - -@nose.with_setup(set_up, tear_down) -@patch('os.remove', side_effect=OSError) -def test_purge_restored_backup_silent_fail(remove): - """should fail silently if remote files do not exist after restore""" - run_restore() - remove.assert_called_once_with(os.path.expanduser(DB_PATH)) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_back_up(): - """should wait for each process to finish when backing up""" - run_back_up() - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_restore(): - """should wait for each process to finish when restoring""" - run_restore() - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) + swb.purge_downloaded_backup('abc') + swb.os.remove.assert_called_once_with('abc') diff --git a/tests/test_remote_db.py b/tests/test_remote_db.py new file mode 100644 index 0000000..40687b0 --- /dev/null +++ b/tests/test_remote_db.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import os +import os.path +import subprocess +import nose.tools as nose +import swb.remote as swb +from mock import ANY, patch +from tests.fixtures.remote import set_up, tear_down + + +WP_PATH = 'tests/files/mysite' +BACKUP_COMPRESSOR = 'bzip2 -v' +BACKUP_DECOMPRESSOR = 'bzip2 -d' +BACKUP_PATH = '~/backups/mysite.sql.bz2' +FULL_BACKUP = 'False' + + +def run_back_up(): + swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) + + +def run_restore(): + swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) + + +@nose.with_setup(set_up, tear_down) +def test_create_dir_structure(): + """should create intermediate directories""" + run_back_up() + swb.os.makedirs.assert_called_with( + os.path.expanduser(os.path.dirname(BACKUP_PATH))) + + +@nose.with_setup(set_up, tear_down) +@patch('os.makedirs', side_effect=OSError) +def test_create_dir_structure_silent_fail(makedirs): + """should fail silently if intermediate directories already exist""" + run_back_up() + makedirs.assert_called_with( + os.path.expanduser(os.path.dirname(BACKUP_PATH))) + + +@nose.with_setup(set_up, tear_down) +def test_dump_db(): + """should dump database to stdout""" + run_back_up() + swb.subprocess.Popen.assert_any_call([ + 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', + '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) + + +@nose.with_setup(set_up, tear_down) +def test_compress_db(): + """should compress dumped database""" + run_back_up() + swb.subprocess.Popen.assert_any_call(['bzip2', '-v'], + stdin=ANY, stdout=ANY) + + +@nose.with_setup(set_up, tear_down) +@patch('os.path.getsize', return_value=20) +def test_corrupted_backup(getsize): + """should raise OSError if backup is corrupted""" + with nose.assert_raises(OSError): + run_back_up() + + +@nose.with_setup(set_up, tear_down) +def test_restore_verify(): + """should verify backup on restore""" + run_restore() + swb.os.path.getsize.assert_called_once_with( + os.path.expanduser(BACKUP_PATH)) + + +@nose.with_setup(set_up, tear_down) +def test_decompress_backup(): + """should decompress backup on restore""" + run_restore() + swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( + BACKUP_PATH)]) + + +@nose.with_setup(set_up, tear_down) +def test_replace_db(): + """should replace database with decompressed revision""" + run_restore() + swb.subprocess.Popen.assert_any_call([ + 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', + '-pmypassword'], stdin=swb.open()) + + +@nose.with_setup(set_up, tear_down) +def test_purge_restored_backup(): + """should purge remote backup/database after restore""" + run_restore() + swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) + swb.os.remove.assert_any_call(os.path.expanduser( + os.path.splitext(BACKUP_PATH)[0])) + + +@nose.with_setup(set_up, tear_down) +@patch('os.remove', side_effect=OSError) +def test_purge_restored_backup_silent_fail(remove): + """should fail silently if remote files do not exist after restore""" + run_restore() + remove.assert_called_once_with(os.path.expanduser( + os.path.splitext(BACKUP_PATH)[0])) + + +@nose.with_setup(set_up, tear_down) +def test_process_wait_back_up(): + """should wait for each process to finish when backing up""" + run_back_up() + nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) + + +@nose.with_setup(set_up, tear_down) +def test_process_wait_restore(): + """should wait for each process to finish when restoring""" + run_restore() + nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) diff --git a/tests/test_remote_full.py b/tests/test_remote_full.py new file mode 100644 index 0000000..4ea7d9c --- /dev/null +++ b/tests/test_remote_full.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import os +import os.path +import subprocess +import tempfile +import nose.tools as nose +import swb.remote as swb +from mock import patch +from tests.fixtures.remote import set_up, tear_down + + +temp_dir = tempfile.gettempdir() +WP_PATH = 'tests/files/mysite' +BACKUP_COMPRESSOR = 'bzip2 -v' +BACKUP_DECOMPRESSOR = 'bzip2 -d' +BACKUP_PATH = os.path.join(temp_dir, 'mysite.tar.bz2') +FULL_BACKUP = 'True' + + +def run_back_up(): + swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) + + +def run_restore(): + swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) + + +@nose.with_setup(set_up, tear_down) +def test_create_dir_structure(): + """should create intermediate directories""" + run_back_up() + swb.os.makedirs.assert_called_with( + os.path.expanduser(os.path.dirname(BACKUP_PATH))) + + +@nose.with_setup(set_up, tear_down) +@patch('os.makedirs', side_effect=OSError) +def test_create_dir_structure_silent_fail(makedirs): + """should fail silently if intermediate directories already exist""" + run_back_up() + makedirs.assert_called_with( + os.path.expanduser(os.path.dirname(BACKUP_PATH))) + + +@nose.with_setup(set_up, tear_down) +def test_dump_db(): + """should dump database to stdout""" + run_back_up() + swb.subprocess.Popen.assert_any_call([ + 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', + '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) + + +@nose.with_setup(set_up, tear_down) +def test_compress_db(): + """should compress dumped database""" + run_back_up() + print(swb.subprocess.Popen.call_args_list) + swb.subprocess.Popen.assert_any_call([ + 'bzip2', '-v', + os.path.expanduser(BACKUP_PATH), + os.path.splitext(os.path.expanduser(BACKUP_PATH))[0]]) + + +@nose.with_setup(set_up, tear_down) +@patch('os.path.getsize', return_value=20) +def test_corrupted_backup(getsize): + """should raise OSError if backup is corrupted""" + with nose.assert_raises(OSError): + run_back_up() + + +@nose.with_setup(set_up, tear_down) +def test_purge_downloaded_backup(): + """should purge remote backup after download""" + swb.purge_downloaded_backup(BACKUP_PATH) + swb.os.remove.assert_called_once_with(BACKUP_PATH) + + +@nose.with_setup(set_up, tear_down) +def test_restore_verify(): + """should verify backup on restore""" + run_restore() + swb.os.path.getsize.assert_called_once_with( + os.path.expanduser(BACKUP_PATH)) + + +@nose.with_setup(set_up, tear_down) +def test_decompress_backup(): + """should decompress backup on restore""" + run_restore() + swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( + BACKUP_PATH)]) + + +@nose.with_setup(set_up, tear_down) +def test_replace_db(): + """should replace database with decompressed revision""" + run_restore() + swb.subprocess.Popen.assert_any_call([ + 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', + '-pmypassword'], stdin=swb.open()) + + +@nose.with_setup(set_up, tear_down) +def test_purge_restored_backup(): + """should purge remote backup/database after restore""" + run_restore() + swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) + swb.os.remove.assert_any_call(os.path.expanduser( + os.path.splitext(BACKUP_PATH)[0])) + + +@nose.with_setup(set_up, tear_down) +@patch('os.remove', side_effect=OSError) +def test_purge_restored_backup_silent_fail(remove): + """should fail silently if remote files do not exist after restore""" + run_restore() + remove.assert_called_once_with(os.path.expanduser( + os.path.splitext(BACKUP_PATH)[0])) + + +@nose.with_setup(set_up, tear_down) +def test_process_wait_back_up(): + """should wait for each process to finish when backing up""" + run_back_up() + nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) + + +@nose.with_setup(set_up, tear_down) +def test_process_wait_restore(): + """should wait for each process to finish when restoring""" + run_restore() + nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) From 69b1c997b0db664f4126b29d540bde8d2c31cc08 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 19:14:07 -0800 Subject: [PATCH 08/39] Conolidate constants used for remote tests --- tests/fixtures/remote.py | 29 ++++++++++++++++++++++++++++ tests/test_remote_db.py | 36 +++++++++++++---------------------- tests/test_remote_full.py | 40 ++++++++++++++------------------------- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/tests/fixtures/remote.py b/tests/fixtures/remote.py index 02f99fa..c2e9432 100644 --- a/tests/fixtures/remote.py +++ b/tests/fixtures/remote.py @@ -1,9 +1,30 @@ #!/usr/bin/env python3 +import os +import os.path +import shutil import subprocess +import tempfile +import swb.remote as swb from mock import Mock, mock_open, patch +TEMP_DIR = os.path.join(tempfile.gettempdir(), 'swb-remote') +WP_PATH = 'tests/files/mysite' +BACKUP_COMPRESSOR = 'bzip2 -v' +BACKUP_DECOMPRESSOR = 'bzip2 -d' + + +def run_back_up(backup_path, full_backup, + wp_path=WP_PATH, backup_compressor=BACKUP_COMPRESSOR): + swb.back_up(wp_path, backup_compressor, backup_path, full_backup) + + +def run_restore(backup_path, full_backup, + wp_path=WP_PATH, backup_decompressor=BACKUP_DECOMPRESSOR): + swb.restore(wp_path, backup_path, backup_decompressor) + + WP_CONFIG_PATH = 'tests/files/mysite/wp-config.php' with open(WP_CONFIG_PATH, 'r') as wp_config: WP_CONFIG_CONTENTS = wp_config.read() @@ -22,6 +43,10 @@ def set_up(): global patch_open + try: + os.makedirs(TEMP_DIR) + except OSError: + pass patch_makedirs.start() patch_remove.start() patch_rmdir.start() @@ -41,3 +66,7 @@ def tear_down(): subprocess.Popen.reset_mock() patch_popen.stop() patch_open.stop() + try: + shutil.rmtree(TEMP_DIR) + except OSError: + pass diff --git a/tests/test_remote_db.py b/tests/test_remote_db.py index 40687b0..e715e80 100644 --- a/tests/test_remote_db.py +++ b/tests/test_remote_db.py @@ -6,28 +6,18 @@ import nose.tools as nose import swb.remote as swb from mock import ANY, patch +from tests.fixtures.remote import run_back_up, run_restore from tests.fixtures.remote import set_up, tear_down -WP_PATH = 'tests/files/mysite' -BACKUP_COMPRESSOR = 'bzip2 -v' -BACKUP_DECOMPRESSOR = 'bzip2 -d' BACKUP_PATH = '~/backups/mysite.sql.bz2' FULL_BACKUP = 'False' -def run_back_up(): - swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) - - -def run_restore(): - swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) - - @nose.with_setup(set_up, tear_down) def test_create_dir_structure(): """should create intermediate directories""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.makedirs.assert_called_with( os.path.expanduser(os.path.dirname(BACKUP_PATH))) @@ -36,7 +26,7 @@ def test_create_dir_structure(): @patch('os.makedirs', side_effect=OSError) def test_create_dir_structure_silent_fail(makedirs): """should fail silently if intermediate directories already exist""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) makedirs.assert_called_with( os.path.expanduser(os.path.dirname(BACKUP_PATH))) @@ -44,7 +34,7 @@ def test_create_dir_structure_silent_fail(makedirs): @nose.with_setup(set_up, tear_down) def test_dump_db(): """should dump database to stdout""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call([ 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) @@ -53,7 +43,7 @@ def test_dump_db(): @nose.with_setup(set_up, tear_down) def test_compress_db(): """should compress dumped database""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call(['bzip2', '-v'], stdin=ANY, stdout=ANY) @@ -63,13 +53,13 @@ def test_compress_db(): def test_corrupted_backup(getsize): """should raise OSError if backup is corrupted""" with nose.assert_raises(OSError): - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) @nose.with_setup(set_up, tear_down) def test_restore_verify(): """should verify backup on restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.path.getsize.assert_called_once_with( os.path.expanduser(BACKUP_PATH)) @@ -77,7 +67,7 @@ def test_restore_verify(): @nose.with_setup(set_up, tear_down) def test_decompress_backup(): """should decompress backup on restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( BACKUP_PATH)]) @@ -85,7 +75,7 @@ def test_decompress_backup(): @nose.with_setup(set_up, tear_down) def test_replace_db(): """should replace database with decompressed revision""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call([ 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword'], stdin=swb.open()) @@ -94,7 +84,7 @@ def test_replace_db(): @nose.with_setup(set_up, tear_down) def test_purge_restored_backup(): """should purge remote backup/database after restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) swb.os.remove.assert_any_call(os.path.expanduser( os.path.splitext(BACKUP_PATH)[0])) @@ -104,7 +94,7 @@ def test_purge_restored_backup(): @patch('os.remove', side_effect=OSError) def test_purge_restored_backup_silent_fail(remove): """should fail silently if remote files do not exist after restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) remove.assert_called_once_with(os.path.expanduser( os.path.splitext(BACKUP_PATH)[0])) @@ -112,12 +102,12 @@ def test_purge_restored_backup_silent_fail(remove): @nose.with_setup(set_up, tear_down) def test_process_wait_back_up(): """should wait for each process to finish when backing up""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) @nose.with_setup(set_up, tear_down) def test_process_wait_restore(): """should wait for each process to finish when restoring""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) diff --git a/tests/test_remote_full.py b/tests/test_remote_full.py index 4ea7d9c..ae9da52 100644 --- a/tests/test_remote_full.py +++ b/tests/test_remote_full.py @@ -3,33 +3,21 @@ import os import os.path import subprocess -import tempfile import nose.tools as nose import swb.remote as swb from mock import patch +from tests.fixtures.remote import TEMP_DIR, run_back_up, run_restore from tests.fixtures.remote import set_up, tear_down -temp_dir = tempfile.gettempdir() -WP_PATH = 'tests/files/mysite' -BACKUP_COMPRESSOR = 'bzip2 -v' -BACKUP_DECOMPRESSOR = 'bzip2 -d' -BACKUP_PATH = os.path.join(temp_dir, 'mysite.tar.bz2') +BACKUP_PATH = os.path.join(TEMP_DIR, 'mysite.tar.bz2') FULL_BACKUP = 'True' -def run_back_up(): - swb.back_up(WP_PATH, BACKUP_COMPRESSOR, BACKUP_PATH, FULL_BACKUP) - - -def run_restore(): - swb.restore(WP_PATH, BACKUP_PATH, BACKUP_DECOMPRESSOR) - - @nose.with_setup(set_up, tear_down) def test_create_dir_structure(): """should create intermediate directories""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.makedirs.assert_called_with( os.path.expanduser(os.path.dirname(BACKUP_PATH))) @@ -38,7 +26,7 @@ def test_create_dir_structure(): @patch('os.makedirs', side_effect=OSError) def test_create_dir_structure_silent_fail(makedirs): """should fail silently if intermediate directories already exist""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) makedirs.assert_called_with( os.path.expanduser(os.path.dirname(BACKUP_PATH))) @@ -46,7 +34,7 @@ def test_create_dir_structure_silent_fail(makedirs): @nose.with_setup(set_up, tear_down) def test_dump_db(): """should dump database to stdout""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call([ 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) @@ -55,7 +43,7 @@ def test_dump_db(): @nose.with_setup(set_up, tear_down) def test_compress_db(): """should compress dumped database""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) print(swb.subprocess.Popen.call_args_list) swb.subprocess.Popen.assert_any_call([ 'bzip2', '-v', @@ -68,7 +56,7 @@ def test_compress_db(): def test_corrupted_backup(getsize): """should raise OSError if backup is corrupted""" with nose.assert_raises(OSError): - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) @nose.with_setup(set_up, tear_down) @@ -81,7 +69,7 @@ def test_purge_downloaded_backup(): @nose.with_setup(set_up, tear_down) def test_restore_verify(): """should verify backup on restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.path.getsize.assert_called_once_with( os.path.expanduser(BACKUP_PATH)) @@ -89,7 +77,7 @@ def test_restore_verify(): @nose.with_setup(set_up, tear_down) def test_decompress_backup(): """should decompress backup on restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( BACKUP_PATH)]) @@ -97,7 +85,7 @@ def test_decompress_backup(): @nose.with_setup(set_up, tear_down) def test_replace_db(): """should replace database with decompressed revision""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.subprocess.Popen.assert_any_call([ 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword'], stdin=swb.open()) @@ -106,7 +94,7 @@ def test_replace_db(): @nose.with_setup(set_up, tear_down) def test_purge_restored_backup(): """should purge remote backup/database after restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) swb.os.remove.assert_any_call(os.path.expanduser( os.path.splitext(BACKUP_PATH)[0])) @@ -116,7 +104,7 @@ def test_purge_restored_backup(): @patch('os.remove', side_effect=OSError) def test_purge_restored_backup_silent_fail(remove): """should fail silently if remote files do not exist after restore""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) remove.assert_called_once_with(os.path.expanduser( os.path.splitext(BACKUP_PATH)[0])) @@ -124,12 +112,12 @@ def test_purge_restored_backup_silent_fail(remove): @nose.with_setup(set_up, tear_down) def test_process_wait_back_up(): """should wait for each process to finish when backing up""" - run_back_up() + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) @nose.with_setup(set_up, tear_down) def test_process_wait_restore(): """should wait for each process to finish when restoring""" - run_restore() + run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) From 7d04854180f5b01d4fa7926f43587e35c6463fb9 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 19:43:21 -0800 Subject: [PATCH 09/39] Add tests for tar --- swb/remote.py | 6 +++--- tests/test_remote_full.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index d39dd8a..357fc2a 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -147,18 +147,17 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): backup_path = os.path.expanduser(backup_path) create_dir_structure(backup_path) + db_info = get_db_info(wordpress_path) if full_backup == 'True': # backup_path is assumed to refer to entire site directory backup - db_info = get_db_info(wordpress_path) db_contents = dump_uncompressed_db( db_info['name'], db_info['host'], db_info['user'], db_info['password']) create_full_backup( wordpress_path, db_contents, backup_path, backup_compressor) - verify_backup_integrity(backup_path) else: @@ -168,7 +167,8 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): db_info['name'], db_info['host'], db_info['user'], db_info['password'], backup_compressor, backup_path) - verify_backup_integrity(backup_path) + + verify_backup_integrity(backup_path) # Decompress the given backup file to a database file in the same directory diff --git a/tests/test_remote_full.py b/tests/test_remote_full.py index ae9da52..e1637b0 100644 --- a/tests/test_remote_full.py +++ b/tests/test_remote_full.py @@ -5,8 +5,8 @@ import subprocess import nose.tools as nose import swb.remote as swb -from mock import patch -from tests.fixtures.remote import TEMP_DIR, run_back_up, run_restore +from mock import ANY, patch +from tests.fixtures.remote import TEMP_DIR, WP_PATH, run_back_up, run_restore from tests.fixtures.remote import set_up, tear_down @@ -121,3 +121,15 @@ def test_process_wait_restore(): """should wait for each process to finish when restoring""" run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) + + +@nose.with_setup(set_up, tear_down) +@patch('tarfile.open') +def test_tar(tarfile_open): + """should wait for each process to finish when restoring""" + run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) + tar_path = os.path.splitext(BACKUP_PATH)[0] + tarfile_open.assert_called_once_with(tar_path, 'w') + tarfile_open.return_value.add.assert_called_once_with( + WP_PATH, arcname='mysite') + tarfile_open.return_value.addfile.assert_called_once_with(ANY, ANY) From 4faac3fcc8508e42789025c4af2e6663b75227b7 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Wed, 6 Jan 2016 21:01:01 -0800 Subject: [PATCH 10/39] Begin rewriting test suite, starting with local tests --- tests/test_compliance.py | 62 +++++----- tests/test_local.py | 243 ++++++-------------------------------- tests/test_remote.py | 34 ------ tests/test_remote_db.py | 113 ------------------ tests/test_remote_full.py | 135 --------------------- 5 files changed, 66 insertions(+), 521 deletions(-) delete mode 100644 tests/test_remote_db.py delete mode 100644 tests/test_remote_full.py diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 37dc4dd..2cfc819 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -1,31 +1,31 @@ -#!/usr/bin/env python - -import glob -import itertools -import nose.tools as nose -import pep8 -import radon.complexity as radon - - -def test_pep8(): - file_paths = glob.iglob('*/*.py') - for file_path in file_paths: - style_guide = pep8.StyleGuide(quiet=True) - total_errors = style_guide.input_file(file_path) - test_pep8.__doc__ = '{} should comply with PEP 8'.format(file_path) - fail_msg = '{} does not comply with PEP 8'.format(file_path) - yield nose.assert_equal, total_errors, 0, fail_msg - - -def test_complexity(): - file_paths = itertools.chain(glob.iglob('*/*.py'), glob.iglob('*/*/*.py')) - for file_path in file_paths: - with open(file_path, 'r') as file: - blocks = radon.cc_visit(file.read()) - for block in blocks: - test_doc = '{} ({}) should have a low cyclomatic complexity score' - test_complexity.__doc__ = test_doc.format( - block.name, file_path) - fail_msg = '{} ({}) has a cyclomatic complexity of {}'.format( - block.name, file_path, block.complexity) - yield nose.assert_less_equal, block.complexity, 10, fail_msg +# #!/usr/bin/env python +# +# import glob +# import itertools +# import nose.tools as nose +# import pep8 +# import radon.complexity as radon +# +# +# def test_pep8(): +# file_paths = glob.iglob('*/*.py') +# for file_path in file_paths: +# style_guide = pep8.StyleGuide(quiet=True) +# total_errors = style_guide.input_file(file_path) +# test_pep8.__doc__ = '{} should comply with PEP 8'.format(file_path) +# fail_msg = '{} does not comply with PEP 8'.format(file_path) +# yield nose.assert_equal, total_errors, 0, fail_msg +# +# +# def test_complexity(): +# file_paths = itertools.chain(glob.iglob('*/*.py'), glob.iglob('*/*/*.py')) +# for file_path in file_paths: +# with open(file_path, 'r') as file: +# blocks = radon.cc_visit(file.read()) +# for block in blocks: +# test_doc = '{} ({}) should have a low cyclomatic complexity score' +# test_complexity.__doc__ = test_doc.format( +# block.name, file_path) +# fail_msg = '{} ({}) has a cyclomatic complexity of {}'.format( +# block.name, file_path, block.complexity) +# yield nose.assert_less_equal, block.complexity, 10, fail_msg diff --git a/tests/test_local.py b/tests/test_local.py index 1c1a39b..05c1259 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,231 +1,58 @@ #!/usr/bin/env python3 import configparser -import os +# import os import nose.tools as nose import swb.local as swb -from mock import ANY, mock_open, patch -from tests.fixtures.local import set_up, tear_down, mock_backups +from mock import patch +# from tests.fixtures.local import set_up, tear_down, mock_backups -CONFIG_PATH = 'tests/files/config.ini' -BACKUP_PATH = '~/Backups/mysite.sql.bz2' -BACKUP_PATH_TIMESTAMPED = '~/Backups/%Y/%m/%d/mysite.sql.bz2' +def test_parse_config(): + """should correctly parse the supplied configuration file""" + config_path = 'tests/files/config.ini' + config = swb.parse_config(config_path) + expected_config = configparser.RawConfigParser() + expected_config.read(config_path) + nose.assert_equal(config, expected_config) -def get_config(): - config = configparser.RawConfigParser() - config.read(CONFIG_PATH) - return config +@patch('os.makedirs') +def test_create_dir_structure(makedirs): + """should create the directory structure for the supplied path""" + swb.create_dir_structure('a/b c/d') + makedirs.assert_called_once_with('a/b c') -def test_config_parser(): - """should parse configuration file correctly""" - config = swb.parse_config(CONFIG_PATH) - nose.assert_is_instance(config, configparser.RawConfigParser) - - -@nose.with_setup(set_up, tear_down) -def test_create_remote_backup(): - """should create remote backup via SSH""" - config = get_config() - swb.back_up(config) - swb.subprocess.Popen.assert_any_call([ - 'ssh', '-p 2222', 'myname@mysite.com', 'python3', '-', 'back-up', - '~/\'public_html/mysite\'', 'bzip2', '~/\'backups/mysite.sql.bz2\'', - 'False'], stdin=ANY, stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -def test_download_remote_backup(): - """should download remote backup via SCP""" - config = get_config() - swb.back_up(config) - swb.subprocess.Popen.assert_any_call([ - 'scp', '-P 2222', 'myname@mysite.com:~/\'backups/mysite.sql.bz2\'', - os.path.expanduser(BACKUP_PATH)], - stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -def test_create_dir_structure(): - """should create intermediate directories""" - config = get_config() - swb.back_up(config) - swb.os.makedirs.assert_called_once_with( - os.path.expanduser(os.path.dirname(BACKUP_PATH))) - - -@nose.with_setup(set_up, tear_down) @patch('os.makedirs', side_effect=OSError) def test_create_dir_structure_silent_fail(makedirs): - """should fail silently if intermediate directories already exist""" - config = get_config() - swb.back_up(config) - makedirs.assert_called_once_with(os.path.expanduser( - os.path.dirname(BACKUP_PATH))) + """should fail silently if directory structure already exists""" + swb.create_dir_structure('a/b c/d') + makedirs.assert_called_once_with('a/b c') -@nose.with_setup(set_up, tear_down) -def test_purge_remote_backup(): - """should purge remote backup after download""" - config = get_config() - swb.back_up(config) - swb.subprocess.Popen.assert_any_call([ - 'ssh', '-p 2222', 'myname@mysite.com', 'python3', '-', 'purge-backup', - '~/\'backups/mysite.sql.bz2\''], - stdin=ANY, stdout=None, stderr=None) +def test_unquote_home_dir_path_with_tilde(): + """should unquote the tilde (~) in the supplied quoted path""" + unquoted_path = swb.unquote_home_dir('\'~/a/b c/d\'') + nose.assert_equal(unquoted_path, '~/\'a/b c/d\'') -@nose.with_setup(set_up, tear_down) -def test_purge_oldest_backups(): - """should purge oldest local backups after download""" - config = get_config() - config.set('paths', 'local_backup', BACKUP_PATH_TIMESTAMPED) - swb.back_up(config) - nose.assert_equal(swb.os.remove.call_count, 2) - for path in mock_backups[:-3]: - swb.os.remove.assert_any_call(path) +def test_unquote_home_dir_path_without_tilde(): + """should return the supplied quoted path""" + unquoted_path = swb.unquote_home_dir('\'/a/b c/d\'') + nose.assert_equal(unquoted_path, '\'/a/b c/d\'') -@nose.with_setup(set_up, tear_down) -def test_null_max_local_backups(): - """should keep all backups if max_local_backups is not set""" - config = get_config() - config.set('paths', 'local_backup', BACKUP_PATH_TIMESTAMPED) - config.remove_option('backup', 'max_local_backups') - swb.back_up(config) - nose.assert_equal(swb.os.remove.call_count, 0) +def test_quote_arg(): + """should correctly quote arguments passed to the shell""" + quoted_arg = swb.quote_arg('a/b c/d') + nose.assert_equal(quoted_arg, '\'a/b c/d\'') -@nose.with_setup(set_up, tear_down) -def test_purge_empty_dirs(): - """should purge empty timestamped directories""" - config = get_config() - config.set('paths', 'local_backup', BACKUP_PATH_TIMESTAMPED) - swb.back_up(config) - nose.assert_equal(swb.os.remove.call_count, 2) - for path in mock_backups[:-3]: - swb.os.rmdir.assert_any_call(path) - - -@nose.with_setup(set_up, tear_down) -@patch('os.rmdir', side_effect=OSError) -def test_keep_nonempty_dirs(rmdir): - """should not purge nonempty timestamped directories""" - config = get_config() - config.set('paths', 'local_backup', BACKUP_PATH_TIMESTAMPED) - swb.back_up(config) - nose.assert_equal(swb.os.remove.call_count, 2) - for path in mock_backups[:-3]: - rmdir.assert_any_call(path) - - -@nose.with_setup(set_up, tear_down) -@patch('swb.local.back_up') -@patch('sys.argv', [swb.__file__, CONFIG_PATH]) -def test_main_back_up(back_up): - """should call back_up() when config path is passed to main()""" - config = get_config() - swb.main() - back_up.assert_called_once_with(config, stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) @patch('swb.local.shlex') -def test_missing_shlex_quote(shlex): - """should use pipes.quote() if shlex.quote() is missing (<3.3)""" +def test_quote_arg_py32(shlex): + """should correctly quote arguments passed to the shell on Python 3.2""" del shlex.quote - config = get_config() - swb.back_up(config) - nose.assert_equal(shlex.call_count, 0) - swb.subprocess.Popen.assert_any_call( - # Only check if path is quoted (ignore preceding arguments) - ([ANY] * 6) + ['~/\'backups/mysite.sql.bz2\''], - stdin=ANY, stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -@patch('sys.exit') -def test_ssh_error(exit): - """should exit if SSH process returns non-zero exit code""" - config = get_config() - swb.subprocess.Popen.return_value.returncode = 3 - swb.back_up(config) - exit.assert_called_with(3) - - -@nose.with_setup(set_up, tear_down) -@patch('sys.argv', [swb.__file__, '-q', CONFIG_PATH]) -def test_quiet_mode(): - """should silence SSH output in quiet mode""" - file_obj = mock_open() - devnull = file_obj() - with patch('swb.local.open', file_obj, create=True): - swb.main() - file_obj.assert_any_call(os.devnull, 'w') - swb.subprocess.Popen.assert_any_call(ANY, stdout=devnull, - stderr=devnull) - - -@nose.with_setup(set_up, tear_down) -@patch('sys.argv', [swb.__file__, CONFIG_PATH, '-r', BACKUP_PATH]) -@patch('swb.local.restore') -def test_main_restore(restore): - """should call restore() when config path is passed to main()""" - config = get_config() - swb.main() - nose.assert_equal(swb.input.call_count, 1) - swb.input.assert_called_once_with(ANY) - restore.assert_called_once_with(config, BACKUP_PATH, - stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -@patch('swb.local.restore') -@patch('sys.argv', [swb.__file__, '-f', CONFIG_PATH, '-r', BACKUP_PATH]) -def test_force_mode(restore): - """should bypass restore confirmation in force mode""" - config = get_config() - swb.main() - nose.assert_equal(swb.input.call_count, 0) - restore.assert_called_once_with(config, BACKUP_PATH, - stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -@patch('sys.argv', [swb.__file__, CONFIG_PATH, '-r', BACKUP_PATH]) -def test_restore_confirm_cancel(): - """should exit script when user cancels restore confirmation""" - responses = ['n', 'N', ' n ', ''] - for response in responses: - swb.input.return_value = response - with nose.assert_raises(Exception): - swb.main() - - -@nose.with_setup(set_up, tear_down) -def test_upload_local_backup(): - """should upload local backup to remote for restoration""" - config = get_config() - swb.restore(config, BACKUP_PATH, stdout=None, stderr=None) - swb.subprocess.Popen.assert_any_call([ - 'scp', '-P 2222', BACKUP_PATH, - 'myname@mysite.com:~/\'backups/mysite.sql.bz2\''], - stdout=None, stderr=None) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_back_up(): - """should wait for each process to finish when backing up""" - config = get_config() - swb.back_up(config) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 3) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_restore(): - """should wait for each process to finish when restoring""" - config = get_config() - swb.restore(config, BACKUP_PATH, stdout=None, stderr=None) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) + nose.assert_false(hasattr(shlex, 'quote')) + quoted_arg = swb.quote_arg('a/b c/d') + nose.assert_equal(quoted_arg, '\'a/b c/d\'') diff --git a/tests/test_remote.py b/tests/test_remote.py index 4ec1d29..a00d080 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,37 +4,3 @@ import swb.remote as swb from mock import patch from tests.fixtures.remote import set_up, tear_down - - -@nose.with_setup(set_up, tear_down) -@patch('swb.remote.back_up') -@patch('sys.argv', [swb.__file__, 'back-up', 'a', 'b', 'c']) -def test_route_back_up(back_up): - """should call back_up() with args if respective action is passed""" - swb.main() - back_up.assert_called_once_with('a', 'b', 'c') - - -@nose.with_setup(set_up, tear_down) -@patch('swb.remote.restore') -@patch('sys.argv', [swb.__file__, 'restore', 'a', 'b', 'c']) -def test_route_restore(restore): - """should call restore() with args if respective action is passed""" - swb.main() - restore.assert_called_once_with('a', 'b', 'c') - - -@nose.with_setup(set_up, tear_down) -@patch('swb.remote.purge_downloaded_backup') -@patch('sys.argv', [swb.__file__, 'purge-backup', 'a', 'b', 'c']) -def test_route_purge_backup(purge): - """should call purge_backup() with args if respective action is passed""" - swb.main() - purge.assert_called_once_with('a', 'b', 'c') - - -@nose.with_setup(set_up, tear_down) -def test_purge_downloaded_backup(): - """should purge remote backup after download""" - swb.purge_downloaded_backup('abc') - swb.os.remove.assert_called_once_with('abc') diff --git a/tests/test_remote_db.py b/tests/test_remote_db.py deleted file mode 100644 index e715e80..0000000 --- a/tests/test_remote_db.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 - -import os -import os.path -import subprocess -import nose.tools as nose -import swb.remote as swb -from mock import ANY, patch -from tests.fixtures.remote import run_back_up, run_restore -from tests.fixtures.remote import set_up, tear_down - - -BACKUP_PATH = '~/backups/mysite.sql.bz2' -FULL_BACKUP = 'False' - - -@nose.with_setup(set_up, tear_down) -def test_create_dir_structure(): - """should create intermediate directories""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.makedirs.assert_called_with( - os.path.expanduser(os.path.dirname(BACKUP_PATH))) - - -@nose.with_setup(set_up, tear_down) -@patch('os.makedirs', side_effect=OSError) -def test_create_dir_structure_silent_fail(makedirs): - """should fail silently if intermediate directories already exist""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - makedirs.assert_called_with( - os.path.expanduser(os.path.dirname(BACKUP_PATH))) - - -@nose.with_setup(set_up, tear_down) -def test_dump_db(): - """should dump database to stdout""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call([ - 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) - - -@nose.with_setup(set_up, tear_down) -def test_compress_db(): - """should compress dumped database""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call(['bzip2', '-v'], - stdin=ANY, stdout=ANY) - - -@nose.with_setup(set_up, tear_down) -@patch('os.path.getsize', return_value=20) -def test_corrupted_backup(getsize): - """should raise OSError if backup is corrupted""" - with nose.assert_raises(OSError): - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - - -@nose.with_setup(set_up, tear_down) -def test_restore_verify(): - """should verify backup on restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.path.getsize.assert_called_once_with( - os.path.expanduser(BACKUP_PATH)) - - -@nose.with_setup(set_up, tear_down) -def test_decompress_backup(): - """should decompress backup on restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( - BACKUP_PATH)]) - - -@nose.with_setup(set_up, tear_down) -def test_replace_db(): - """should replace database with decompressed revision""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call([ - 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword'], stdin=swb.open()) - - -@nose.with_setup(set_up, tear_down) -def test_purge_restored_backup(): - """should purge remote backup/database after restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) - swb.os.remove.assert_any_call(os.path.expanduser( - os.path.splitext(BACKUP_PATH)[0])) - - -@nose.with_setup(set_up, tear_down) -@patch('os.remove', side_effect=OSError) -def test_purge_restored_backup_silent_fail(remove): - """should fail silently if remote files do not exist after restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - remove.assert_called_once_with(os.path.expanduser( - os.path.splitext(BACKUP_PATH)[0])) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_back_up(): - """should wait for each process to finish when backing up""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_restore(): - """should wait for each process to finish when restoring""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) diff --git a/tests/test_remote_full.py b/tests/test_remote_full.py deleted file mode 100644 index e1637b0..0000000 --- a/tests/test_remote_full.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 - -import os -import os.path -import subprocess -import nose.tools as nose -import swb.remote as swb -from mock import ANY, patch -from tests.fixtures.remote import TEMP_DIR, WP_PATH, run_back_up, run_restore -from tests.fixtures.remote import set_up, tear_down - - -BACKUP_PATH = os.path.join(TEMP_DIR, 'mysite.tar.bz2') -FULL_BACKUP = 'True' - - -@nose.with_setup(set_up, tear_down) -def test_create_dir_structure(): - """should create intermediate directories""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.makedirs.assert_called_with( - os.path.expanduser(os.path.dirname(BACKUP_PATH))) - - -@nose.with_setup(set_up, tear_down) -@patch('os.makedirs', side_effect=OSError) -def test_create_dir_structure_silent_fail(makedirs): - """should fail silently if intermediate directories already exist""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - makedirs.assert_called_with( - os.path.expanduser(os.path.dirname(BACKUP_PATH))) - - -@nose.with_setup(set_up, tear_down) -def test_dump_db(): - """should dump database to stdout""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call([ - 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) - - -@nose.with_setup(set_up, tear_down) -def test_compress_db(): - """should compress dumped database""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - print(swb.subprocess.Popen.call_args_list) - swb.subprocess.Popen.assert_any_call([ - 'bzip2', '-v', - os.path.expanduser(BACKUP_PATH), - os.path.splitext(os.path.expanduser(BACKUP_PATH))[0]]) - - -@nose.with_setup(set_up, tear_down) -@patch('os.path.getsize', return_value=20) -def test_corrupted_backup(getsize): - """should raise OSError if backup is corrupted""" - with nose.assert_raises(OSError): - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - - -@nose.with_setup(set_up, tear_down) -def test_purge_downloaded_backup(): - """should purge remote backup after download""" - swb.purge_downloaded_backup(BACKUP_PATH) - swb.os.remove.assert_called_once_with(BACKUP_PATH) - - -@nose.with_setup(set_up, tear_down) -def test_restore_verify(): - """should verify backup on restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.path.getsize.assert_called_once_with( - os.path.expanduser(BACKUP_PATH)) - - -@nose.with_setup(set_up, tear_down) -def test_decompress_backup(): - """should decompress backup on restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call(['bzip2', '-d', os.path.expanduser( - BACKUP_PATH)]) - - -@nose.with_setup(set_up, tear_down) -def test_replace_db(): - """should replace database with decompressed revision""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.subprocess.Popen.assert_any_call([ - 'mysql', 'mydb', '-h', 'myhost', '-u', 'myname', - '-pmypassword'], stdin=swb.open()) - - -@nose.with_setup(set_up, tear_down) -def test_purge_restored_backup(): - """should purge remote backup/database after restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - swb.os.remove.assert_any_call(os.path.expanduser(BACKUP_PATH)) - swb.os.remove.assert_any_call(os.path.expanduser( - os.path.splitext(BACKUP_PATH)[0])) - - -@nose.with_setup(set_up, tear_down) -@patch('os.remove', side_effect=OSError) -def test_purge_restored_backup_silent_fail(remove): - """should fail silently if remote files do not exist after restore""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - remove.assert_called_once_with(os.path.expanduser( - os.path.splitext(BACKUP_PATH)[0])) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_back_up(): - """should wait for each process to finish when backing up""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) - - -@nose.with_setup(set_up, tear_down) -def test_process_wait_restore(): - """should wait for each process to finish when restoring""" - run_restore(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - nose.assert_equal(swb.subprocess.Popen.return_value.wait.call_count, 2) - - -@nose.with_setup(set_up, tear_down) -@patch('tarfile.open') -def test_tar(tarfile_open): - """should wait for each process to finish when restoring""" - run_back_up(backup_path=BACKUP_PATH, full_backup=FULL_BACKUP) - tar_path = os.path.splitext(BACKUP_PATH)[0] - tarfile_open.assert_called_once_with(tar_path, 'w') - tarfile_open.return_value.add.assert_called_once_with( - WP_PATH, arcname='mysite') - tarfile_open.return_value.addfile.assert_called_once_with(ANY, ANY) From 115574a79a2867bc91841abe2ca306160dca7a31 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 11:04:40 -0800 Subject: [PATCH 11/39] Add exec_on_remote tests --- tests/test_local.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 05c1259..3b22b8a 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 import configparser -# import os import nose.tools as nose import swb.local as swb -from mock import patch +from mock import NonCallableMock, patch # from tests.fixtures.local import set_up, tear_down, mock_backups @@ -56,3 +55,29 @@ def test_quote_arg_py32(shlex): nose.assert_false(hasattr(shlex, 'quote')) quoted_arg = swb.quote_arg('a/b c/d') nose.assert_equal(quoted_arg, '\'a/b c/d\'') + + +@patch('subprocess.Popen', return_value=NonCallableMock(returncode=0)) +@patch('swb.local.open') +def test_exec_on_remote(local_open, popen): + """should execute script on remote server""" + swb.exec_on_remote( + 'myname', 'mysite.com', '2222', + 'back-up', ['~/public_html/mysite', 'bzip2 -v', 'a/b c/d', 'False'], + stdout=1, stderr=2) + popen.assert_called_once_with([ + 'ssh', '-p 2222', 'myname@mysite.com', + 'python3', '-', 'back-up', '~/\'public_html/mysite\'', + '\'bzip2 -v\'', '\'a/b c/d\'', 'False'], + stdin=local_open.return_value.__enter__(), stdout=1, stderr=2) + popen.return_value.wait.assert_called_once_with() + + +@patch('subprocess.Popen', return_value=NonCallableMock(returncode=3)) +@patch('swb.local.open') +@patch('sys.exit') +def test_exec_on_remote_nonzero_return(exit, local_open, popen): + """should exit script if nonzero status code is returned""" + swb.exec_on_remote( + 'a', 'b.com', '2222', 'c', ['d', 'e', 'f', 'g'], stdout=1, stderr=2) + exit.assert_called_once_with(3) From 9183ba64993eaf93385c1b4caad17541d31d98e7 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 11:19:23 -0800 Subject: [PATCH 12/39] Add transfer_file tests (both download and upload) --- tests/test_local.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 3b22b8a..7eae30c 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -81,3 +81,27 @@ def test_exec_on_remote_nonzero_return(exit, local_open, popen): swb.exec_on_remote( 'a', 'b.com', '2222', 'c', ['d', 'e', 'f', 'g'], stdout=1, stderr=2) exit.assert_called_once_with(3) + + +@patch('subprocess.Popen') +def test_transfer_file_download(popen): + """should download backup from remote server when backing up""" + swb.transfer_file( + 'myname', 'mysite.com', '2222', 'a/b c/d', 'e/f g/h', + action='download', stdout=1, stderr=2) + popen.assert_called_once_with( + ['scp', '-P 2222', 'myname@mysite.com:\'a/b c/d\'', 'e/f g/h'], + stdout=1, stderr=2) + popen.return_value.wait.assert_called_once_with() + + +@patch('subprocess.Popen') +def test_transfer_file_upload(popen): + """should upload backup to remote server when restoring""" + swb.transfer_file( + 'myname', 'mysite.com', '2222', 'a/b c/d', 'e/f g/h', + action='upload', stdout=1, stderr=2) + popen.assert_called_once_with( + ['scp', '-P 2222', 'a/b c/d', 'myname@mysite.com:\'e/f g/h\''], + stdout=1, stderr=2) + popen.return_value.wait.assert_called_once_with() From 959d1d37796181310d731baca0f2c77b8a87df88 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 11:38:22 -0800 Subject: [PATCH 13/39] Add create/download/purge remote backup tests --- tests/test_local.py | 53 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 7eae30c..f140479 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -42,10 +42,12 @@ def test_unquote_home_dir_path_without_tilde(): nose.assert_equal(unquoted_path, '\'/a/b c/d\'') -def test_quote_arg(): +@patch('swb.local.unquote_home_dir', side_effect=lambda x: x) +def test_quote_arg(unquote_home_dir): """should correctly quote arguments passed to the shell""" quoted_arg = swb.quote_arg('a/b c/d') nose.assert_equal(quoted_arg, '\'a/b c/d\'') + unquote_home_dir.assert_called_once_with('\'a/b c/d\'') @patch('swb.local.shlex') @@ -62,8 +64,9 @@ def test_quote_arg_py32(shlex): def test_exec_on_remote(local_open, popen): """should execute script on remote server""" swb.exec_on_remote( - 'myname', 'mysite.com', '2222', - 'back-up', ['~/public_html/mysite', 'bzip2 -v', 'a/b c/d', 'False'], + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + action='back-up', + action_args=['~/public_html/mysite', 'bzip2 -v', 'a/b c/d', 'False'], stdout=1, stderr=2) popen.assert_called_once_with([ 'ssh', '-p 2222', 'myname@mysite.com', @@ -87,7 +90,8 @@ def test_exec_on_remote_nonzero_return(exit, local_open, popen): def test_transfer_file_download(popen): """should download backup from remote server when backing up""" swb.transfer_file( - 'myname', 'mysite.com', '2222', 'a/b c/d', 'e/f g/h', + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + src_path='a/b c/d', dest_path='e/f g/h', action='download', stdout=1, stderr=2) popen.assert_called_once_with( ['scp', '-P 2222', 'myname@mysite.com:\'a/b c/d\'', 'e/f g/h'], @@ -99,9 +103,48 @@ def test_transfer_file_download(popen): def test_transfer_file_upload(popen): """should upload backup to remote server when restoring""" swb.transfer_file( - 'myname', 'mysite.com', '2222', 'a/b c/d', 'e/f g/h', + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + src_path='a/b c/d', dest_path='e/f g/h', action='upload', stdout=1, stderr=2) popen.assert_called_once_with( ['scp', '-P 2222', 'a/b c/d', 'myname@mysite.com:\'e/f g/h\''], stdout=1, stderr=2) popen.return_value.wait.assert_called_once_with() + + +@patch('swb.local.exec_on_remote') +def test_create_remote_backup(exec_on_remote): + """should execute remote script when creating remote backup""" + swb.create_remote_backup( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + wordpress_path='a/b c/d', remote_backup_path='e/f g/h', + backup_compressor='bzip2 -v', full_backup=False, + stdout=1, stderr=2) + exec_on_remote.assert_called_once_with( + 'myname', 'mysite.com', '2222', 'back-up', + ['a/b c/d', 'bzip2 -v', 'e/f g/h', False], + stdout=1, stderr=2) + + +@patch('swb.local.transfer_file') +def test_download_remote_backup(transfer_file): + """should download remote backup after creation""" + swb.download_remote_backup( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + remote_backup_path='a/b c/d', local_backup_path='e/f g/h', + stdout=1, stderr=2) + transfer_file.assert_called_once_with( + 'myname', 'mysite.com', '2222', + 'a/b c/d', 'e/f g/h', 'download', + stdout=1, stderr=2) + + +@patch('swb.local.exec_on_remote') +def test_purge_remote_backup(exec_on_remote): + """should purge remote backup after download""" + swb.purge_remote_backup( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + remote_backup_path='a/b c/d', stdout=1, stderr=2) + exec_on_remote.assert_called_once_with( + 'myname', 'mysite.com', '2222', 'purge-backup', ['a/b c/d'], + stdout=1, stderr=2) From 38a4be71ee0b4e7456312f9bd8ec8eb4ad3a01e9 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 11:44:59 -0800 Subject: [PATCH 14/39] Correctly patch open() via builtins module --- tests/test_local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index f140479..3de5793 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -60,7 +60,7 @@ def test_quote_arg_py32(shlex): @patch('subprocess.Popen', return_value=NonCallableMock(returncode=0)) -@patch('swb.local.open') +@patch('builtins.open') def test_exec_on_remote(local_open, popen): """should execute script on remote server""" swb.exec_on_remote( @@ -77,7 +77,7 @@ def test_exec_on_remote(local_open, popen): @patch('subprocess.Popen', return_value=NonCallableMock(returncode=3)) -@patch('swb.local.open') +@patch('builtins.open') @patch('sys.exit') def test_exec_on_remote_nonzero_return(exit, local_open, popen): """should exit script if nonzero status code is returned""" From 97f8c8b9799fd596f5ad39b801a1367a4f39d7f9 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 12:13:18 -0800 Subject: [PATCH 15/39] Add get_last_modified_time test --- tests/test_local.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 3de5793..bffd015 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import configparser +import os import nose.tools as nose import swb.local as swb from mock import NonCallableMock, patch @@ -148,3 +149,10 @@ def test_purge_remote_backup(exec_on_remote): exec_on_remote.assert_called_once_with( 'myname', 'mysite.com', '2222', 'purge-backup', ['a/b c/d'], stdout=1, stderr=2) + + +def test_get_last_modified_time(): + """should retrieve correct last modified time of the supplied path""" + nose.assert_equal( + swb.get_last_modified_time('swb/local.py'), + os.stat('swb/local.py').st_mtime) From 7fd68c9dcd46fea46002562d3c5aab89a039c25c Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 12:22:54 -0800 Subject: [PATCH 16/39] Make optional backup.full_backup config option --- swb/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swb/local.py b/swb/local.py index 6997d30..8c164f5 100755 --- a/swb/local.py +++ b/swb/local.py @@ -231,7 +231,7 @@ def back_up(config, *, stdout=None, stderr=None): config.get('paths', 'wordpress'), expanded_remote_backup_path, config.get('backup', 'compressor'), - config.getboolean('backup', 'full_backup'), + config.getboolean('backup', 'full_backup', fallback=False), stdout=stdout, stderr=stderr) create_dir_structure(expanded_local_backup_path) From 89a7a73360ace3c61280c787bf4d9eaa778a86ae Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 12:29:26 -0800 Subject: [PATCH 17/39] Add purge_empty_dirs test --- tests/test_local.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_local.py b/tests/test_local.py index bffd015..c0f6714 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,7 +4,7 @@ import os import nose.tools as nose import swb.local as swb -from mock import NonCallableMock, patch +from mock import call, NonCallableMock, patch # from tests.fixtures.local import set_up, tear_down, mock_backups @@ -156,3 +156,27 @@ def test_get_last_modified_time(): nose.assert_equal( swb.get_last_modified_time('swb/local.py'), os.stat('swb/local.py').st_mtime) + + +@patch('glob.iglob', side_effect=[ + ['/a/b/2011/02/03'], + ['/a/b/2011/02'], + ['/a/b/2011'], + ['/a/b'], + ['/a'] +]) +@patch('os.rmdir') +def test_purge_empty_dirs(rmdir, iglob): + """should purge empty timestamped directories""" + swb.purge_empty_dirs('/a/b/*/*/*/c') + nose.assert_equal(iglob.call_count, 3) + nose.assert_equal(iglob.call_args_list, [ + call('/a/b/*/*/*'), + call('/a/b/*/*'), + call('/a/b/*') + ]) + nose.assert_equal(rmdir.call_args_list, [ + call('/a/b/2011/02/03'), + call('/a/b/2011/02'), + call('/a/b/2011') + ]) From ffe5dc9502e955aa3bb556d51a762ed477e00de5 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 12:59:39 -0800 Subject: [PATCH 18/39] Make purge_empty_dirs test more complete --- tests/test_local.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index c0f6714..26b3c83 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -159,9 +159,9 @@ def test_get_last_modified_time(): @patch('glob.iglob', side_effect=[ - ['/a/b/2011/02/03'], + ['/a/b/2011/02/03', '/a/b/2011/02/04'], ['/a/b/2011/02'], - ['/a/b/2011'], + ['/a/b/2010', '/a/b/2011'], ['/a/b'], ['/a'] ]) @@ -177,6 +177,8 @@ def test_purge_empty_dirs(rmdir, iglob): ]) nose.assert_equal(rmdir.call_args_list, [ call('/a/b/2011/02/03'), + call('/a/b/2011/02/04'), call('/a/b/2011/02'), + call('/a/b/2010'), call('/a/b/2011') ]) From 5166dbcc1c5fdbd4cdd62a3386a24d0e5a9ad224 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 13:37:15 -0800 Subject: [PATCH 19/39] Add test for purge_empty_dirs silent fail --- tests/test_local.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 26b3c83..56bf641 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -182,3 +182,18 @@ def test_purge_empty_dirs(rmdir, iglob): call('/a/b/2010'), call('/a/b/2011') ]) + + +@patch('glob.iglob', side_effect=[ + ['/a/b/2011/02/03', '/a/b/2011/02/04'], + ['/a/b/2011/02'], + ['/a/b/2010', '/a/b/2011'], + ['/a/b'], + ['/a'] +]) +@patch('os.rmdir', side_effect=OSError) +def test_purge_empty_dirs_silent_fail(rmdir, iglob): + """should ignore non-empty directories when purging empty directories""" + swb.purge_empty_dirs('/a/b/*/*/*/c') + nose.assert_equal(iglob.call_count, 3) + nose.assert_equal(rmdir.call_count, 5) From cb6941cb9ba6c304eda000b50366b9930ec4f69e Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 13:38:01 -0800 Subject: [PATCH 20/39] Add purge_oldest_backups test --- tests/test_local.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 56bf641..fb4ed29 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -197,3 +197,24 @@ def test_purge_empty_dirs_silent_fail(rmdir, iglob): swb.purge_empty_dirs('/a/b/*/*/*/c') nose.assert_equal(iglob.call_count, 3) nose.assert_equal(rmdir.call_count, 5) + + +@patch('glob.iglob', return_value=[ + 'a/2015/02/03/b', + 'a/2013/04/05/b', + 'a/2016/01/02/b', + 'a/2012/05/06/b', + 'a/2014/03/04/b' +]) +@patch('os.remove') +@patch('swb.local.get_last_modified_time', side_effect=[5, 3, 6, 2, 4]) +@patch('swb.local.purge_empty_dirs') +def test_purge_oldest_backups(purge_empty_dirs, get_last_modified_time, + remove, iglob): + """should purge oldest local backups""" + swb.purge_oldest_backups('a/%y/%m/%d/b', max_local_backups=3) + nose.assert_equal(remove.call_args_list, [ + call('a/2012/05/06/b'), + call('a/2013/04/05/b') + ]) + purge_empty_dirs.assert_called_once_with('a/*/*/*/b') From 075390b9a754a47d01c11a8d35d5b54e60289f7d Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 14:15:21 -0800 Subject: [PATCH 21/39] Add back_up tests --- tests/files/config.ini | 5 ++--- tests/test_local.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/files/config.ini b/tests/files/config.ini index f88ab76..4efbeac 100644 --- a/tests/files/config.ini +++ b/tests/files/config.ini @@ -5,11 +5,10 @@ port = 2222 [paths] wordpress = ~/public_html/mysite -remote_backup = ~/backups/mysite.sql.bz2 -local_backup = ~/Backups/mysite.sql.bz2 +remote_backup = ~/backups/%y/%m/%d/mysite.sql.bz2 +local_backup = ~/Backups/%y/%m/%d/mysite.sql.bz2 [backup] compressor = bzip2 decompressor = bzip2 -d -max_local_backups = 3 full_backup = no diff --git a/tests/test_local.py b/tests/test_local.py index fb4ed29..7502936 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -2,8 +2,10 @@ import configparser import os +import os.path import nose.tools as nose import swb.local as swb +from time import strftime from mock import call, NonCallableMock, patch # from tests.fixtures.local import set_up, tear_down, mock_backups @@ -218,3 +220,49 @@ def test_purge_oldest_backups(purge_empty_dirs, get_last_modified_time, call('a/2013/04/05/b') ]) purge_empty_dirs.assert_called_once_with('a/*/*/*/b') + + +@patch('swb.local.create_remote_backup') +@patch('swb.local.create_dir_structure') +@patch('swb.local.download_remote_backup') +@patch('swb.local.purge_remote_backup') +@patch('swb.local.purge_oldest_backups') +def test_back_up(purge_oldest_backups, purge_remote_backup, + download_remote_backup, create_dir_structure, + create_remote_backup): + """should run correct backup procedure""" + config = configparser.RawConfigParser() + config.read('tests/files/config.ini') + swb.back_up(config, stdout=1, stderr=2) + expanded_local_backup_path = os.path.expanduser(strftime( + '~/Backups/%y/%m/%d/mysite.sql.bz2')) + expanded_remote_backup_path = strftime('~/backups/%y/%m/%d/mysite.sql.bz2') + create_remote_backup.assert_called_once_with( + 'myname', 'mysite.com', '2222', '~/public_html/mysite', + expanded_remote_backup_path, 'bzip2', False, + stdout=1, stderr=2) + create_dir_structure.assert_called_once_with(expanded_local_backup_path) + download_remote_backup.assert_called_once_with( + 'myname', 'mysite.com', '2222', + expanded_remote_backup_path, expanded_local_backup_path, + stdout=1, stderr=2) + purge_remote_backup.assert_called_once_with( + 'myname', 'mysite.com', '2222', expanded_remote_backup_path, + stdout=1, stderr=2) + + +@patch('swb.local.create_remote_backup') +@patch('swb.local.create_dir_structure') +@patch('swb.local.download_remote_backup') +@patch('swb.local.purge_remote_backup') +@patch('swb.local.purge_oldest_backups') +def test_back_up_max_local_backups(purge_oldest_backups, purge_remote_backup, + download_remote_backup, + create_dir_structure, create_remote_backup): + """should purge oldest backups if max_local_backups option is set""" + config = configparser.RawConfigParser() + config.read('tests/files/config.ini') + config.set('backup', 'max_local_backups', 3) + swb.back_up(config) + purge_oldest_backups.assert_called_once_with( + os.path.expanduser('~/Backups/%y/%m/%d/mysite.sql.bz2'), 3) From 95ed01f7b49bae39dac5728c52f505d32a4ceee4 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 17:49:56 -0800 Subject: [PATCH 22/39] Utilize keyword arguments to reduce ambiguity --- swb/local.py | 177 ++++++++++++++++++++++++++++---------------- tests/test_local.py | 66 +++++++++++++---- 2 files changed, 164 insertions(+), 79 deletions(-) diff --git a/swb/local.py b/swb/local.py index 6997d30..6f1a70b 100755 --- a/swb/local.py +++ b/swb/local.py @@ -58,10 +58,10 @@ def parse_config(config_path): # Create intermediate directories in local backup path if necessary -def create_dir_structure(path): +def create_dir_structure(local_backup_path): try: - os.makedirs(os.path.dirname(path)) + os.makedirs(os.path.dirname(local_backup_path)) except OSError: pass @@ -87,8 +87,8 @@ def quote_arg(arg): # Connect to remote via SSH and execute remote script -def exec_on_remote(ssh_user, ssh_hostname, ssh_port, action, action_args, - *, stdout, stderr): +def exec_on_remote(ssh_user, ssh_hostname, ssh_port, *, + action, action_args, stdout, stderr): # Read remote script so as to pass contents to SSH session with open(remote_driver_path, 'r') as remote_script: @@ -116,8 +116,8 @@ def exec_on_remote(ssh_user, ssh_hostname, ssh_port, action, action_args, # Transfer a file from remote to local (or vice-versa) using SCP -def transfer_file(ssh_user, ssh_hostname, ssh_port, - src_path, dest_path, action, *, stdout, stderr): +def transfer_file(ssh_user, ssh_hostname, ssh_port, *, + src_path, dest_path, action, stdout, stderr): scp_args = ['scp', '-P {}'.format(ssh_port)] @@ -139,35 +139,83 @@ def transfer_file(ssh_user, ssh_hostname, ssh_port, # Execute remote backup script to create remote backup -def create_remote_backup(ssh_user, ssh_hostname, ssh_port, +def create_remote_backup(ssh_user, ssh_hostname, ssh_port, *, wordpress_path, remote_backup_path, - backup_compressor, full_backup, *, stdout, stderr): + backup_compressor, full_backup, stdout, stderr): - exec_on_remote(ssh_user, ssh_hostname, ssh_port, 'back-up', [ - wordpress_path, - backup_compressor, - remote_backup_path, - full_backup - ], stdout=stdout, stderr=stderr) + exec_on_remote( + ssh_user=ssh_user, + ssh_hostname=ssh_hostname, + ssh_port=ssh_port, + action='back-up', + action_args=[ + wordpress_path, + backup_compressor, + remote_backup_path, + full_backup + ], + stdout=stdout, stderr=stderr) # Download remote backup to local system -def download_remote_backup(ssh_user, ssh_hostname, ssh_port, +def download_remote_backup(ssh_user, ssh_hostname, ssh_port, *, remote_backup_path, local_backup_path, - *, stdout, stderr): + stdout, stderr): + + transfer_file( + ssh_user=ssh_user, + ssh_hostname=ssh_hostname, + ssh_port=ssh_port, + src_path=remote_backup_path, + dest_path=local_backup_path, + action='download', + stdout=stdout, stderr=stderr) + + +# Uploads the given local backup to the given remote destination +def upload_local_backup(ssh_user, ssh_hostname, ssh_port, *, + remote_backup_path, local_backup_path, + stdout, stderr): + + transfer_file( + ssh_user=ssh_user, + ssh_hostname=ssh_hostname, + ssh_port=ssh_port, + src_path=local_backup_path, + dest_path=remote_backup_path, + action='upload', + stdout=stdout, stderr=stderr) - transfer_file(ssh_user, ssh_hostname, ssh_port, - remote_backup_path, local_backup_path, - 'download', stdout=stdout, stderr=stderr) + +# Restores the local backup after upload to remote +def restore_remote_backup(ssh_user, ssh_hostname, ssh_port, *, + wordpress_path, remote_backup_path, + backup_decompressor, stdout, stderr): + + exec_on_remote( + ssh_user=ssh_user, + ssh_hostname=ssh_hostname, + ssh_port=ssh_port, + action='restore', + action_args=[ + wordpress_path, + remote_backup_path, + backup_decompressor + ], + stdout=stdout, stderr=stderr) # Forcefully remove backup from remote -def purge_remote_backup(ssh_user, ssh_hostname, ssh_port, - remote_backup_path, *, stdout, stderr): +def purge_remote_backup(ssh_user, ssh_hostname, ssh_port, *, + remote_backup_path, stdout, stderr): - exec_on_remote(ssh_user, ssh_hostname, ssh_port, - 'purge-backup', [remote_backup_path], - stdout=stdout, stderr=stderr) + exec_on_remote( + ssh_user=ssh_user, + ssh_hostname=ssh_hostname, + ssh_port=ssh_port, + action='purge-backup', + action_args=[remote_backup_path], + stdout=stdout, stderr=stderr) # Retrieve a file's last modified time in seconds @@ -194,7 +242,7 @@ def purge_empty_dirs(dir_path): # Purge oldest backups to keep number of backups within specified limit -def purge_oldest_backups(local_backup_path, max_local_backups): +def purge_oldest_backups(local_backup_path, *, max_local_backups): # Convert date format sequences to wildcards local_backup_path = re.sub(r'%\-?[A-Za-z]', '*', local_backup_path) @@ -225,65 +273,62 @@ def back_up(config, *, stdout=None, stderr=None): config.get('paths', 'remote_backup')) create_remote_backup( - config.get('ssh', 'user'), - config.get('ssh', 'hostname'), - config.get('ssh', 'port'), - config.get('paths', 'wordpress'), - expanded_remote_backup_path, - config.get('backup', 'compressor'), - config.getboolean('backup', 'full_backup'), + ssh_user=config.get('ssh', 'user'), + ssh_hostname=config.get('ssh', 'hostname'), + ssh_port=config.get('ssh', 'port'), + wordpress_path=config.get('paths', 'wordpress'), + remote_backup_path=expanded_remote_backup_path, + backup_compressor=config.get('backup', 'compressor'), + full_backup=config.getboolean('backup', 'full_backup'), stdout=stdout, stderr=stderr) - create_dir_structure(expanded_local_backup_path) + create_dir_structure(local_backup_path=expanded_local_backup_path) download_remote_backup( - config.get('ssh', 'user'), - config.get('ssh', 'hostname'), - config.get('ssh', 'port'), - expanded_remote_backup_path, - expanded_local_backup_path, + ssh_user=config.get('ssh', 'user'), + ssh_hostname=config.get('ssh', 'hostname'), + ssh_port=config.get('ssh', 'port'), + remote_backup_path=expanded_remote_backup_path, + local_backup_path=expanded_local_backup_path, stdout=stdout, stderr=stderr) purge_remote_backup( - config.get('ssh', 'user'), - config.get('ssh', 'hostname'), - config.get('ssh', 'port'), - expanded_remote_backup_path, + ssh_user=config.get('ssh', 'user'), + ssh_hostname=config.get('ssh', 'hostname'), + ssh_port=config.get('ssh', 'port'), + remote_backup_path=expanded_remote_backup_path, stdout=stdout, stderr=stderr) if config.has_option('backup', 'max_local_backups'): purge_oldest_backups( - config.get('paths', 'local_backup'), - config.getint('backup', 'max_local_backups')) + local_backup_path=config.get('paths', 'local_backup'), + max_local_backups=config.getint('backup', 'max_local_backups')) # Restore the chosen database revision to the Wordpress install on remote -def restore(config, local_backup_path, *, stdout=None, stderr=None): +def restore(config, *, local_backup_path, stdout=None, stderr=None): expanded_remote_backup_path = time.strftime( config.get('paths', 'remote_backup')) - # Copy local backup to remote so it can be used for restoration - transfer_file( - config.get('ssh', 'user'), - config.get('ssh', 'hostname'), - config.get('ssh', 'port'), - local_backup_path, - expanded_remote_backup_path, - 'upload', + upload_local_backup( + ssh_user=config.get('ssh', 'user'), + ssh_hostname=config.get('ssh', 'hostname'), + ssh_port=config.get('ssh', 'port'), + remote_backup_path=expanded_remote_backup_path, + local_backup_path=config.get('paths', 'local_backup'), + backup_compressor=config.get('backup', 'compressor'), + full_backup=config.getboolean('backup', 'full_backup'), stdout=stdout, stderr=stderr) - action_args = [ - config.get('paths', 'wordpress'), - expanded_remote_backup_path, - config.get('backup', 'decompressor') - ] - exec_on_remote( - config.get('ssh', 'user'), - config.get('ssh', 'hostname'), - config.get('ssh', 'port'), - 'restore', action_args, - stdout=stdout, stderr=stderr) + restore_remote_backup( + ssh_user=config.get('ssh', 'user'), + ssh_hostname=config.get('ssh', 'hostname'), + ssh_port=config.get('ssh', 'port'), + action='restore', + wordpress_path=config.get('paths', 'wordpress'), + remote_backup_path=config.get('paths', 'remote_backup'), + backup_decompressor=config.get('backup', 'decompressor')) def main(): @@ -306,7 +351,9 @@ def main(): answer = input('Do you want to continue? (y/n) ') if not answer.lower().lstrip().startswith('y'): raise Exception('User canceled. Aborting.') - restore(config, cli_args.restore, stdout=stdout, stderr=stderr) + restore( + config, local_backup_path=cli_args.restore, + stdout=stdout, stderr=stderr) else: back_up(config, stdout=stdout, stderr=stderr) diff --git a/tests/test_local.py b/tests/test_local.py index 7502936..4cb1a1d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -85,7 +85,9 @@ def test_exec_on_remote(local_open, popen): def test_exec_on_remote_nonzero_return(exit, local_open, popen): """should exit script if nonzero status code is returned""" swb.exec_on_remote( - 'a', 'b.com', '2222', 'c', ['d', 'e', 'f', 'g'], stdout=1, stderr=2) + ssh_user='a', ssh_hostname='b.com', ssh_port='2222', + action='c', action_args=['d', 'e', 'f', 'g'], + stdout=1, stderr=2) exit.assert_called_once_with(3) @@ -121,11 +123,11 @@ def test_create_remote_backup(exec_on_remote): swb.create_remote_backup( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', wordpress_path='a/b c/d', remote_backup_path='e/f g/h', - backup_compressor='bzip2 -v', full_backup=False, + backup_compressor='bzip2 -v', full_backup=True, stdout=1, stderr=2) exec_on_remote.assert_called_once_with( - 'myname', 'mysite.com', '2222', 'back-up', - ['a/b c/d', 'bzip2 -v', 'e/f g/h', False], + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + action='back-up', action_args=['a/b c/d', 'bzip2 -v', 'e/f g/h', True], stdout=1, stderr=2) @@ -137,8 +139,35 @@ def test_download_remote_backup(transfer_file): remote_backup_path='a/b c/d', local_backup_path='e/f g/h', stdout=1, stderr=2) transfer_file.assert_called_once_with( - 'myname', 'mysite.com', '2222', - 'a/b c/d', 'e/f g/h', 'download', + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + src_path='a/b c/d', dest_path='e/f g/h', + action='download', stdout=1, stderr=2) + + +@patch('swb.local.transfer_file') +def test_upload_local_backup(transfer_file): + """should upload local backup when restoring""" + swb.upload_local_backup( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + remote_backup_path='a/b c/d', local_backup_path='e/f g/h', + stdout=1, stderr=2) + transfer_file.assert_called_once_with( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + src_path='e/f g/h', dest_path='a/b c/d', + action='upload', stdout=1, stderr=2) + + +@patch('swb.local.exec_on_remote') +def test_restore_remote_backup(exec_on_remote): + """should restore remote backup after upload""" + swb.restore_remote_backup( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + wordpress_path='a/b c/d', remote_backup_path='e/f g/h', + backup_decompressor='bzip2 -v', + stdout=1, stderr=2) + exec_on_remote.assert_called_once_with( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + action='restore', action_args=['a/b c/d', 'e/f g/h', 'bzip2 -v'], stdout=1, stderr=2) @@ -149,7 +178,8 @@ def test_purge_remote_backup(exec_on_remote): ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', remote_backup_path='a/b c/d', stdout=1, stderr=2) exec_on_remote.assert_called_once_with( - 'myname', 'mysite.com', '2222', 'purge-backup', ['a/b c/d'], + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + action='purge-backup', action_args=['a/b c/d'], stdout=1, stderr=2) @@ -238,16 +268,22 @@ def test_back_up(purge_oldest_backups, purge_remote_backup, '~/Backups/%y/%m/%d/mysite.sql.bz2')) expanded_remote_backup_path = strftime('~/backups/%y/%m/%d/mysite.sql.bz2') create_remote_backup.assert_called_once_with( - 'myname', 'mysite.com', '2222', '~/public_html/mysite', - expanded_remote_backup_path, 'bzip2', False, + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + wordpress_path='~/public_html/mysite', + remote_backup_path=expanded_remote_backup_path, + backup_compressor='bzip2', + full_backup=False, stdout=1, stderr=2) - create_dir_structure.assert_called_once_with(expanded_local_backup_path) + create_dir_structure.assert_called_once_with( + local_backup_path=expanded_local_backup_path) download_remote_backup.assert_called_once_with( - 'myname', 'mysite.com', '2222', - expanded_remote_backup_path, expanded_local_backup_path, + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + remote_backup_path=expanded_remote_backup_path, + local_backup_path=expanded_local_backup_path, stdout=1, stderr=2) purge_remote_backup.assert_called_once_with( - 'myname', 'mysite.com', '2222', expanded_remote_backup_path, + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + remote_backup_path=expanded_remote_backup_path, stdout=1, stderr=2) @@ -265,4 +301,6 @@ def test_back_up_max_local_backups(purge_oldest_backups, purge_remote_backup, config.set('backup', 'max_local_backups', 3) swb.back_up(config) purge_oldest_backups.assert_called_once_with( - os.path.expanduser('~/Backups/%y/%m/%d/mysite.sql.bz2'), 3) + local_backup_path=os.path.expanduser( + '~/Backups/%y/%m/%d/mysite.sql.bz2'), + max_local_backups=3) From bc7a37cc89577b723d295b2b7ee1f2c463cae5bf Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 18:40:43 -0800 Subject: [PATCH 23/39] Move main helper functions closer to main --- swb/local.py | 78 ++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/swb/local.py b/swb/local.py index 6f1a70b..e4677b2 100755 --- a/swb/local.py +++ b/swb/local.py @@ -18,45 +18,6 @@ remote_driver_path = os.path.join(program_dir, 'remote.py') -# Parse command line arguments passed to the local driver -def parse_cli_args(): - - parser = argparse.ArgumentParser() - - parser.add_argument( - '--quiet', - '-q', - action='store_true', - help='silences stdout and stderr') - - parser.add_argument( - 'config_path', - help='the path to a configuration file (.ini)') - - parser.add_argument( - '--restore', - '-r', - help='the path to a compressed backup file from which to restore') - - parser.add_argument( - '--force', - '-f', - action='store_true', - help='bypasses the confirmation prompt when restoring from backup') - - cli_args = parser.parse_args() - return cli_args - - -# Parse configuration files at given paths into object -def parse_config(config_path): - - config = configparser.RawConfigParser() - config.read(config_path) - - return config - - # Create intermediate directories in local backup path if necessary def create_dir_structure(local_backup_path): @@ -331,6 +292,45 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): backup_decompressor=config.get('backup', 'decompressor')) +# Parse command line arguments passed to the local driver +def parse_cli_args(): + + parser = argparse.ArgumentParser() + + parser.add_argument( + '--quiet', + '-q', + action='store_true', + help='silences stdout and stderr') + + parser.add_argument( + 'config_path', + help='the path to a configuration file (.ini)') + + parser.add_argument( + '--restore', + '-r', + help='the path to a compressed backup file from which to restore') + + parser.add_argument( + '--force', + '-f', + action='store_true', + help='bypasses the confirmation prompt when restoring from backup') + + cli_args = parser.parse_args() + return cli_args + + +# Parse configuration files at given paths into object +def parse_config(config_path): + + config = configparser.RawConfigParser() + config.read(config_path) + + return config + + def main(): cli_args = parse_cli_args() From 47d2f93ef11660bdbfab877169d8158558f24b04 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 19:54:36 -0800 Subject: [PATCH 24/39] Add tests for main() back_up route --- tests/test_local.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 4cb1a1d..2cd8d98 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -64,7 +64,7 @@ def test_quote_arg_py32(shlex): @patch('subprocess.Popen', return_value=NonCallableMock(returncode=0)) @patch('builtins.open') -def test_exec_on_remote(local_open, popen): +def test_exec_on_remote(builtin_open, popen): """should execute script on remote server""" swb.exec_on_remote( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', @@ -75,14 +75,14 @@ def test_exec_on_remote(local_open, popen): 'ssh', '-p 2222', 'myname@mysite.com', 'python3', '-', 'back-up', '~/\'public_html/mysite\'', '\'bzip2 -v\'', '\'a/b c/d\'', 'False'], - stdin=local_open.return_value.__enter__(), stdout=1, stderr=2) + stdin=builtin_open.return_value.__enter__(), stdout=1, stderr=2) popen.return_value.wait.assert_called_once_with() @patch('subprocess.Popen', return_value=NonCallableMock(returncode=3)) @patch('builtins.open') @patch('sys.exit') -def test_exec_on_remote_nonzero_return(exit, local_open, popen): +def test_exec_on_remote_nonzero_return(exit, builtin_open, popen): """should exit script if nonzero status code is returned""" swb.exec_on_remote( ssh_user='a', ssh_hostname='b.com', ssh_port='2222', @@ -304,3 +304,26 @@ def test_back_up_max_local_backups(purge_oldest_backups, purge_remote_backup, local_backup_path=os.path.expanduser( '~/Backups/%y/%m/%d/mysite.sql.bz2'), max_local_backups=3) + + +@patch('sys.argv', [swb.__file__, 'tests/files/config.ini']) +@patch('swb.local.back_up') +@patch('swb.local.parse_config') +def test_main_back_up(parse_config, back_up): + """should correctly run main driver""" + swb.main() + back_up.assert_called_once_with( + parse_config.return_value, stdout=None, stderr=None) + + +@patch('sys.argv', [swb.__file__, '-q', 'tests/files/config.ini']) +@patch('builtins.open') +@patch('swb.local.back_up') +@patch('swb.local.parse_config') +def test_main_quiet(parse_config, back_up, builtin_open): + """should silent stdout/stderr when utility is run in quiet mode""" + swb.main() + devnull = builtin_open.return_value.__enter__() + back_up.assert_called_once_with( + parse_config.return_value, + stdout=devnull, stderr=devnull) From c5ce0b968205d4ece999d25e57fd1e02259b0b74 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 20:21:14 -0800 Subject: [PATCH 25/39] Order patches consistently --- tests/test_local.py | 58 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 2cd8d98..5641290 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -79,10 +79,10 @@ def test_exec_on_remote(builtin_open, popen): popen.return_value.wait.assert_called_once_with() +@patch('sys.exit') @patch('subprocess.Popen', return_value=NonCallableMock(returncode=3)) @patch('builtins.open') -@patch('sys.exit') -def test_exec_on_remote_nonzero_return(exit, builtin_open, popen): +def test_exec_on_remote_nonzero_return(builtin_open, popen, exit): """should exit script if nonzero status code is returned""" swb.exec_on_remote( ssh_user='a', ssh_hostname='b.com', ssh_port='2222', @@ -190,6 +190,7 @@ def test_get_last_modified_time(): os.stat('swb/local.py').st_mtime) +@patch('os.rmdir') @patch('glob.iglob', side_effect=[ ['/a/b/2011/02/03', '/a/b/2011/02/04'], ['/a/b/2011/02'], @@ -197,8 +198,7 @@ def test_get_last_modified_time(): ['/a/b'], ['/a'] ]) -@patch('os.rmdir') -def test_purge_empty_dirs(rmdir, iglob): +def test_purge_empty_dirs(iglob, rmdir): """should purge empty timestamped directories""" swb.purge_empty_dirs('/a/b/*/*/*/c') nose.assert_equal(iglob.call_count, 3) @@ -216,6 +216,7 @@ def test_purge_empty_dirs(rmdir, iglob): ]) +@patch('os.rmdir', side_effect=OSError) @patch('glob.iglob', side_effect=[ ['/a/b/2011/02/03', '/a/b/2011/02/04'], ['/a/b/2011/02'], @@ -223,14 +224,16 @@ def test_purge_empty_dirs(rmdir, iglob): ['/a/b'], ['/a'] ]) -@patch('os.rmdir', side_effect=OSError) -def test_purge_empty_dirs_silent_fail(rmdir, iglob): +def test_purge_empty_dirs_silent_fail(iglob, rmdir): """should ignore non-empty directories when purging empty directories""" swb.purge_empty_dirs('/a/b/*/*/*/c') nose.assert_equal(iglob.call_count, 3) nose.assert_equal(rmdir.call_count, 5) +@patch('swb.local.get_last_modified_time', side_effect=[5, 3, 6, 2, 4]) +@patch('swb.local.purge_empty_dirs') +@patch('os.remove') @patch('glob.iglob', return_value=[ 'a/2015/02/03/b', 'a/2013/04/05/b', @@ -238,11 +241,8 @@ def test_purge_empty_dirs_silent_fail(rmdir, iglob): 'a/2012/05/06/b', 'a/2014/03/04/b' ]) -@patch('os.remove') -@patch('swb.local.get_last_modified_time', side_effect=[5, 3, 6, 2, 4]) -@patch('swb.local.purge_empty_dirs') -def test_purge_oldest_backups(purge_empty_dirs, get_last_modified_time, - remove, iglob): +def test_purge_oldest_backups(iglob, remove, purge_empty_dirs, + get_last_modified_time): """should purge oldest local backups""" swb.purge_oldest_backups('a/%y/%m/%d/b', max_local_backups=3) nose.assert_equal(remove.call_args_list, [ @@ -252,14 +252,14 @@ def test_purge_oldest_backups(purge_empty_dirs, get_last_modified_time, purge_empty_dirs.assert_called_once_with('a/*/*/*/b') -@patch('swb.local.create_remote_backup') -@patch('swb.local.create_dir_structure') -@patch('swb.local.download_remote_backup') @patch('swb.local.purge_remote_backup') @patch('swb.local.purge_oldest_backups') -def test_back_up(purge_oldest_backups, purge_remote_backup, - download_remote_backup, create_dir_structure, - create_remote_backup): +@patch('swb.local.download_remote_backup') +@patch('swb.local.create_remote_backup') +@patch('swb.local.create_dir_structure') +def test_back_up(create_dir_structure, create_remote_backup, + download_remote_backup, purge_oldest_backups, + purge_remote_backup): """should run correct backup procedure""" config = configparser.RawConfigParser() config.read('tests/files/config.ini') @@ -287,14 +287,14 @@ def test_back_up(purge_oldest_backups, purge_remote_backup, stdout=1, stderr=2) -@patch('swb.local.create_remote_backup') -@patch('swb.local.create_dir_structure') -@patch('swb.local.download_remote_backup') @patch('swb.local.purge_remote_backup') @patch('swb.local.purge_oldest_backups') -def test_back_up_max_local_backups(purge_oldest_backups, purge_remote_backup, - download_remote_backup, - create_dir_structure, create_remote_backup): +@patch('swb.local.download_remote_backup') +@patch('swb.local.create_remote_backup') +@patch('swb.local.create_dir_structure') +def test_back_up_purge_oldest(create_dir_structure, create_remote_backup, + download_remote_backup, purge_oldest_backups, + purge_remote_backup): """should purge oldest backups if max_local_backups option is set""" config = configparser.RawConfigParser() config.read('tests/files/config.ini') @@ -306,21 +306,21 @@ def test_back_up_max_local_backups(purge_oldest_backups, purge_remote_backup, max_local_backups=3) -@patch('sys.argv', [swb.__file__, 'tests/files/config.ini']) -@patch('swb.local.back_up') @patch('swb.local.parse_config') -def test_main_back_up(parse_config, back_up): +@patch('swb.local.back_up') +@patch('sys.argv', [swb.__file__, 'tests/files/config.ini']) +def test_main_back_up(back_up, parse_config): """should correctly run main driver""" swb.main() back_up.assert_called_once_with( parse_config.return_value, stdout=None, stderr=None) +@patch('swb.local.parse_config') +@patch('swb.local.back_up') @patch('sys.argv', [swb.__file__, '-q', 'tests/files/config.ini']) @patch('builtins.open') -@patch('swb.local.back_up') -@patch('swb.local.parse_config') -def test_main_quiet(parse_config, back_up, builtin_open): +def test_main_quiet(builtin_open, back_up, parse_config): """should silent stdout/stderr when utility is run in quiet mode""" swb.main() devnull = builtin_open.return_value.__enter__() From 635572d576c064e1ba177b784751ec748d76d1f9 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 21:39:05 -0800 Subject: [PATCH 26/39] Add tests for main() restore branch --- tests/test_local.py | 51 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 5641290..c65fcf8 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -6,8 +6,7 @@ import nose.tools as nose import swb.local as swb from time import strftime -from mock import call, NonCallableMock, patch -# from tests.fixtures.local import set_up, tear_down, mock_backups +from mock import ANY, call, NonCallableMock, patch def test_parse_config(): @@ -310,7 +309,7 @@ def test_back_up_purge_oldest(create_dir_structure, create_remote_backup, @patch('swb.local.back_up') @patch('sys.argv', [swb.__file__, 'tests/files/config.ini']) def test_main_back_up(back_up, parse_config): - """should correctly run main driver""" + """should run backup procedure by default when utility is run""" swb.main() back_up.assert_called_once_with( parse_config.return_value, stdout=None, stderr=None) @@ -321,9 +320,53 @@ def test_main_back_up(back_up, parse_config): @patch('sys.argv', [swb.__file__, '-q', 'tests/files/config.ini']) @patch('builtins.open') def test_main_quiet(builtin_open, back_up, parse_config): - """should silent stdout/stderr when utility is run in quiet mode""" + """should silence stdout/stderr when utility is run in quiet mode""" swb.main() devnull = builtin_open.return_value.__enter__() back_up.assert_called_once_with( parse_config.return_value, stdout=devnull, stderr=devnull) + + +@patch('swb.local.restore') +@patch('swb.local.parse_config') +@patch('sys.argv', [swb.__file__, 'tests/files/config.ini', '-r', 'a.tar.bz2']) +@patch('builtins.print') +@patch('builtins.input', return_value='y') +def test_main_restore(builtin_input, builtin_print, parse_config, restore): + """should run restore procedure when -r/--restore is passed to utility""" + swb.main() + builtin_input.assert_called_once_with(ANY) + restore.assert_called_once_with( + parse_config.return_value, local_backup_path='a.tar.bz2', + stdout=None, stderr=None) + + +@patch('swb.local.restore') +@patch('swb.local.parse_config') +@patch('sys.argv', [swb.__file__, 'tests/files/config.ini', '-r', 'a.tar.bz2']) +@patch('builtins.print') +@patch('builtins.input', return_value='n') +def test_main_restore_cancel(builtin_input, builtin_print, + parse_config, restore): + """should cancel restore procedure when user cancels confirmation""" + with nose.assert_raises(Exception): + swb.main() + restore.assert_not_called() + + +@patch('swb.local.restore') +@patch('swb.local.parse_config') +@patch('sys.argv', [ + swb.__file__, 'tests/files/config.ini', + '-fr', 'a.tar.bz2']) +@patch('builtins.print') +@patch('builtins.input', return_value='y') +def test_main_restore_force(builtin_input, builtin_print, + parse_config, restore): + """should force restore procedure when -fr is passed to utility""" + swb.main() + builtin_input.assert_not_called() + restore.assert_called_once_with( + parse_config.return_value, local_backup_path='a.tar.bz2', + stdout=None, stderr=None) From 409d48a249248c0e2fa79dde7a71502d8badc30b Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Thu, 7 Jan 2016 21:59:21 -0800 Subject: [PATCH 27/39] Add restore test --- swb/local.py | 4 ++-- tests/files/config.ini | 2 +- tests/test_local.py | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/swb/local.py b/swb/local.py index e4677b2..7f507f5 100755 --- a/swb/local.py +++ b/swb/local.py @@ -135,7 +135,7 @@ def download_remote_backup(ssh_user, ssh_hostname, ssh_port, *, # Uploads the given local backup to the given remote destination def upload_local_backup(ssh_user, ssh_hostname, ssh_port, *, - remote_backup_path, local_backup_path, + local_backup_path, remote_backup_path, stdout, stderr): transfer_file( @@ -276,8 +276,8 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): ssh_user=config.get('ssh', 'user'), ssh_hostname=config.get('ssh', 'hostname'), ssh_port=config.get('ssh', 'port'), + local_backup_path=local_backup_path, remote_backup_path=expanded_remote_backup_path, - local_backup_path=config.get('paths', 'local_backup'), backup_compressor=config.get('backup', 'compressor'), full_backup=config.getboolean('backup', 'full_backup'), stdout=stdout, stderr=stderr) diff --git a/tests/files/config.ini b/tests/files/config.ini index 4efbeac..df8308a 100644 --- a/tests/files/config.ini +++ b/tests/files/config.ini @@ -9,6 +9,6 @@ remote_backup = ~/backups/%y/%m/%d/mysite.sql.bz2 local_backup = ~/Backups/%y/%m/%d/mysite.sql.bz2 [backup] -compressor = bzip2 +compressor = bzip2 -v decompressor = bzip2 -d full_backup = no diff --git a/tests/test_local.py b/tests/test_local.py index c65fcf8..dd7d2d2 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -270,7 +270,7 @@ def test_back_up(create_dir_structure, create_remote_backup, ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', wordpress_path='~/public_html/mysite', remote_backup_path=expanded_remote_backup_path, - backup_compressor='bzip2', + backup_compressor='bzip2 -v', full_backup=False, stdout=1, stderr=2) create_dir_structure.assert_called_once_with( @@ -305,6 +305,24 @@ def test_back_up_purge_oldest(create_dir_structure, create_remote_backup, max_local_backups=3) +@patch('swb.local.upload_local_backup') +@patch('swb.local.restore_remote_backup') +def test_restore(restore_remote_backup, upload_local_backup): + """should run correct restore procedure""" + config = configparser.RawConfigParser() + config.read('tests/files/config.ini') + swb.restore( + config, local_backup_path='a/b/c.tar.bz2', + stdout=1, stderr=2) + expanded_remote_backup_path = strftime( + config.get('paths', 'remote_backup')) + upload_local_backup.assert_called_once_with( + ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', + local_backup_path='a/b/c.tar.bz2', + remote_backup_path=expanded_remote_backup_path, + backup_compressor='bzip2 -v', full_backup=False, stdout=1, stderr=2) + + @patch('swb.local.parse_config') @patch('swb.local.back_up') @patch('sys.argv', [swb.__file__, 'tests/files/config.ini']) From 863342a82e50f4d63c4c2e617491bd55c1c2e646 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 10:33:30 -0800 Subject: [PATCH 28/39] Remove test fixtures --- tests/fixtures/__init__.py | 0 tests/fixtures/local.py | 50 -------------------------- tests/fixtures/remote.py | 72 -------------------------------------- tests/test_remote.py | 1 - 4 files changed, 123 deletions(-) delete mode 100644 tests/fixtures/__init__.py delete mode 100644 tests/fixtures/local.py delete mode 100644 tests/fixtures/remote.py diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/fixtures/local.py b/tests/fixtures/local.py deleted file mode 100644 index c6c0dcb..0000000 --- a/tests/fixtures/local.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -from mock import Mock, patch - - -mock_backups = [ - '~/Backups/2011/02/03/mysite.sql.bz2', - '~/Backups/2012/03/04/mysite.sql.bz2', - '~/Backups/2013/04/05/mysite.sql.bz2', - '~/Backups/2014/05/06/mysite.sql.bz2', - '~/Backups/2015/06/07/mysite.sql.bz2' -] - - -def mock_stat(path): - if path in mock_backups: - return Mock(st_mtime=mock_backups.index(path)) - - -patch_makedirs = patch('os.makedirs') -patch_remove = patch('os.remove') -patch_rmdir = patch('os.rmdir') -patch_stat = patch('os.stat', mock_stat) -patch_iglob = patch('glob.iglob', - return_value=mock_backups) -patch_input = patch('swb.local.input', create=True) -patch_popen = patch('subprocess.Popen', - return_value=Mock(returncode=0)) - - -def set_up(): - patch_makedirs.start() - patch_remove.start() - patch_rmdir.start() - patch_stat.start() - patch_iglob.start() - patch_input.start() - patch_popen.start() - - -def tear_down(): - patch_makedirs.stop() - patch_remove.stop() - patch_rmdir.stop() - patch_stat.stop() - patch_iglob.stop() - patch_input.stop() - subprocess.Popen.reset_mock() - patch_popen.stop() diff --git a/tests/fixtures/remote.py b/tests/fixtures/remote.py deleted file mode 100644 index c2e9432..0000000 --- a/tests/fixtures/remote.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 - -import os -import os.path -import shutil -import subprocess -import tempfile -import swb.remote as swb -from mock import Mock, mock_open, patch - - -TEMP_DIR = os.path.join(tempfile.gettempdir(), 'swb-remote') -WP_PATH = 'tests/files/mysite' -BACKUP_COMPRESSOR = 'bzip2 -v' -BACKUP_DECOMPRESSOR = 'bzip2 -d' - - -def run_back_up(backup_path, full_backup, - wp_path=WP_PATH, backup_compressor=BACKUP_COMPRESSOR): - swb.back_up(wp_path, backup_compressor, backup_path, full_backup) - - -def run_restore(backup_path, full_backup, - wp_path=WP_PATH, backup_decompressor=BACKUP_DECOMPRESSOR): - swb.restore(wp_path, backup_path, backup_decompressor) - - -WP_CONFIG_PATH = 'tests/files/mysite/wp-config.php' -with open(WP_CONFIG_PATH, 'r') as wp_config: - WP_CONFIG_CONTENTS = wp_config.read() - - -patch_makedirs = patch('os.makedirs') -patch_remove = patch('os.remove') -patch_rmdir = patch('os.rmdir') -patch_getsize = patch('os.path.getsize', return_value=54321) -mock_communicate = Mock(return_value=[b'db contents', b'no errors']) -patch_popen = patch('subprocess.Popen', - return_value=Mock( - returncode=0, communicate=mock_communicate)) -patch_open = None - - -def set_up(): - global patch_open - try: - os.makedirs(TEMP_DIR) - except OSError: - pass - patch_makedirs.start() - patch_remove.start() - patch_rmdir.start() - patch_getsize.start() - patch_popen.start() - patch_open = patch('swb.remote.open', - mock_open(read_data=WP_CONFIG_CONTENTS), create=True) - patch_open.start() - - -def tear_down(): - global patch_open - patch_makedirs.stop() - patch_remove.stop() - patch_rmdir.stop() - patch_getsize.stop() - subprocess.Popen.reset_mock() - patch_popen.stop() - patch_open.stop() - try: - shutil.rmtree(TEMP_DIR) - except OSError: - pass diff --git a/tests/test_remote.py b/tests/test_remote.py index a00d080..cc66521 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -3,4 +3,3 @@ import nose.tools as nose import swb.remote as swb from mock import patch -from tests.fixtures.remote import set_up, tear_down From c31158f4c20e9793e33595a9502af20ebccc97fa Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 11:32:39 -0800 Subject: [PATCH 29/39] Begin writing remote module tests --- tests/files/mysite/wp-config.php | 6 +-- tests/test_remote.py | 73 +++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/tests/files/mysite/wp-config.php b/tests/files/mysite/wp-config.php index 870a899..b579b3c 100644 --- a/tests/files/mysite/wp-config.php +++ b/tests/files/mysite/wp-config.php @@ -1,8 +1,8 @@ Date: Fri, 8 Jan 2016 11:52:36 -0800 Subject: [PATCH 30/39] Add db tests and verify_backup_integrity tests --- swb/remote.py | 4 ++-- tests/test_remote.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index cbdf0d0..db1d33c 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -79,7 +79,7 @@ def dump_compressed_db(db_name, db_host, db_user, db_password, # Dump MySQL database to uncompressed file at the given path -def dump_uncompressed_db(db_name, db_host, db_user, db_password): +def get_uncompressed_db(db_name, db_host, db_user, db_password): mysqldump = get_mysqldump(db_name, db_host, db_user, db_password) db_contents = mysqldump.communicate()[0] @@ -152,7 +152,7 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): if full_backup == 'True': # backup_path is assumed to refer to entire site directory backup - db_contents = dump_uncompressed_db( + db_contents = get_uncompressed_db( db_info['name'], db_info['host'], db_info['user'], db_info['password']) create_full_backup( diff --git a/tests/test_remote.py b/tests/test_remote.py index 36532b7..7df9352 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -74,3 +74,36 @@ def test_dump_compressed_db(builtin_open): nose.assert_equal( manager.mock_calls[3], call.popen().wait()) + + +@patch('swb.remote.get_mysqldump', return_value=Mock(communicate=Mock( + return_value=[b'db output', b'db error']))) +def test_get_uncompressed_db(get_mysqldump): + """should return uncompressed database to designated location on remote""" + db_contents = swb.get_uncompressed_db( + db_name='mydb', db_host='myhost', + db_user='myname', db_password='mypassword') + nose.assert_equal(db_contents, b'db output') + nose.assert_list_equal(get_mysqldump.return_value.mock_calls, [ + call.communicate(), call.wait()]) + + +@patch('os.path.getsize', return_value=20480) +def test_verify_backup_integrity_valid(getsize): + """should validate a given valid backup file""" + swb.verify_backup_integrity('a/b c/d') + getsize.assert_called_once_with('a/b c/d') + + +@patch('os.path.getsize', return_value=20) +def test_verify_backup_integrity_invalid(getsize): + """should invalidate a given corrupted backup file""" + with nose.assert_raises(OSError): + swb.verify_backup_integrity('a/b c/d') + + +@patch('os.remove') +def test_purge_downloaded_backup(remove): + """should purge the downloaded backup file by removing it""" + swb.purge_downloaded_backup('a/b c/d') + remove.assert_called_once_with('a/b c/d') From d69084dbeda1acd854dff015001f8badac117ff8 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 12:27:07 -0800 Subject: [PATCH 31/39] Write tar contents to stdout rather than file --- swb/remote.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index db1d33c..7e5a96e 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -92,6 +92,7 @@ def get_uncompressed_db(db_name, db_host, db_user, db_password): def verify_backup_integrity(backup_path): if os.path.getsize(backup_path) < 1024: + os.remove(backup_path) raise OSError('Backup is corrupted (too small). Aborting.') @@ -102,11 +103,15 @@ def purge_downloaded_backup(backup_path): # Compressed an existing tar backup file using the chosen compressor -def compress_tar(tar_path, backup_path, backup_compressor): +def compress_tar(tar_out, backup_path, backup_compressor): - compressor = subprocess.Popen( - shlex.split(backup_compressor) + [backup_path, tar_path]) - compressor.wait() + with open(backup_path, 'w') as backup_file: + + compressor = subprocess.Popen( + shlex.split(backup_compressor), + stdin=subprocess.PIPE, stdout=backup_file) + compressor.communicate(input=tar_out.getvalue()) + compressor.wait() # Add a database to the given tar file under the given name @@ -125,21 +130,15 @@ def add_db_to_tar(tar_file, db_file_name, db_contents): def create_full_backup(wordpress_path, db_contents, backup_path, backup_compressor): - backup_name = os.path.basename(backup_path) - tar_name = os.path.splitext(backup_name)[0] - wordpress_site_name = os.path.basename(wordpress_path) db_file_name = '{}.sql'.format(wordpress_site_name) - backup_pwd_path = os.path.dirname(backup_path) - tar_path = os.path.join(backup_pwd_path, tar_name) - - tar_file = tarfile.open(tar_path, 'w') - tar_file.add(wordpress_path, arcname=wordpress_site_name) - add_db_to_tar(tar_file, db_file_name, db_contents) - tar_file.close() + tar_out = io.BytesIO() + with tarfile.open(fileobj=tar_out, mode='w') as tar_file: - compress_tar(tar_path, backup_path, backup_compressor) + tar_file.add(wordpress_path, arcname=wordpress_site_name) + add_db_to_tar(tar_file, db_file_name, db_contents) + compress_tar(tar_out, backup_path, backup_compressor) # Back up WordPress database or installation From 4f6f9237b5e543e151bd25c9d30185029bac86d0 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 12:29:22 -0800 Subject: [PATCH 32/39] Set default value for backup.full_backup option --- swb/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swb/local.py b/swb/local.py index 7f507f5..066f79a 100755 --- a/swb/local.py +++ b/swb/local.py @@ -240,7 +240,7 @@ def back_up(config, *, stdout=None, stderr=None): wordpress_path=config.get('paths', 'wordpress'), remote_backup_path=expanded_remote_backup_path, backup_compressor=config.get('backup', 'compressor'), - full_backup=config.getboolean('backup', 'full_backup'), + full_backup=config.getboolean('backup', 'full_backup', fallback=False), stdout=stdout, stderr=stderr) create_dir_structure(local_backup_path=expanded_local_backup_path) From 4a3e0472566398fd1841b7dc741c3ab9f94c295b Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 14:03:34 -0800 Subject: [PATCH 33/39] Add tarfile tests; add specs to appropriate mocks --- tests/test_local.py | 13 ++++++++----- tests/test_remote.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index dd7d2d2..07755d5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -3,10 +3,11 @@ import configparser import os import os.path +import subprocess import nose.tools as nose import swb.local as swb from time import strftime -from mock import ANY, call, NonCallableMock, patch +from mock import ANY, call, patch def test_parse_config(): @@ -61,10 +62,11 @@ def test_quote_arg_py32(shlex): nose.assert_equal(quoted_arg, '\'a/b c/d\'') -@patch('subprocess.Popen', return_value=NonCallableMock(returncode=0)) +@patch('subprocess.Popen', spec=subprocess.Popen) @patch('builtins.open') def test_exec_on_remote(builtin_open, popen): """should execute script on remote server""" + popen.return_value.returncode = 0 swb.exec_on_remote( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', action='back-up', @@ -79,10 +81,11 @@ def test_exec_on_remote(builtin_open, popen): @patch('sys.exit') -@patch('subprocess.Popen', return_value=NonCallableMock(returncode=3)) +@patch('subprocess.Popen', spec=subprocess.Popen) @patch('builtins.open') def test_exec_on_remote_nonzero_return(builtin_open, popen, exit): """should exit script if nonzero status code is returned""" + popen.return_value.returncode = 3 swb.exec_on_remote( ssh_user='a', ssh_hostname='b.com', ssh_port='2222', action='c', action_args=['d', 'e', 'f', 'g'], @@ -90,7 +93,7 @@ def test_exec_on_remote_nonzero_return(builtin_open, popen, exit): exit.assert_called_once_with(3) -@patch('subprocess.Popen') +@patch('subprocess.Popen', spec=subprocess.Popen) def test_transfer_file_download(popen): """should download backup from remote server when backing up""" swb.transfer_file( @@ -103,7 +106,7 @@ def test_transfer_file_download(popen): popen.return_value.wait.assert_called_once_with() -@patch('subprocess.Popen') +@patch('subprocess.Popen', spec=subprocess.Popen) def test_transfer_file_upload(popen): """should upload backup to remote server when restoring""" swb.transfer_file( diff --git a/tests/test_remote.py b/tests/test_remote.py index 7df9352..c8d35f6 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import io import os import os.path import subprocess +import tarfile import nose.tools as nose import swb.remote as swb from mock import call, Mock, patch @@ -39,7 +41,7 @@ def test_get_db_info(read_wp_config): nose.assert_equal(db_info['collate'], '') -@patch('subprocess.Popen') +@patch('subprocess.Popen', spec=subprocess.Popen) def test_get_mysqldump(popen): """should retrieve correct mysqldump subprocess object""" mysqldump = swb.get_mysqldump( @@ -49,6 +51,7 @@ def test_get_mysqldump(popen): 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword', '--add-drop-table'], stdout=subprocess.PIPE) nose.assert_equal(mysqldump, popen.return_value) + popen.return_value.wait.assert_not_called() @patch('builtins.open') @@ -107,3 +110,31 @@ def test_purge_downloaded_backup(remove): """should purge the downloaded backup file by removing it""" swb.purge_downloaded_backup('a/b c/d') remove.assert_called_once_with('a/b c/d') + + +@patch('subprocess.Popen', spec=subprocess.Popen) +@patch('builtins.open') +def test_compress_tar(builtin_open, popen): + """should write a compressed tar file with the given contents""" + tar_out = io.BytesIO() + tar_out.write(b'tar contents') + swb.compress_tar(tar_out, 'a/b c/d', 'bzip2 -v') + popen.assert_called_once_with( + ['bzip2', '-v'], + stdin=subprocess.PIPE, + stdout=builtin_open.return_value.__enter__()) + nose.assert_list_equal(popen.return_value.mock_calls, [ + call.communicate(input=b'tar contents'), call.wait()]) + + +@patch('tarfile.TarInfo', spec=tarfile.TarInfo) +def test_add_db_to_tar(tarinfo): + """should add database contents to tar file under the given name""" + tar_file = Mock(spec=tarfile.TarFile) + swb.add_db_to_tar(tar_file, 'mysite.sql', b'db contents') + nose.assert_equal(tarinfo.return_value.size, 11) + nose.assert_equal(tar_file.addfile.call_count, 1) + nose.assert_equal( + tar_file.addfile.call_args_list[0][0][0], tarinfo.return_value) + nose.assert_equal( + tar_file.addfile.call_args_list[0][0][1].getvalue(), b'db contents') From b898586f53f4b569748f2a811c586decfb71942d Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 14:36:29 -0800 Subject: [PATCH 34/39] Finish tests for backup procedure of remote module --- swb/remote.py | 18 +++++++-------- tests/test_remote.py | 54 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index 7e5a96e..b3a703d 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -150,22 +150,22 @@ def back_up(wordpress_path, backup_compressor, backup_path, full_backup): if full_backup == 'True': - # backup_path is assumed to refer to entire site directory backup db_contents = get_uncompressed_db( - db_info['name'], db_info['host'], - db_info['user'], db_info['password']) + db_name=db_info['name'], db_host=db_info['host'], + db_user=db_info['user'], db_password=db_info['password']) + # backup_path is assumed to refer to entire site directory backup create_full_backup( - wordpress_path, db_contents, - backup_path, backup_compressor) + wordpress_path=wordpress_path, db_contents=db_contents, + backup_path=backup_path, backup_compressor=backup_compressor) else: # backup_path is assumed to refer to SQL database file backup - db_info = get_db_info(wordpress_path) dump_compressed_db( - db_info['name'], db_info['host'], - db_info['user'], db_info['password'], - backup_compressor, backup_path) + db_name=db_info['name'], db_host=db_info['host'], + db_user=db_info['user'], db_password=db_info['password'], + backup_compressor=backup_compressor, + backup_path=backup_path) verify_backup_integrity(backup_path) diff --git a/tests/test_remote.py b/tests/test_remote.py index c8d35f6..87639d5 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -138,3 +138,57 @@ def test_add_db_to_tar(tarinfo): tar_file.addfile.call_args_list[0][0][0], tarinfo.return_value) nose.assert_equal( tar_file.addfile.call_args_list[0][0][1].getvalue(), b'db contents') + + +@patch('swb.remote.compress_tar') +@patch('swb.remote.add_db_to_tar') +@patch('tarfile.open', spec=tarfile.TarFile) +@patch('io.BytesIO', spec=io.BytesIO) +def test_create_full_backup(bytesio, tarfile_open, + add_db_to_tar, compress_tar): + """should create compressed full backup""" + swb.create_full_backup( + wordpress_path='path/to/my site', + db_contents=b'db contents', + backup_path='my backup.tar.bz2', + backup_compressor='bzip2 -v') + tar_file_obj = tarfile.open.return_value.__enter__() + tarfile.open.assert_called_once_with( + fileobj=bytesio.return_value, mode='w') + tar_file_obj.add.assert_called_once_with( + 'path/to/my site', arcname='my site') + add_db_to_tar.assert_called_once_with( + tar_file_obj, 'my site.sql', b'db contents') + + +@patch('swb.remote.verify_backup_integrity') +@patch('swb.remote.get_uncompressed_db', return_value=b'db contents') +@patch('swb.remote.get_db_info', return_value={ + 'name': 'mydb', + 'host': 'myhost', + 'user': 'myname', + 'password': 'mypassword' +}) +@patch('swb.remote.create_full_backup') +@patch('swb.remote.create_dir_structure') +def test_back_up_full(create_dir_structure, create_full_backup, + get_db_info, get_uncompressed_db, + verify_backup_integrity): + """should perform a full backup""" + swb.back_up( + wordpress_path='path/to/my site', + backup_compressor='bzip2 -v', + backup_path='path/to/my backup.tar.bz2', + full_backup='True') + create_dir_structure.assert_called_once_with( + 'path/to/my backup.tar.bz2') + get_uncompressed_db.assert_called_once_with( + db_name='mydb', db_host='myhost', + db_user='myname', db_password='mypassword') + create_full_backup.assert_called_once_with( + wordpress_path='path/to/my site', + db_contents=b'db contents', + backup_path='path/to/my backup.tar.bz2', + backup_compressor='bzip2 -v') + verify_backup_integrity.assert_called_once_with( + 'path/to/my backup.tar.bz2') From e23891acb84fc15df508efa51ef20cf61353bf35 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 15:11:14 -0800 Subject: [PATCH 35/39] Finish writing remote restore tests --- swb/remote.py | 23 +++++++------- tests/test_remote.py | 75 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/swb/remote.py b/swb/remote.py index b3a703d..32ccb65 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -178,12 +178,6 @@ def decompress_backup(backup_path, backup_decompressor): compressor.wait() -# Construct path to decompressed database file from given backup file -def get_db_path(backup_path): - - return os.path.splitext(backup_path)[0] - - # Replace a WordPress database with the database at the given path def replace_db(db_name, db_host, db_user, db_password, db_path): @@ -206,6 +200,9 @@ def purge_restored_backup(backup_path, db_path): try: os.remove(db_path) + except OSError: + pass + try: os.remove(backup_path) except OSError: pass @@ -214,18 +211,22 @@ def purge_restored_backup(backup_path, db_path): # Restore WordPress database using the given remote backup def restore(wordpress_path, backup_path, backup_decompressor): + wordpress_path = os.path.expanduser(wordpress_path) backup_path = os.path.expanduser(backup_path) verify_backup_integrity(backup_path) - decompress_backup(backup_path, backup_decompressor) + decompress_backup( + backup_path=backup_path, + backup_decompressor=backup_decompressor) db_info = get_db_info(wordpress_path) - db_path = get_db_path(backup_path) + db_path = os.path.splitext(backup_path)[0] replace_db( - db_info['name'], db_info['host'], - db_info['user'], db_info['password'], db_path) + db_name=db_info['name'], db_host=db_info['host'], + db_user=db_info['user'], db_password=db_info['password'], + db_path=db_path) - purge_restored_backup(backup_path, db_path) + purge_restored_backup(backup_path=backup_path, db_path=db_path) def main(): diff --git a/tests/test_remote.py b/tests/test_remote.py index 87639d5..eb9927e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -192,3 +192,78 @@ def test_back_up_full(create_dir_structure, create_full_backup, backup_compressor='bzip2 -v') verify_backup_integrity.assert_called_once_with( 'path/to/my backup.tar.bz2') + + +@patch('subprocess.Popen', spec=subprocess.Popen) +def test_decompress_backup(popen): + """should decompress the given backup file using the given decompressor""" + swb.decompress_backup( + backup_path='path/to/my backup.sql.bz2', + backup_decompressor='bzip2 -d') + popen.assert_called_once_with( + ['bzip2', '-d', 'path/to/my backup.sql.bz2']) + popen.return_value.wait.assert_called_once_with() + + +@patch('subprocess.Popen', spec=subprocess.Popen) +@patch('builtins.open') +def test_replace_db(builtin_open, popen): + """should replace the MySQL database when restoring from backup""" + swb.replace_db( + db_name='mydb', db_host='myhost', + db_user='myname', db_password='mypassword', + db_path='path/to/my backup.sql') + builtin_open.assert_called_once_with( + 'path/to/my backup.sql', 'r') + popen.assert_called_once_with( + ['mysql', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword'], + stdin=builtin_open.return_value.__enter__()) + popen.return_value.wait.assert_called_once_with() + + +@patch('os.remove') +def test_purge_restored_backup(remove): + """should purge restored backup and database file after restore""" + swb.purge_restored_backup( + backup_path='path/to/my backup.sql.bz2', + db_path='path/to/my backup.sql') + remove.assert_any_call('path/to/my backup.sql') + remove.assert_any_call('path/to/my backup.sql.bz2') + + +@patch('os.remove', side_effect=OSError) +def test_purge_restored_backup_silent_fail(remove): + """should silently fail if restored database/backup file does not exist""" + swb.purge_restored_backup( + backup_path='path/to/my backup.sql.bz2', + db_path='path/to/my backup.sql') + nose.assert_equal(remove.call_count, 2) + + +@patch('swb.remote.verify_backup_integrity') +@patch('swb.remote.replace_db') +@patch('swb.remote.purge_restored_backup') +@patch('swb.remote.get_db_info', return_value={ + 'name': 'mydb', + 'host': 'myhost', + 'user': 'myname', + 'password': 'mypassword' +}) +@patch('swb.remote.decompress_backup') +def test_restore(decompress_backup, get_db_info, purge_restored_backup, + replace_db, verify_backup_integrity): + """should run restore procedure""" + swb.restore( + wordpress_path='~/path/to/my site', + backup_path='~/path/to/my site.sql.bz2', + backup_decompressor='bzip2 -d') + decompress_backup.assert_called_once_with( + backup_path=os.path.expanduser('~/path/to/my site.sql.bz2'), + backup_decompressor='bzip2 -d') + replace_db.assert_called_once_with( + db_name='mydb', db_host='myhost', + db_user='myname', db_password='mypassword', + db_path=os.path.expanduser('~/path/to/my site.sql')) + purge_restored_backup.assert_called_once_with( + backup_path=os.path.expanduser('~/path/to/my site.sql.bz2'), + db_path=os.path.expanduser('~/path/to/my site.sql')) From 61a97c8d037c85a6593ce17fec0f76f2d2695e03 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 15:25:08 -0800 Subject: [PATCH 36/39] Finish remote tests; bring coverage to 100% --- tests/test_remote.py | 68 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index eb9927e..bd47c42 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -28,6 +28,13 @@ def test_create_dir_structure(makedirs): makedirs.assert_called_once_with('a/b c') +@patch('os.makedirs', side_effect=OSError) +def test_create_dir_structure_silent_fail(makedirs): + """should fail silently if directory structure already exists""" + swb.create_dir_structure('a/b c/d') + makedirs.assert_called_once_with('a/b c') + + @patch('swb.remote.read_wp_config', return_value=WP_CONFIG_CONTENTS) def test_get_db_info(read_wp_config): """should parsedatabase info from wp-config.php""" @@ -99,10 +106,12 @@ def test_verify_backup_integrity_valid(getsize): @patch('os.path.getsize', return_value=20) -def test_verify_backup_integrity_invalid(getsize): +@patch('os.remove') +def test_verify_backup_integrity_invalid(remove, getsize): """should invalidate a given corrupted backup file""" with nose.assert_raises(OSError): swb.verify_backup_integrity('a/b c/d') + remove.assert_called_once_with('a/b c/d') @patch('os.remove') @@ -194,6 +203,36 @@ def test_back_up_full(create_dir_structure, create_full_backup, 'path/to/my backup.tar.bz2') +@patch('swb.remote.verify_backup_integrity') +@patch('swb.remote.dump_compressed_db') +@patch('swb.remote.get_db_info', return_value={ + 'name': 'mydb', + 'host': 'myhost', + 'user': 'myname', + 'password': 'mypassword' +}) +@patch('swb.remote.create_full_backup') +@patch('swb.remote.create_dir_structure') +def test_back_up_db(create_dir_structure, create_full_backup, + get_db_info, dump_compressed_db, + verify_backup_integrity): + """should perform a full backup""" + swb.back_up( + wordpress_path='path/to/my site', + backup_compressor='bzip2 -v', + backup_path='path/to/my backup.sql.bz2', + full_backup='False') + create_dir_structure.assert_called_once_with( + 'path/to/my backup.sql.bz2') + dump_compressed_db.assert_called_once_with( + db_name='mydb', db_host='myhost', + db_user='myname', db_password='mypassword', + backup_path='path/to/my backup.sql.bz2', + backup_compressor='bzip2 -v') + verify_backup_integrity.assert_called_once_with( + 'path/to/my backup.sql.bz2') + + @patch('subprocess.Popen', spec=subprocess.Popen) def test_decompress_backup(popen): """should decompress the given backup file using the given decompressor""" @@ -267,3 +306,30 @@ def test_restore(decompress_backup, get_db_info, purge_restored_backup, purge_restored_backup.assert_called_once_with( backup_path=os.path.expanduser('~/path/to/my site.sql.bz2'), db_path=os.path.expanduser('~/path/to/my site.sql')) + + +@patch('swb.remote.back_up') +@patch('sys.argv', [swb.__file__, 'back-up', 'a', 'b', 'c', 'd']) +@patch('builtins.print') +def test_main_back_up(builtin_print, back_up): + """should run backup procedure by default when remote script is run""" + swb.main() + back_up.assert_called_once_with('a', 'b', 'c', 'd') + + +@patch('swb.remote.restore') +@patch('sys.argv', [swb.__file__, 'restore', 'a', 'b', 'c', 'd']) +@patch('builtins.print') +def test_main_restore(builtin_print, restore): + """should run restore procedure when remote script is run""" + swb.main() + restore.assert_called_once_with('a', 'b', 'c', 'd') + + +@patch('swb.remote.purge_downloaded_backup') +@patch('sys.argv', [swb.__file__, 'purge-backup', 'a', 'b', 'c', 'd']) +@patch('builtins.print') +def test_main_purge_downloaded(builtin_print, purge_downloaded_backup): + """should run purge procedure when remote script is run""" + swb.main() + purge_downloaded_backup.assert_called_once_with('a', 'b', 'c', 'd') From 623ef67d42fc8df81c938bfae94d3bed0166295a Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 16:02:44 -0800 Subject: [PATCH 37/39] Pass full_backup option to remote script --- swb/local.py | 10 ++++++---- tests/test_local.py | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/swb/local.py b/swb/local.py index 066f79a..59d85d1 100755 --- a/swb/local.py +++ b/swb/local.py @@ -151,7 +151,7 @@ def upload_local_backup(ssh_user, ssh_hostname, ssh_port, *, # Restores the local backup after upload to remote def restore_remote_backup(ssh_user, ssh_hostname, ssh_port, *, wordpress_path, remote_backup_path, - backup_decompressor, stdout, stderr): + backup_decompressor, full_backup, stdout, stderr): exec_on_remote( ssh_user=ssh_user, @@ -161,7 +161,8 @@ def restore_remote_backup(ssh_user, ssh_hostname, ssh_port, *, action_args=[ wordpress_path, remote_backup_path, - backup_decompressor + backup_decompressor, + full_backup ], stdout=stdout, stderr=stderr) @@ -279,7 +280,6 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): local_backup_path=local_backup_path, remote_backup_path=expanded_remote_backup_path, backup_compressor=config.get('backup', 'compressor'), - full_backup=config.getboolean('backup', 'full_backup'), stdout=stdout, stderr=stderr) restore_remote_backup( @@ -289,7 +289,9 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): action='restore', wordpress_path=config.get('paths', 'wordpress'), remote_backup_path=config.get('paths', 'remote_backup'), - backup_decompressor=config.get('backup', 'decompressor')) + backup_decompressor=config.get('backup', 'decompressor'), + full_backup=config.getboolean('backup', 'full_backup', fallback=False), + stdout=stdout, stderr=stderr) # Parse command line arguments passed to the local driver diff --git a/tests/test_local.py b/tests/test_local.py index 07755d5..fe0b4f5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -53,13 +53,23 @@ def test_quote_arg(unquote_home_dir): unquote_home_dir.assert_called_once_with('\'a/b c/d\'') +@patch('swb.local.unquote_home_dir', side_effect=lambda x: x) +def test_quote_arg_str(unquote_home_dir): + """should correctly quote arguments passed to the shell""" + quoted_arg = swb.quote_arg('a/b c/d') + nose.assert_equal(quoted_arg, '\'a/b c/d\'') + unquote_home_dir.assert_called_once_with('\'a/b c/d\'') + + +@patch('swb.local.unquote_home_dir', side_effect=lambda x: x) @patch('swb.local.shlex') -def test_quote_arg_py32(shlex): +def test_quote_arg_py32(shlex, unquote_home_dir): """should correctly quote arguments passed to the shell on Python 3.2""" del shlex.quote nose.assert_false(hasattr(shlex, 'quote')) quoted_arg = swb.quote_arg('a/b c/d') nose.assert_equal(quoted_arg, '\'a/b c/d\'') + unquote_home_dir.assert_called_once_with('\'a/b c/d\'') @patch('subprocess.Popen', spec=subprocess.Popen) @@ -165,11 +175,12 @@ def test_restore_remote_backup(exec_on_remote): swb.restore_remote_backup( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', wordpress_path='a/b c/d', remote_backup_path='e/f g/h', - backup_decompressor='bzip2 -v', + backup_decompressor='bzip2 -v', full_backup=False, stdout=1, stderr=2) exec_on_remote.assert_called_once_with( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', - action='restore', action_args=['a/b c/d', 'e/f g/h', 'bzip2 -v'], + action='restore', + action_args=['a/b c/d', 'e/f g/h', 'bzip2 -v', False], stdout=1, stderr=2) @@ -323,7 +334,7 @@ def test_restore(restore_remote_backup, upload_local_backup): ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', local_backup_path='a/b/c.tar.bz2', remote_backup_path=expanded_remote_backup_path, - backup_compressor='bzip2 -v', full_backup=False, stdout=1, stderr=2) + backup_compressor='bzip2 -v', stdout=1, stderr=2) @patch('swb.local.parse_config') From aa413c267dc12c1cf567d91c7b93e2822f9d1654 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 16:29:14 -0800 Subject: [PATCH 38/39] Fix bugs pertaining to function arguments --- swb/local.py | 2 -- swb/remote.py | 2 +- tests/test_local.py | 2 +- tests/test_remote.py | 3 ++- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/swb/local.py b/swb/local.py index 59d85d1..da4975c 100755 --- a/swb/local.py +++ b/swb/local.py @@ -279,14 +279,12 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): ssh_port=config.get('ssh', 'port'), local_backup_path=local_backup_path, remote_backup_path=expanded_remote_backup_path, - backup_compressor=config.get('backup', 'compressor'), stdout=stdout, stderr=stderr) restore_remote_backup( ssh_user=config.get('ssh', 'user'), ssh_hostname=config.get('ssh', 'hostname'), ssh_port=config.get('ssh', 'port'), - action='restore', wordpress_path=config.get('paths', 'wordpress'), remote_backup_path=config.get('paths', 'remote_backup'), backup_decompressor=config.get('backup', 'decompressor'), diff --git a/swb/remote.py b/swb/remote.py index 32ccb65..38f6aff 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -209,7 +209,7 @@ def purge_restored_backup(backup_path, db_path): # Restore WordPress database using the given remote backup -def restore(wordpress_path, backup_path, backup_decompressor): +def restore(wordpress_path, backup_path, backup_decompressor, full_backup): wordpress_path = os.path.expanduser(wordpress_path) backup_path = os.path.expanduser(backup_path) diff --git a/tests/test_local.py b/tests/test_local.py index fe0b4f5..51c013d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -334,7 +334,7 @@ def test_restore(restore_remote_backup, upload_local_backup): ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', local_backup_path='a/b/c.tar.bz2', remote_backup_path=expanded_remote_backup_path, - backup_compressor='bzip2 -v', stdout=1, stderr=2) + stdout=1, stderr=2) @patch('swb.local.parse_config') diff --git a/tests/test_remote.py b/tests/test_remote.py index bd47c42..27ac033 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -295,7 +295,8 @@ def test_restore(decompress_backup, get_db_info, purge_restored_backup, swb.restore( wordpress_path='~/path/to/my site', backup_path='~/path/to/my site.sql.bz2', - backup_decompressor='bzip2 -d') + backup_decompressor='bzip2 -d', + full_backup='False') decompress_backup.assert_called_once_with( backup_path=os.path.expanduser('~/path/to/my site.sql.bz2'), backup_decompressor='bzip2 -d') From 96ed0a9d3a187c9380ea3a9e4093d45079f1e579 Mon Sep 17 00:00:00 2001 From: Caleb Evans Date: Fri, 8 Jan 2016 21:15:16 -0800 Subject: [PATCH 39/39] Remove full backup components (stick to db only) --- swb/local.py | 12 ++-- swb/remote.py | 93 +++---------------------- tests/test_local.py | 13 ++-- tests/test_remote.py | 159 ++++++------------------------------------- 4 files changed, 41 insertions(+), 236 deletions(-) diff --git a/swb/local.py b/swb/local.py index da4975c..99f122d 100755 --- a/swb/local.py +++ b/swb/local.py @@ -102,7 +102,7 @@ def transfer_file(ssh_user, ssh_hostname, ssh_port, *, # Execute remote backup script to create remote backup def create_remote_backup(ssh_user, ssh_hostname, ssh_port, *, wordpress_path, remote_backup_path, - backup_compressor, full_backup, stdout, stderr): + backup_compressor, stdout, stderr): exec_on_remote( ssh_user=ssh_user, @@ -112,8 +112,7 @@ def create_remote_backup(ssh_user, ssh_hostname, ssh_port, *, action_args=[ wordpress_path, backup_compressor, - remote_backup_path, - full_backup + remote_backup_path ], stdout=stdout, stderr=stderr) @@ -151,7 +150,7 @@ def upload_local_backup(ssh_user, ssh_hostname, ssh_port, *, # Restores the local backup after upload to remote def restore_remote_backup(ssh_user, ssh_hostname, ssh_port, *, wordpress_path, remote_backup_path, - backup_decompressor, full_backup, stdout, stderr): + backup_decompressor, stdout, stderr): exec_on_remote( ssh_user=ssh_user, @@ -161,8 +160,7 @@ def restore_remote_backup(ssh_user, ssh_hostname, ssh_port, *, action_args=[ wordpress_path, remote_backup_path, - backup_decompressor, - full_backup + backup_decompressor ], stdout=stdout, stderr=stderr) @@ -241,7 +239,6 @@ def back_up(config, *, stdout=None, stderr=None): wordpress_path=config.get('paths', 'wordpress'), remote_backup_path=expanded_remote_backup_path, backup_compressor=config.get('backup', 'compressor'), - full_backup=config.getboolean('backup', 'full_backup', fallback=False), stdout=stdout, stderr=stderr) create_dir_structure(local_backup_path=expanded_local_backup_path) @@ -288,7 +285,6 @@ def restore(config, *, local_backup_path, stdout=None, stderr=None): wordpress_path=config.get('paths', 'wordpress'), remote_backup_path=config.get('paths', 'remote_backup'), backup_decompressor=config.get('backup', 'decompressor'), - full_backup=config.getboolean('backup', 'full_backup', fallback=False), stdout=stdout, stderr=stderr) diff --git a/swb/remote.py b/swb/remote.py index 38f6aff..28824c7 100755 --- a/swb/remote.py +++ b/swb/remote.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -import io import os import os.path import re import shlex import subprocess import sys -import tarfile # Read contents of wp-config.php for a WordPress installation @@ -45,8 +43,9 @@ def get_db_info(wordpress_path): return db_info -# Dump MySQL database to file and return subprocess -def get_mysqldump(db_name, db_host, db_user, db_password): +# Dump MySQL database to compressed file +def dump_compressed_db(db_name, db_host, db_user, db_password, + backup_compressor, backup_path): mysqldump = subprocess.Popen([ 'mysqldump', @@ -57,15 +56,6 @@ def get_mysqldump(db_name, db_host, db_user, db_password): '--add-drop-table' ], stdout=subprocess.PIPE) - return mysqldump - - -# Dump MySQL database to compressed file -def dump_compressed_db(db_name, db_host, db_user, db_password, - backup_compressor, backup_path): - - mysqldump = get_mysqldump(db_name, db_host, db_user, db_password) - # Create remote backup so as to write output of dump/compress to file with open(backup_path, 'w') as backup_file: @@ -78,16 +68,6 @@ def dump_compressed_db(db_name, db_host, db_user, db_password, compressor.wait() -# Dump MySQL database to uncompressed file at the given path -def get_uncompressed_db(db_name, db_host, db_user, db_password): - - mysqldump = get_mysqldump(db_name, db_host, db_user, db_password) - db_contents = mysqldump.communicate()[0] - mysqldump.wait() - - return db_contents - - # Verify integrity of remote backup by checking its size def verify_backup_integrity(backup_path): @@ -102,70 +82,19 @@ def purge_downloaded_backup(backup_path): os.remove(backup_path) -# Compressed an existing tar backup file using the chosen compressor -def compress_tar(tar_out, backup_path, backup_compressor): - - with open(backup_path, 'w') as backup_file: - - compressor = subprocess.Popen( - shlex.split(backup_compressor), - stdin=subprocess.PIPE, stdout=backup_file) - compressor.communicate(input=tar_out.getvalue()) - compressor.wait() - - -# Add a database to the given tar file under the given name -def add_db_to_tar(tar_file, db_file_name, db_contents): - - db_file_obj = io.BytesIO() - db_file_obj.write(db_contents) - db_tar_info = tarfile.TarInfo(db_file_name) - db_tar_info.size = len(db_file_obj.getvalue()) - db_file_obj.seek(0) - tar_file.addfile(db_tar_info, db_file_obj) - - -# Create the full backup file by tar'ing both the wordpress site directory and -# the the dumped database contents -def create_full_backup(wordpress_path, db_contents, - backup_path, backup_compressor): - - wordpress_site_name = os.path.basename(wordpress_path) - db_file_name = '{}.sql'.format(wordpress_site_name) - - tar_out = io.BytesIO() - with tarfile.open(fileobj=tar_out, mode='w') as tar_file: - - tar_file.add(wordpress_path, arcname=wordpress_site_name) - add_db_to_tar(tar_file, db_file_name, db_contents) - compress_tar(tar_out, backup_path, backup_compressor) - - # Back up WordPress database or installation -def back_up(wordpress_path, backup_compressor, backup_path, full_backup): +def back_up(wordpress_path, backup_compressor, backup_path): backup_path = os.path.expanduser(backup_path) create_dir_structure(backup_path) db_info = get_db_info(wordpress_path) - if full_backup == 'True': - - db_contents = get_uncompressed_db( - db_name=db_info['name'], db_host=db_info['host'], - db_user=db_info['user'], db_password=db_info['password']) - # backup_path is assumed to refer to entire site directory backup - create_full_backup( - wordpress_path=wordpress_path, db_contents=db_contents, - backup_path=backup_path, backup_compressor=backup_compressor) - - else: - - # backup_path is assumed to refer to SQL database file backup - dump_compressed_db( - db_name=db_info['name'], db_host=db_info['host'], - db_user=db_info['user'], db_password=db_info['password'], - backup_compressor=backup_compressor, - backup_path=backup_path) + # backup_path is assumed to refer to SQL database file backup + dump_compressed_db( + db_name=db_info['name'], db_host=db_info['host'], + db_user=db_info['user'], db_password=db_info['password'], + backup_compressor=backup_compressor, + backup_path=backup_path) verify_backup_integrity(backup_path) @@ -209,7 +138,7 @@ def purge_restored_backup(backup_path, db_path): # Restore WordPress database using the given remote backup -def restore(wordpress_path, backup_path, backup_decompressor, full_backup): +def restore(wordpress_path, backup_path, backup_decompressor): wordpress_path = os.path.expanduser(wordpress_path) backup_path = os.path.expanduser(backup_path) diff --git a/tests/test_local.py b/tests/test_local.py index 51c013d..5cdffa9 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -80,12 +80,12 @@ def test_exec_on_remote(builtin_open, popen): swb.exec_on_remote( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', action='back-up', - action_args=['~/public_html/mysite', 'bzip2 -v', 'a/b c/d', 'False'], + action_args=['~/public_html/mysite', 'bzip2 -v', 'a/b c/d'], stdout=1, stderr=2) popen.assert_called_once_with([ 'ssh', '-p 2222', 'myname@mysite.com', 'python3', '-', 'back-up', '~/\'public_html/mysite\'', - '\'bzip2 -v\'', '\'a/b c/d\'', 'False'], + '\'bzip2 -v\'', '\'a/b c/d\''], stdin=builtin_open.return_value.__enter__(), stdout=1, stderr=2) popen.return_value.wait.assert_called_once_with() @@ -135,11 +135,11 @@ def test_create_remote_backup(exec_on_remote): swb.create_remote_backup( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', wordpress_path='a/b c/d', remote_backup_path='e/f g/h', - backup_compressor='bzip2 -v', full_backup=True, + backup_compressor='bzip2 -v', stdout=1, stderr=2) exec_on_remote.assert_called_once_with( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', - action='back-up', action_args=['a/b c/d', 'bzip2 -v', 'e/f g/h', True], + action='back-up', action_args=['a/b c/d', 'bzip2 -v', 'e/f g/h'], stdout=1, stderr=2) @@ -175,12 +175,12 @@ def test_restore_remote_backup(exec_on_remote): swb.restore_remote_backup( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', wordpress_path='a/b c/d', remote_backup_path='e/f g/h', - backup_decompressor='bzip2 -v', full_backup=False, + backup_decompressor='bzip2 -v', stdout=1, stderr=2) exec_on_remote.assert_called_once_with( ssh_user='myname', ssh_hostname='mysite.com', ssh_port='2222', action='restore', - action_args=['a/b c/d', 'e/f g/h', 'bzip2 -v', False], + action_args=['a/b c/d', 'e/f g/h', 'bzip2 -v'], stdout=1, stderr=2) @@ -285,7 +285,6 @@ def test_back_up(create_dir_structure, create_remote_backup, wordpress_path='~/public_html/mysite', remote_backup_path=expanded_remote_backup_path, backup_compressor='bzip2 -v', - full_backup=False, stdout=1, stderr=2) create_dir_structure.assert_called_once_with( local_backup_path=expanded_local_backup_path) diff --git a/tests/test_remote.py b/tests/test_remote.py index 27ac033..abf910e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -import io import os import os.path import subprocess -import tarfile import nose.tools as nose import swb.remote as swb -from mock import call, Mock, patch +from mock import patch WP_PATH = 'tests/files/mysite' @@ -48,54 +46,23 @@ def test_get_db_info(read_wp_config): nose.assert_equal(db_info['collate'], '') -@patch('subprocess.Popen', spec=subprocess.Popen) -def test_get_mysqldump(popen): - """should retrieve correct mysqldump subprocess object""" - mysqldump = swb.get_mysqldump( - db_name='mydb', db_host='myhost', - db_user='myname', db_password='mypassword') - popen.assert_called_once_with([ - 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword', - '--add-drop-table'], stdout=subprocess.PIPE) - nose.assert_equal(mysqldump, popen.return_value) - popen.return_value.wait.assert_not_called() - - +@patch('subprocess.Popen') @patch('builtins.open') -def test_dump_compressed_db(builtin_open): +def test_dump_compressed_db(builtin_open, popen): """should dump compressed database to designated location on remote""" - manager = Mock() - with patch('subprocess.Popen', manager.popen): - with patch('swb.remote.get_mysqldump', manager.get_mysqldump): - swb.dump_compressed_db( - db_name='mydb', db_host='myhost', - db_user='myname', db_password='mypassword', - backup_compressor='bzip2 -v', backup_path='a/b c/d') - manager.get_mysqldump.assert_called_once_with( - 'mydb', 'myhost', 'myname', 'mypassword',) - builtin_open.assert_called_once_with('a/b c/d', 'w') - manager.popen.assert_called_once_with( - ['bzip2', '-v'], - stdin=manager.get_mysqldump.return_value.stdout, - stdout=builtin_open.return_value.__enter__()) - nose.assert_equal( - manager.mock_calls[2], - call.get_mysqldump().wait()) - nose.assert_equal( - manager.mock_calls[3], - call.popen().wait()) - - -@patch('swb.remote.get_mysqldump', return_value=Mock(communicate=Mock( - return_value=[b'db output', b'db error']))) -def test_get_uncompressed_db(get_mysqldump): - """should return uncompressed database to designated location on remote""" - db_contents = swb.get_uncompressed_db( + swb.dump_compressed_db( db_name='mydb', db_host='myhost', - db_user='myname', db_password='mypassword') - nose.assert_equal(db_contents, b'db output') - nose.assert_list_equal(get_mysqldump.return_value.mock_calls, [ - call.communicate(), call.wait()]) + db_user='myname', db_password='mypassword', + backup_compressor='bzip2 -v', backup_path='a/b c/d') + popen.assert_any_call([ + 'mysqldump', 'mydb', '-h', 'myhost', '-u', 'myname', '-pmypassword', + '--add-drop-table'], stdout=subprocess.PIPE) + builtin_open.assert_called_once_with('a/b c/d', 'w') + popen.assert_any_call( + ['bzip2', '-v'], + stdin=popen.return_value.stdout, + stdout=builtin_open.return_value.__enter__()) + nose.assert_equal(popen.return_value.wait.call_count, 2) @patch('os.path.getsize', return_value=20480) @@ -121,88 +88,6 @@ def test_purge_downloaded_backup(remove): remove.assert_called_once_with('a/b c/d') -@patch('subprocess.Popen', spec=subprocess.Popen) -@patch('builtins.open') -def test_compress_tar(builtin_open, popen): - """should write a compressed tar file with the given contents""" - tar_out = io.BytesIO() - tar_out.write(b'tar contents') - swb.compress_tar(tar_out, 'a/b c/d', 'bzip2 -v') - popen.assert_called_once_with( - ['bzip2', '-v'], - stdin=subprocess.PIPE, - stdout=builtin_open.return_value.__enter__()) - nose.assert_list_equal(popen.return_value.mock_calls, [ - call.communicate(input=b'tar contents'), call.wait()]) - - -@patch('tarfile.TarInfo', spec=tarfile.TarInfo) -def test_add_db_to_tar(tarinfo): - """should add database contents to tar file under the given name""" - tar_file = Mock(spec=tarfile.TarFile) - swb.add_db_to_tar(tar_file, 'mysite.sql', b'db contents') - nose.assert_equal(tarinfo.return_value.size, 11) - nose.assert_equal(tar_file.addfile.call_count, 1) - nose.assert_equal( - tar_file.addfile.call_args_list[0][0][0], tarinfo.return_value) - nose.assert_equal( - tar_file.addfile.call_args_list[0][0][1].getvalue(), b'db contents') - - -@patch('swb.remote.compress_tar') -@patch('swb.remote.add_db_to_tar') -@patch('tarfile.open', spec=tarfile.TarFile) -@patch('io.BytesIO', spec=io.BytesIO) -def test_create_full_backup(bytesio, tarfile_open, - add_db_to_tar, compress_tar): - """should create compressed full backup""" - swb.create_full_backup( - wordpress_path='path/to/my site', - db_contents=b'db contents', - backup_path='my backup.tar.bz2', - backup_compressor='bzip2 -v') - tar_file_obj = tarfile.open.return_value.__enter__() - tarfile.open.assert_called_once_with( - fileobj=bytesio.return_value, mode='w') - tar_file_obj.add.assert_called_once_with( - 'path/to/my site', arcname='my site') - add_db_to_tar.assert_called_once_with( - tar_file_obj, 'my site.sql', b'db contents') - - -@patch('swb.remote.verify_backup_integrity') -@patch('swb.remote.get_uncompressed_db', return_value=b'db contents') -@patch('swb.remote.get_db_info', return_value={ - 'name': 'mydb', - 'host': 'myhost', - 'user': 'myname', - 'password': 'mypassword' -}) -@patch('swb.remote.create_full_backup') -@patch('swb.remote.create_dir_structure') -def test_back_up_full(create_dir_structure, create_full_backup, - get_db_info, get_uncompressed_db, - verify_backup_integrity): - """should perform a full backup""" - swb.back_up( - wordpress_path='path/to/my site', - backup_compressor='bzip2 -v', - backup_path='path/to/my backup.tar.bz2', - full_backup='True') - create_dir_structure.assert_called_once_with( - 'path/to/my backup.tar.bz2') - get_uncompressed_db.assert_called_once_with( - db_name='mydb', db_host='myhost', - db_user='myname', db_password='mypassword') - create_full_backup.assert_called_once_with( - wordpress_path='path/to/my site', - db_contents=b'db contents', - backup_path='path/to/my backup.tar.bz2', - backup_compressor='bzip2 -v') - verify_backup_integrity.assert_called_once_with( - 'path/to/my backup.tar.bz2') - - @patch('swb.remote.verify_backup_integrity') @patch('swb.remote.dump_compressed_db') @patch('swb.remote.get_db_info', return_value={ @@ -211,17 +96,14 @@ def test_back_up_full(create_dir_structure, create_full_backup, 'user': 'myname', 'password': 'mypassword' }) -@patch('swb.remote.create_full_backup') @patch('swb.remote.create_dir_structure') -def test_back_up_db(create_dir_structure, create_full_backup, - get_db_info, dump_compressed_db, - verify_backup_integrity): - """should perform a full backup""" +def test_back_up_db(create_dir_structure, get_db_info, + dump_compressed_db, verify_backup_integrity): + """should perform a WordPress database backup""" swb.back_up( wordpress_path='path/to/my site', backup_compressor='bzip2 -v', - backup_path='path/to/my backup.sql.bz2', - full_backup='False') + backup_path='path/to/my backup.sql.bz2') create_dir_structure.assert_called_once_with( 'path/to/my backup.sql.bz2') dump_compressed_db.assert_called_once_with( @@ -295,8 +177,7 @@ def test_restore(decompress_backup, get_db_info, purge_restored_backup, swb.restore( wordpress_path='~/path/to/my site', backup_path='~/path/to/my site.sql.bz2', - backup_decompressor='bzip2 -d', - full_backup='False') + backup_decompressor='bzip2 -d') decompress_backup.assert_called_once_with( backup_path=os.path.expanduser('~/path/to/my site.sql.bz2'), backup_decompressor='bzip2 -d')