From 46bad96b0b289f2f1cf31baaaa29bd933766a372 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Sun, 30 Apr 2017 15:31:04 +0200 Subject: [PATCH 1/6] Implement support for --chmod and --chown rsync flags. By default, afl-sync behaves the same as before, running rsync in archive mode (-a) and trasferring local permissions and owner/group. However, --chmod PERMS and --chown USER:GROUP can now be specified, which passes these flags to rsync, enabling the user to specify the permissions, owner and group of the remote files. This is useful when ensuring the remote storage dir is readable and/or writable on the server for all processes that need to access it. The new flags are only supported when pushing and in the push phase of syncing, not pulling. The support for the flags was implemented by the introduction of a new parameter for AflRsync, which represents the options for rsync, both its get and put modes. --- afl_utils/afl_sync.py | 47 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/afl_utils/afl_sync.py b/afl_utils/afl_sync.py index 8057606..87ac630 100644 --- a/afl_utils/afl_sync.py +++ b/afl_utils/afl_sync.py @@ -24,20 +24,20 @@ _rsync_default_options = ['-racz'] - class AflBaseSync(object): - def __init__(self, server_config, fuzzer_config): + def __init__(self, server_config, fuzzer_config, rsync_config): self.server_config = server_config self.fuzzer_config = fuzzer_config + self.rsync_config = rsync_config class AflRsync(AflBaseSync): - def __init__(self, server_config, fuzzer_config): + def __init__(self, server_config, fuzzer_config, rsync_config): # default excludes self.__excludes = ['*.cur_input'] - super(AflRsync, self).__init__(server_config, fuzzer_config) + super(AflRsync, self).__init__(server_config, fuzzer_config, rsync_config) - def __prepare_rsync_commandline(self, local_path, remote_path, rsync_options=list(_rsync_default_options), + def __prepare_rsync_commandline(self, local_path, remote_path, rsync_options, rsync_excludes=list([]), rsync_get=False): cmd = ['rsync'] @@ -66,11 +66,11 @@ def __get_fuzzers(self): fuzzers = (fuzzer for fuzzer in fuzzers if not fuzzer.endswith('.sync')) return fuzzers - def rsync_put(self, local_path, remote_path, rsync_options=list(_rsync_default_options), rsync_excludes=list([])): + def rsync_put(self, local_path, remote_path, rsync_options, rsync_excludes=list([])): cmd = self.__prepare_rsync_commandline(local_path, remote_path, rsync_options, rsync_excludes) return self.__invoke_rsync(cmd) - def rsync_get(self, remote_path, local_path, rsync_options=list(_rsync_default_options), rsync_excludes=list([])): + def rsync_get(self, remote_path, local_path, rsync_options, rsync_excludes=list([])): cmd = self.__prepare_rsync_commandline(local_path, remote_path, rsync_options, rsync_excludes, True) return self.__invoke_rsync(cmd) @@ -93,7 +93,7 @@ def push(self): local_path = os.path.join(self.fuzzer_config['sync_dir'], f) remote_path = os.path.join(self.server_config['remote_path'], f) print_ok('Pushing {} -> {}.sync'.format(local_path, remote_path)) - self.rsync_put(local_path, remote_path, rsync_excludes=excludes) + self.rsync_put(local_path, remote_path, self.rsync_config['put'], rsync_excludes=excludes) def pull(self): fuzzers = self.__get_fuzzers() @@ -101,7 +101,7 @@ def pull(self): local_path = self.fuzzer_config['sync_dir'] remote_path = self.server_config['remote_path'] - options = list(_rsync_default_options) + options = self.rsync_config['get'][:] excludes = self.__excludes # exclude our previously pushed fuzzer states from being pulled again @@ -150,6 +150,12 @@ def main(argv): help='Source afl synchronisation directory containing state directories of afl instances.') parser.add_argument('dst_storage_dir', help='Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!') + parser.add_argument('--chmod', + help='Affect destination\'s file and directory permissions, e.g. --chmod=g+rw to add ' + 'read/write group permissions.', metavar="PERMS") + parser.add_argument('--chown', + help='Affect destination\'s file and directory user and group, e.g. --chmod=foo:bar to ' + 'let the files be owned by user foo and group bar.', metavar="USER:GROUP") parser.add_argument('-S', '--session', dest='session', default=None, help='Name of an afl-multicore session. If provided, only fuzzers belonging to ' 'the specified session will be synced with the destination. Otherwise state ' @@ -163,6 +169,22 @@ def main(argv): print_err('Sorry, unknown command requested!') sys.exit(1) + rsync_put_options = _rsync_default_options[:] + rsync_get_options = _rsync_default_options[:] + + if args.chmod or args.chown: + # these arguments are meaningless with pull since they should only + # affect the remote side + if args.cmd == 'pull': + print_err('Sorry, --chmod and --chown are meaningless with pull.') + sys.exit(1) + + if args.chmod: + rsync_put_options.append("--chmod={}".format(args.chmod)) + + if args.chown: + rsync_put_options.append("--chown={}".format(args.chown)) + if not os.path.exists(args.src_sync_dir): if args.cmd in ['pull', 'sync']: print_warn('Local afl sync dir does not exist! Will create it for you!') @@ -182,7 +204,12 @@ def main(argv): 'exclude_hangs': False, } - rsyncEngine = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': rsync_get_options, + 'put': rsync_put_options, + } + + rsyncEngine = AflRsync(server_config, fuzzer_config, rsync_config) if args.cmd == 'push': rsyncEngine.push() From 239658c3d8e9d68050d122b10cf277775e55d817 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Thu, 4 May 2017 10:35:45 +0200 Subject: [PATCH 2/6] Fix typo in help message. --- afl_utils/afl_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afl_utils/afl_sync.py b/afl_utils/afl_sync.py index 87ac630..97e4152 100644 --- a/afl_utils/afl_sync.py +++ b/afl_utils/afl_sync.py @@ -154,7 +154,7 @@ def main(argv): help='Affect destination\'s file and directory permissions, e.g. --chmod=g+rw to add ' 'read/write group permissions.', metavar="PERMS") parser.add_argument('--chown', - help='Affect destination\'s file and directory user and group, e.g. --chmod=foo:bar to ' + help='Affect destination\'s file and directory user and group, e.g. --chown=foo:bar to ' 'let the files be owned by user foo and group bar.', metavar="USER:GROUP") parser.add_argument('-S', '--session', dest='session', default=None, help='Name of an afl-multicore session. If provided, only fuzzers belonging to ' From 0926bbb2e4a4c434bd40447cb6f62bf4cd1c7394 Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Thu, 4 May 2017 13:11:12 +0200 Subject: [PATCH 3/6] Prevent local globbing when --chown is passed to rsync. rsync seems to internally convert --chown=foo:bar to --usermap=*:foo --groupmap=*:bar. Due to a bug[*], it passes this to a shell at some point without properly escaping it. The shell then globs the wildcard, which is not correct and likely makes the shell complain about nonexistent matches. A workaround is to pass --protect-args to rsync which makes makes it work as intended but may have other consequences. [*] https://bugzilla.samba.org/show_bug.cgi?id=10705 --- afl_utils/afl_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afl_utils/afl_sync.py b/afl_utils/afl_sync.py index 97e4152..2094ea5 100644 --- a/afl_utils/afl_sync.py +++ b/afl_utils/afl_sync.py @@ -183,7 +183,7 @@ def main(argv): rsync_put_options.append("--chmod={}".format(args.chmod)) if args.chown: - rsync_put_options.append("--chown={}".format(args.chown)) + rsync_put_options.append(["--protect-args", "--chown={}".format(args.chown)]) if not os.path.exists(args.src_sync_dir): if args.cmd in ['pull', 'sync']: From b9c661fe772e739c64f763476907a80646b6565c Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Fri, 5 May 2017 20:59:01 +0200 Subject: [PATCH 4/6] Make --chmod and --chown with pull non-fatal. --- afl_utils/afl_sync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/afl_utils/afl_sync.py b/afl_utils/afl_sync.py index 2094ea5..94f1433 100644 --- a/afl_utils/afl_sync.py +++ b/afl_utils/afl_sync.py @@ -176,8 +176,7 @@ def main(argv): # these arguments are meaningless with pull since they should only # affect the remote side if args.cmd == 'pull': - print_err('Sorry, --chmod and --chown are meaningless with pull.') - sys.exit(1) + print_warn('--chmod and --chown have no effect with pull and will be ignored.') if args.chmod: rsync_put_options.append("--chmod={}".format(args.chmod)) From 4da84269b5fb414cd84e691b767ee25f2f1d117e Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Tue, 9 May 2017 13:20:20 +0200 Subject: [PATCH 5/6] Fix tests. --- tests/test_afl_sync.py | 62 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/tests/test_afl_sync.py b/tests/test_afl_sync.py index 746aa1b..f8bda89 100644 --- a/tests/test_afl_sync.py +++ b/tests/test_afl_sync.py @@ -77,13 +77,19 @@ def test_afl_rsync_init(self): 'exclude_hangs': True, } - afl_rsync = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(server_config, fuzzer_config, rsync_config) self.assertDictEqual(server_config, afl_rsync.server_config) self.assertDictEqual(fuzzer_config, afl_rsync.fuzzer_config) + self.assertDictEqual(rsync_config, afl_rsync.rsync_config) def test_afl_rsync_prepare_sync_command(self): - afl_rsync = AflRsync(None, None) + afl_rsync = AflRsync(None, None, None) expected_put_cmdline = [ 'rsync', @@ -102,15 +108,17 @@ def test_afl_rsync_prepare_sync_command(self): ] self.assertListEqual(expected_put_cmdline, afl_rsync._AflRsync__prepare_rsync_commandline('src', 'dst', + list(afl_sync._rsync_default_options), rsync_excludes=[ 'exclude'])) self.assertListEqual(expected_get_cmdline, afl_rsync._AflRsync__prepare_rsync_commandline('src', 'dst', + list(afl_sync._rsync_default_options), rsync_excludes=['exclude'], rsync_get=True)) def test_afl_rsync_invoke_rsync(self): rsync_cmdline = ['rsync', '--help'] - afl_rsync = AflRsync(None, None) + afl_rsync = AflRsync(None, None, None) self.assertTrue(afl_rsync._AflRsync__invoke_rsync(rsync_cmdline)) self.assertFalse(afl_rsync._AflRsync__invoke_rsync(['rsync'])) @@ -130,7 +138,7 @@ def test_afl_rsync_get_fuzzers(self): 'invalid_fuzz001' ] - afl_rsync = AflRsync(None, fuzzer_config) + afl_rsync = AflRsync(None, fuzzer_config, None) self.assertListEqual(sorted(expected_fuzzers), sorted(afl_rsync._AflRsync__get_fuzzers())) def test_afl_rsync_put(self): @@ -138,8 +146,13 @@ def test_afl_rsync_put(self): remote_path = 'testdata/rsync_tmp_store/fuzz000' excludes = ['crashes*/', 'hangs*/'] - afl_rsync = AflRsync(None, None) - self.assertTrue(afl_rsync.rsync_put(local_path, remote_path, rsync_excludes=excludes)) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(None, None, rsync_config) + self.assertTrue(afl_rsync.rsync_put(local_path, remote_path, afl_rsync.rsync_config['put'], rsync_excludes=excludes)) self.assertTrue(os.path.exists(remote_path + '.sync/fuzzer_stats')) self.assertTrue(os.path.exists(remote_path + '.sync/.cur_input')) self.assertFalse(os.path.exists(remote_path + '.sync/crashes')) @@ -150,8 +163,13 @@ def test_afl_rsync_get(self): remote_path = 'testdata/sync/fuzz000' excludes = ['crashes*/', 'hangs*/'] - afl_rsync = AflRsync(None, None) - self.assertTrue(afl_rsync.rsync_get(remote_path, local_path, rsync_excludes=excludes)) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(None, None, rsync_config) + self.assertTrue(afl_rsync.rsync_get(remote_path, local_path, afl_rsync.rsync_config['get'], rsync_excludes=excludes)) self.assertTrue(os.path.exists(local_path + '/fuzzer_stats')) self.assertFalse(os.path.exists(local_path + '/crashes')) self.assertFalse(os.path.exists(local_path + '/hangs')) @@ -168,7 +186,12 @@ def test_afl_rsync_push(self): 'exclude_hangs': True, } - afl_rsync = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(server_config, fuzzer_config, rsync_config) self.assertIsNone(afl_rsync.push()) self.assertTrue(os.path.exists('testdata/rsync_output_push/fuzz000.sync')) self.assertFalse(os.path.exists('testdata/rsync_output_push/fuzz000.sync/.cur_input')) @@ -191,7 +214,12 @@ def test_afl_rsync_pull_session(self): 'exclude_hangs': True, } - afl_rsync = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(server_config, fuzzer_config, rsync_config) self.assertIsNone(afl_rsync.pull()) self.assertTrue(os.path.exists('testdata/sync/other_fuzz000.sync')) self.assertTrue(os.path.exists('testdata/sync/other_fuzz000.sync/crashes')) @@ -214,7 +242,12 @@ def test_afl_rsync_pull_all(self): 'exclude_hangs': True, } - afl_rsync = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(server_config, fuzzer_config, rsync_config) self.assertIsNone(afl_rsync.pull()) self.assertTrue(os.path.exists('testdata/sync/other_fuzz000.sync')) self.assertTrue(os.path.exists('testdata/sync/other_fuzz001.sync')) @@ -236,7 +269,12 @@ def test_afl_rsync_sync(self): 'exclude_hangs': True, } - afl_rsync = AflRsync(server_config, fuzzer_config) + rsync_config = { + 'get': afl_sync._rsync_default_options[:], + 'put': afl_sync._rsync_default_options[:], + } + + afl_rsync = AflRsync(server_config, fuzzer_config, rsync_config) self.assertIsNone(afl_rsync.sync()) # pull assertions From 8da548c1484233e1c549015f397666e3d6d5917c Mon Sep 17 00:00:00 2001 From: Denis Kasak Date: Wed, 17 May 2017 22:11:27 +0200 Subject: [PATCH 6/6] Remove duplicate assertion. --- tests/test_afl_sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_afl_sync.py b/tests/test_afl_sync.py index f8bda89..b7ae2f5 100644 --- a/tests/test_afl_sync.py +++ b/tests/test_afl_sync.py @@ -196,7 +196,6 @@ def test_afl_rsync_push(self): self.assertTrue(os.path.exists('testdata/rsync_output_push/fuzz000.sync')) self.assertFalse(os.path.exists('testdata/rsync_output_push/fuzz000.sync/.cur_input')) self.assertTrue(os.path.exists('testdata/rsync_output_push/fuzz001.sync')) - self.assertFalse(os.path.exists('testdata/rsync_output_push/fuzz000.sync/.cur_input')) self.assertFalse(os.path.exists('testdata/rsync_output_push/fuzz002.sync')) self.assertFalse(os.path.exists('testdata/rsync_output_push/fuzz002.sync.sync')) self.assertFalse(os.path.exists('testdata/rsync_output_push/invalid_fuzz000.sync'))