From 5174f493fda1ac164b011961e91f82d6e9edf18c Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Thu, 22 Nov 2018 01:19:10 +0100 Subject: [PATCH 01/12] new appendable extensions --- w3af/plugins/crawl/url_fuzzer.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index ab16447cbd..fe3935ba8b 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -41,11 +41,18 @@ class url_fuzzer(CrawlPlugin): Try to find backups, and other related files. :author: Andres Riancho (andres.riancho@gmail.com) """ - _appendables = ('~', '.tar.gz', '.gz', '.7z', '.cab', '.tgz', - '.gzip', '.bzip2', '.inc', '.zip', '.rar', '.jar', '.java', - '.class', '.properties', '.bak', '.bak1', '.bkp', '.back', - '.backup', '.backup1', '.old', '.old1', '.$$$' - ) + _appendables = ('~', '~~', '_', '.', '.tar.gz', '.gz', '.7z', '.cab', + '.tgz', '.gzip', '.bzip2', '.inc', '.zip', '.rar', + '.tar', '.jar', '.java', '.class', '.properties', + '.bak', '.bak1', '_bak', '-bak', '.bk', '.bkp', '.back', + '.bac', '.backup', '.backup1', '.old', '.old1', '_old', + '.$$$', '.sav', '.save', '.saved', '.swp', '.swo', + '.copy', '.original', '.orig', '.org', '.txt', '.default', + '.tpl', '.tmp', '.temp', '.conf', '.nsx', '.cs', '.csproj', + '.vb', '.0', '.1', '.2', '.arc', '.lst', '.inc', '::$DATA', + '.sql.gz', '.bak.sql', '.bak.sql.gz', '.bak.sql.bz2', + '.bak.sql.tar.gz' + ) _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', 'bzip2', 'zip', 'rar' ) From e58fe702bc4489f666c51a82b28f43a4e7162c80 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Thu, 22 Nov 2018 01:20:53 +0100 Subject: [PATCH 02/12] new prepandable patterns --- w3af/plugins/crawl/url_fuzzer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index fe3935ba8b..fa9b565411 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -53,6 +53,9 @@ class url_fuzzer(CrawlPlugin): '.sql.gz', '.bak.sql', '.bak.sql.gz', '.bak.sql.bz2', '.bak.sql.tar.gz' ) + _prependables = ('_', '.', '~', '.~', 'Copy_', 'Copy_of_', 'Copy_(1)_of_', + 'Copy_(2)_of_', 'Copy ', 'Copy of ', 'backup-' + ) _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', 'bzip2', 'zip', 'rar' ) From 5fa90325246f9e937869be35c042b98a32ee46fd Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Thu, 22 Nov 2018 01:21:42 +0100 Subject: [PATCH 03/12] new backup extensions --- w3af/plugins/crawl/url_fuzzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index fa9b565411..2413c99100 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -57,8 +57,8 @@ class url_fuzzer(CrawlPlugin): 'Copy_(2)_of_', 'Copy ', 'Copy of ', 'backup-' ) _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', - 'bzip2', 'zip', 'rar' - ) + 'bzip2', 'zip', 'rar', 'tar' + ) _file_types = ( 'inc', 'fla', 'jar', 'war', 'java', 'class', 'properties', 'bak', 'bak1', 'backup', 'backup1', 'old', 'old1', 'c', 'cpp', From 9e7235a699bae9005ee68fea3f4d54e67bd025f4 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Thu, 22 Nov 2018 01:22:17 +0100 Subject: [PATCH 04/12] new file types --- w3af/plugins/crawl/url_fuzzer.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 2413c99100..4e39f18a90 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -59,11 +59,12 @@ class url_fuzzer(CrawlPlugin): _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', 'bzip2', 'zip', 'rar', 'tar' ) - _file_types = ( - 'inc', 'fla', 'jar', 'war', 'java', 'class', 'properties', - 'bak', 'bak1', 'backup', 'backup1', 'old', 'old1', 'c', 'cpp', - 'cs', 'vb', 'phps', 'disco', 'ori', 'orig', 'original' - ) + _file_types = ('inc', 'fla', 'jar', 'war', 'java', 'class', 'properties', + 'bak', 'bak1', 'backup', 'backup1', 'old', 'old1', 'c', 'cpp', + 'cs', 'vb', 'phps', 'disco', 'ori', 'orig', 'original', 'save', + 'saved', 'bkp', 'txt', 'tpl', 'tmp', 'temp', 'bakup', 'bakup1', + 'sql' + ) def __init__(self): CrawlPlugin.__init__(self) From d827ec776a619566dead6b28abfadee83754f3db Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Thu, 22 Nov 2018 01:24:47 +0100 Subject: [PATCH 05/12] mutate by prepending patterns to file names --- w3af/plugins/crawl/url_fuzzer.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 4e39f18a90..75bf52a24f 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -248,6 +248,37 @@ def _mutate_by_appending(self, url): url_copy.set_file_name(filename) yield url_copy + def _mutate_by_prepending(self, url): + """ + Adds something before the file name of the url (mutate the file being requested) + + :param url: A URL to transform. + :return: A list of URL's that mutate the original url passed + as parameter. + + >>> from w3af.core.data.parsers.doc.url import URL + >>> u = url_fuzzer() + >>> url = URL( 'http://www.w3af.com/' ) + >>> mutants = u._mutate_by_prepending( url ) + >>> list(mutants) + [] + + >>> url = URL( 'http://www.w3af.com/foo.html' ) + >>> mutants = u._mutate_by_prepending( url ) + >>> URL( 'http://www.w3af.com/.foo.html' ) in mutants + True + >>> URL( 'http://www.w3af.com/Copy_of_foo.html' ) in mutants + True + + """ + if not url.url_string.endswith('/') and url.url_string.count('/') >= 3: + for to_prepend in self._prependables: + url_copy = url.copy() + filename = url_copy.get_file_name() + filename = to_prepend + filename + url_copy.set_file_name(filename) + yield url_copy + def _mutate_file_type(self, url): """ If the url is : "http://www.foobar.com/asd.txt" this method returns: From 73bc9324b11c7eb6af682f2f908837af4dc799a7 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Fri, 23 Nov 2018 00:40:46 +0100 Subject: [PATCH 06/12] remove .inc (duplicate) and add .$ --- w3af/plugins/crawl/url_fuzzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 75bf52a24f..0dd4773212 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -49,11 +49,11 @@ class url_fuzzer(CrawlPlugin): '.$$$', '.sav', '.save', '.saved', '.swp', '.swo', '.copy', '.original', '.orig', '.org', '.txt', '.default', '.tpl', '.tmp', '.temp', '.conf', '.nsx', '.cs', '.csproj', - '.vb', '.0', '.1', '.2', '.arc', '.lst', '.inc', '::$DATA', + '.vb', '.0', '.1', '.2', '.arc', '.lst', '::$DATA', '.sql.gz', '.bak.sql', '.bak.sql.gz', '.bak.sql.bz2', '.bak.sql.tar.gz' ) - _prependables = ('_', '.', '~', '.~', 'Copy_', 'Copy_of_', 'Copy_(1)_of_', + _prependables = ('_', '.', '~', '.~', '.$', 'Copy_', 'Copy_of_', 'Copy_(1)_of_', 'Copy_(2)_of_', 'Copy ', 'Copy of ', 'backup-' ) _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', From ab85c79096e8d835249da363ece1b2df04a6410b Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Fri, 23 Nov 2018 00:42:39 +0100 Subject: [PATCH 07/12] mutate by prepending and/or appending to file names --- w3af/plugins/crawl/url_fuzzer.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 0dd4773212..a564e37791 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -311,6 +311,33 @@ def _mutate_file_type(self, url): url_copy.set_extension(filetype) yield url_copy + def _mutate_file_name(self, url): + filename = url.get_file_name() + if filename: + domain = url.get_domain_path() + url_string = domain.url_string + name = filename[:filename.rfind(u'.')] + extension = url.get_extension() + + mutate_name_testing = ( + url_string + '#' + filename + '#', + url_string + name + ' (copy).' + extension, + url_string + name + ' - Copy.' + extension, + url_string + name + ' copy.' + extension, + url_string + '.~lock.' + filename + '#', + url_string + name + '-backup.' + extension, + url_string + name + '-bkp.' + extension, + url_string + '.' + name + '.swp', + url_string + '_' + name + '.swp', + url_string + '.' + name + '.swo', + url_string + '_' + name + '.swo', + url_string + '~' + name + '.tmp' + ) + + for change in mutate_name_testing: + newurl = URL(change) + yield newurl + def _mutate_path(self, url): """ Mutate the path instead of the file. From 6f75847ff6e61729e3f20910f828015552fccce5 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Fri, 23 Nov 2018 00:43:40 +0100 Subject: [PATCH 08/12] call to new mutate functions in _do_request --- w3af/plugins/crawl/url_fuzzer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index a564e37791..e7d772456b 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -107,7 +107,10 @@ def crawl(self, fuzzable_request): mutants_chain = chain(self._mutate_by_appending(url), self._mutate_path(url), self._mutate_file_type(url), - self._mutate_domain_name(url)) + self._mutate_domain_name(url), + self._mutate_by_prepending(url), + self._mutate_file_name(url) + ) url_repeater = repeat(url) args = izip(url_repeater, mutants_chain) From 3a5fd89501d723df2359cbe750095adef0c190f7 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Fri, 23 Nov 2018 01:23:29 +0100 Subject: [PATCH 09/12] put the extension inside few transformations like in bfac --- w3af/plugins/crawl/url_fuzzer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index e7d772456b..d9d7291553 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -330,11 +330,11 @@ def _mutate_file_name(self, url): url_string + '.~lock.' + filename + '#', url_string + name + '-backup.' + extension, url_string + name + '-bkp.' + extension, - url_string + '.' + name + '.swp', - url_string + '_' + name + '.swp', - url_string + '.' + name + '.swo', - url_string + '_' + name + '.swo', - url_string + '~' + name + '.tmp' + url_string + '.' + filename + '.swp', + url_string + '_' + filename + '.swp', + url_string + '.' + filename + '.swo', + url_string + '_' + filename + '.swo', + url_string + '~' + filename + '.tmp' ) for change in mutate_name_testing: From ba8a1d2c4f80d724cb2921418a3ac63db7611a85 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Fri, 23 Nov 2018 22:54:15 +0100 Subject: [PATCH 10/12] tuple to set to avoid duplicates --- w3af/plugins/crawl/url_fuzzer.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index d9d7291553..08043fd41f 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -41,7 +41,7 @@ class url_fuzzer(CrawlPlugin): Try to find backups, and other related files. :author: Andres Riancho (andres.riancho@gmail.com) """ - _appendables = ('~', '~~', '_', '.', '.tar.gz', '.gz', '.7z', '.cab', + _appendables = {'~', '~~', '_', '.', '.tar.gz', '.gz', '.7z', '.cab', '.tgz', '.gzip', '.bzip2', '.inc', '.zip', '.rar', '.tar', '.jar', '.java', '.class', '.properties', '.bak', '.bak1', '_bak', '-bak', '.bk', '.bkp', '.back', @@ -52,19 +52,19 @@ class url_fuzzer(CrawlPlugin): '.vb', '.0', '.1', '.2', '.arc', '.lst', '::$DATA', '.sql.gz', '.bak.sql', '.bak.sql.gz', '.bak.sql.bz2', '.bak.sql.tar.gz' - ) - _prependables = ('_', '.', '~', '.~', '.$', 'Copy_', 'Copy_of_', 'Copy_(1)_of_', + } + _prependables = {'_', '.', '~', '.~', '.$', 'Copy_', 'Copy_of_', 'Copy_(1)_of_', 'Copy_(2)_of_', 'Copy ', 'Copy of ', 'backup-' - ) - _backup_exts = ('tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', + } + _backup_exts = {'tar.gz', '7z', 'gz', 'cab', 'tgz', 'gzip', 'bzip2', 'zip', 'rar', 'tar' - ) - _file_types = ('inc', 'fla', 'jar', 'war', 'java', 'class', 'properties', + } + _file_types = {'inc', 'fla', 'jar', 'war', 'java', 'class', 'properties', 'bak', 'bak1', 'backup', 'backup1', 'old', 'old1', 'c', 'cpp', 'cs', 'vb', 'phps', 'disco', 'ori', 'orig', 'original', 'save', 'saved', 'bkp', 'txt', 'tpl', 'tmp', 'temp', 'bakup', 'bakup1', 'sql' - ) + } def __init__(self): CrawlPlugin.__init__(self) @@ -322,7 +322,7 @@ def _mutate_file_name(self, url): name = filename[:filename.rfind(u'.')] extension = url.get_extension() - mutate_name_testing = ( + mutate_name_testing = { url_string + '#' + filename + '#', url_string + name + ' (copy).' + extension, url_string + name + ' - Copy.' + extension, @@ -335,7 +335,7 @@ def _mutate_file_name(self, url): url_string + '.' + filename + '.swo', url_string + '_' + filename + '.swo', url_string + '~' + filename + '.tmp' - ) + } for change in mutate_name_testing: newurl = URL(change) From 4668c0daa468a3749da799535d52fe087dd7349f Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Sat, 24 Nov 2018 12:58:54 +0100 Subject: [PATCH 11/12] proper handling of files like foo, foo. and .foo --- w3af/plugins/crawl/url_fuzzer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 08043fd41f..494d7a0bce 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -319,17 +319,22 @@ def _mutate_file_name(self, url): if filename: domain = url.get_domain_path() url_string = domain.url_string - name = filename[:filename.rfind(u'.')] extension = url.get_extension() + if extension: + extension = '.' + extension + name = filename[:filename.rfind(u'.')] + else: + name = filename + mutate_name_testing = { url_string + '#' + filename + '#', - url_string + name + ' (copy).' + extension, - url_string + name + ' - Copy.' + extension, - url_string + name + ' copy.' + extension, + url_string + name + ' (copy)' + extension, + url_string + name + ' - Copy' + extension, + url_string + name + ' copy' + extension, url_string + '.~lock.' + filename + '#', - url_string + name + '-backup.' + extension, - url_string + name + '-bkp.' + extension, + url_string + name + '-backup' + extension, + url_string + name + '-bkp' + extension, url_string + '.' + filename + '.swp', url_string + '_' + filename + '.swp', url_string + '.' + filename + '.swo', From 5fa27476a05d6ad07ca8282ecdeb9fa05ecdddf7 Mon Sep 17 00:00:00 2001 From: Alexandre Borgo Date: Wed, 9 Jan 2019 20:27:38 +0100 Subject: [PATCH 12/12] add try/except around new url's generation --- w3af/plugins/crawl/url_fuzzer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/w3af/plugins/crawl/url_fuzzer.py b/w3af/plugins/crawl/url_fuzzer.py index 494d7a0bce..863a6d0094 100644 --- a/w3af/plugins/crawl/url_fuzzer.py +++ b/w3af/plugins/crawl/url_fuzzer.py @@ -343,8 +343,12 @@ def _mutate_file_name(self, url): } for change in mutate_name_testing: - newurl = URL(change) - yield newurl + try : + newurl = URL(change) + yield newurl + except ValueError as exception: + om.out.information('Error while generating a new URL: '\ + + exception) def _mutate_path(self, url): """