fix: download default wordlists if missing (#261)

secator/cli.py

@@ -12,10 +12,12 @@
 
 from secator.config import ConfigLoader
 from secator.decorators import OrderedGroup, register_runner
-from secator.definitions import (ASCII, BUILD_ADDON_ENABLED, CVES_FOLDER, DATA_FOLDER, DEV_ADDON_ENABLED,  # noqa: F401
-								 DEV_PACKAGE, GOOGLE_ADDON_ENABLED, VERSION_LATEST, LIB_FOLDER, MONGODB_ADDON_ENABLED,
-								 VERSION_OBSOLETE, OPT_NOT_SUPPORTED, PAYLOADS_FOLDER, REDIS_ADDON_ENABLED, REVSHELLS_FOLDER, ROOT_FOLDER,
-								 TRACE_ADDON_ENABLED, VERSION, VERSION_STR, WORKER_ADDON_ENABLED)
+from secator.definitions import (ASCII, BUILD_ADDON_ENABLED, CVES_FOLDER, DATA_FOLDER,  # noqa: F401
+								 DEFAULT_DNS_WORDLIST, DEFAULT_DNS_WORDLIST_URL, DEFAULT_HTTP_WORDLIST,
+								 DEFAULT_HTTP_WORDLIST_URL, DEV_ADDON_ENABLED, DEV_PACKAGE, GOOGLE_ADDON_ENABLED,
+								 LIB_FOLDER, MONGODB_ADDON_ENABLED, OPT_NOT_SUPPORTED, PAYLOADS_FOLDER,
+								 REDIS_ADDON_ENABLED, REVSHELLS_FOLDER, ROOT_FOLDER, TRACE_ADDON_ENABLED, VERSION,
+								 VERSION_LATEST, VERSION_OBSOLETE, VERSION_STR, WORKER_ADDON_ENABLED)
 from secator.installer import ToolInstaller
 from secator.rich import console
 from secator.runners import Command

secator/definitions.py

@@ -6,6 +6,8 @@
 from dotenv import find_dotenv, load_dotenv
 from pkg_resources import get_distribution, parse_version
 
+from secator.rich import console
+
 load_dotenv(find_dotenv(usecwd=True), override=False)
 
 
@@ -16,7 +18,8 @@ def get_latest_version():
 		resp.raise_for_status()
 		latest_version = resp.json()['name'].lstrip('v')
 		return latest_version
-	except (requests.exceptions.RequestException):
+	except requests.exceptions.RequestException as e:
+		console.print(f'[bold red]Failed to get latest version from GitHub: {type(e).__name__}.')
 		return None
 
 
@@ -50,20 +53,10 @@ def get_latest_version():
 PAYLOADS_FOLDER = f'{DATA_FOLDER}/payloads'
 REVSHELLS_FOLDER = f'{DATA_FOLDER}/revshells'
 TESTS_FOLDER = f'{ROOT_FOLDER}/tests'
-os.makedirs(BIN_FOLDER, exist_ok=True)
-os.makedirs(DATA_FOLDER, exist_ok=True)
-os.makedirs(REPORTS_FOLDER, exist_ok=True)
-os.makedirs(WORDLISTS_FOLDER, exist_ok=True)
-os.makedirs(SCRIPTS_FOLDER, exist_ok=True)
-os.makedirs(CVES_FOLDER, exist_ok=True)
-os.makedirs(PAYLOADS_FOLDER, exist_ok=True)
-os.makedirs(REVSHELLS_FOLDER, exist_ok=True)
 
 # Celery local fs folders
 CELERY_DATA_FOLDER = f'{DATA_FOLDER}/celery/data'
 CELERY_RESULTS_FOLDER = f'{DATA_FOLDER}/celery/results'
-os.makedirs(CELERY_DATA_FOLDER, exist_ok=True)
-os.makedirs(CELERY_RESULTS_FOLDER, exist_ok=True)
 
 # Environment variables
 DEBUG = int(os.environ.get('DEBUG', '0'))
@@ -99,8 +92,10 @@ def get_latest_version():
 DEFAULT_SKIP_CVE_SEARCH = bool(int(os.environ.get('DEFAULT_SKIP_CVE_SEARCH', 0)))
 
 # Default wordlists
-DEFAULT_HTTP_WORDLIST = os.environ.get('DEFAULT_HTTP_WORDLIST', f'{WORDLISTS_FOLDER}/Fuzzing/fuzz-Bo0oM.txt')
-DEFAULT_DNS_WORDLIST = os.environ.get('DEFAULT_DNS_WORDLIST', f'{WORDLISTS_FOLDER}/Discovery/DNS/combined_subdomains.txt')  # noqa:E501
+DEFAULT_HTTP_WORDLIST = os.environ.get('DEFAULT_HTTP_WORDLIST', f'{WORDLISTS_FOLDER}/fuzz-Bo0oM.txt')
+DEFAULT_HTTP_WORDLIST_URL = 'https://raw.githubusercontent.com/Bo0oM/fuzz.txt/master/fuzz.txt'
+DEFAULT_DNS_WORDLIST = os.environ.get('DEFAULT_DNS_WORDLIST', f'{WORDLISTS_FOLDER}/combined_subdomains.txt')
+DEFAULT_DNS_WORDLIST_URL = 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/combined_subdomains.txt'  # noqa: E501
 
 # Constants
 OPT_NOT_SUPPORTED = -1
@@ -175,6 +170,30 @@ def get_latest_version():
 WORDLIST = 'wordlist'
 WORDS = 'words'
 
+
+# Create all folders
+for folder in [BIN_FOLDER, DATA_FOLDER, REPORTS_FOLDER, WORDLISTS_FOLDER, SCRIPTS_FOLDER, CVES_FOLDER, PAYLOADS_FOLDER,
+			   REVSHELLS_FOLDER, CELERY_DATA_FOLDER, CELERY_RESULTS_FOLDER]:
+	if not os.path.exists(folder):
+		os.makedirs(folder)
+		console.print(f'[bold turquoise4]Created folder[/] {folder}.')
+
+
+# Download default wordlists
+for wordlist in ['HTTP', 'DNS']:
+	wordlist_path = globals()[f'DEFAULT_{wordlist}_WORDLIST']
+	wordlist_url = globals()[f'DEFAULT_{wordlist}_WORDLIST_URL']
+	if not os.path.exists(wordlist_path):
+		try:
+			resp = requests.get(wordlist_url)
+			with open(wordlist_path, 'w') as f:
+				f.write(resp.text)
+				console.print(f'[bold turquoise4]Downloaded default {wordlist} wordlist[/] {wordlist_path}.')
+		except requests.exceptions.RequestException as e:
+			console.print(f'[bold red]Failed to download default {wordlist} wordlist: {type(e).__name__}.')
+			pass
+
+
 # Check worker addon
 try:
 	import eventlet  # noqa: F401

secator/rich.py

@@ -1,19 +1,13 @@
 import operator
 
-import click
-import rich_click
 import yaml
 from rich import box
 from rich.console import Console
 from rich.table import Table
-from rich.traceback import install
 
-from secator.definitions import DEBUG, RECORD
-
-console = Console(stderr=True, record=RECORD, color_system='truecolor')
+console = Console(stderr=True, color_system='truecolor')
 console_stdout = Console(record=True)
 # handler = RichHandler(rich_tracebacks=True)  # TODO: add logging handler
-install(show_locals=DEBUG > 2, suppress=[click, rich_click])
 
 
 def criticity_to_color(value):
@@ -73,7 +67,7 @@ def build_table(items, output_fields=[], exclude_fields=[], sort_by=None):
 		items = sorted(items, key=operator.attrgetter(*sort_by))
 
 	# Create rich table
-	box_style = box.DOUBLE if RECORD else box.ROUNDED
+	box_style = box.ROUNDED
 	table = Table(show_lines=True, box=box_style)
 
 	# Get table schema if any, default to first item keys

secator/runners/_base.py

@@ -171,7 +171,7 @@ def __init__(self, config, targets, results=[], run_opts={}, hooks={}, context={
 			user_hooks.extend(hooks.get(key, []))
 			for hook in user_hooks:
 				name = f'{self.__class__.__name__}.{key}'
-				fun = f'{hook.__module__}.{hook.__name__}'
+				fun = self.get_func_path(hook)
 				debug('', obj={name + ' [dim yellow]->[/] ' + fun: 'registered (user)'}, sub='hooks', level=3)
 			self.hooks[key].extend(user_hooks)
 

secator/tasks/_categories.py

@@ -6,17 +6,12 @@
 from bs4 import BeautifulSoup
 from cpe import CPE
 
-from secator.definitions import (CIDR_RANGE, CONFIDENCE, CVSS_SCORE,
-							   DEFAULT_HTTP_WORDLIST, DEFAULT_SKIP_CVE_SEARCH, DELAY, DEPTH, DESCRIPTION,
-							   FILTER_CODES, FILTER_REGEX, FILTER_SIZE,
-							   FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID,
-							   MATCH_CODES, MATCH_REGEX, MATCH_SIZE,
-							   MATCH_WORDS, METHOD, NAME, PATH, PROVIDER,
-							   PROXY, RATE_LIMIT, REFERENCES, RETRIES,
-							   SEVERITY, TAGS, DATA_FOLDER, THREADS, TIMEOUT,
-							   URL, USER_AGENT, USERNAME, WORDLIST)
-from secator.output_types import (Ip, Port, Subdomain, Tag, Url, UserAccount,
-								Vulnerability)
+from secator.definitions import (CIDR_RANGE, CONFIDENCE, CVSS_SCORE, DATA_FOLDER, DEFAULT_HTTP_WORDLIST,
+								 DEFAULT_SKIP_CVE_SEARCH, DELAY, DEPTH, DESCRIPTION, FILTER_CODES, FILTER_REGEX,
+								 FILTER_SIZE, FILTER_WORDS, FOLLOW_REDIRECT, HEADER, HOST, ID, MATCH_CODES, MATCH_REGEX,
+								 MATCH_SIZE, MATCH_WORDS, METHOD, NAME, PATH, PROVIDER, PROXY, RATE_LIMIT, REFERENCES,
+								 RETRIES, SEVERITY, TAGS, THREADS, TIMEOUT, URL, USER_AGENT, USERNAME, WORDLIST)
+from secator.output_types import Ip, Port, Subdomain, Tag, Url, UserAccount, Vulnerability
 from secator.rich import console
 from secator.runners import Command