diff --git a/README.md b/README.md index 24e4b3f..4a05b50 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

- version + version python codacy pypi diff --git a/das/common.py b/das/common.py index 9b3842f..c18f6e9 100644 --- a/das/common.py +++ b/das/common.py @@ -2,7 +2,7 @@ __author__ = '@snovvcrash' __site__ = 'https://github.com/snovvcrash/DivideAndScan' -__version__ = '1.0.0' +__version__ = '1.0.1' import time import shlex diff --git a/das/db.py b/das/db.py new file mode 100644 index 0000000..1cefff1 --- /dev/null +++ b/das/db.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os +import tempfile +from pathlib import Path +from collections import defaultdict + +from tinydb import TinyDB + +from das.common import Logger, run_command + + +class DB: + """Class for utilities that serve for manual DB manipulations.""" + + def __init__(self, db_path): + """ + Constructor. + + :param db_path: a TinyDB database file path + :type db_path: str + """ + self.db = TinyDB(db_path) + + def create_generic(self, scan_path, domains_path=None): + """Create TinyDB from a generic scan output and a list of domain names (projectdiscovery/dnsx must be in PATH). + + :param scan_path: an input file path with newline-separated scan output + :type scan_path: pathlib.PosixPath + :param domains_path: an input file path with newline-separated domain names + :type domains_path: pathlib.PosixPath + :return: number of hosts added to DB + :rtype: int + """ + with open(scan_path) as f: + # fmt -> 127.0.0.1:1337 + scan = f.read().splitlines() + + if domains_path: + with open(domains_path) as f: + domains = set(f.read().splitlines()) + + with tempfile.NamedTemporaryFile('w', suffix='.txt') as tmp: + dnsx_path = Path(tempfile.gettempdir()) / 'dnsx.txt' + domains_punycode = [domain.encode('idna').decode() + '\n' for domain in domains] + tmp.writelines(domains_punycode) + run_command(f'dnsx -l {tmp.name} -re -silent -o {dnsx_path}') + + with open(dnsx_path) as f: + dnsx = f.read().splitlines() + + Logger.print_info(f'Resolved {len(dnsx)} DNS records') + os.remove(dnsx_path) + + domains = defaultdict(set) + for line in dnsx: + domain, ip = line.split() + domain = domain.encode().decode('idna') + ip = ip.replace('[', '').replace(']', '') + domains[ip].add(domain) + + items = [{'ip': ip, 'port': int(port), 'domains': list(domains[ip])} for ip, port in (line.split(':') for line in scan)] + else: + items = [{'ip': ip, 'port': int(port), 'domains': []} for ip, port in (line.split(':') for line in scan)] + + self.db.truncate() + self.db.insert_multiple(items) + + return len(set([i['ip'] for i in items])) diff --git a/das/parsers/masscan_import.py b/das/parsers/masscan_import.py new file mode 100644 index 0000000..f0ed027 --- /dev/null +++ b/das/parsers/masscan_import.py @@ -0,0 +1,28 @@ +from das.parsers import IAddPortscanOutput + + +class AddPortscanOutput(IAddPortscanOutput): + """Child class for processing Masscan output (import).""" + + def parse(self): + """ + Masscan (import) raw output parser. + + :return: a pair of values (portscan raw output filename, number of hosts added to DB) + :rtype: tuple + """ + items, hosts = [], set() + for line in self.portscan_raw: + try: + ip = line.split()[3] + port, _, proto = line.split()[-1].split('/')[0:3] + except Exception: + pass + else: + if proto == 'tcp': + items.append({'ip': ip, 'port': int(port), 'domains': []}) + hosts.add(ip) + + self.db.insert_multiple(items) + + return (self.portscan_out, len(hosts)) diff --git a/poetry.lock b/poetry.lock index 47be65d..37901bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -311,7 +311,7 @@ plotly = ">=5.0.0" [package.extras] celery = ["celery[redis] (>=5.1.2)", "importlib-metadata (<5)", "redis (>=3.5.3)"] -ci = ["black (==21.6b0)", "black (==22.3.0)", "dash-dangerously-set-inner-html", "dash-flow-example (==0.0.5)", "flake8 (==3.9.2)", "flaky (==3.7.0)", "flask-talisman (==1.0.0)", "isort (==4.3.21)", "mimesis", "mock (==4.0.3)", "numpy", "openpyxl", "orjson (==3.5.4)", "orjson (==3.6.7)", "pandas (==1.1.5)", "pandas (>=1.4.0)", "preconditions", "pyarrow", "pyarrow (<3)", "pylint (==2.13.5)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "xlrd (<2)", "xlrd (>=2.0.1)"] +ci = ["black (==21.6b0)", "black (==22.3.0)", "dash-dangerously-set-inner-html", "dash-flow-example (==0.0.5)", "flake8 (==3.9.2)", "flaky (==3.7.0)", "flask-talisman (==1.0.1)", "isort (==4.3.21)", "mimesis", "mock (==4.0.3)", "numpy", "openpyxl", "orjson (==3.5.4)", "orjson (==3.6.7)", "pandas (==1.1.5)", "pandas (>=1.4.0)", "preconditions", "pyarrow", "pyarrow (<3)", "pylint (==2.13.5)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "xlrd (<2)", "xlrd (>=2.0.1)"] compress = ["flask-compress"] dev = ["PyYAML (>=5.4.1)", "coloredlogs (>=15.0.1)", "fire (>=0.4.0)"] diskcache = ["diskcache (>=5.2.1)", "multiprocess (>=0.70.12)", "psutil (>=5.8.0)"] diff --git a/pyproject.toml b/pyproject.toml index a3909c2..2fcd616 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "divideandscan" -version = "1.0.0" +version = "1.0.1" description = "Divide full port scan results and use it for targeted Nmap runs" authors = ["Sam Freeside "] license = "BSD-2-Clause" @@ -42,5 +42,5 @@ pylint = "^2.12.2" twine = "^3.8.0" [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.0.1"] build-backend = "poetry.core.masonry.api"