diff --git a/README.md b/README.md index 24e4b3f..4a05b50 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@
-
+
diff --git a/das/common.py b/das/common.py
index 9b3842f..c18f6e9 100644
--- a/das/common.py
+++ b/das/common.py
@@ -2,7 +2,7 @@
__author__ = '@snovvcrash'
__site__ = 'https://github.com/snovvcrash/DivideAndScan'
-__version__ = '1.0.0'
+__version__ = '1.0.1'
import time
import shlex
diff --git a/das/db.py b/das/db.py
new file mode 100644
index 0000000..1cefff1
--- /dev/null
+++ b/das/db.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import os
+import tempfile
+from pathlib import Path
+from collections import defaultdict
+
+from tinydb import TinyDB
+
+from das.common import Logger, run_command
+
+
+class DB:
+ """Class for utilities that serve for manual DB manipulations."""
+
+ def __init__(self, db_path):
+ """
+ Constructor.
+
+ :param db_path: a TinyDB database file path
+ :type db_path: str
+ """
+ self.db = TinyDB(db_path)
+
+ def create_generic(self, scan_path, domains_path=None):
+ """Create TinyDB from a generic scan output and a list of domain names (projectdiscovery/dnsx must be in PATH).
+
+ :param scan_path: an input file path with newline-separated scan output
+ :type scan_path: pathlib.PosixPath
+ :param domains_path: an input file path with newline-separated domain names
+ :type domains_path: pathlib.PosixPath
+ :return: number of hosts added to DB
+ :rtype: int
+ """
+ with open(scan_path) as f:
+ # fmt -> 127.0.0.1:1337
+ scan = f.read().splitlines()
+
+ if domains_path:
+ with open(domains_path) as f:
+ domains = set(f.read().splitlines())
+
+ with tempfile.NamedTemporaryFile('w', suffix='.txt') as tmp:
+ dnsx_path = Path(tempfile.gettempdir()) / 'dnsx.txt'
+ domains_punycode = [domain.encode('idna').decode() + '\n' for domain in domains]
+ tmp.writelines(domains_punycode)
+ run_command(f'dnsx -l {tmp.name} -re -silent -o {dnsx_path}')
+
+ with open(dnsx_path) as f:
+ dnsx = f.read().splitlines()
+
+ Logger.print_info(f'Resolved {len(dnsx)} DNS records')
+ os.remove(dnsx_path)
+
+ domains = defaultdict(set)
+ for line in dnsx:
+ domain, ip = line.split()
+ domain = domain.encode().decode('idna')
+ ip = ip.replace('[', '').replace(']', '')
+ domains[ip].add(domain)
+
+ items = [{'ip': ip, 'port': int(port), 'domains': list(domains[ip])} for ip, port in (line.split(':') for line in scan)]
+ else:
+ items = [{'ip': ip, 'port': int(port), 'domains': []} for ip, port in (line.split(':') for line in scan)]
+
+ self.db.truncate()
+ self.db.insert_multiple(items)
+
+ return len(set([i['ip'] for i in items]))
diff --git a/das/parsers/masscan_import.py b/das/parsers/masscan_import.py
new file mode 100644
index 0000000..f0ed027
--- /dev/null
+++ b/das/parsers/masscan_import.py
@@ -0,0 +1,28 @@
+from das.parsers import IAddPortscanOutput
+
+
+class AddPortscanOutput(IAddPortscanOutput):
+ """Child class for processing Masscan output (import)."""
+
+ def parse(self):
+ """
+ Masscan (import) raw output parser.
+
+ :return: a pair of values (portscan raw output filename, number of hosts added to DB)
+ :rtype: tuple
+ """
+ items, hosts = [], set()
+ for line in self.portscan_raw:
+ try:
+ ip = line.split()[3]
+ port, _, proto = line.split()[-1].split('/')[0:3]
+ except Exception:
+ pass
+ else:
+ if proto == 'tcp':
+ items.append({'ip': ip, 'port': int(port), 'domains': []})
+ hosts.add(ip)
+
+ self.db.insert_multiple(items)
+
+ return (self.portscan_out, len(hosts))
diff --git a/poetry.lock b/poetry.lock
index 47be65d..37901bd 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -311,7 +311,7 @@ plotly = ">=5.0.0"
[package.extras]
celery = ["celery[redis] (>=5.1.2)", "importlib-metadata (<5)", "redis (>=3.5.3)"]
-ci = ["black (==21.6b0)", "black (==22.3.0)", "dash-dangerously-set-inner-html", "dash-flow-example (==0.0.5)", "flake8 (==3.9.2)", "flaky (==3.7.0)", "flask-talisman (==1.0.0)", "isort (==4.3.21)", "mimesis", "mock (==4.0.3)", "numpy", "openpyxl", "orjson (==3.5.4)", "orjson (==3.6.7)", "pandas (==1.1.5)", "pandas (>=1.4.0)", "preconditions", "pyarrow", "pyarrow (<3)", "pylint (==2.13.5)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "xlrd (<2)", "xlrd (>=2.0.1)"]
+ci = ["black (==21.6b0)", "black (==22.3.0)", "dash-dangerously-set-inner-html", "dash-flow-example (==0.0.5)", "flake8 (==3.9.2)", "flaky (==3.7.0)", "flask-talisman (==1.0.1)", "isort (==4.3.21)", "mimesis", "mock (==4.0.3)", "numpy", "openpyxl", "orjson (==3.5.4)", "orjson (==3.6.7)", "pandas (==1.1.5)", "pandas (>=1.4.0)", "preconditions", "pyarrow", "pyarrow (<3)", "pylint (==2.13.5)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "xlrd (<2)", "xlrd (>=2.0.1)"]
compress = ["flask-compress"]
dev = ["PyYAML (>=5.4.1)", "coloredlogs (>=15.0.1)", "fire (>=0.4.0)"]
diskcache = ["diskcache (>=5.2.1)", "multiprocess (>=0.70.12)", "psutil (>=5.8.0)"]
diff --git a/pyproject.toml b/pyproject.toml
index a3909c2..2fcd616 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "divideandscan"
-version = "1.0.0"
+version = "1.0.1"
description = "Divide full port scan results and use it for targeted Nmap runs"
authors = ["Sam Freeside