diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 0000000..0a192a3
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,16 @@
+version: 2
+
+build:
+ os: ubuntu-20.04
+ tools:
+ python: "3.10"
+ jobs:
+ pre_build:
+ - pip install mkdocs mkdocs-material
+
+python:
+ install:
+ - requirements: requirements.txt
+
+mkdocs:
+ configuration: docs/dpulse-docs/mkdocs.yml
diff --git a/README.md b/README.md
index 0ade92b..b20b09c 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# DPULSE - Domain Public Data Collection Service
- 
+ 
@@ -9,7 +9,7 @@
-
+
@@ -20,15 +20,16 @@
> Attention! DPULSE is a research tool. It is not intended for criminal activities! Use DPULSE only on allowed domains and for legal purposes!
+> Please, before creating an issue or DMing developer about DPULSE, make sure that your problem or question is not covered with [DPULSE documentation](https://dpulse.readthedocs.io)
+
# Repository map
## What to visit?
| What do you want to see? | Link |
| --- | --- |
-| What is DPULSE? | [See "About DPULSE" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#about-dpulse) |
-| Where I can find some demo and use-cases? | [See "Demo and use-cases" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#dpulse-demo-and-use-cases) |
-| I want to read project documentation | [See DPULSE wiki](https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-WIKI) |
+| I want to read project documentation | [See DPULSE Readthedocs Page](https://dpulse.readthedocs.io) |
+| I want to see developer's contacts | [See "Contact developer" page on Readthedocs](https://dpulse.readthedocs.io/en/latest/contact_dev/#) |
| I want to see project roadmap and future development plans | [See DPULSE roadmap](https://github.com/users/OSINT-TECHNOLOGIES/projects/1) |
## What to download?
@@ -36,16 +37,10 @@
| Your expectations | Version and link for you |
| --- | --- |
| I want to use only tested and stable version of DPULSE | [DPULSE stable ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/main.zip) |
-| I don't mind to use DPULSE with latest changes and you're OK with bugs and issues | [DPULSE rolling ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/rolling.zip) |
+| I don't mind to use DPULSE with latest changes and I'm OK with bugs and issues | [DPULSE rolling ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/rolling.zip) |
| I want to use only one specific version of DPULSE | [See DPULSE releases page](https://github.com/OSINT-TECHNOLOGIES/dpulse/releases) |
| I want to see more detailed installation instructions | [See DPULSE installation guides](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#how-to-install-and-run-dpulse)
-## How can I contact the developer?
-
-| Reasons to contact | Links & addresses |
-| --- | --- |
-| I want to talk with developer in person | DM to osint.technologies@gmail.com |
-| I want to report some bug or issue, or maybe I have some good idea for developer | [Make a new issue page](https://github.com/OSINT-TECHNOLOGIES/dpulse/issues/new/choose) |
# About DPULSE
@@ -157,11 +152,11 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
### You can start DPULSE and see the main menu on the screen using one of the recommended commands in DPULSE root folder. Don't forget to install all requirements before starting DPULSE
-
+
### After choosing first menu point, you will be able to enter target's URL and case comment, and then you will see scanning progress
-
+
### Finally, DPULSE will create report folder which contains case name (basically URL of target), date and time of scan. All report folders are contained in DPULSE root folder
@@ -194,6 +189,12 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
### [by @UndeadSec](https://x.com/UndeadSec/status/1827692406797689032)
+### [by @akaclandestine](https://x.com/akaclandestine/status/1875980998418002151?t=Ac-fZ9oe2FLKkTTCClss8g&s=19)
+
+### [by @hdH4dg8](https://x.com/hdH4dg8/status/1876106586869104906?t=Awfas25ukXkblPRt4krSzA&s=19)
+
+### [by @wy88215534](https://x.com/wy88215534/status/1876522251828494794?t=cmwoNCDZv0kMEDMKokcVFA&s=19)
+
## Facebook mentions:
### [by Inteligência Cibernética](https://www.facebook.com/osintbrasil/posts/pfbid037ibycZcBWe2MjtV4HiWvRWxyKei8TJ5Ycfxai4TDNHXuwrYkDGuyjDsPow8WUNbyl)
diff --git a/datagather_modules/crawl_processor.py b/datagather_modules/crawl_processor.py
index 5c3777f..b0930ed 100644
--- a/datagather_modules/crawl_processor.py
+++ b/datagather_modules/crawl_processor.py
@@ -103,6 +103,7 @@ def subdomains_gather(url, short_domain):
print(Fore.RED + f"Cannot gather subdomains due to error. See journal for details" + Style.RESET_ALL)
logging.error(f'SUBDOMAINS GATHERING: ERROR. REASON: {e}')
pass
+ return ['No subdomains were found'], 0
def sm_gather(url):
response = requests.get(url)
@@ -173,7 +174,7 @@ def domains_reverse_research(subdomains, report_file_type):
for subdomain in subdomains:
subdomain_url = "http://" + subdomain + "/"
subdomain_urls.append(subdomain_url)
- except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e:
+ except Exception as e:
print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL)
logging.error(f'SUBDOMAINS URL FORMING: ERROR. REASON: {e}')
pass
@@ -183,7 +184,7 @@ def domains_reverse_research(subdomains, report_file_type):
subdomains_ip = ip_gather(subdomain)
subdomain_ip.append(subdomains_ip)
subdomain_ip = list(set(subdomain_ip))
- except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e:
+ except Exception as e:
print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL)
logging.error(f'SUBDOMAINS IP GATHERING: ERROR. REASON: {e}')
pass
@@ -194,7 +195,7 @@ def domains_reverse_research(subdomains, report_file_type):
subdomain_mails.append(subdomain_mail)
subdomain_social = sm_gather(subdomain_url)
subdomain_socials.append(subdomain_social)
- except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e:
+ except Exception as e:
print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL)
logging.error(f'SUBDOMAINS MAIL/SOCIALS GATHERING: ERROR. REASON: {e}')
pass
diff --git a/datagather_modules/data_assembler.py b/datagather_modules/data_assembler.py
index a8cd5fa..8bb7c14 100644
--- a/datagather_modules/data_assembler.py
+++ b/datagather_modules/data_assembler.py
@@ -81,7 +81,12 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
print(Fore.GREEN + 'Processing subdomain gathering' + Style.RESET_ALL)
subdomains, subdomains_amount = cp.subdomains_gather(url, short_domain)
print(Fore.GREEN + 'Processing social medias gathering' + Style.RESET_ALL)
- social_medias = cp.sm_gather(url)
+ try:
+ social_medias = cp.sm_gather(url)
+ except:
+ print(Fore.RED + "Social medias were not gathered because of error" + Style.RESET_ALL)
+ social_medias = ['Social medias were not extracted because of error']
+ pass
print(Fore.GREEN + 'Processing subdomain analysis' + Style.RESET_ALL)
if report_file_type == 'xlsx':
subdomain_urls, subdomain_mails, subdomain_ip, sd_socials = cp.domains_reverse_research(subdomains, report_file_type)
@@ -164,17 +169,8 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
api_scan_db.append('No')
pass
- #data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
- # subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
- # robots_txt_result, sitemap_xml_result, sitemap_links_status,
- # web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports,
- # hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return,
- # accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter,
- # website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats,
- # vt_deturls, vt_detsamples, vt_undetsamples]
cleaned_dorking = [item.strip() for item in dorking_results if item.strip()]
- #print(cleaned_dorking)
data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
diff --git a/datagather_modules/networking_processor.py b/datagather_modules/networking_processor.py
index 1d770a7..e5641f8 100644
--- a/datagather_modules/networking_processor.py
+++ b/datagather_modules/networking_processor.py
@@ -51,7 +51,7 @@ def get_ssl_certificate(short_domain, port=443):
commonName = str(cert['issuer'][2][0][1]) + ', version: ' + str(cert['version'])
serialNumber = cert['serialNumber']
return issuer, subject, notBefore, notAfter, commonName, serialNumber
- except (ssl.CertificateError, ssl.SSLError, socket.gaierror, ConnectionRefusedError) as e:
+ except Exception as e:
print(Fore.RED + "Error while gathering info about SSL certificate. See journal for details")
logging.error(f'SSL CERTIFICATE GATHERING: ERROR. REASON: {e}')
issuer = subject = notBefore = notAfter = commonName = serialNumber = "No information about SSL certificate was gathered"
diff --git a/docs/__init__.py b/docs/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/docs/__init__.py
@@ -0,0 +1 @@
+
diff --git a/docs/dpulse-docs/docs/api.md b/docs/dpulse-docs/docs/api.md
new file mode 100644
index 0000000..9ce09bf
--- /dev/null
+++ b/docs/dpulse-docs/docs/api.md
@@ -0,0 +1,41 @@
+# Third-party API scan mode
+
+Currently DPULSE supports two third-party APIs:
+
+* SecurityTrails API (securitytrails.com) for deep subdomains and DNS enumeration
+* VirusTotal API (virustotal.com) for brief domain information gathering
+
+## SecurityTrails API
+
+SecurityTrails API is used to gather information about a specified domain. It retrieves various types of DNS records, subdomains, and other details. SecurityTrails API in DPULSE returns these details about target domain:
+
+* Alexa rank
+* Apex domain
+* Hostname
+* A/MX/NS/SOA/TXT records
+* All subdomains list
+* Alive (pingable) subdomains list
+
+## VirusTotal API
+
+VirusTotal API is used to interact with the VirusTotal service programmatically and analyze files and URLs using multiple antivirus engines and website scanners, providing insights into whether they are malicious. VirusTotal API in DPULSE returns these details about target domain:
+
+* Categories
+* Detected samples
+* Undetected samples
+* Detected URLs
+
+## API Keys database
+
+In order to ensure the functioning of API services individually for each DPULSE user, API keys storage database was created. Similar to report storage database, it is lightweight .db extension database with simple structure shown below:
+
+
+
+Since every API key is individual for each user, you can see fillers instead of actual keys when you start DPULSE for the first time, so until you replace filler with a real API key, you can't start using API in scans. You can enter your actual API keys using DPULSE CLI. You can see full process on the screenshot below:
+
+
+
+For the first time you will see red-colored API key field, which means that scan is not available with this API. After changing filler for actual key, you will see that color changed, which indicates that you can use your API key for scanning. Be advised that every free API service provided with some limitations (you can see them in DPULSE CLI for all supported API), so keep in mind that frequent usage of free API plans is not possible.
+
+In case if you want to fully replace API keys, you can use reference API keys database. You can see menu point for this action on the screenshot above. This action will delete your actual API keys database, copy reference database and rename it. This action is very optional because you can change your API keys by just using first menu point in API Keys DB Manager (according to the screenshot above)
+
diff --git a/docs/dpulse-docs/docs/basic_scan.md b/docs/dpulse-docs/docs/basic_scan.md
new file mode 100644
index 0000000..4d9bebc
--- /dev/null
+++ b/docs/dpulse-docs/docs/basic_scan.md
@@ -0,0 +1,18 @@
+# Basic scan
+
+Basic scan is default and the most basic scanning mode which starts after all preparation steps. It always starts first and you can't start scanning without it.
+
+## Basic scan results
+
+Basic scan returns some basic information open information about target domain, such as:
+
+1. WHOIS information which contains domain name, full URL, IP address, registrar info, creation and expiration dates, organization name and contact e-mails.
+2. Subdomains list
+3. E-mail addresses list (gathered from subdomains)
+4. IP addresses list (gathered from subdomains)
+5. Social medias links, posts and profiles. Supported social medias are Facebook, Twitter (X.com), Instagram, Telegram, TikTok, LinkedIn, VKontakte, YouTube, Odnoklassniki, WeChat
+6. DNS and SSL information. DNS information contains name servers and MX addresses. SSL certificate information contains issuer, subject, creation and expiration dates, certificate name and serial number.
+7. Basic pre-pentest information such as possible vulnerabilities (CVEs), open ports and hostnames.
+8. Development and deployment services and frameworks information, such as CMS, web servers, used programming languages and web frameworks, analytics services, tags and so on.
+9. Downloaded copies of sitemap.xml and robots.txt files from a domain
+
diff --git a/docs/dpulse-docs/docs/contact_dev.md b/docs/dpulse-docs/docs/contact_dev.md
new file mode 100644
index 0000000..4135c9b
--- /dev/null
+++ b/docs/dpulse-docs/docs/contact_dev.md
@@ -0,0 +1,10 @@
+## Contact developer
+
+DPULSE developer will be glad to see your messages with feedback, questions and suggestions. Feel free to contact developer with most convenient ways for you:
+
+* OSINT-TECHNOLOGIES e-mail: osint.technologies@gmail.com
+* [Make an issue page on DPULSE GitHub repository](https://github.com/OSINT-TECHNOLOGIES/dpulse/issues/new/choose)
+
+
+
+
diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md
new file mode 100644
index 0000000..637b025
--- /dev/null
+++ b/docs/dpulse-docs/docs/demo.md
@@ -0,0 +1,16 @@
+# DPULSE demos & use-cases
+
+On this page you can see some actual DPULSE demos and use-cases in some cybersecurity and OSINT scenarios
+
+## Demo №1. Starting DPULSE
+
+Yes, starting DPULSE is as simple as that. Don't forget to install all requirements before starting DPULSE
+
+
+
+## Demo №2. DPULSE basic scan workflow
+
+Here you can see complete process of DPULSE basic scan from the beginning to the end. Remember that sometimes your scan might not be ideal and errors may appear. However, they will not interrupt scanning process, but will only affect the number and variety of results in the final report.
+
+
+
diff --git a/docs/dpulse-docs/docs/dorking.md b/docs/dpulse-docs/docs/dorking.md
new file mode 100644
index 0000000..62610bf
--- /dev/null
+++ b/docs/dpulse-docs/docs/dorking.md
@@ -0,0 +1,198 @@
+# Automatic Google Dorking scan mode
+
+Automatic Google Dorking scan is an extended domain research function with prepared Google Dorking databases for different purposes.
+
+## Prepared Dorking databases description
+
+At the moment DPULSE offers the following prepared databases for automatic Google Dorking:
+
+1. IoT dorking
+2. Files dorking
+3. Admin panels dorking
+4. Web elements dorking
+
+IoT dorking table contains following 20 dorks:
+```
+inurl:":8080" site:{}
+inurl:":1883" site:{}
+inurl:":8883" site:{}
+inurl:":554" site:{}
+inurl:":81" site:{}
+inurl:":5000" site:{}
+inurl:":9000" site:{}
+inurl:":10000" site:{}
+inurl:debug site:{}
+inurl:device site:{}
+inurl:control site:{}
+inurl:status site:{}
+inurl:service site:{}
+inurl:monitor site:{}
+inurl:stream site:{}
+inurl:video site:{}
+inurl:camera site:{}
+inurl:sensor site:{}
+inurl:api site:{}
+inurl:firmware site:{}
+```
+
+Files dorking table contains following 30 dorks:
+```
+filetype:pdf site:{}
+filetype:doc site:{}
+filetype:docx site:{}
+filetype:xlsx site:{}
+filetype:xls site:{}
+filetype:ppt site:{}
+filetype:pptx site:{}
+filetype:txt site:{}
+filetype:csv site:{}
+filetype:xml site:{}
+filetype:json site:{}
+filetype:html site:{}
+filetype:php site:{}
+filetype:asp site:{}
+filetype:aspx site:{}
+filetype:js site:{}
+filetype:css site:{}
+filetype:jpg site:{}
+filetype:jpeg site:{}
+filetype:png site:{}
+filetype:gif site:{}
+filetype:mp3 site:{}
+filetype:mp4 site:{}
+filetype:avi site:{}
+filetype:zip site:{}
+filetype:rar site:{}
+filetype:sql site:{}
+filetype:db site:{}
+filetype:conf site:{}
+filetype:ini site:{}
+```
+
+Admin panels dorking table contains following 72 dorks:
+```
+site:{} intitle:"WordPress Login"
+site:{} inurl:/wp-admin/
+site:{} intext:"Войти в WordPress"
+site:{} intitle:"Dashboard" "WordPress"
+site:{} intitle:"Joomla! Administrator Login"
+site:{} inurl:/administrator/
+site:{} intitle:"Joomla! 3.x" "Login"
+site:{} intitle:"Drupal login"
+site:{} inurl:/user/login
+site:{} intitle:"Drupal 8" "Login"
+site:{} intitle:"phpMyAdmin"
+site:{} inurl:/phpmyadmin/
+site:{} intitle:"phpMyAdmin 4.x"
+site:{} intitle:"Magento Admin"
+site:{} inurl:/admin/
+site:{} intitle:"Magento 2" "Admin"
+site:{} intitle:"vBulletin Admin CP"
+site:{} inurl:/admincp/
+site:{} intitle:"vBulletin 4.x" "Admin"
+site:{} intitle:"osCommerce Administration"
+site:{} intitle:"osCommerce 2.x" "Admin"
+site:{} intitle:"PrestaShop Back Office"
+site:{} inurl:/admin-dev/
+site:{} intitle:"PrestaShop 1.7" "Back Office"
+site:{} intitle:"OpenCart Admin Panel"
+site:{} intitle:"OpenCart 3.x" "Admin"
+site:{} intitle:"Zen Cart Admin"
+site:{} intitle:"Zen Cart 1.5" "Admin"
+site:{} intitle:"MediaWiki" "Special:UserLogin"
+site:{} inurl:/mediawiki/index.php/Special:UserLogin
+site:{} intitle:"Moodle" "Log in to the site"
+site:{} inurl:/login/index.php
+site:{} intitle:"Concrete5" "Sign In"
+site:{} inurl:/index.php/dashboard/
+site:{} intitle:"TYPO3" "Backend Login"
+site:{} inurl:/typo3/
+site:{} intitle:"Plone" "Log in"
+site:{} inurl:/login_form
+site:{} intitle:"Django" "Site administration"
+site:{} inurl:/rails/admin/
+site:{} intitle:"Ruby on Rails" "Admin"
+site:{} intitle:"Craft CMS" "Control Panel"
+site:{} inurl:/admin/
+site:{} intitle:"ExpressionEngine" "Control Panel"
+site:{} inurl:/admin.php
+site:{} intitle:"Kentico" "CMS Desk"
+site:{} inurl:/cmsdesk/
+site:{} intitle:"Umbraco" "Backoffice"
+site:{} inurl:/umbraco/
+site:{} intitle:"Sitecore" "Launchpad"
+site:{} inurl:/sitecore/
+site:{} intitle:"DotNetNuke" "Host"
+site:{} inurl:/host/
+site:{} intitle:"SharePoint" "Sign In"
+site:{} inurl:/_layouts/15/
+site:{} intitle:"Plesk" "Login"
+site:{} inurl:login.php?user=admin
+site:{} inurl:dashboard
+site:{} intitle:"admin login"
+site:{} intitle:"administrator login"
+site:{} "admin panel"
+site:{} inurl:panel
+site:{} inurl:cp
+site:{} inurl:controlpanel
+site:{} inurl:backend
+site:{} inurl:management
+site:{} inurl:administration
+site:{} intitle:"admin access"
+site:{} intitle:"control panel"
+site:{} "admin login" +directory
+site:{} "administrator login" +password
+site:{} inurl:/plesk-login/
+```
+
+Web elements dorking table contains following 25 dorks:
+```
+site:{} intext:"index of"
+site:{} inurl:admin
+site:{} inurl:login
+site:{} inurl:dashboard
+site:{} inurl:wp-content
+site:{} inurl:backup
+site:{} inurl:old
+site:{} inurl:temp
+site:{} inurl:upload
+site:{} inurl:download
+site:{} inurl:config
+site:{} inurl:setup
+site:{} inurl:install
+site:{} inurl:database
+site:{} inurl:log
+site:{} inurl:debug
+site:{} inurl:api
+site:{} inurl:secret
+site:{} inurl:private
+site:{} inurl:secure
+site:{} inurl:password
+site:{} inurl:auth
+site:{} inurl:token
+site:{} inurl:session
+site:{} inurl:panel
+```
+
+## Creating custom Dorking database
+
+DPULSE allows you to create your own custom Google Dorking database. You can do it using DPULSE CLI by selecting menus as shown below:
+
+
+
+After you select this menu point you will be welcomed with custom Dorking DB generator. It's very simple to use. First you should enter your new custom Dorking DB name without any extensions. Then you'll be prompted to enter id of your first dork (first id in custom DB is always 1, and every next dork gives +1 to id) and dork itself. There's a rule DPULSE requires from you when inputting dorks: when it comes to define domain in dork, put {} instead of it so the program code will replace these brackets with actual domain you'll enter lately.
+
+Example of custom Dorking DB generator interaction is shown below:
+
+
+
+In result, new .db file will appear in dorking folder, which can be selected later to use in scan:
+
+
+
+And how it looks inside:
+
+
+
+
+
diff --git a/docs/dpulse-docs/docs/getting_started.md b/docs/dpulse-docs/docs/getting_started.md
new file mode 100644
index 0000000..1ec28a8
--- /dev/null
+++ b/docs/dpulse-docs/docs/getting_started.md
@@ -0,0 +1,109 @@
+## Installing DPULSE
+
+You can install DPULSE in several ways, use the way you like the most. But since DPULSE repository is using Poetry to manage dependencies, it is higly recommended to install and start DPULSE using Poetry, especially on Linux systems where a lot of Python packages which DPULSE requires are preinstalled. More information about Poetry you can find on [Poetry official documentation page](https://python-poetry.org/docs/#ci-recommendations)
+
+### Install and start DPULSE. Way №1
+
+Just download DPULSE using fast-access links at the top of the README:
+
+
+
+Then just unpack downloaded archive, open terminal in DPULSE root folder and use `pip install -r requirements.txt` command to install requirements. Then type `python dpulse.py` in terminal, and that's where program starts.
+
+If `pip install -r requirements.txt` doesn't work, then just use `poetry install` command. After that, start DPULSE with `poetry run python dpulse.py`
+
+### Install and start DPULSE. Way №2
+
+Use this set of commands to install DPULSE stable versions:
+
+```
+git clone https://github.com/OSINT-TECHNOLOGIES/dpulse
+cd dpulse
+poetry install
+```
+
+Use this set of commands to install DPULSE rolling versions:
+
+```
+git clone --branch rolling --single-branch https://github.com/OSINT-TECHNOLOGIES/dpulse.git
+cd dpulse
+poetry install
+```
+
+After installation, you simply start DPULSE using `poetry run python dpulse.py`
+
+### Install and start DPULSE. Way №3
+
+You also can install DPULSE using pip manager. It'll install DPULSE and necessery dependencies in one command: `pip install dpulse`. Then you just locate DPULSE root folder and type `python dpulse.py` to start program.
+
+### Installers usage
+
+DPULSE has two pre-written installation scripts, both for Windows (installer.bat) and for Linux (installer.sh). You can use them to clone repository and install dependencies or only for dependencies installation. Keep in mind that installer.bat (Windows installer) requires installed Git to clone repository.
+
+#### Windows installer usage
+
+You can start installer.bat from terminal by typing `./installer.bat` in terminal. Then you choose menu item which you want to start.
+
+If you have problems with starting installer.bat, you should try to start it in admin terminal.
+
+#### Linux installer usage
+
+To start installer.sh in Linux you should follow these steps in your terminal:
+
+```
+sudo chmod +x installer.sh
+sudo bash installer.sh
+```
+
+Then you choose menu item which you want to start.
+
+If you have problems with starting installer.sh, you should try to use `dos2unix installer.sh` or `sed -i 's/\r//' installer.sh` commands.
+
+### Conduct your first scan
+
+#### Meet DPULSE main menu
+
+After you successfully start DPULSE, you will see its main menu which contains some menu points. In order to conduct your first scan, you should select 1st point:
+
+
+
+#### Correctly input your target
+
+After selecting 1st menu point, you'll be asked to enter your target's domain name as shown below:
+
+
+
+DPULSE works only with domain names. Enter domain name when you are asked to determine target resource. When it is necessary, DPULSE will transform domain name in URL.
+
+A domain name is a unique web address that can be acquired through domain registration. It’s the website’s equivalent to a physical address. It consists of a name and an extension, and it helps users easily find your website and eliminates the need to memorize the site’s Internet Protocol (IP) address. In the context of the internet, a domain name identifies a realm of administrative autonomy, authority, or control. Domain names are often used to identify services provided through the internet, such as websites, email services, and more.
+
+| URL | Domain name |
+| ------------- | ------------- |
+| Example: https://www.bing.com/ | Example: bing.com |
+| String that provides the complete address of a web page | Human-friendly text form of an IP address |
+| Contains the following parts: method, protocol, hostname, port, and path of the file | Part of the URL and is mostly used for branding of the organization |
+
+#### Select the modifiers
+
+After you've enter correct domain name of your target, you'll be asked to set some modifiers for your scan. These contain:
+
+* Case comment: some brief description of what you are scanning and what do you want to find (actually you can enter anything here)
+* Report file extension: type of your report file, as you can see it can be XLSX or HTML report
+* PageSearch usage: determines if PageSearch will be used during scan
+* PageSearch keywords (if previous modifier was set to Y): keywords which DPULSE will try to find in found PDF documents
+* Dorking mode: determines which Dorking database will be used during scan, but can be declined with N flag
+* Third-party APIs usage: determines which API will be used during scan, but can be declined with N flag
+
+Example of modifiers selection is shown below:
+
+
+
+#### Get the report!
+
+After your first scan will be successfully ended, you will see this post-scan messages:
+
+
+
+This means that scan was ended, report was generated and scan results were written in report storage database. Congratulations, you successfully conducted your first scan using DPULSE!
+
+
diff --git a/docs/dpulse-docs/docs/index.md b/docs/dpulse-docs/docs/index.md
new file mode 100644
index 0000000..7fa7b9e
--- /dev/null
+++ b/docs/dpulse-docs/docs/index.md
@@ -0,0 +1,19 @@
+# Welcome to DPULSE documentation
+
+Convenient, fast and user-friendly collector of domain information from open-sources
+
+## What is DPULSE
+
+DPULSE is a software solution for conducting OSINT research in relation to a certain domain. In general, it provides you with a certain set of functions, such as:
+
+1. ***Basic scan:*** extracts general information about domain such as WHOIS information, subdomains, e-mail addresses, IP addresses, social medias links/posts/profiles, SSL certificate info, possible vulnerabilities, open ports, CPEs, used web-technologies and so on. It also can download sitemap.xml and robots.txt files from a domain
+
+2. ***PageSearch standard scan:*** extended subdomains deep search function, which starts in addition to basic scan and which can find more e-mail addresses, API keys, exposed passwords, cookies, hidden forms of data and other web page elements, documents, config files, databases files (and PageSearch can download them!), specified words by user in PDF files
+
+3. ***PageSearch Sitemap inspection scan:*** sitemap links crawler which starts in addition to basic scan and which can find even more e-mails
+
+4. ***Dorking scan:*** extended domain research function with prepared Google Dorking databases for different purposes, such as IoT dorking, files dorking, admin panels dorking, web elements dorking. Moreover, this mode allows you to create your own custom Google Dorking database
+
+5. ***API scan:*** extended domain research function with prepared functions for 3rd party APIs usage. Currently DPULSE supports VirusTotal API (for brief domain information gathering) and SecurityTrails API (deep subdomains and DNS enumeration)
+
+Finally, DPULSE compiles all found data into an easy-to-read HTML or XLSX report by category. It also saves all information about scan in local report storage database, which can be restored later.
diff --git a/docs/dpulse-docs/docs/pagesearch.md b/docs/dpulse-docs/docs/pagesearch.md
new file mode 100644
index 0000000..02c04f0
--- /dev/null
+++ b/docs/dpulse-docs/docs/pagesearch.md
@@ -0,0 +1,16 @@
+# PageSearch mode
+
+PageSearch is an additional extended subdomains deep search function, which starts in addition to basic scan. User can choose whether to use PageSearch or not during pre-scan preparation steps.
+
+## PageSearch results
+
+PageSearch returns extended information about found subdomains during basic scan. Extended information contains following:
+
+1. More e-mail addresses
+2. API keys
+3. Exposed passwords
+4. Cookies
+5. Hidden forms of data and other web page elements
+6. Documents, config files, databases files (and PageSearch can download them!)
+7. Specified words by user in PDF files
+
diff --git a/docs/dpulse-docs/docs/pagesearch_sitemap.md b/docs/dpulse-docs/docs/pagesearch_sitemap.md
new file mode 100644
index 0000000..3f29b9f
--- /dev/null
+++ b/docs/dpulse-docs/docs/pagesearch_sitemap.md
@@ -0,0 +1,9 @@
+# PageSearch Sitemap Inspection mode
+
+PageSearch Sitemap Inspection is an additional extended sitemap links inspection function, which basically crawls into them and extract all e-mails from them. User can choose whether to use PageSearch Sitemap Inspection or not during pre-scan preparation steps.
+
+## PageSearch Sitemap Inspection results
+
+Basically PageSearch Sitemap Inspection mode returns lots of e-mails from all links, extracted from sitemap.xml
+
+
diff --git a/docs/dpulse-docs/docs/reporting.md b/docs/dpulse-docs/docs/reporting.md
new file mode 100644
index 0000000..5d09fa5
--- /dev/null
+++ b/docs/dpulse-docs/docs/reporting.md
@@ -0,0 +1,59 @@
+# About reporting system
+
+DPULSE as every OSINT tool is highly dependent on reporting system. User-friendly reports are crucial for detailed results presentation and further storage. DPULSE supports the most common types of reports: HTML and XLSX. Moreover, DPULSE provides you with reports storage database, which contains some information about scan, report and extracted data and gives you opportunity for long-term reports storage in one place. Also you can move this database between different DPULSE versions, which brings a little bit better user-experience.
+
+## HTML report
+
+HTML report was the first supported type of report. HTML is a widely supported format that can be opened in any web browser, allowing for the creation of visually appealing reports using tables, charts, diagrams, and other elements. It supports links and hyperlinks that can be used to create navigation within the report and link to external resources, and enables creating dynamic content that can be updated in real-time. HTML is often used in web applications, making it easy to integrate reports with existing web systems. And, in general, this report format is more user-friendly, which makes it more convenient for sharing with investigation customers, OSINT teams and usage in presentations. Moreover, unlike PDF report generation, HTML is easier to handle when both developing and delievering, as it doesn't require to install 3rd party applications (like wkhtmltopdf). You can see example of DPULSE generated HTML report [here](https://github.com/OSINT-TECHNOLOGIES/dpulse/tree/rolling/report_examples/html_report_example).
+
+## XLSX report
+
+XLSX is a widely supported format that can be opened in most spreadsheet and office applications, including Microsoft Excel, Google Sheets, and LibreOffice Calc, making it easy to analyze and process data. It allows storing data in a structured format, supports formulas and functions that can be used to automate calculations and data analysis, and enables creating charts and diagrams to visualize data. Additionally, XLSX is often used in business applications, making it easy to integrate reports with existing systems. You can see example of DPULSE generated XLSX report [here](https://github.com/OSINT-TECHNOLOGIES/dpulse/tree/rolling/report_examples/xlsx_report_example).
+
+## Side files
+
+As you may have noticed in report examples on GitHub page, there are also some side files except for report file. These files may be the following:
+
+* target's robots.txt file (if accessible)
+* target's sitemap.xml file (if accessible)
+* ps_documents folder with extracted documents from domain and its subdomains (if PageSearch was selected for scan)
+
+## Report storage database
+
+As said above, report storage database contains key information about scan, report and extracted data. DPULSE generates this database when DPULSE is first launched or if database file was not found in the root directory, so users don't need to worry about it's manual creation. Report storage database is a simple .db file (with hard-coded report_storage.db name) with structure which shown below:
+
+
+
+Lets describe these fields in more detailed way:
+
+* id - integer value that displays the number of reports generated and the order in which they are generated
+* report_file_extension - string which shows main report file extension, in current DPULSE version this value could be xlsx, pdf or html
+* report_content - BLOB or HTML data which contains main report file's copy
+* comment - string which shows comment to your cases, which you can enter before each scan
+* target - string which shows domain which you've scanned
+* creation_date - string which shows when your report was generated (YYYYMMDD format)
+* dorks_results - text array which contains a copy of Google Dorking results (if this mode was selected before scan)
+* robots_text - text array which contains a copy of robots.txt file from scanned domain
+* sitemap_text - text array which contains all sitemap.xml links file from scanned domain
+* sitemap_file - text array which contains a copy of sitemap.xml file from scanned domain
+* api_scan - string which indicates whether API scanning was activated or not, and if it was activated - contains used APIs
+
+Interacting with report storage database is a very simple process. First of all, after each scan you can see several messages which indicate that your report was successfully saved in report storage database:
+
+
+
+Also, you have separate menu item in DPULSE CLI to work with report storage database which named "Report storage DB manager":
+
+
+
+As you can see, there are menu points for both seeing DB content and recreating reports. Lets see what DPULSE will return if we select first menu item:
+
+
+
+Report recreating process is shown below:
+
+
+
+And that's how recreated report looks like inside:
+
+
diff --git a/docs/dpulse-docs/mkdocs.yml b/docs/dpulse-docs/mkdocs.yml
new file mode 100644
index 0000000..c9618f9
--- /dev/null
+++ b/docs/dpulse-docs/mkdocs.yml
@@ -0,0 +1,15 @@
+site_name: DPULSE Docs
+theme:
+ name: readthedocs
+nav:
+ - Home: index.md
+ - Getting started: getting_started.md
+ - Basic scan: basic_scan.md
+ - PageSearch: pagesearch.md
+ - PageSearch Sitemap Inspection: pagesearch_sitemap.md
+ - Built-in automatic Dorking: dorking.md
+ - Built-in API scanning: api.md
+ - Reporting and report types: reporting.md
+ - Demo and use-cases: demo.md
+ - Contact developer: contact_dev.md
+
diff --git a/dpulse.py b/dpulse.py
index 48c02ea..34a1127 100644
--- a/dpulse.py
+++ b/dpulse.py
@@ -25,7 +25,12 @@
from logs_processing import logging
from db_creator import get_columns_amount
-db.db_creation('report_storage.db')
+rsdb_presence = db.check_rsdb_presence('report_storage.db')
+if rsdb_presence is True:
+ print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL)
+else:
+ db.db_creation('report_storage.db')
+ print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL)
dorks_files_check()
@@ -236,27 +241,11 @@ def run():
elif choice_dorking == '2':
continue
elif choice == "6":
- cli.print_help_menu()
- choice_help = input(Fore.YELLOW + "Enter your choice >> ")
- if choice_help == '1':
- webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse')
- elif choice_help == '2':
- webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-WIKI')
- elif choice_help == '3':
- webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/How-to-correctly-input-your-targets-address-in-DPULSE')
- elif choice_help == '4':
- webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-PageSearch-function-guide')
- elif choice_help == '5':
- webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-report-storage-database')
- elif choice_help == '6':
- continue
- else:
- print(Fore.RED + "\nInvalid menu item. Please select between existing menu items")
+ webbrowser.open('https://dpulse.readthedocs.io/en/latest/')
elif choice == '5':
cli.api_manager()
- print('\n')
- choice_api = input(Fore.YELLOW + "Enter your choice >> ")
+ choice_api = input(Fore.YELLOW + "\nEnter your choice >> ")
if choice_api == '1':
print(Fore.GREEN + "\nSupported APIs and your keys:\n")
cursor, conn = db.select_api_keys('updating')
@@ -294,18 +283,18 @@ def run():
elif choice == "4":
cli.print_db_menu()
- print('\n')
- db.db_creation('report_storage.db')
- print('\n')
+ rsdb_presence = db.check_rsdb_presence('report_storage.db')
+ if rsdb_presence is True:
+ print(Fore.GREEN + "\nReport storage database presence: OK\n" + Style.RESET_ALL)
+ else:
+ db.db_creation('report_storage.db')
+ print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL)
choice_db = input(Fore.YELLOW + "Enter your choice >> ")
- if choice_db == '1':
- db.db_select()
+ if choice_db == "1":
+ cursor, sqlite_connection, data_presence_flag = db.db_select()
elif choice_db == "2":
- if db.db_select() is None:
- pass
- else:
- print(Fore.LIGHTMAGENTA_EX + "\n[DATABASE'S CONTENT]\n" + Style.RESET_ALL)
- db.db_select()
+ cursor, sqlite_connection, data_presence_flag = db.db_select()
+ if data_presence_flag is True:
id_to_extract = int(input(Fore.YELLOW + "\nEnter report ID you want to extract >> "))
extracted_folder_name = 'report_recreated_ID#{}'.format(id_to_extract)
try:
@@ -315,7 +304,8 @@ def run():
print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL)
except Exception as e:
print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL)
-
+ else:
+ pass
elif choice_db == "3":
print(Fore.GREEN + "\nDatabase connection is successfully closed")
continue
diff --git a/service/cli_init.py b/service/cli_init.py
index ea1c436..05f1de6 100644
--- a/service/cli_init.py
+++ b/service/cli_init.py
@@ -20,8 +20,8 @@ def welcome_menu(self):
fig = Figlet(font=wm_font)
print('\n')
self.console.print(fig.renderText('DPULSE'), style=preview_style)
- print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.1.6 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL)
- print(Fore.MAGENTA + Style.BRIGHT + 'Visit our pages:\nhttps://github.com/OSINT-TECHNOLOGIES\nhttps://pypi.org/project/dpulse/' + Style.RESET_ALL)
+ print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL)
+ print(Fore.MAGENTA + Style.BRIGHT + '[Visit our pages]\nGitHub repository: https://github.com/OSINT-TECHNOLOGIES\nPyPi page: https://pypi.org/project/dpulse/\nDocumentation: https://dpulse.readthedocs.io' + Style.RESET_ALL)
def print_main_menu(self):
print('\n')
@@ -31,7 +31,7 @@ def print_main_menu(self):
print(Fore.CYAN + "3. Dorking module manager")
print(Fore.CYAN + "4. Report storage DB manager")
print(Fore.CYAN + "5. API modules manager")
- print(Fore.CYAN + "6. Help")
+ print(Fore.CYAN + "6. Help (browser will be opened!)")
print(Fore.LIGHTRED_EX + "7. Exit DPULSE" + Style.RESET_ALL + '\n')
def print_settings_menu(self):
@@ -42,16 +42,6 @@ def print_settings_menu(self):
print(Fore.CYAN + "3. Clear journal content")
print(Fore.LIGHTRED_EX + "4. Return to main menu" + Style.RESET_ALL + '\n')
- def print_help_menu(self):
- print(Fore.MAGENTA + Back.WHITE + '[HELP MENU]' + Style.RESET_ALL)
- print(Fore.BLACK + Back.WHITE + '\nBe advised that choosing any of points below will open your web browser!' + Style.RESET_ALL)
- print(Fore.CYAN + "1. Open DPULSE repository")
- print(Fore.CYAN + "2. Open DPULSE wiki")
- print(Fore.CYAN + "3. How to correctly input your targets in DPULSE")
- print(Fore.CYAN + "4. PageSearch user guide")
- print(Fore.CYAN + "5. Report storage database user guide")
- print(Fore.LIGHTRED_EX + "6. Return to main menu" + Style.RESET_ALL + '\n')
-
def print_db_menu(self):
print('\n')
print(Fore.MAGENTA + Back.WHITE + '[REPORTS DATABASE MANAGER]' + Style.RESET_ALL)
@@ -63,8 +53,7 @@ def dorking_db_manager(self):
print('\n')
print(Fore.MAGENTA + Back.WHITE + '[DORKING DB MANAGER]' + Style.RESET_ALL)
print(Fore.CYAN + "1. Generate custom Dorking DB")
- print(Fore.LIGHTRED_EX + "2. Return to main menu" + Style.RESET_ALL)
- print('\n')
+ print(Fore.LIGHTRED_EX + "2. Return to main menu\n" + Style.RESET_ALL)
def api_manager(self):
print('\n')
@@ -83,8 +72,7 @@ def print_prescan_summary(short_domain, report_filetype, pagesearch_ui_mark, dor
print(Fore.GREEN + "Case comment: " + Fore.LIGHTCYAN_EX + Style.BRIGHT + case_comment + Style.RESET_ALL + "\n")
def print_api_db_msg():
- print('\n')
- print(Fore.GREEN + "You've entered custom Dorking DB generator!\n" + Style.RESET_ALL)
+ print(Fore.GREEN + "\nYou've entered custom Dorking DB generator!\n" + Style.RESET_ALL)
print(Fore.GREEN + "Remember some rules in order to successfully create your custom Dorking DB:" + Style.RESET_ALL)
print(Fore.GREEN + "[1] - dork_id variable must be unique, starting with 1 and then +1 every new dork" + Style.RESET_ALL)
print(Fore.GREEN + "[2] - When it comes to define domain in dork, put {} in it\n" + Style.RESET_ALL)
diff --git a/service/db_processing.py b/service/db_processing.py
index fab120d..b179977 100644
--- a/service/db_processing.py
+++ b/service/db_processing.py
@@ -8,15 +8,19 @@
def db_connect():
sqlite_connection = sqlite3.connect('report_storage.db')
cursor = sqlite_connection.cursor()
- print(Fore.GREEN + "Successfully established SQLite3 DB connection")
return cursor, sqlite_connection
-def db_creation(db_path):
+def check_rsdb_presence(db_path):
if not os.path.exists(db_path):
print(Fore.RED + "Report storage database was not found. DPULSE will create it in a second" + Style.RESET_ALL)
- cursor, sqlite_connection = db_connect()
- create_table_sql = """
- CREATE TABLE "report_storage" (
+ return False
+ else:
+ return True
+
+def db_creation(db_path):
+ cursor, sqlite_connection = db_connect()
+ create_table_sql = """
+ CREATE TABLE "report_storage" (
"id" INTEGER NOT NULL UNIQUE,
"report_file_extension" TEXT NOT NULL,
"report_content" BLOB NOT NULL,
@@ -31,15 +35,11 @@ def db_creation(db_path):
PRIMARY KEY("id" AUTOINCREMENT)
);
"""
- cursor.execute(create_table_sql)
- sqlite_connection.commit()
- sqlite_connection.close()
- print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL)
- else:
- print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL)
+ cursor.execute(create_table_sql)
+ sqlite_connection.commit()
+ sqlite_connection.close()
def db_select():
- db_creation('report_storage.db')
cursor, sqlite_connection = db_connect()
if_rows = "SELECT * FROM report_storage"
cursor.execute(if_rows)
@@ -49,6 +49,7 @@ def db_select():
select_query = "SELECT creation_date, report_file_extension, target, id, comment, dorks_results, robots_text, sitemap_text, sitemap_file, api_scan FROM report_storage;"
cursor.execute(select_query)
records = cursor.fetchall()
+ print(Fore.LIGHTMAGENTA_EX + "\n[DATABASE'S CONTENT]\n" + Style.RESET_ALL)
for row in records:
dorks_presence = robots_presence = sitemap_presence = "None"
if len(row[4]) > 1:
@@ -58,17 +59,34 @@ def db_select():
if len(row[6]) > 1:
sitemap_presence = "In DB"
print(Fore.LIGHTBLUE_EX + f"Case ID: {row[3]} | Case name: {row[2]} | Case file extension: {row[1]} | Case comment: {row[4]} | Case creation date: {row[0]} | Dorking: {dorks_presence} | robots.txt: {robots_presence} | sitemap.xml: {sitemap_presence} | API scan: {row[-1]}" + Style.RESET_ALL)
+ data_presence_flag = True
except sqlite3.Error as e:
print(Fore.RED + "Failed to see storage database's content. Reason: {}".format(e))
sqlite_connection.close()
+ data_presence_flag = False
else:
print(Fore.RED + 'No data found in report storage database')
sqlite_connection.close()
- return None
+ data_presence_flag = False
+ return cursor, sqlite_connection, data_presence_flag
+
+def db_select_silent():
+ cursor, sqlite_connection = db_connect()
+ if_rows = "SELECT * FROM report_storage"
+ cursor.execute(if_rows)
+ rows = cursor.fetchall()
+ if rows:
+ try:
+ select_query = "SELECT creation_date, report_file_extension, target, id, comment, dorks_results, robots_text, sitemap_text, sitemap_file, api_scan FROM report_storage;"
+ cursor.execute(select_query)
+ except sqlite3.Error as e:
+ sqlite_connection.close()
+ else:
+ sqlite_connection.close()
return cursor, sqlite_connection
def db_report_recreate(extracted_folder_name, id_to_extract):
- cursor, sqlite_connection = db_select()
+ cursor, sqlite_connection = db_select_silent()
cursor.execute("SELECT report_content FROM report_storage WHERE id=?", (id_to_extract,))
try:
blob = cursor.fetchone()