From 14b0db32529cb1823ea336dd2e368bb5872d008e Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sat, 28 Dec 2024 10:08:59 +0300 Subject: [PATCH 01/59] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ade92b..3185249 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

-Static Badge Static Badge +Static Badge Static Badge Static Badge Static Badge

From c51aaf28a2dd3549780be493451350b801a4ff5a Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sat, 28 Dec 2024 10:09:29 +0300 Subject: [PATCH 02/59] Bumped version --- service/cli_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/cli_init.py b/service/cli_init.py index ea1c436..d8e2180 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -20,7 +20,7 @@ def welcome_menu(self): fig = Figlet(font=wm_font) print('\n') self.console.print(fig.renderText('DPULSE'), style=preview_style) - print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.1.6 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) + print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2 rolling] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) print(Fore.MAGENTA + Style.BRIGHT + 'Visit our pages:\nhttps://github.com/OSINT-TECHNOLOGIES\nhttps://pypi.org/project/dpulse/' + Style.RESET_ALL) def print_main_menu(self): From 8e305c27cdf24e3e40cad961848f11a824a0f6f3 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:43:03 +0300 Subject: [PATCH 03/59] Create __init__.py --- docs/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/__init__.py diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/__init__.py @@ -0,0 +1 @@ + From 8213e2b0c8b3fc8c2189d91113ee43a963edb09d Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:44:07 +0300 Subject: [PATCH 04/59] Created mkdocs.yml for future RTD documentation --- docs/dpulse-docs/mkdocs.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/dpulse-docs/mkdocs.yml diff --git a/docs/dpulse-docs/mkdocs.yml b/docs/dpulse-docs/mkdocs.yml new file mode 100644 index 0000000..777f785 --- /dev/null +++ b/docs/dpulse-docs/mkdocs.yml @@ -0,0 +1,14 @@ +site_name: DPULSE Docs +theme: + name: readthedocs +nav: + - Home: index.md + - Getting started: getting_started.md + - Basic scan: basic_scan.md + - PageSearch: pagesearch.md + - PageSearch Sitemap Inspection: pagesearch_sitemap.md + - Built-in automatic Dorking: dorking.md + - Built-in API scanning: api.md + - Reports types: reporting.md + - Contact developer: contact_dev.md + From 6912d28c618c9aac63506f9aeea207db3d021afd Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:45:04 +0300 Subject: [PATCH 05/59] Create index.md --- docs/dpulse-docs/docs/index.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/dpulse-docs/docs/index.md diff --git a/docs/dpulse-docs/docs/index.md b/docs/dpulse-docs/docs/index.md new file mode 100644 index 0000000..7fa7b9e --- /dev/null +++ b/docs/dpulse-docs/docs/index.md @@ -0,0 +1,19 @@ +# Welcome to DPULSE documentation + +Convenient, fast and user-friendly collector of domain information from open-sources + +## What is DPULSE + +DPULSE is a software solution for conducting OSINT research in relation to a certain domain. In general, it provides you with a certain set of functions, such as: + +1. ***Basic scan:*** extracts general information about domain such as WHOIS information, subdomains, e-mail addresses, IP addresses, social medias links/posts/profiles, SSL certificate info, possible vulnerabilities, open ports, CPEs, used web-technologies and so on. It also can download sitemap.xml and robots.txt files from a domain + +2. ***PageSearch standard scan:*** extended subdomains deep search function, which starts in addition to basic scan and which can find more e-mail addresses, API keys, exposed passwords, cookies, hidden forms of data and other web page elements, documents, config files, databases files (and PageSearch can download them!), specified words by user in PDF files + +3. ***PageSearch Sitemap inspection scan:*** sitemap links crawler which starts in addition to basic scan and which can find even more e-mails + +4. ***Dorking scan:*** extended domain research function with prepared Google Dorking databases for different purposes, such as IoT dorking, files dorking, admin panels dorking, web elements dorking. Moreover, this mode allows you to create your own custom Google Dorking database + +5. ***API scan:*** extended domain research function with prepared functions for 3rd party APIs usage. Currently DPULSE supports VirusTotal API (for brief domain information gathering) and SecurityTrails API (deep subdomains and DNS enumeration) + +Finally, DPULSE compiles all found data into an easy-to-read HTML or XLSX report by category. It also saves all information about scan in local report storage database, which can be restored later. From a9177e1640b1a2456ad149f0b4e7f787af153494 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:45:26 +0300 Subject: [PATCH 06/59] Create dorking.md --- docs/dpulse-docs/docs/dorking.md | 181 +++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 docs/dpulse-docs/docs/dorking.md diff --git a/docs/dpulse-docs/docs/dorking.md b/docs/dpulse-docs/docs/dorking.md new file mode 100644 index 0000000..a11a03a --- /dev/null +++ b/docs/dpulse-docs/docs/dorking.md @@ -0,0 +1,181 @@ +# Automatic Google Dorking scan mode + +Automatic Google Dorking scan is an extended domain research function with prepared Google Dorking databases for different purposes. + +## Prepared Dorking databases description + +At the moment DPULSE offers the following prepared databases for automatic Google Dorking: + +1. IoT dorking +2. Files dorking +3. Admin panels dorking +4. Web elements dorking + +IoT dorking table contains following 20 dorks: +``` +inurl:":8080" site:{} +inurl:":1883" site:{} +inurl:":8883" site:{} +inurl:":554" site:{} +inurl:":81" site:{} +inurl:":5000" site:{} +inurl:":9000" site:{} +inurl:":10000" site:{} +inurl:debug site:{} +inurl:device site:{} +inurl:control site:{} +inurl:status site:{} +inurl:service site:{} +inurl:monitor site:{} +inurl:stream site:{} +inurl:video site:{} +inurl:camera site:{} +inurl:sensor site:{} +inurl:api site:{} +inurl:firmware site:{} +``` + +Files dorking table contains following 30 dorks: +``` +filetype:pdf site:{} +filetype:doc site:{} +filetype:docx site:{} +filetype:xlsx site:{} +filetype:xls site:{} +filetype:ppt site:{} +filetype:pptx site:{} +filetype:txt site:{} +filetype:csv site:{} +filetype:xml site:{} +filetype:json site:{} +filetype:html site:{} +filetype:php site:{} +filetype:asp site:{} +filetype:aspx site:{} +filetype:js site:{} +filetype:css site:{} +filetype:jpg site:{} +filetype:jpeg site:{} +filetype:png site:{} +filetype:gif site:{} +filetype:mp3 site:{} +filetype:mp4 site:{} +filetype:avi site:{} +filetype:zip site:{} +filetype:rar site:{} +filetype:sql site:{} +filetype:db site:{} +filetype:conf site:{} +filetype:ini site:{} +``` + +Admin panels dorking table contains following 72 dorks: +``` +site:{} intitle:"WordPress Login" +site:{} inurl:/wp-admin/ +site:{} intext:"Войти в WordPress" +site:{} intitle:"Dashboard" "WordPress" +site:{} intitle:"Joomla! Administrator Login" +site:{} inurl:/administrator/ +site:{} intitle:"Joomla! 3.x" "Login" +site:{} intitle:"Drupal login" +site:{} inurl:/user/login +site:{} intitle:"Drupal 8" "Login" +site:{} intitle:"phpMyAdmin" +site:{} inurl:/phpmyadmin/ +site:{} intitle:"phpMyAdmin 4.x" +site:{} intitle:"Magento Admin" +site:{} inurl:/admin/ +site:{} intitle:"Magento 2" "Admin" +site:{} intitle:"vBulletin Admin CP" +site:{} inurl:/admincp/ +site:{} intitle:"vBulletin 4.x" "Admin" +site:{} intitle:"osCommerce Administration" +site:{} intitle:"osCommerce 2.x" "Admin" +site:{} intitle:"PrestaShop Back Office" +site:{} inurl:/admin-dev/ +site:{} intitle:"PrestaShop 1.7" "Back Office" +site:{} intitle:"OpenCart Admin Panel" +site:{} intitle:"OpenCart 3.x" "Admin" +site:{} intitle:"Zen Cart Admin" +site:{} intitle:"Zen Cart 1.5" "Admin" +site:{} intitle:"MediaWiki" "Special:UserLogin" +site:{} inurl:/mediawiki/index.php/Special:UserLogin +site:{} intitle:"Moodle" "Log in to the site" +site:{} inurl:/login/index.php +site:{} intitle:"Concrete5" "Sign In" +site:{} inurl:/index.php/dashboard/ +site:{} intitle:"TYPO3" "Backend Login" +site:{} inurl:/typo3/ +site:{} intitle:"Plone" "Log in" +site:{} inurl:/login_form +site:{} intitle:"Django" "Site administration" +site:{} inurl:/rails/admin/ +site:{} intitle:"Ruby on Rails" "Admin" +site:{} intitle:"Craft CMS" "Control Panel" +site:{} inurl:/admin/ +site:{} intitle:"ExpressionEngine" "Control Panel" +site:{} inurl:/admin.php +site:{} intitle:"Kentico" "CMS Desk" +site:{} inurl:/cmsdesk/ +site:{} intitle:"Umbraco" "Backoffice" +site:{} inurl:/umbraco/ +site:{} intitle:"Sitecore" "Launchpad" +site:{} inurl:/sitecore/ +site:{} intitle:"DotNetNuke" "Host" +site:{} inurl:/host/ +site:{} intitle:"SharePoint" "Sign In" +site:{} inurl:/_layouts/15/ +site:{} intitle:"Plesk" "Login" +site:{} inurl:login.php?user=admin +site:{} inurl:dashboard +site:{} intitle:"admin login" +site:{} intitle:"administrator login" +site:{} "admin panel" +site:{} inurl:panel +site:{} inurl:cp +site:{} inurl:controlpanel +site:{} inurl:backend +site:{} inurl:management +site:{} inurl:administration +site:{} intitle:"admin access" +site:{} intitle:"control panel" +site:{} "admin login" +directory +site:{} "administrator login" +password +site:{} inurl:/plesk-login/ +``` + +Web elements dorking table contains following 25 dorks: +``` +site:{} intext:"index of" +site:{} inurl:admin +site:{} inurl:login +site:{} inurl:dashboard +site:{} inurl:wp-content +site:{} inurl:backup +site:{} inurl:old +site:{} inurl:temp +site:{} inurl:upload +site:{} inurl:download +site:{} inurl:config +site:{} inurl:setup +site:{} inurl:install +site:{} inurl:database +site:{} inurl:log +site:{} inurl:debug +site:{} inurl:api +site:{} inurl:secret +site:{} inurl:private +site:{} inurl:secure +site:{} inurl:password +site:{} inurl:auth +site:{} inurl:token +site:{} inurl:session +site:{} inurl:panel +``` + +## Creating custom Dorking database + +DPULSE allows you to create your own custom Google Dorking database + + From 852f5839f0b5d4125e99d4d427789562dfd648e0 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:45:41 +0300 Subject: [PATCH 07/59] Create pagesearch.md --- docs/dpulse-docs/docs/pagesearch.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/dpulse-docs/docs/pagesearch.md diff --git a/docs/dpulse-docs/docs/pagesearch.md b/docs/dpulse-docs/docs/pagesearch.md new file mode 100644 index 0000000..02c04f0 --- /dev/null +++ b/docs/dpulse-docs/docs/pagesearch.md @@ -0,0 +1,16 @@ +# PageSearch mode + +PageSearch is an additional extended subdomains deep search function, which starts in addition to basic scan. User can choose whether to use PageSearch or not during pre-scan preparation steps. + +## PageSearch results + +PageSearch returns extended information about found subdomains during basic scan. Extended information contains following: + +1. More e-mail addresses +2. API keys +3. Exposed passwords +4. Cookies +5. Hidden forms of data and other web page elements +6. Documents, config files, databases files (and PageSearch can download them!) +7. Specified words by user in PDF files + From acc2ee36c7351341b4cea9891d8373e496985e65 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:46:00 +0300 Subject: [PATCH 08/59] Create basic_scan.md --- docs/dpulse-docs/docs/basic_scan.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/dpulse-docs/docs/basic_scan.md diff --git a/docs/dpulse-docs/docs/basic_scan.md b/docs/dpulse-docs/docs/basic_scan.md new file mode 100644 index 0000000..4d9bebc --- /dev/null +++ b/docs/dpulse-docs/docs/basic_scan.md @@ -0,0 +1,18 @@ +# Basic scan + +Basic scan is default and the most basic scanning mode which starts after all preparation steps. It always starts first and you can't start scanning without it. + +## Basic scan results + +Basic scan returns some basic information open information about target domain, such as: + +1. WHOIS information which contains domain name, full URL, IP address, registrar info, creation and expiration dates, organization name and contact e-mails. +2. Subdomains list +3. E-mail addresses list (gathered from subdomains) +4. IP addresses list (gathered from subdomains) +5. Social medias links, posts and profiles. Supported social medias are Facebook, Twitter (X.com), Instagram, Telegram, TikTok, LinkedIn, VKontakte, YouTube, Odnoklassniki, WeChat +6. DNS and SSL information. DNS information contains name servers and MX addresses. SSL certificate information contains issuer, subject, creation and expiration dates, certificate name and serial number. +7. Basic pre-pentest information such as possible vulnerabilities (CVEs), open ports and hostnames. +8. Development and deployment services and frameworks information, such as CMS, web servers, used programming languages and web frameworks, analytics services, tags and so on. +9. Downloaded copies of sitemap.xml and robots.txt files from a domain + From 496f36bf828ca415ec89b6001491abdcfc2bc827 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:46:17 +0300 Subject: [PATCH 09/59] Create api.md --- docs/dpulse-docs/docs/api.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/dpulse-docs/docs/api.md diff --git a/docs/dpulse-docs/docs/api.md b/docs/dpulse-docs/docs/api.md new file mode 100644 index 0000000..bad8c72 --- /dev/null +++ b/docs/dpulse-docs/docs/api.md @@ -0,0 +1,10 @@ +# Third-party API scan mode + +Currently DPULSE supports two third-party APIs: +1. SecurityTrails API (securitytrails.com) for deep subdomains and DNS enumeration +2. VirusTotal API (virustotal.com) for brief domain information gathering + +## SecurityTrails API + +## VirusTotal API + From 48402e4fe0697e03641817b5e559fab14f14a083 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:46:37 +0300 Subject: [PATCH 10/59] Create reporting.md --- docs/dpulse-docs/docs/reporting.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dpulse-docs/docs/reporting.md diff --git a/docs/dpulse-docs/docs/reporting.md b/docs/dpulse-docs/docs/reporting.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/dpulse-docs/docs/reporting.md @@ -0,0 +1 @@ + From 2b198b1189e585356064e857894efe775e2ae283 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:47:00 +0300 Subject: [PATCH 11/59] Create pagesearch_sitemap.md --- docs/dpulse-docs/docs/pagesearch_sitemap.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/dpulse-docs/docs/pagesearch_sitemap.md diff --git a/docs/dpulse-docs/docs/pagesearch_sitemap.md b/docs/dpulse-docs/docs/pagesearch_sitemap.md new file mode 100644 index 0000000..3f29b9f --- /dev/null +++ b/docs/dpulse-docs/docs/pagesearch_sitemap.md @@ -0,0 +1,9 @@ +# PageSearch Sitemap Inspection mode + +PageSearch Sitemap Inspection is an additional extended sitemap links inspection function, which basically crawls into them and extract all e-mails from them. User can choose whether to use PageSearch Sitemap Inspection or not during pre-scan preparation steps. + +## PageSearch Sitemap Inspection results + +Basically PageSearch Sitemap Inspection mode returns lots of e-mails from all links, extracted from sitemap.xml + + From d04fa3214dddf8dc5a0764328ee02206bb653f99 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:47:13 +0300 Subject: [PATCH 12/59] Create getting_started.md --- docs/dpulse-docs/docs/getting_started.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dpulse-docs/docs/getting_started.md diff --git a/docs/dpulse-docs/docs/getting_started.md b/docs/dpulse-docs/docs/getting_started.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/dpulse-docs/docs/getting_started.md @@ -0,0 +1 @@ + From 55413ec0e2d1669eff7fbae8915a75862da16dae Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:47:28 +0300 Subject: [PATCH 13/59] Create contact_dev.md --- docs/dpulse-docs/docs/contact_dev.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dpulse-docs/docs/contact_dev.md diff --git a/docs/dpulse-docs/docs/contact_dev.md b/docs/dpulse-docs/docs/contact_dev.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/dpulse-docs/docs/contact_dev.md @@ -0,0 +1 @@ + From e211efaf5d0aeadd53403fbf6f2092db87f8c7d6 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 29 Dec 2024 02:50:21 +0300 Subject: [PATCH 14/59] Create .readthedocs.yaml (empty) --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1 @@ + From 593b0ee495ac467d3af5a33c909dada058aab06f Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 00:51:39 +0300 Subject: [PATCH 15/59] Added some new paragraphs --- docs/dpulse-docs/mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/dpulse-docs/mkdocs.yml b/docs/dpulse-docs/mkdocs.yml index 777f785..582908e 100644 --- a/docs/dpulse-docs/mkdocs.yml +++ b/docs/dpulse-docs/mkdocs.yml @@ -10,5 +10,7 @@ nav: - Built-in automatic Dorking: dorking.md - Built-in API scanning: api.md - Reports types: reporting.md + - Reports storage database: report_storage.md + - Demo and use-cases: demo.md - Contact developer: contact_dev.md From 05f29d677be1a4d65d79d9f92dfb9887a2cc2ffd Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 00:52:24 +0300 Subject: [PATCH 16/59] Updated getting_started.md --- docs/dpulse-docs/docs/getting_started.md | 108 +++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/docs/dpulse-docs/docs/getting_started.md b/docs/dpulse-docs/docs/getting_started.md index 8b13789..1ec28a8 100644 --- a/docs/dpulse-docs/docs/getting_started.md +++ b/docs/dpulse-docs/docs/getting_started.md @@ -1 +1,109 @@ +## Installing DPULSE + +You can install DPULSE in several ways, use the way you like the most. But since DPULSE repository is using Poetry to manage dependencies, it is higly recommended to install and start DPULSE using Poetry, especially on Linux systems where a lot of Python packages which DPULSE requires are preinstalled. More information about Poetry you can find on [Poetry official documentation page](https://python-poetry.org/docs/#ci-recommendations) + +### Install and start DPULSE. Way №1 + +Just download DPULSE using fast-access links at the top of the README: + +![image1](https://github.com/user-attachments/assets/bd1d9627-950b-40d4-91c4-6751476d7b65) + +Then just unpack downloaded archive, open terminal in DPULSE root folder and use `pip install -r requirements.txt` command to install requirements. Then type `python dpulse.py` in terminal, and that's where program starts. + +If `pip install -r requirements.txt` doesn't work, then just use `poetry install` command. After that, start DPULSE with `poetry run python dpulse.py` + +### Install and start DPULSE. Way №2 + +Use this set of commands to install DPULSE stable versions: + +``` +git clone https://github.com/OSINT-TECHNOLOGIES/dpulse +cd dpulse +poetry install +``` + +Use this set of commands to install DPULSE rolling versions: + +``` +git clone --branch rolling --single-branch https://github.com/OSINT-TECHNOLOGIES/dpulse.git +cd dpulse +poetry install +``` + +After installation, you simply start DPULSE using `poetry run python dpulse.py` + +### Install and start DPULSE. Way №3 + +You also can install DPULSE using pip manager. It'll install DPULSE and necessery dependencies in one command: `pip install dpulse`. Then you just locate DPULSE root folder and type `python dpulse.py` to start program. + +### Installers usage + +DPULSE has two pre-written installation scripts, both for Windows (installer.bat) and for Linux (installer.sh). You can use them to clone repository and install dependencies or only for dependencies installation. Keep in mind that installer.bat (Windows installer) requires installed Git to clone repository. + +#### Windows installer usage + +You can start installer.bat from terminal by typing `./installer.bat` in terminal. Then you choose menu item which you want to start. + +If you have problems with starting installer.bat, you should try to start it in admin terminal. + +#### Linux installer usage + +To start installer.sh in Linux you should follow these steps in your terminal: + +``` +sudo chmod +x installer.sh +sudo bash installer.sh +``` + +Then you choose menu item which you want to start. + +If you have problems with starting installer.sh, you should try to use `dos2unix installer.sh` or `sed -i 's/\r//' installer.sh` commands. + +### Conduct your first scan + +#### Meet DPULSE main menu + +After you successfully start DPULSE, you will see its main menu which contains some menu points. In order to conduct your first scan, you should select 1st point: + +![image](https://github.com/user-attachments/assets/5b45d4f0-9fad-4e17-8d74-96989037a66a) + +#### Correctly input your target + +After selecting 1st menu point, you'll be asked to enter your target's domain name as shown below: + +![image2](https://github.com/user-attachments/assets/cc5676d5-e11c-4aeb-b0b4-dd4c23fa228a) + +DPULSE works only with domain names. Enter domain name when you are asked to determine target resource. When it is necessary, DPULSE will transform domain name in URL. + +A domain name is a unique web address that can be acquired through domain registration. It’s the website’s equivalent to a physical address. It consists of a name and an extension, and it helps users easily find your website and eliminates the need to memorize the site’s Internet Protocol (IP) address. In the context of the internet, a domain name identifies a realm of administrative autonomy, authority, or control. Domain names are often used to identify services provided through the internet, such as websites, email services, and more. + +| URL | Domain name | +| ------------- | ------------- | +| Example: https://www.bing.com/ | Example: bing.com | +| String that provides the complete address of a web page | Human-friendly text form of an IP address | +| Contains the following parts: method, protocol, hostname, port, and path of the file | Part of the URL and is mostly used for branding of the organization | + +#### Select the modifiers + +After you've enter correct domain name of your target, you'll be asked to set some modifiers for your scan. These contain: + +* Case comment: some brief description of what you are scanning and what do you want to find (actually you can enter anything here) +* Report file extension: type of your report file, as you can see it can be XLSX or HTML report +* PageSearch usage: determines if PageSearch will be used during scan +* PageSearch keywords (if previous modifier was set to Y): keywords which DPULSE will try to find in found PDF documents +* Dorking mode: determines which Dorking database will be used during scan, but can be declined with N flag +* Third-party APIs usage: determines which API will be used during scan, but can be declined with N flag + +Example of modifiers selection is shown below: + +![image_mods](https://github.com/user-attachments/assets/9470350f-edf3-4692-b9bd-7c327cea2017) + +#### Get the report! + +After your first scan will be successfully ended, you will see this post-scan messages: + +![image_report](https://github.com/user-attachments/assets/4e16f1e6-df60-441c-b730-79ea69134bb7) + +This means that scan was ended, report was generated and scan results were written in report storage database. Congratulations, you successfully conducted your first scan using DPULSE! + From 2a8ada51b134005275ce2b6eeb6658765fe68ac0 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:00:58 +0300 Subject: [PATCH 17/59] Added configuration for RTD --- .readthedocs.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b13789..ab235d3 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1 +1,16 @@ +version: 2 +build: + os: ubuntu-20.04 + tools: + python: "3.10" + jobs: + pre_build: + - pip install mkdocs mkdocs-material + +python: + install: + - requirements: requirements.txt + +mkdocs: + configuration: docs/dpulse-docs/docs/mkdocs.yml From 647ce417ca126916b404575e35b19a5255ab34e6 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:05:36 +0300 Subject: [PATCH 18/59] Fixed wrong path to mkdocs.yml --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ab235d3..0a192a3 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,4 +13,4 @@ python: - requirements: requirements.txt mkdocs: - configuration: docs/dpulse-docs/docs/mkdocs.yml + configuration: docs/dpulse-docs/mkdocs.yml From 95abf0d6632901febb3a59f6d2ec856aa336a1b2 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:12:36 +0300 Subject: [PATCH 19/59] Updated README.md with Readthedocs link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3185249..2795308 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ | --- | --- | | What is DPULSE? | [See "About DPULSE" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#about-dpulse) | | Where I can find some demo and use-cases? | [See "Demo and use-cases" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#dpulse-demo-and-use-cases) | -| I want to read project documentation | [See DPULSE wiki](https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-WIKI) | +| I want to read project documentation | [See DPULSE Readthedocs Page](https://dpulse.readthedocs.io) | | I want to see project roadmap and future development plans | [See DPULSE roadmap](https://github.com/users/OSINT-TECHNOLOGIES/projects/1) | ## What to download? From e8439cbbe711130fcbbeff16d367cf9a7ca3a870 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:19:40 +0300 Subject: [PATCH 20/59] Update contact_dev.md --- docs/dpulse-docs/docs/contact_dev.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/dpulse-docs/docs/contact_dev.md b/docs/dpulse-docs/docs/contact_dev.md index 8b13789..4135c9b 100644 --- a/docs/dpulse-docs/docs/contact_dev.md +++ b/docs/dpulse-docs/docs/contact_dev.md @@ -1 +1,10 @@ +## Contact developer + +DPULSE developer will be glad to see your messages with feedback, questions and suggestions. Feel free to contact developer with most convenient ways for you: + +* OSINT-TECHNOLOGIES e-mail: osint.technologies@gmail.com +* [Make an issue page on DPULSE GitHub repository](https://github.com/OSINT-TECHNOLOGIES/dpulse/issues/new/choose) + + + From db77d8532d2c5c47ca37cd9f816594978b6625f7 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:35:01 +0300 Subject: [PATCH 21/59] Update dorking.md --- docs/dpulse-docs/docs/dorking.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/dpulse-docs/docs/dorking.md b/docs/dpulse-docs/docs/dorking.md index a11a03a..ad73f66 100644 --- a/docs/dpulse-docs/docs/dorking.md +++ b/docs/dpulse-docs/docs/dorking.md @@ -176,6 +176,15 @@ site:{} inurl:panel ## Creating custom Dorking database -DPULSE allows you to create your own custom Google Dorking database +DPULSE allows you to create your own custom Google Dorking database. You can do it using DPULSE CLI by selecting menus as shown below: + +![dorking_start](https://github.com/user-attachments/assets/fc8fe1ba-1845-46d1-a9b9-d09d3dc03ce6) + +After you select this menu point you will be welcomed with custom Dorking DB generator. It's very simple to use. First you should enter your new custom Dorking DB name without any extensions. Then you'll be prompted to enter id of your first dork (first id in custom DB is always 1, and every next dork gives +1 to id) and dork itself. There's a rule DPULSE requires from you when inputting dorks: when it comes to define domain in dork, put {} instead of it so the program code will replace these brackets with actual domain you'll enter lately. + +Example of custom Dorking DB generator interaction is shown below: + +![customdork](https://github.com/user-attachments/assets/8f3e8ca5-feec-4bf5-add8-048f54931b67) + From 38452670569c37075620774357da46f15a8bf4e7 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 01:45:07 +0300 Subject: [PATCH 22/59] Update dorking.md --- docs/dpulse-docs/docs/dorking.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/dpulse-docs/docs/dorking.md b/docs/dpulse-docs/docs/dorking.md index ad73f66..62610bf 100644 --- a/docs/dpulse-docs/docs/dorking.md +++ b/docs/dpulse-docs/docs/dorking.md @@ -186,5 +186,13 @@ Example of custom Dorking DB generator interaction is shown below: ![customdork](https://github.com/user-attachments/assets/8f3e8ca5-feec-4bf5-add8-048f54931b67) +In result, new .db file will appear in dorking folder, which can be selected later to use in scan: + +![dorking_customdbresult](https://github.com/user-attachments/assets/0cd4facc-215b-4e56-ab56-aa23cb5136db) + +And how it looks inside: + +![look_inside](https://github.com/user-attachments/assets/023467c2-008b-451f-8e14-88b7e54a8c3c) + From 03512c4af4134545fa71cd75d118ff8fa6fd78ac Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 30 Dec 2024 02:15:10 +0300 Subject: [PATCH 23/59] Update api.md --- docs/dpulse-docs/docs/api.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/dpulse-docs/docs/api.md b/docs/dpulse-docs/docs/api.md index bad8c72..0fbf3de 100644 --- a/docs/dpulse-docs/docs/api.md +++ b/docs/dpulse-docs/docs/api.md @@ -1,10 +1,29 @@ # Third-party API scan mode Currently DPULSE supports two third-party APIs: -1. SecurityTrails API (securitytrails.com) for deep subdomains and DNS enumeration -2. VirusTotal API (virustotal.com) for brief domain information gathering + +* SecurityTrails API (securitytrails.com) for deep subdomains and DNS enumeration +* VirusTotal API (virustotal.com) for brief domain information gathering ## SecurityTrails API +SecurityTrails API is used to gather information about a specified domain. It retrieves various types of DNS records, subdomains, and other details. SecurityTrails API in DPULSE returns these details about target domain: + +* Alexa rank +* Apex domain +* Hostname +* A/MX/NS/SOA/TXT records +* All subdomains list +* Alive (pingable) subdomains list + ## VirusTotal API +VirusTotal API is used to interact with the VirusTotal service programmatically and analyze files and URLs using multiple antivirus engines and website scanners, providing insights into whether they are malicious. VirusTotal API in DPULSE returns these details about target domain: + +* Categories +* Detected samples +* Undetected samples +* Detected URLs + +## API Keys database + From a21271c38f928d06125085f51c4d53b1fe2ca666 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 5 Jan 2025 20:58:29 +0300 Subject: [PATCH 24/59] Create report_storage.md (blank) --- docs/dpulse-docs/docs/report_storage.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dpulse-docs/docs/report_storage.md diff --git a/docs/dpulse-docs/docs/report_storage.md b/docs/dpulse-docs/docs/report_storage.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/dpulse-docs/docs/report_storage.md @@ -0,0 +1 @@ + From 5d4207f434ac1ab0ed41bb8c41b9901929ca591b Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 5 Jan 2025 20:58:54 +0300 Subject: [PATCH 25/59] Create demo.md (blank) --- docs/dpulse-docs/docs/demo.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/dpulse-docs/docs/demo.md diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/docs/dpulse-docs/docs/demo.md @@ -0,0 +1 @@ + From 1d2865d6571a0ca1ba491536e5da035c9910e925 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:55:42 +0300 Subject: [PATCH 26/59] Update api.md (finished) --- docs/dpulse-docs/docs/api.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/dpulse-docs/docs/api.md b/docs/dpulse-docs/docs/api.md index 0fbf3de..9ce09bf 100644 --- a/docs/dpulse-docs/docs/api.md +++ b/docs/dpulse-docs/docs/api.md @@ -27,3 +27,15 @@ VirusTotal API is used to interact with the VirusTotal service programmatically ## API Keys database +In order to ensure the functioning of API services individually for each DPULSE user, API keys storage database was created. Similar to report storage database, it is lightweight .db extension database with simple structure shown below: + +![apistordb](https://github.com/user-attachments/assets/02233813-781e-4bf8-be7c-76ec7627be06) + +Since every API key is individual for each user, you can see fillers instead of actual keys when you start DPULSE for the first time, so until you replace filler with a real API key, you can't start using API in scans. You can enter your actual API keys using DPULSE CLI. You can see full process on the screenshot below: + +![apiproc](https://github.com/user-attachments/assets/effb27ab-dd4b-4470-b90c-34c6f9a43d8c) + +For the first time you will see red-colored API key field, which means that scan is not available with this API. After changing filler for actual key, you will see that color changed, which indicates that you can use your API key for scanning. Be advised that every free API service provided with some limitations (you can see them in DPULSE CLI for all supported API), so keep in mind that frequent usage of free API plans is not possible. + +In case if you want to fully replace API keys, you can use reference API keys database. You can see menu point for this action on the screenshot above. This action will delete your actual API keys database, copy reference database and rename it. This action is very optional because you can change your API keys by just using first menu point in API Keys DB Manager (according to the screenshot above) + From 85d987a42aa705823a6305f6ee21603e2fb93b55 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:58:03 +0300 Subject: [PATCH 27/59] Update mkdocs.yml --- docs/dpulse-docs/mkdocs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/dpulse-docs/mkdocs.yml b/docs/dpulse-docs/mkdocs.yml index 582908e..c9618f9 100644 --- a/docs/dpulse-docs/mkdocs.yml +++ b/docs/dpulse-docs/mkdocs.yml @@ -9,8 +9,7 @@ nav: - PageSearch Sitemap Inspection: pagesearch_sitemap.md - Built-in automatic Dorking: dorking.md - Built-in API scanning: api.md - - Reports types: reporting.md - - Reports storage database: report_storage.md + - Reporting and report types: reporting.md - Demo and use-cases: demo.md - Contact developer: contact_dev.md From e152425131edb8aa87ac0145f1d1a2484dfe9ef0 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:58:32 +0300 Subject: [PATCH 28/59] Delete docs/dpulse-docs/docs/report_storage.md --- docs/dpulse-docs/docs/report_storage.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/dpulse-docs/docs/report_storage.md diff --git a/docs/dpulse-docs/docs/report_storage.md b/docs/dpulse-docs/docs/report_storage.md deleted file mode 100644 index 8b13789..0000000 --- a/docs/dpulse-docs/docs/report_storage.md +++ /dev/null @@ -1 +0,0 @@ - From b6bac354d6d1d833b1392a23c291133e9ed70910 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 00:43:43 +0300 Subject: [PATCH 29/59] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2795308..4ae7711 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ > Attention! DPULSE is a research tool. It is not intended for criminal activities! Use DPULSE only on allowed domains and for legal purposes! +> Please, before creating an issue or DMing developer about DPULSE, make sure that your problem or question is not covered with [DPULSE documentation](https://dpulse.readthedocs.io) + # Repository map ## What to visit? From 9e5a35b79ebd39b10d9e3d6d1e318e5a65b13efa Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 00:50:13 +0300 Subject: [PATCH 30/59] Update reporting.md (not finished) --- docs/dpulse-docs/docs/reporting.md | 59 ++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/dpulse-docs/docs/reporting.md b/docs/dpulse-docs/docs/reporting.md index 8b13789..7f765c7 100644 --- a/docs/dpulse-docs/docs/reporting.md +++ b/docs/dpulse-docs/docs/reporting.md @@ -1 +1,60 @@ +# About reporting system + +DPULSE as every OSINT tool is highly dependent on reporting system. User-friendly reports are crucial for detailed results presentation and further storage. DPULSE supports the most common types of reports: HTML and XLSX. Moreover, DPULSE provides you with reports storage database, which contains some information about scan, report and extracted data and gives you opportunity for long-term reports storage in one place. Also you can move this database between different DPULSE versions, which brings a little bit better user-experience. + +## HTML report + +HTML report was the first supported type of report. HTML is a widely supported format that can be opened in any web browser, allowing for the creation of visually appealing reports using tables, charts, diagrams, and other elements. It supports links and hyperlinks that can be used to create navigation within the report and link to external resources, and enables creating dynamic content that can be updated in real-time. HTML is often used in web applications, making it easy to integrate reports with existing web systems. And, in general, this report format is more user-friendly, which makes it more convenient for sharing with investigation customers, OSINT teams and usage in presentations. Moreover, unlike PDF report generation, HTML is easier to handle when both developing and delievering, as it doesn't require to install 3rd party applications (like wkhtmltopdf). You can see example of DPULSE generated HTML report [here](https://github.com/OSINT-TECHNOLOGIES/dpulse/tree/rolling/report_examples/html_report_example). + +## XLSX report + +XLSX is a widely supported format that can be opened in most spreadsheet and office applications, including Microsoft Excel, Google Sheets, and LibreOffice Calc, making it easy to analyze and process data. It allows storing data in a structured format, supports formulas and functions that can be used to automate calculations and data analysis, and enables creating charts and diagrams to visualize data. Additionally, XLSX is often used in business applications, making it easy to integrate reports with existing systems. You can see example of DPULSE generated XLSX report [here](https://github.com/OSINT-TECHNOLOGIES/dpulse/tree/rolling/report_examples/xlsx_report_example). + +## Side files + +As you may have noticed in report examples on GitHub page, there are also some side files except for report file. These files may be the following: + +* target's robots.txt file (if accessible) +* target's sitemap.xml file (if accessible) +* ps_documents folder with extracted documents from domain and its subdomains (if PageSearch was selected for scan) + +## Report storage database + +As said above, report storage database contains key information about scan, report and extracted data. DPULSE generates this database when DPULSE is first launched or if database file was not found in the root directory, so users don't need to worry about it's manual creation. Report storage database is a simple .db file (with hard-coded report_storage.db name) with structure which shown below: + +![rsdbstr](https://github.com/user-attachments/assets/491d1147-78ca-47a8-a405-5e351dc2730e) + +Lets describe these fields in more detailed way: + +* id - integer value that displays the number of reports generated and the order in which they are generated +* report_file_extension - string which shows main report file extension, in current DPULSE version this value could be xlsx, pdf or html +* report_content - BLOB or HTML data which contains main report file's copy +* comment - string which shows comment to your cases, which you can enter before each scan +* target - string which shows domain which you've scanned +* creation_date - string which shows when your report was generated (YYYYMMDD format) +* dorks_results - text array which contains a copy of Google Dorking results (if this mode was selected before scan) +* robots_text - text array which contains a copy of robots.txt file from scanned domain +* sitemap_text - text array which contains all sitemap.xml links file from scanned domain +* sitemap_file - text array which contains a copy of sitemap.xml file from scanned domain +* api_scan - string which indicates whether API scanning was activated or not, and if it was activated - contains used APIs + +Interacting with report storage database is a very simple process. First of all, after each scan you can see several messages which indicate that your report was successfully saved in report storage database: + +![rsdb1](https://github.com/user-attachments/assets/db3b22f8-1e74-4095-8ab7-99fd5837aa0a) + +Also, you have separate menu item in DPULSE CLI to work with report storage database which named "Report storage DB manager": + +![rsdb2](https://github.com/user-attachments/assets/519682dc-5d01-4844-8dcd-67e1914bb765) + +As you can see, there are menu points for both seeing DB content and recreating reports. Lets see what DPULSE will return if we select first menu item: + +![rsdb3](https://github.com/user-attachments/assets/6778cf83-e9cf-4580-b46d-7c187cbdde9d) + +Report recreating process is shown below: + + + + + + From abbccdb6b9fe9b981354a6518da4883b4b864b57 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:02:53 +0300 Subject: [PATCH 31/59] Removed help menu printing function --- service/cli_init.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/service/cli_init.py b/service/cli_init.py index d8e2180..0d5337f 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -31,7 +31,7 @@ def print_main_menu(self): print(Fore.CYAN + "3. Dorking module manager") print(Fore.CYAN + "4. Report storage DB manager") print(Fore.CYAN + "5. API modules manager") - print(Fore.CYAN + "6. Help") + print(Fore.CYAN + "6. Help (browser will be opened!)") print(Fore.LIGHTRED_EX + "7. Exit DPULSE" + Style.RESET_ALL + '\n') def print_settings_menu(self): @@ -42,16 +42,6 @@ def print_settings_menu(self): print(Fore.CYAN + "3. Clear journal content") print(Fore.LIGHTRED_EX + "4. Return to main menu" + Style.RESET_ALL + '\n') - def print_help_menu(self): - print(Fore.MAGENTA + Back.WHITE + '[HELP MENU]' + Style.RESET_ALL) - print(Fore.BLACK + Back.WHITE + '\nBe advised that choosing any of points below will open your web browser!' + Style.RESET_ALL) - print(Fore.CYAN + "1. Open DPULSE repository") - print(Fore.CYAN + "2. Open DPULSE wiki") - print(Fore.CYAN + "3. How to correctly input your targets in DPULSE") - print(Fore.CYAN + "4. PageSearch user guide") - print(Fore.CYAN + "5. Report storage database user guide") - print(Fore.LIGHTRED_EX + "6. Return to main menu" + Style.RESET_ALL + '\n') - def print_db_menu(self): print('\n') print(Fore.MAGENTA + Back.WHITE + '[REPORTS DATABASE MANAGER]' + Style.RESET_ALL) From c3ee90ea79c9bb30b1b00473e36556493897f5dd Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:03:28 +0300 Subject: [PATCH 32/59] Made --- dpulse.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/dpulse.py b/dpulse.py index 48c02ea..c11f156 100644 --- a/dpulse.py +++ b/dpulse.py @@ -236,22 +236,7 @@ def run(): elif choice_dorking == '2': continue elif choice == "6": - cli.print_help_menu() - choice_help = input(Fore.YELLOW + "Enter your choice >> ") - if choice_help == '1': - webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse') - elif choice_help == '2': - webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-WIKI') - elif choice_help == '3': - webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/How-to-correctly-input-your-targets-address-in-DPULSE') - elif choice_help == '4': - webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-PageSearch-function-guide') - elif choice_help == '5': - webbrowser.open('https://github.com/OSINT-TECHNOLOGIES/dpulse/wiki/DPULSE-report-storage-database') - elif choice_help == '6': - continue - else: - print(Fore.RED + "\nInvalid menu item. Please select between existing menu items") + webbrowser.open('https://dpulse.readthedocs.io/en/latest/') elif choice == '5': cli.api_manager() @@ -298,7 +283,7 @@ def run(): db.db_creation('report_storage.db') print('\n') choice_db = input(Fore.YELLOW + "Enter your choice >> ") - if choice_db == '1': + if choice_db == "1": db.db_select() elif choice_db == "2": if db.db_select() is None: @@ -315,7 +300,6 @@ def run(): print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL) except Exception as e: print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL) - elif choice_db == "3": print(Fore.GREEN + "\nDatabase connection is successfully closed") continue From 21332076c6dab82e8e65b2bb38f955e6f52cdec3 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:11:12 +0300 Subject: [PATCH 33/59] Added documentation page link --- service/cli_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/cli_init.py b/service/cli_init.py index 0d5337f..3561533 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -21,7 +21,7 @@ def welcome_menu(self): print('\n') self.console.print(fig.renderText('DPULSE'), style=preview_style) print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2 rolling] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) - print(Fore.MAGENTA + Style.BRIGHT + 'Visit our pages:\nhttps://github.com/OSINT-TECHNOLOGIES\nhttps://pypi.org/project/dpulse/' + Style.RESET_ALL) + print(Fore.MAGENTA + Style.BRIGHT + '[Visit our pages]\nGitHub repository: https://github.com/OSINT-TECHNOLOGIES\nPyPi page: https://pypi.org/project/dpulse/\nDocumentation: https://dpulse.readthedocs.io' + Style.RESET_ALL) def print_main_menu(self): print('\n') From c565026dadba3213e40eb248cbdddede0c52489a Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:26:43 +0300 Subject: [PATCH 34/59] Update reporting.md (finished) --- docs/dpulse-docs/docs/reporting.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/dpulse-docs/docs/reporting.md b/docs/dpulse-docs/docs/reporting.md index 7f765c7..5d09fa5 100644 --- a/docs/dpulse-docs/docs/reporting.md +++ b/docs/dpulse-docs/docs/reporting.md @@ -52,9 +52,8 @@ As you can see, there are menu points for both seeing DB content and recreating Report recreating process is shown below: +![rsdb4](https://github.com/user-attachments/assets/d7af9b03-703e-46b2-846b-05d99b33b900) +And that's how recreated report looks like inside: - - - - +![rsdb5](https://github.com/user-attachments/assets/799d45cb-bc51-43ca-8b06-14e236d21912) From 8e285d6ba2a1b54aad736415ed010edb3484affa Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:27:51 +0300 Subject: [PATCH 35/59] Fixed bug with spamming report db's content with every action in manager --- service/db_processing.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/service/db_processing.py b/service/db_processing.py index fab120d..259ef31 100644 --- a/service/db_processing.py +++ b/service/db_processing.py @@ -8,7 +8,6 @@ def db_connect(): sqlite_connection = sqlite3.connect('report_storage.db') cursor = sqlite_connection.cursor() - print(Fore.GREEN + "Successfully established SQLite3 DB connection") return cursor, sqlite_connection def db_creation(db_path): @@ -49,6 +48,7 @@ def db_select(): select_query = "SELECT creation_date, report_file_extension, target, id, comment, dorks_results, robots_text, sitemap_text, sitemap_file, api_scan FROM report_storage;" cursor.execute(select_query) records = cursor.fetchall() + print(Fore.LIGHTMAGENTA_EX + "\n[DATABASE'S CONTENT]\n" + Style.RESET_ALL) for row in records: dorks_presence = robots_presence = sitemap_presence = "None" if len(row[4]) > 1: @@ -64,11 +64,26 @@ def db_select(): else: print(Fore.RED + 'No data found in report storage database') sqlite_connection.close() - return None + return cursor, sqlite_connection + +def db_select_silent(): + cursor, sqlite_connection = db_connect() + if_rows = "SELECT * FROM report_storage" + cursor.execute(if_rows) + rows = cursor.fetchall() + if rows: + try: + select_query = "SELECT creation_date, report_file_extension, target, id, comment, dorks_results, robots_text, sitemap_text, sitemap_file, api_scan FROM report_storage;" + cursor.execute(select_query) + #records = cursor.fetchall() + except sqlite3.Error as e: + sqlite_connection.close() + else: + sqlite_connection.close() return cursor, sqlite_connection def db_report_recreate(extracted_folder_name, id_to_extract): - cursor, sqlite_connection = db_select() + cursor, sqlite_connection = db_select_silent() cursor.execute("SELECT report_content FROM report_storage WHERE id=?", (id_to_extract,)) try: blob = cursor.fetchone() From 17e87a45128542afe5d73df17393edf64dd0bf2c Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:28:17 +0300 Subject: [PATCH 36/59] Fixed bug with spamming report db's content with every action in manager --- dpulse.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/dpulse.py b/dpulse.py index c11f156..44173df 100644 --- a/dpulse.py +++ b/dpulse.py @@ -286,20 +286,16 @@ def run(): if choice_db == "1": db.db_select() elif choice_db == "2": - if db.db_select() is None: - pass - else: - print(Fore.LIGHTMAGENTA_EX + "\n[DATABASE'S CONTENT]\n" + Style.RESET_ALL) - db.db_select() - id_to_extract = int(input(Fore.YELLOW + "\nEnter report ID you want to extract >> ")) - extracted_folder_name = 'report_recreated_ID#{}'.format(id_to_extract) - try: - os.makedirs(extracted_folder_name) - db.db_report_recreate(extracted_folder_name, id_to_extract) - except FileExistsError: - print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL) - except Exception as e: - print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL) + db.db_select() + id_to_extract = int(input(Fore.YELLOW + "\nEnter report ID you want to extract >> ")) + extracted_folder_name = 'report_recreated_ID#{}'.format(id_to_extract) + try: + os.makedirs(extracted_folder_name) + db.db_report_recreate(extracted_folder_name, id_to_extract) + except FileExistsError: + print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL) + except Exception as e: + print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL) elif choice_db == "3": print(Fore.GREEN + "\nDatabase connection is successfully closed") continue From 48b8681114189cba695c559ecf3ec16feaba414f Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:50:02 +0300 Subject: [PATCH 37/59] Added 3 new mentions --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 4ae7711..07ea0b5 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,12 @@ If you have problems with starting installer.sh, you should try to use `dos2unix ### [by @UndeadSec](https://x.com/UndeadSec/status/1827692406797689032) +### [by @akaclandestine](https://x.com/akaclandestine/status/1875980998418002151?t=Ac-fZ9oe2FLKkTTCClss8g&s=19) + +### [by @hdH4dg8](https://x.com/hdH4dg8/status/1876106586869104906?t=Awfas25ukXkblPRt4krSzA&s=19) + +### [by @@wy88215534](https://x.com/wy88215534/status/1876522251828494794?t=cmwoNCDZv0kMEDMKokcVFA&s=19) + ## Facebook mentions: ### [by Inteligência Cibernética](https://www.facebook.com/osintbrasil/posts/pfbid037ibycZcBWe2MjtV4HiWvRWxyKei8TJ5Ycfxai4TDNHXuwrYkDGuyjDsPow8WUNbyl) From 7a19dd7db05b4d7d261bf949d3f65c5e494ece20 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 02:48:28 +0300 Subject: [PATCH 38/59] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 07ea0b5..0bc5844 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix ### [by @hdH4dg8](https://x.com/hdH4dg8/status/1876106586869104906?t=Awfas25ukXkblPRt4krSzA&s=19) -### [by @@wy88215534](https://x.com/wy88215534/status/1876522251828494794?t=cmwoNCDZv0kMEDMKokcVFA&s=19) +### [by @wy88215534](https://x.com/wy88215534/status/1876522251828494794?t=cmwoNCDZv0kMEDMKokcVFA&s=19) ## Facebook mentions: From 9bc7018203e9d454767696cfe2df698a7e842dcd Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:05:00 +0300 Subject: [PATCH 39/59] Improved handling of report storage database Added exception handling for trying to recreate report from empty DB, fixed multiple DB creation tries --- dpulse.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/dpulse.py b/dpulse.py index 44173df..6d0d80b 100644 --- a/dpulse.py +++ b/dpulse.py @@ -25,7 +25,12 @@ from logs_processing import logging from db_creator import get_columns_amount -db.db_creation('report_storage.db') +rsdb_presence = db.check_rsdb_presence('report_storage.db') +if rsdb_presence is True: + print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL) +else: + db.db_creation('report_storage.db') + print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL) dorks_files_check() @@ -280,22 +285,30 @@ def run(): elif choice == "4": cli.print_db_menu() print('\n') - db.db_creation('report_storage.db') + rsdb_presence = db.check_rsdb_presence('report_storage.db') + if rsdb_presence is True: + print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL) + else: + db.db_creation('report_storage.db') + print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL) print('\n') choice_db = input(Fore.YELLOW + "Enter your choice >> ") if choice_db == "1": - db.db_select() + cursor, sqlite_connection, data_presence_flag = db.db_select() elif choice_db == "2": - db.db_select() - id_to_extract = int(input(Fore.YELLOW + "\nEnter report ID you want to extract >> ")) - extracted_folder_name = 'report_recreated_ID#{}'.format(id_to_extract) - try: - os.makedirs(extracted_folder_name) - db.db_report_recreate(extracted_folder_name, id_to_extract) - except FileExistsError: - print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL) - except Exception as e: - print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL) + cursor, sqlite_connection, data_presence_flag = db.db_select() + if data_presence_flag is True: + id_to_extract = int(input(Fore.YELLOW + "\nEnter report ID you want to extract >> ")) + extracted_folder_name = 'report_recreated_ID#{}'.format(id_to_extract) + try: + os.makedirs(extracted_folder_name) + db.db_report_recreate(extracted_folder_name, id_to_extract) + except FileExistsError: + print(Fore.RED + "Report with the same recreated folder already exists. Please check its content or delete it and try again" + Style.RESET_ALL) + except Exception as e: + print(Fore.RED + "Error appeared when trying to recreate report from DB. See journal for details" + Style.RESET_ALL) + else: + pass elif choice_db == "3": print(Fore.GREEN + "\nDatabase connection is successfully closed") continue From eea9b943c87c410b37228a83f52cdcf7ea5406a3 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:05:35 +0300 Subject: [PATCH 40/59] Improved handling of report storage database Added exception handling for trying to recreate report from empty DB, fixed multiple DB creation tries --- service/db_processing.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/service/db_processing.py b/service/db_processing.py index 259ef31..6a098a0 100644 --- a/service/db_processing.py +++ b/service/db_processing.py @@ -10,12 +10,17 @@ def db_connect(): cursor = sqlite_connection.cursor() return cursor, sqlite_connection -def db_creation(db_path): +def check_rsdb_presence(db_path): if not os.path.exists(db_path): print(Fore.RED + "Report storage database was not found. DPULSE will create it in a second" + Style.RESET_ALL) - cursor, sqlite_connection = db_connect() - create_table_sql = """ - CREATE TABLE "report_storage" ( + return False + else: + return True + +def db_creation(db_path): + cursor, sqlite_connection = db_connect() + create_table_sql = """ + CREATE TABLE "report_storage" ( "id" INTEGER NOT NULL UNIQUE, "report_file_extension" TEXT NOT NULL, "report_content" BLOB NOT NULL, @@ -30,15 +35,12 @@ def db_creation(db_path): PRIMARY KEY("id" AUTOINCREMENT) ); """ - cursor.execute(create_table_sql) - sqlite_connection.commit() - sqlite_connection.close() - print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL) - else: - print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL) + cursor.execute(create_table_sql) + sqlite_connection.commit() + sqlite_connection.close() def db_select(): - db_creation('report_storage.db') + #db_creation('report_storage.db') cursor, sqlite_connection = db_connect() if_rows = "SELECT * FROM report_storage" cursor.execute(if_rows) @@ -58,13 +60,16 @@ def db_select(): if len(row[6]) > 1: sitemap_presence = "In DB" print(Fore.LIGHTBLUE_EX + f"Case ID: {row[3]} | Case name: {row[2]} | Case file extension: {row[1]} | Case comment: {row[4]} | Case creation date: {row[0]} | Dorking: {dorks_presence} | robots.txt: {robots_presence} | sitemap.xml: {sitemap_presence} | API scan: {row[-1]}" + Style.RESET_ALL) + data_presence_flag = True except sqlite3.Error as e: print(Fore.RED + "Failed to see storage database's content. Reason: {}".format(e)) sqlite_connection.close() + data_presence_flag = False else: print(Fore.RED + 'No data found in report storage database') sqlite_connection.close() - return cursor, sqlite_connection + data_presence_flag = False + return cursor, sqlite_connection, data_presence_flag def db_select_silent(): cursor, sqlite_connection = db_connect() From 43bc3fcf160e5425bfdce9c3a3f275d1e87687a5 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:13:39 +0300 Subject: [PATCH 41/59] Some CLI visual improvements --- service/cli_init.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/service/cli_init.py b/service/cli_init.py index 3561533..7a2c2d7 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -53,8 +53,7 @@ def dorking_db_manager(self): print('\n') print(Fore.MAGENTA + Back.WHITE + '[DORKING DB MANAGER]' + Style.RESET_ALL) print(Fore.CYAN + "1. Generate custom Dorking DB") - print(Fore.LIGHTRED_EX + "2. Return to main menu" + Style.RESET_ALL) - print('\n') + print(Fore.LIGHTRED_EX + "2. Return to main menu\n" + Style.RESET_ALL) def api_manager(self): print('\n') @@ -73,8 +72,7 @@ def print_prescan_summary(short_domain, report_filetype, pagesearch_ui_mark, dor print(Fore.GREEN + "Case comment: " + Fore.LIGHTCYAN_EX + Style.BRIGHT + case_comment + Style.RESET_ALL + "\n") def print_api_db_msg(): - print('\n') - print(Fore.GREEN + "You've entered custom Dorking DB generator!\n" + Style.RESET_ALL) + print(Fore.GREEN + "\nYou've entered custom Dorking DB generator!\n" + Style.RESET_ALL) print(Fore.GREEN + "Remember some rules in order to successfully create your custom Dorking DB:" + Style.RESET_ALL) print(Fore.GREEN + "[1] - dork_id variable must be unique, starting with 1 and then +1 every new dork" + Style.RESET_ALL) print(Fore.GREEN + "[2] - When it comes to define domain in dork, put {} in it\n" + Style.RESET_ALL) From 9fca8b2b89face5eafa376832452ce82abe57be6 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:14:02 +0300 Subject: [PATCH 42/59] Some CLI visual improvements --- dpulse.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dpulse.py b/dpulse.py index 6d0d80b..34a1127 100644 --- a/dpulse.py +++ b/dpulse.py @@ -245,8 +245,7 @@ def run(): elif choice == '5': cli.api_manager() - print('\n') - choice_api = input(Fore.YELLOW + "Enter your choice >> ") + choice_api = input(Fore.YELLOW + "\nEnter your choice >> ") if choice_api == '1': print(Fore.GREEN + "\nSupported APIs and your keys:\n") cursor, conn = db.select_api_keys('updating') @@ -284,14 +283,12 @@ def run(): elif choice == "4": cli.print_db_menu() - print('\n') rsdb_presence = db.check_rsdb_presence('report_storage.db') if rsdb_presence is True: - print(Fore.GREEN + "Report storage database presence: OK" + Style.RESET_ALL) + print(Fore.GREEN + "\nReport storage database presence: OK\n" + Style.RESET_ALL) else: db.db_creation('report_storage.db') print(Fore.GREEN + "Successfully created report storage database" + Style.RESET_ALL) - print('\n') choice_db = input(Fore.YELLOW + "Enter your choice >> ") if choice_db == "1": cursor, sqlite_connection, data_presence_flag = db.db_select() From 8a79cd6de200c19fe6c542a1397167b32bd6ff8a Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:18:04 +0300 Subject: [PATCH 43/59] Update README.md --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 0bc5844..f4d3b94 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,8 @@ | What do you want to see? | Link | | --- | --- | -| What is DPULSE? | [See "About DPULSE" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#about-dpulse) | -| Where I can find some demo and use-cases? | [See "Demo and use-cases" page](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#dpulse-demo-and-use-cases) | | I want to read project documentation | [See DPULSE Readthedocs Page](https://dpulse.readthedocs.io) | +| I want to see developer's contacts | [See "Contact developer" page on Readthedocs](https://dpulse.readthedocs.io/en/latest/contact_dev/#) | | I want to see project roadmap and future development plans | [See DPULSE roadmap](https://github.com/users/OSINT-TECHNOLOGIES/projects/1) | ## What to download? @@ -42,12 +41,6 @@ | I want to use only one specific version of DPULSE | [See DPULSE releases page](https://github.com/OSINT-TECHNOLOGIES/dpulse/releases) | | I want to see more detailed installation instructions | [See DPULSE installation guides](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#how-to-install-and-run-dpulse) -## How can I contact the developer? - -| Reasons to contact | Links & addresses | -| --- | --- | -| I want to talk with developer in person | DM to osint.technologies@gmail.com | -| I want to report some bug or issue, or maybe I have some good idea for developer | [Make a new issue page](https://github.com/OSINT-TECHNOLOGIES/dpulse/issues/new/choose) | # About DPULSE From 1cf1fe8817578ab31413f0cce61e8340fed8754e Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:18:49 +0300 Subject: [PATCH 44/59] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4d3b94..6927f75 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ | Your expectations | Version and link for you | | --- | --- | | I want to use only tested and stable version of DPULSE | [DPULSE stable ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/main.zip) | -| I don't mind to use DPULSE with latest changes and you're OK with bugs and issues | [DPULSE rolling ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/rolling.zip) | +| I don't mind to use DPULSE with latest changes and I'm OK with bugs and issues | [DPULSE rolling ZIP archive](https://github.com/OSINT-TECHNOLOGIES/dpulse/archive/refs/heads/rolling.zip) | | I want to use only one specific version of DPULSE | [See DPULSE releases page](https://github.com/OSINT-TECHNOLOGIES/dpulse/releases) | | I want to see more detailed installation instructions | [See DPULSE installation guides](https://github.com/OSINT-TECHNOLOGIES/dpulse?tab=readme-ov-file#how-to-install-and-run-dpulse) From bd7ff6ca697caf0dddc967a341bebd7d14e2dc64 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:38:47 +0300 Subject: [PATCH 45/59] Updated DPULSE logo --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6927f75..516dbbd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # DPULSE - Domain Public Data Collection Service +

- +

From 05e0a8bc1f96cb63e13a562d808d216a0b3de91d Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Fri, 10 Jan 2025 03:40:19 +0300 Subject: [PATCH 46/59] Updated logo --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 516dbbd..55bdd90 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # DPULSE - Domain Public Data Collection Service -

- +

From 76b99ab62592fccb01b727adb64de7573dfb5e27 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:07:48 +0300 Subject: [PATCH 47/59] Code clean-up --- service/db_processing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/service/db_processing.py b/service/db_processing.py index 6a098a0..b179977 100644 --- a/service/db_processing.py +++ b/service/db_processing.py @@ -40,7 +40,6 @@ def db_creation(db_path): sqlite_connection.close() def db_select(): - #db_creation('report_storage.db') cursor, sqlite_connection = db_connect() if_rows = "SELECT * FROM report_storage" cursor.execute(if_rows) @@ -80,7 +79,6 @@ def db_select_silent(): try: select_query = "SELECT creation_date, report_file_extension, target, id, comment, dorks_results, robots_text, sitemap_text, sitemap_file, api_scan FROM report_storage;" cursor.execute(select_query) - #records = cursor.fetchall() except sqlite3.Error as e: sqlite_connection.close() else: From 0e1448ed6d0edfa952c1b9753e7b25de5279b360 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:09:40 +0300 Subject: [PATCH 48/59] Code clean-up --- datagather_modules/data_assembler.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/datagather_modules/data_assembler.py b/datagather_modules/data_assembler.py index a8cd5fa..aba4437 100644 --- a/datagather_modules/data_assembler.py +++ b/datagather_modules/data_assembler.py @@ -164,17 +164,8 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k api_scan_db.append('No') pass - #data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials, - # subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records, - # robots_txt_result, sitemap_xml_result, sitemap_links_status, - # web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports, - # hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return, - # accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, - # website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats, - # vt_deturls, vt_detsamples, vt_undetsamples] cleaned_dorking = [item.strip() for item in dorking_results if item.strip()] - #print(cleaned_dorking) data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials, subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records, From b576a29cc51f965a32ad6a1ad0ba1a02b94266f9 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:10:47 +0300 Subject: [PATCH 49/59] Bumped version --- service/cli_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/cli_init.py b/service/cli_init.py index 7a2c2d7..05f1de6 100644 --- a/service/cli_init.py +++ b/service/cli_init.py @@ -20,7 +20,7 @@ def welcome_menu(self): fig = Figlet(font=wm_font) print('\n') self.console.print(fig.renderText('DPULSE'), style=preview_style) - print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2 rolling] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) + print(Fore.MAGENTA + Style.BRIGHT + '[DPULSE-CLI] - [v1.2 stable] - [OSINT-TECHNOLOGIES]\n' + Style.RESET_ALL) print(Fore.MAGENTA + Style.BRIGHT + '[Visit our pages]\nGitHub repository: https://github.com/OSINT-TECHNOLOGIES\nPyPi page: https://pypi.org/project/dpulse/\nDocumentation: https://dpulse.readthedocs.io' + Style.RESET_ALL) def print_main_menu(self): From 45a6ff1cc2a47e7d9749fb3f3d9f1d010ce7c3ca Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 26 Jan 2025 02:20:02 +0300 Subject: [PATCH 50/59] Updated GIF --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55bdd90..202a3a4 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix ### You can start DPULSE and see the main menu on the screen using one of the recommended commands in DPULSE root folder. Don't forget to install all requirements before starting DPULSE -![dpulse_start](https://github.com/user-attachments/assets/49cb476c-d9a5-4ff6-999f-afa0badf0e0d) +![dpulse_start](https://github.com/user-attachments/assets/9ec0ab73-2206-4d38-bae6-e88656e17f95) ### After choosing first menu point, you will be able to enter target's URL and case comment, and then you will see scanning progress From 3d1aa54964162f29a0385a8086238de2d75727b2 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 26 Jan 2025 02:24:04 +0300 Subject: [PATCH 51/59] Update demo.md --- docs/dpulse-docs/docs/demo.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md index 8b13789..af4233b 100644 --- a/docs/dpulse-docs/docs/demo.md +++ b/docs/dpulse-docs/docs/demo.md @@ -1 +1,9 @@ +# DPULSE demos & use-cases +On this page you can see some actual DPULSE demos and use-cases in some cybersecurity and OSINT scenarios + +## Demo №1. Starting DPULSE + +Yes, starting DPULSE is as simple as that: + +![dpulse_start](https://github.com/user-attachments/assets/9ec0ab73-2206-4d38-bae6-e88656e17f95) From 8890acfb566dee263a842e7487f10a46220b03ef Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Sun, 26 Jan 2025 02:31:07 +0300 Subject: [PATCH 52/59] Update demo.md --- docs/dpulse-docs/docs/demo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md index af4233b..0981c2f 100644 --- a/docs/dpulse-docs/docs/demo.md +++ b/docs/dpulse-docs/docs/demo.md @@ -4,6 +4,6 @@ On this page you can see some actual DPULSE demos and use-cases in some cybersec ## Demo №1. Starting DPULSE -Yes, starting DPULSE is as simple as that: +Yes, starting DPULSE is as simple as that. Don't forget to install all requirements before starting DPULSE ![dpulse_start](https://github.com/user-attachments/assets/9ec0ab73-2206-4d38-bae6-e88656e17f95) From a5a1de4719a645956e9b15189f87dfceceee2d6f Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:30:44 +0300 Subject: [PATCH 53/59] Added requests.exceptions.ReadTimeout exception to handle --- datagather_modules/crawl_processor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datagather_modules/crawl_processor.py b/datagather_modules/crawl_processor.py index 5c3777f..6769489 100644 --- a/datagather_modules/crawl_processor.py +++ b/datagather_modules/crawl_processor.py @@ -173,7 +173,7 @@ def domains_reverse_research(subdomains, report_file_type): for subdomain in subdomains: subdomain_url = "http://" + subdomain + "/" subdomain_urls.append(subdomain_url) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e: + except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS URL FORMING: ERROR. REASON: {e}') pass @@ -183,7 +183,7 @@ def domains_reverse_research(subdomains, report_file_type): subdomains_ip = ip_gather(subdomain) subdomain_ip.append(subdomains_ip) subdomain_ip = list(set(subdomain_ip)) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e: + except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS IP GATHERING: ERROR. REASON: {e}') pass @@ -194,7 +194,7 @@ def domains_reverse_research(subdomains, report_file_type): subdomain_mails.append(subdomain_mail) subdomain_social = sm_gather(subdomain_url) subdomain_socials.append(subdomain_social) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError) as e: + except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS MAIL/SOCIALS GATHERING: ERROR. REASON: {e}') pass From 500315fb7ce7c23d2ca959062c425b30d4e1e57d Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:10:02 +0300 Subject: [PATCH 54/59] Added exception handler for social medias gathering function --- datagather_modules/data_assembler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datagather_modules/data_assembler.py b/datagather_modules/data_assembler.py index aba4437..8bb7c14 100644 --- a/datagather_modules/data_assembler.py +++ b/datagather_modules/data_assembler.py @@ -81,7 +81,12 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k print(Fore.GREEN + 'Processing subdomain gathering' + Style.RESET_ALL) subdomains, subdomains_amount = cp.subdomains_gather(url, short_domain) print(Fore.GREEN + 'Processing social medias gathering' + Style.RESET_ALL) - social_medias = cp.sm_gather(url) + try: + social_medias = cp.sm_gather(url) + except: + print(Fore.RED + "Social medias were not gathered because of error" + Style.RESET_ALL) + social_medias = ['Social medias were not extracted because of error'] + pass print(Fore.GREEN + 'Processing subdomain analysis' + Style.RESET_ALL) if report_file_type == 'xlsx': subdomain_urls, subdomain_mails, subdomain_ip, sd_socials = cp.domains_reverse_research(subdomains, report_file_type) From bab9241ad37d561894cee060e246604f6167e0e8 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:10:20 +0300 Subject: [PATCH 55/59] Widened pool of exceptions to handle --- datagather_modules/crawl_processor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datagather_modules/crawl_processor.py b/datagather_modules/crawl_processor.py index 6769489..b0930ed 100644 --- a/datagather_modules/crawl_processor.py +++ b/datagather_modules/crawl_processor.py @@ -103,6 +103,7 @@ def subdomains_gather(url, short_domain): print(Fore.RED + f"Cannot gather subdomains due to error. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS GATHERING: ERROR. REASON: {e}') pass + return ['No subdomains were found'], 0 def sm_gather(url): response = requests.get(url) @@ -173,7 +174,7 @@ def domains_reverse_research(subdomains, report_file_type): for subdomain in subdomains: subdomain_url = "http://" + subdomain + "/" subdomain_urls.append(subdomain_url) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: + except Exception as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS URL FORMING: ERROR. REASON: {e}') pass @@ -183,7 +184,7 @@ def domains_reverse_research(subdomains, report_file_type): subdomains_ip = ip_gather(subdomain) subdomain_ip.append(subdomains_ip) subdomain_ip = list(set(subdomain_ip)) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: + except Exception as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS IP GATHERING: ERROR. REASON: {e}') pass @@ -194,7 +195,7 @@ def domains_reverse_research(subdomains, report_file_type): subdomain_mails.append(subdomain_mail) subdomain_social = sm_gather(subdomain_url) subdomain_socials.append(subdomain_social) - except (socket.gaierror, requests.exceptions.SSLError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e: + except Exception as e: print(Fore.RED + "Some URL seems unreachable! DPULSE will continue to work, but the URL causing the error won't be included in report. See journal for details" + Style.RESET_ALL) logging.error(f'SUBDOMAINS MAIL/SOCIALS GATHERING: ERROR. REASON: {e}') pass From a458a9dd34cf777efa860e891dd3d79ef0881672 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:10:45 +0300 Subject: [PATCH 56/59] Widened pool of exception to handle (SSL) --- datagather_modules/networking_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagather_modules/networking_processor.py b/datagather_modules/networking_processor.py index 1d770a7..e5641f8 100644 --- a/datagather_modules/networking_processor.py +++ b/datagather_modules/networking_processor.py @@ -51,7 +51,7 @@ def get_ssl_certificate(short_domain, port=443): commonName = str(cert['issuer'][2][0][1]) + ', version: ' + str(cert['version']) serialNumber = cert['serialNumber'] return issuer, subject, notBefore, notAfter, commonName, serialNumber - except (ssl.CertificateError, ssl.SSLError, socket.gaierror, ConnectionRefusedError) as e: + except Exception as e: print(Fore.RED + "Error while gathering info about SSL certificate. See journal for details") logging.error(f'SSL CERTIFICATE GATHERING: ERROR. REASON: {e}') issuer = subject = notBefore = notAfter = commonName = serialNumber = "No information about SSL certificate was gathered" From 1753c979def142b2b6c4a8ecbef483af62d85554 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:18:01 +0300 Subject: [PATCH 57/59] Update demo GIF --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 202a3a4..b20b09c 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix ### After choosing first menu point, you will be able to enter target's URL and case comment, and then you will see scanning progress -![dpulse_running](https://github.com/user-attachments/assets/9a3cd8c2-3281-4d58-b12e-91ced6dbbd91) +![dpulse_bs](https://github.com/user-attachments/assets/b0ad7827-6dac-4f82-a369-4447a0e1c878) ### Finally, DPULSE will create report folder which contains case name (basically URL of target), date and time of scan. All report folders are contained in DPULSE root folder From bec83f4765822ce340482f0bcce107fae2835188 Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:24:46 +0300 Subject: [PATCH 58/59] Update demo.md (added new GIF) --- docs/dpulse-docs/docs/demo.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md index 0981c2f..3d8411b 100644 --- a/docs/dpulse-docs/docs/demo.md +++ b/docs/dpulse-docs/docs/demo.md @@ -7,3 +7,10 @@ On this page you can see some actual DPULSE demos and use-cases in some cybersec Yes, starting DPULSE is as simple as that. Don't forget to install all requirements before starting DPULSE ![dpulse_start](https://github.com/user-attachments/assets/9ec0ab73-2206-4d38-bae6-e88656e17f95) + +## Demo №2. DPULSE scan workflow + +Here you can see complete process of DPULSE basic scan from the beginning to the end. Remember that sometimes your scan might not be ideal and errors may appear. However, they will not interrupt scanning process, but will only affect the number and variety of results in the final report. + +![dpulse_bs](https://github.com/user-attachments/assets/b0ad7827-6dac-4f82-a369-4447a0e1c878) + From 6c3ee98c94fb80fb1d69da65c8889eb4ed32acbf Mon Sep 17 00:00:00 2001 From: OSINT-TECHNOLOGIES <77023667+OSINT-TECHNOLOGIES@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:28:09 +0300 Subject: [PATCH 59/59] Update demo.md --- docs/dpulse-docs/docs/demo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dpulse-docs/docs/demo.md b/docs/dpulse-docs/docs/demo.md index 3d8411b..637b025 100644 --- a/docs/dpulse-docs/docs/demo.md +++ b/docs/dpulse-docs/docs/demo.md @@ -8,7 +8,7 @@ Yes, starting DPULSE is as simple as that. Don't forget to install all requireme ![dpulse_start](https://github.com/user-attachments/assets/9ec0ab73-2206-4d38-bae6-e88656e17f95) -## Demo №2. DPULSE scan workflow +## Demo №2. DPULSE basic scan workflow Here you can see complete process of DPULSE basic scan from the beginning to the end. Remember that sometimes your scan might not be ideal and errors may appear. However, they will not interrupt scanning process, but will only affect the number and variety of results in the final report.