From 96249fc42d4b025fdcc70ad0585a3dc20620b57e Mon Sep 17 00:00:00 2001 From: G <52905881+giga-a@users.noreply.github.com> Date: Wed, 27 Jan 2021 23:22:49 -0800 Subject: [PATCH] [manual] Added shared detections - ref c9d24b98 --- README.md | 2 +- app.js | 4 +- data/sites.json | 157 +++++++++++-------------------------------- info | 4 +- modules/engine.js | 64 +++++++++++++++--- modules/fast-scan.js | 4 +- modules/helper.js | 1 + modules/slow-scan.js | 7 +- 8 files changed, 105 insertions(+), 138 deletions(-) diff --git a/README.md b/README.md index 109c01a8..17e8b4d0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The analysis and extracted social media information from this OSINT tool could h This project *is currently used by some law enforcement agencies in countries where resources are limited*. ## Updates -* Implementing generic and shared detections +* Added shared detections to NodeJS Web App & CLI 👏 * API, CLIs & Web App produce: * [Detected] -> 0 to 100 (No-Maybe-Yes) * [Unknown] -> Detection did not meet the requirement diff --git a/app.js b/app.js index 50498249..4722f31b 100644 --- a/app.js +++ b/app.js @@ -10,7 +10,7 @@ var argv = require('yargs') .usage('Usage: $0 --cli --mode "fast" --username "johndoe" --websites "youtube tiktok"\nUsage: $0 --cli --mode "fast" --username "johndoe"') - .describe('cli', 'a user or stirng') + .describe('cli', 'enable this cli') .default("cli", false) .boolean('cli') .describe('username', 'E.g. johndoe, john_doe or johndoe9999') @@ -19,7 +19,7 @@ var argv = require('yargs') .default("websites", "all") .describe('mode', 'Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial') .default("mode", "") - .describe('output', 'arg_parser_optional.add_argument("--output", help="Show the output in the following format: json -> json output for integration or pretty -> prettify the output') + .describe('output', 'Show the output in the following format: json -> json output for integration or pretty -> prettify the output') .default("output", "") .describe('options', 'Show the following when a profile is found: link, rate, title or text') .default("options", "") diff --git a/data/sites.json b/data/sites.json index 3c4926fb..ba416241 100644 --- a/data/sites.json +++ b/data/sites.json @@ -2219,8 +2219,6 @@ "type": "normal" }, { - "return": "true", - "string": "", "function": "special_facebook_1", "type": "special" } @@ -2708,30 +2706,14 @@ { "url": "https://fosstodon.org/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for isn", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for isn", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" - }, - { - "return": "true", - "string": "/@{username}", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", "timeout": 0, "implicit": 0, - "type": "Mastodon instance" + "type": "unavailable" }, { "url": "https://fotolog.com/author/{username}", @@ -3158,13 +3140,10 @@ { "url": "https://gmail.com/{username}", "detections": [{ - "return": "true", - "string": "", "function": "special_gmail_1", "type": "special" }], "selected": "false", - "top": "top10", "timeout": 0, "implicit": 0, "type": "Social networking service" @@ -3190,13 +3169,10 @@ { "url": "https://google.com/{username}", "detections": [{ - "return": "true", - "string": "", "function": "special_google_1", "type": "special" }], "selected": "false", - "top": "top10", "timeout": 0, "implicit": 0, "type": "Social networking service" @@ -4294,24 +4270,8 @@ { "url": "https://mastodon.cloud/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for isn", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for isn", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" - }, - { - "return": "true", - "string": "/@{username}", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", @@ -4322,24 +4282,8 @@ { "url": "https://mastodon.social/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for isn", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for isn", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" - }, - { - "return": "true", - "string": "/@{username}", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", @@ -4350,24 +4294,8 @@ { "url": "https://mastodon.technology/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for isn", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for isn", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" - }, - { - "return": "true", - "string": "/@{username}", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", @@ -4378,24 +4306,8 @@ { "url": "https://mastodon.xyz/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for isn", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for isn", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" - }, - { - "return": "true", - "string": "/@{username}", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", @@ -4592,24 +4504,8 @@ { "url": "https://mstdn.io/@{username}", "detections": [{ - "return": "false", - "string": "The page you are looking for", - "type": "ocr" - }, - { - "return": "false", - "string": "The page you are looking for", - "type": "normal" - }, - { - "return": "true", - "string": "og:title", - "type": "normal" - }, - { - "return": "true", - "string": "profile:username", - "type": "normal" + "name": "mastodon", + "type": "shared" } ], "selected": "false", @@ -7654,7 +7550,7 @@ "type": "normal" } ], - "selected": "false", + "selected": "true", "top": "top10", "timeout": 0, "implicit": 0, @@ -7747,6 +7643,29 @@ "type": "unavailable" } ], - "shared_detections": [], + "shared_detections": [{ + "name": "mastodon", + "detections": [{ + "return": "false", + "string": "The page you are looking for isn", + "type": "ocr" + }, + { + "return": "false", + "string": "The page you are looking for isn", + "type": "normal" + }, + { + "return": "true", + "string": "profile:username", + "type": "normal" + }, + { + "return": "true", + "string": "/@{username}", + "type": "normal" + } + ] + }], "generic_detection": [] } diff --git a/info b/info index 22332fdb..457eef11 100644 --- a/info +++ b/info @@ -1,4 +1,4 @@ -{"version":"2021.V.2.22", +{"version":"2021.V.2.23", "build":"pass", "test":"pass", "grid_test":"pass", @@ -11,4 +11,4 @@ "full_scan":"15 workers < 20secs", "max_retries":"3", "awaiting_verification":"22", - "auto_testing":"a420a20b-c7be-49b3-b9ca-c02f418bca79"} + "auto_testing":"aa4326d3-a290-47cb-bc8d-1836c34292de"} diff --git a/modules/engine.js b/modules/engine.js index ca21e9cf..70c737b4 100644 --- a/modules/engine.js +++ b/modules/engine.js @@ -1,12 +1,53 @@ var helper = require('./helper.js') var tesseract = require("tesseract.js") -async function detect(type, uuid, username, options, site, source="", screen_shot="") { +function merge_dicts(temp_dict){ + result = {} + temp_dict.forEach(item => { + for (const [key, value] of Object.entries(item)) { + if (result[key]) { + result[key] += value; + } else { + result[key] = value; + } + } + }); + return result; +} + +async function detect(type, uuid, username, options, site, source = "", screen_shot = "") { + var all_results = []; + var temp_profile = []; + var temp_detected = []; + var detections_count = 0; + await Promise.all(site.detections.map(async detection => { + if (detection.type == "shared") { + var shared_detection = await helper.shared_detections.find(o => o.name === detection.name); + var [val1, val2, val3] = await detect_logic("fast", uuid, username, options, shared_detection, source) + temp_profile.push(val1) + temp_detected.push(val2) + detections_count += val3 + } else if (detection.type == "generic") { + helper.verbose && console.log("None"); + } else if (detection.type == "special") { + helper.verbose && console.log("None"); + } + })); + + var [val1, val2, val3] = await detect_logic("fast", uuid, username, options, site, source) + temp_profile.push(val1) + temp_detected.push(val2) + detections_count += val3 + //console.log(temp_profile,merge_dicts(temp_detected),detections_count) + return [merge_dicts(temp_profile),merge_dicts(temp_detected),detections_count] +} + +async function detect_logic(type, uuid, username, options, site, source = "", screen_shot = "") { var temp_profile = Object.assign({}, helper.profile_template); var temp_detected = Object.assign({}, helper.detected_websites); var detections_count = 0; await Promise.all(site.detections.map(async detection => { - if (source != "" && helper.detection_level[helper.detection_level.current][type].includes(detection.type)) { + if (source != "" && helper.detection_level[helper.detection_level.current][type].includes(detection.type) && detection.type != "shared" && detection.type != "generic" && detection.type != "special") { try { detections_count += 1 temp_detected.count += 1 @@ -23,9 +64,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho if (detection.return == temp_found) { temp_profile.found += 1 temp_detected.ocr += 1 - if (detection.return == 'true'){ + if (detection.return == 'true') { temp_detected.true += 1 - }else{ + } else { temp_detected.false += 1 } } @@ -42,9 +83,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho if (detection.return == temp_found) { temp_profile.found += 1 temp_detected.normal += 1 - if (detection.return == 'true'){ + if (detection.return == 'true') { temp_detected.true += 1 - }else{ + } else { temp_detected.false += 1 } } @@ -56,9 +97,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho if (detection.return == temp_found) { temp_profile.found += 1 temp_detected.advanced += 1 - if (detection.return == 'true'){ + if (detection.return == 'true') { temp_detected.true += 1 - }else{ + } else { temp_detected.false += 1 } } @@ -69,9 +110,12 @@ async function detect(type, uuid, username, options, site, source="", screen_sho } })); - helper.verbose && console.log({"Temp Profile":temp_profile,"Detected":temp_detected}) + helper.verbose && console.log({ + "Temp Profile": temp_profile, + "Detected": temp_detected + }) - return {temp_profile, temp_detected, detections_count} + return [temp_profile, temp_detected, detections_count] } module.exports = { diff --git a/modules/fast-scan.js b/modules/fast-scan.js index 86b60d9a..5be42dfe 100644 --- a/modules/fast-scan.js +++ b/modules/fast-scan.js @@ -90,11 +90,11 @@ async function find_username_site(uuid, username, options, site) { var title = "unavailable"; var language = "unavailable" var good_or_bad = "false" - var { + var [ temp_profile, temp_detected, detections_count - } = await engine.detect("fast", uuid, username, options, site, source) + ] = await engine.detect("fast", uuid, username, options, site, source) if (temp_profile.found >= helper.detection_level[helper.detection_level.current].found && detections_count >= helper.detection_level[helper.detection_level.current].count) { temp_profile.good = "true" } diff --git a/modules/helper.js b/modules/helper.js index 406a61ca..c5c5a863 100644 --- a/modules/helper.js +++ b/modules/helper.js @@ -204,6 +204,7 @@ module.exports = { profile_template, detection_level, detected_websites, + shared_detections, get_language_by_parsing, get_language_by_guessing, websites_entries, diff --git a/modules/slow-scan.js b/modules/slow-scan.js index 9d7688c0..7c9ebe66 100644 --- a/modules/slow-scan.js +++ b/modules/slow-scan.js @@ -89,11 +89,12 @@ async function find_username_site(uuid, username, options, site) { title = await driver.getTitle(); text_only = await driver.findElement(By.tagName("body")).getText(); await driver.quit() - var {temp_profile, temp_detected, detections_count} = await engine.detect("slow", uuid, username, options, site ,source, screen_shot) + var [temp_profile, temp_detected, detections_count] = await engine.detect("slow", uuid, username, options, site ,source, screen_shot) if (options.includes("ShowUserProfilesSlow")) { temp_profile["image"] = "data:image/png;base64,{image}".replace("{image}", screen_shot); } if (temp_profile.found >= helper.detection_level[helper.detection_level.current].found && detections_count >= helper.detection_level[helper.detection_level.current].count){ + temp_profile.good = "true" try { language = helper.get_language_by_parsing(source) if (language == "unavailable") { @@ -106,7 +107,9 @@ async function find_username_site(uuid, username, options, site) { temp_profile.text = sanitizeHtml(text_only); temp_profile.title = sanitizeHtml(title); temp_profile.language = language - temp_profile.rate = "%" + ((temp_profile.found / site.detections.length) * 100).toFixed(2); + if (temp_profile.good == "true") { + temp_profile.rate = "%" + ((temp_profile["found"] / detections_count) * 100).toFixed(2); + } temp_profile.link = site.url.replace("{username}", username); temp_profile.type = site.type resolve(temp_profile);