Skip to content

Commit

Permalink
[manual] Added shared detections - ref c9d24b98
Browse files Browse the repository at this point in the history
  • Loading branch information
@giga-a
giga-a committed Jan 28, 2021
1 parent d2bbf05 commit 96249fc
Show file tree
Hide file tree
Showing 8 changed files with 105 additions and 138 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ The analysis and extracted social media information from this OSINT tool could h
This project *is currently used by some law enforcement agencies in countries where resources are limited*.

## Updates
* Implementing generic and shared detections
* Added shared detections to NodeJS Web App & CLI 👏
* API, CLIs & Web App produce:
* [Detected] -> 0 to 100 (No-Maybe-Yes)
* [Unknown] -> Detection did not meet the requirement
Expand Down
4 changes: 2 additions & 2 deletions app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

var argv = require('yargs')
.usage('Usage: $0 --cli --mode "fast" --username "johndoe" --websites "youtube tiktok"\nUsage: $0 --cli --mode "fast" --username "johndoe"')
.describe('cli', 'a user or stirng')
.describe('cli', 'enable this cli')
.default("cli", false)
.boolean('cli')
.describe('username', 'E.g. johndoe, john_doe or johndoe9999')
Expand All @@ -19,7 +19,7 @@ var argv = require('yargs')
.default("websites", "all")
.describe('mode', 'Analysis mode E.g.fast -> FindUserProfilesFast, slow -> FindUserProfilesSlow or special -> FindUserProfilesSpecial')
.default("mode", "")
.describe('output', 'arg_parser_optional.add_argument("--output", help="Show the output in the following format: json -> json output for integration or pretty -> prettify the output')
.describe('output', 'Show the output in the following format: json -> json output for integration or pretty -> prettify the output')
.default("output", "")
.describe('options', 'Show the following when a profile is found: link, rate, title or text')
.default("options", "")
Expand Down
157 changes: 38 additions & 119 deletions data/sites.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2219,8 +2219,6 @@
"type": "normal"
},
{
"return": "true",
"string": "",
"function": "special_facebook_1",
"type": "special"
}
Expand Down Expand Up @@ -2708,30 +2706,14 @@
{
"url": "https://fosstodon.org/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
"timeout": 0,
"implicit": 0,
"type": "Mastodon instance"
"type": "unavailable"
},
{
"url": "https://fotolog.com/author/{username}",
Expand Down Expand Up @@ -3158,13 +3140,10 @@
{
"url": "https://gmail.com/{username}",
"detections": [{
"return": "true",
"string": "",
"function": "special_gmail_1",
"type": "special"
}],
"selected": "false",
"top": "top10",
"timeout": 0,
"implicit": 0,
"type": "Social networking service"
Expand All @@ -3190,13 +3169,10 @@
{
"url": "https://google.com/{username}",
"detections": [{
"return": "true",
"string": "",
"function": "special_google_1",
"type": "special"
}],
"selected": "false",
"top": "top10",
"timeout": 0,
"implicit": 0,
"type": "Social networking service"
Expand Down Expand Up @@ -4294,24 +4270,8 @@
{
"url": "https://mastodon.cloud/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
Expand All @@ -4322,24 +4282,8 @@
{
"url": "https://mastodon.social/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
Expand All @@ -4350,24 +4294,8 @@
{
"url": "https://mastodon.technology/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
Expand All @@ -4378,24 +4306,8 @@
{
"url": "https://mastodon.xyz/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
Expand Down Expand Up @@ -4592,24 +4504,8 @@
{
"url": "https://mstdn.io/@{username}",
"detections": [{
"return": "false",
"string": "The page you are looking for",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for",
"type": "normal"
},
{
"return": "true",
"string": "og:title",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
"name": "mastodon",
"type": "shared"
}
],
"selected": "false",
Expand Down Expand Up @@ -7654,7 +7550,7 @@
"type": "normal"
}
],
"selected": "false",
"selected": "true",
"top": "top10",
"timeout": 0,
"implicit": 0,
Expand Down Expand Up @@ -7747,6 +7643,29 @@
"type": "unavailable"
}
],
"shared_detections": [],
"shared_detections": [{
"name": "mastodon",
"detections": [{
"return": "false",
"string": "The page you are looking for isn",
"type": "ocr"
},
{
"return": "false",
"string": "The page you are looking for isn",
"type": "normal"
},
{
"return": "true",
"string": "profile:username",
"type": "normal"
},
{
"return": "true",
"string": "/@{username}",
"type": "normal"
}
]
}],
"generic_detection": []
}
4 changes: 2 additions & 2 deletions info
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{"version":"2021.V.2.22",
{"version":"2021.V.2.23",
"build":"pass",
"test":"pass",
"grid_test":"pass",
Expand All @@ -11,4 +11,4 @@
"full_scan":"15 workers < 20secs",
"max_retries":"3",
"awaiting_verification":"22",
"auto_testing":"a420a20b-c7be-49b3-b9ca-c02f418bca79"}
"auto_testing":"aa4326d3-a290-47cb-bc8d-1836c34292de"}
64 changes: 54 additions & 10 deletions modules/engine.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,53 @@
var helper = require('./helper.js')
var tesseract = require("tesseract.js")

async function detect(type, uuid, username, options, site, source="", screen_shot="") {
function merge_dicts(temp_dict){
result = {}
temp_dict.forEach(item => {
for (const [key, value] of Object.entries(item)) {
if (result[key]) {
result[key] += value;
} else {
result[key] = value;
}
}
});
return result;
}

async function detect(type, uuid, username, options, site, source = "", screen_shot = "") {
var all_results = [];
var temp_profile = [];
var temp_detected = [];
var detections_count = 0;
await Promise.all(site.detections.map(async detection => {
if (detection.type == "shared") {
var shared_detection = await helper.shared_detections.find(o => o.name === detection.name);
var [val1, val2, val3] = await detect_logic("fast", uuid, username, options, shared_detection, source)
temp_profile.push(val1)
temp_detected.push(val2)
detections_count += val3
} else if (detection.type == "generic") {
helper.verbose && console.log("None");
} else if (detection.type == "special") {
helper.verbose && console.log("None");
}
}));

var [val1, val2, val3] = await detect_logic("fast", uuid, username, options, site, source)
temp_profile.push(val1)
temp_detected.push(val2)
detections_count += val3
//console.log(temp_profile,merge_dicts(temp_detected),detections_count)
return [merge_dicts(temp_profile),merge_dicts(temp_detected),detections_count]
}

async function detect_logic(type, uuid, username, options, site, source = "", screen_shot = "") {
var temp_profile = Object.assign({}, helper.profile_template);
var temp_detected = Object.assign({}, helper.detected_websites);
var detections_count = 0;
await Promise.all(site.detections.map(async detection => {
if (source != "" && helper.detection_level[helper.detection_level.current][type].includes(detection.type)) {
if (source != "" && helper.detection_level[helper.detection_level.current][type].includes(detection.type) && detection.type != "shared" && detection.type != "generic" && detection.type != "special") {
try {
detections_count += 1
temp_detected.count += 1
Expand All @@ -23,9 +64,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho
if (detection.return == temp_found) {
temp_profile.found += 1
temp_detected.ocr += 1
if (detection.return == 'true'){
if (detection.return == 'true') {
temp_detected.true += 1
}else{
} else {
temp_detected.false += 1
}
}
Expand All @@ -42,9 +83,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho
if (detection.return == temp_found) {
temp_profile.found += 1
temp_detected.normal += 1
if (detection.return == 'true'){
if (detection.return == 'true') {
temp_detected.true += 1
}else{
} else {
temp_detected.false += 1
}
}
Expand All @@ -56,9 +97,9 @@ async function detect(type, uuid, username, options, site, source="", screen_sho
if (detection.return == temp_found) {
temp_profile.found += 1
temp_detected.advanced += 1
if (detection.return == 'true'){
if (detection.return == 'true') {
temp_detected.true += 1
}else{
} else {
temp_detected.false += 1
}
}
Expand All @@ -69,9 +110,12 @@ async function detect(type, uuid, username, options, site, source="", screen_sho
}
}));

helper.verbose && console.log({"Temp Profile":temp_profile,"Detected":temp_detected})
helper.verbose && console.log({
"Temp Profile": temp_profile,
"Detected": temp_detected
})

return {temp_profile, temp_detected, detections_count}
return [temp_profile, temp_detected, detections_count]
}

module.exports = {
Expand Down
Loading

0 comments on commit 96249fc

Please sign in to comment.