forked from s-rah/onionscan
/
get_onion_links.go
75 lines (64 loc) · 2.16 KB
/
get_onion_links.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
package deanonymization
import (
"fmt"
"github.com/s-rah/onionscan/config"
"github.com/s-rah/onionscan/model"
"github.com/s-rah/onionscan/report"
"github.com/s-rah/onionscan/utils"
"net"
"net/url"
"regexp"
"strings"
)
// GetOnionLinks extracts links and urls from the current crawl
func GetOnionLinks(osreport *report.OnionScanReport, anonreport *report.AnonymityReport, osc *config.OnionScanConfig) {
linkmap := make(map[string]bool)
urlmap := make(map[string]bool)
for _, id := range osreport.Crawls {
crawlRecord, _ := osc.Database.GetCrawlRecord(id)
if strings.Contains(crawlRecord.Page.Headers.Get("Content-Type"), "text/html") {
var entities []model.Element
entities = append(entities, crawlRecord.Page.Anchors...)
entities = append(entities, crawlRecord.Page.Links...)
entities = append(entities, crawlRecord.Page.Images...)
entities = append(entities, crawlRecord.Page.Scripts...)
for _, link := range entities {
ref, err := url.Parse(link.Target)
if err == nil {
host := ref.Host
if strings.Contains(ref.Host, ":") {
host, _, _ = net.SplitHostPort(ref.Host)
}
if utils.IsOnion(host) {
if linkmap[host] == false && host != osreport.HiddenService {
osc.LogInfo(fmt.Sprintf("Found Onion %s", host))
linkmap[host] = true
}
} else {
if strings.HasPrefix(ref.String(), "data:") {
// Ignore Data URI
} else {
urlmap[ref.String()] = true
}
}
} // This will ignore [embedded document] URLs
}
// FIXME: This can be smarter
onionregex := regexp.MustCompile(`[a-z2-7]{16}\.onion`)
foundOnions := onionregex.FindAllString(crawlRecord.Page.Snapshot, -1)
for _, host := range foundOnions {
if linkmap[host] == false && host != osreport.HiddenService {
osc.LogInfo(fmt.Sprintf("Found Onion %s", host))
linkmap[host] = true
}
}
}
}
for uri := range urlmap {
osc.Database.InsertRelationship(osreport.HiddenService, "clearnetlink", "uri", uri)
}
for onion := range linkmap {
anonreport.LinkedOnions = append(anonreport.LinkedOnions, onion)
osc.Database.InsertRelationship(osreport.HiddenService, "links", "uri", onion)
}
}