-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrape.go
75 lines (52 loc) · 1.23 KB
/
scrape.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
package request
import (
"net/url"
"strings"
"sync"
"github.com/thisisiliya/go_utils/time"
)
type Data struct {
URL string
Host string
Path string
Subdomain string
}
func Scrape(options *Options) *[]Data {
var result []Data
var wg_task sync.WaitGroup
var wg_time sync.WaitGroup
wg_time.Add(1)
go func() {
time.RandomSleep(options.MinDelay, options.MaxDelay)
wg_time.Done()
}()
for _, opt := range options.Engines {
wg_task.Add(1)
opt := opt
go func() {
defer wg_task.Done()
URLs := request(&opt, &options.Dork, options.Browser)
for _, newURL := range *URLs {
if parsed_URL, err := url.Parse(newURL); err == nil {
if strings.Contains(parsed_URL.Host, options.Dork.Domain) {
result = append(result, Data{
URL: newURL,
Host: parsed_URL.Host,
Path: parsed_URL.Path,
Subdomain: subdomainExt(parsed_URL.Host),
})
}
}
}
}()
}
wg_task.Wait()
wg_time.Wait()
return &result
}
func subdomainExt(host string) string {
if domain := strings.Split(host, "."); len(domain) > 2 {
return strings.Join(domain[0:len(domain)-2], ".")
}
return ""
}