-
Notifications
You must be signed in to change notification settings - Fork 0
/
searchSite.go
96 lines (87 loc) · 2.45 KB
/
searchSite.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package main
import (
"fmt"
"strconv"
"strings"
"github.com/fatih/color"
"github.com/gocolly/colly"
"github.com/urfave/cli"
)
var searches []string
var scripts []string
func searchSite(cl *cli.Context, targetURL string, crawl bool) {
collector := colly.NewCollector(
colly.IgnoreRobotsTxt(),
)
targetURL = handleTargetURL(targetURL)
targetDomain := urlToDomain(targetURL)
handleOutputPath(cl.String("out"), targetDomain)
// crawl website
if crawl {
collector.OnHTML("a", func(e *colly.HTMLElement) {
url := e.Attr("href")
if strings.Contains(url, targetDomain) ||
// or is a relative link
strings.HasPrefix(url, "./") ||
( // or isn't the following
// an external url
!strings.HasPrefix(url, "http") &&
// a javascript url
!strings.HasPrefix(url, "javascript:") &&
!strings.HasPrefix(url, "mailto:") &&
!strings.HasPrefix(url, "tel:")) {
// visit next page
e.Request.Visit(url)
}
})
}
collector.OnHTML("script", func(e *colly.HTMLElement) {
url := e.Attr("src")
if len(url) > 1 && !arrayContains(scripts, url) {
scripts = append(scripts, url)
e.Request.Visit(url)
}
})
collector.OnRequest(func(r *colly.Request) {
// log url
fmt.Println("Searching: ", r.URL)
})
// search page
collector.OnResponse(func(r *colly.Response) {
url := r.Request.URL.String()
content := strings.ToLower(string(r.Body))
search := strings.ToLower(cl.String("text"))
keywords := strings.ToLower(cl.String("keywords"))
searchPage(url, content, search, keywords)
})
collector.Visit(targetURL)
}
func searchPage(url string, content string, search string, keywords string) {
if len(keywords) > 0 {
keys := strings.Split(keywords, ",")
for _, k := range keys {
occurences := strings.Count(content, k)
if occurences >= 1 {
if !arrayContains(searches, url) {
searches = append(searches, url)
output := k + " found " + strconv.Itoa(occurences) + " times at: " + url
color.Green(output)
saveLineToFile("searches.txt", output)
}
}
}
} else if len(search) > 0 {
occurences := strings.Count(content, search)
if occurences >= 1 {
if !arrayContains(searches, url) {
searches = append(searches, url)
output := search + " found " + strconv.Itoa(occurences) + " times at: " + url
color.Green(output)
saveLineToFile("searches.txt", output)
}
}
} else {
color.Red("No search critera was specified")
color.Red("Please specify --text or --keywords")
}
}