-
Notifications
You must be signed in to change notification settings - Fork 2
/
main.go
33 lines (28 loc) · 869 Bytes
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
package main
import (
"flag"
"strings"
"github.com/gocolly/colly/v2"
ss51pwn "github.com/hktalent/scrapysite/lib"
)
func main() {
flag.StringVar(&ss51pwn.ConfigName, "config", "./config/config.json", "config file name")
flag.Parse()
ss51pwn.Init()
var scrapysite = ss51pwn.NewScrapySite()
scrapysite.OnRequest(func(r *colly.Request) {
//fmt.Println("Visiting", r.URL.String())
})
scrapysite.OnResponse(func(r *colly.Response) {
// 非文本,计算sha1、md5
if strings.Index(r.Headers.Get("Content-Type"), "text/") == -1 {
//r.Save("./xx/" + r.FileName())
return
}
})
scrapysite.Start()
// spew.Dump(scrapysite.GetDomainInfo("http://www.gov.cn"))
// md5R, sha1R, sha256R := scrapysite.Hash([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"))
// log.Println(md5R, sha1R, sha256R)
select {}
}