-
Notifications
You must be signed in to change notification settings - Fork 42
/
spider.go
106 lines (102 loc) · 2.3 KB
/
spider.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
package main
import (
"fmt"
"github.com/yaklang/yaklang/common/rpa"
"github.com/yaklang/yaklang/common/rpa/cmd/semi"
"github.com/yaklang/yaklang/common/rpa/core"
"github.com/yaklang/yaklang/common/utils"
"os"
"github.com/urfave/cli"
)
func main() {
app := cli.NewApp()
app.Commands = []cli.Command{
semi.BruteForce,
semi.Test,
}
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "host, H",
Value: "127.0.0.1",
Usage: "spider target host `URL`",
},
cli.IntFlag{
Name: "depth, D",
Value: 3,
Usage: "spider depth",
},
cli.StringFlag{
Name: "proxy",
Usage: "spider proxy",
},
cli.StringFlag{
Name: "proxy-username",
Usage: "spider proxy username",
},
cli.StringFlag{
Name: "proxy-password",
Usage: "spider proxy password",
},
cli.BoolFlag{
Name: "stricturl",
Usage: "strict url scan mode. do not click sensitive url",
},
cli.StringFlag{
Name: "headers",
Usage: "scan headers by headers info file path (recommend) or json string (not recommend)",
},
cli.IntFlag{
Name: "maxurl",
Value: 0,
Usage: "max url scan number. 0 means no limit",
},
cli.IntFlag{
Name: "timeout",
Usage: "page timeout default 20s",
},
}
app.Action = func(c *cli.Context) error {
host := c.String("host")
depth := c.Int("depth")
proxy := c.String("proxy")
username := c.String("username")
password := c.String("password")
stricturl := c.Bool("stricturl")
headers := c.String("headers")
maxurl := c.Int("maxurl")
timeout := c.Int("timeout")
opts := make([]core.ConfigOpt, 0)
opts = append(opts,
core.WithSpiderDepth(depth),
core.WithStrictUrlDetect(stricturl),
core.WithUrlCount(maxurl),
core.WithTimeout(timeout),
)
if headers != "" {
opts = append(opts, core.WithHeader(headers))
}
if proxy != "" {
if username == "" {
opts = append(opts, core.WithBrowserProxy(proxy))
} else {
opts = append(opts, core.WithBrowserProxy(proxy, username, password))
}
}
rsts, err := rpa.Start(host, opts...)
if err != nil {
return utils.Errorf("spider run error:%s", err)
}
// hasPrint filter repeat urls
// hasPrint := filter.NewFilter()
for rst := range rsts {
url := rst.Url()
fmt.Println(url)
}
return nil
}
err := app.Run(os.Args)
if err != nil {
println(err.Error())
return
}
}