-
Notifications
You must be signed in to change notification settings - Fork 27
/
main.go
63 lines (53 loc) · 924 Bytes
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
package main
import (
"flag"
"fmt"
"os"
"github.com/twiny/spidy/crawler"
)
const msg = `
Spidy - Fast Bulk Expired Domain Scraper
Usage:
./Spidy -config /path/to/setting.yaml
`
// usage
func usage() {
print(msg)
os.Exit(0)
}
// main
func main() {
// load configs
config := flag.String("config", "", "config: path to setting.yaml.")
flag.Usage = usage
flag.Parse()
//
if len(*config) == 0 {
fmt.Println("enter config path.")
return
}
//
fmt.Println("Welcome, Spidy is running.")
//
setting, err := crawler.SettingFromFile(*config)
if err != nil {
fmt.Println(err)
return
}
// make directory for cache & log if not exisit
if _, err := os.Stat("./log"); os.IsNotExist(err) {
os.Mkdir("./log", os.ModePerm)
}
//
tool, err := crawler.NewSpider(setting)
if err != nil {
fmt.Println(err)
return
}
// Logger
tool.Logger()
// Run
tool.Run()
// finialy
fmt.Println("done :)")
}