-
Notifications
You must be signed in to change notification settings - Fork 5
/
config.go
49 lines (43 loc) · 885 Bytes
/
config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
package crawler
import (
"net/url"
"gopkg.in/inconshreveable/log15.v2"
"github.com/fanyang01/crawler/queue"
"github.com/fanyang01/crawler/urlx"
)
type Config struct {
Controller Controller
Store Store
Queue queue.WaitQueue
Logger log15.Logger
NormalizeURL func(*url.URL) error
Option *Option
}
var (
DefaultController = NopController{}
)
func initConfig(cfg *Config) *Config {
if cfg == nil {
cfg = &Config{}
}
if cfg.Option == nil {
cfg.Option = DefaultOption
}
if cfg.Store == nil {
cfg.Store = NewMemStore()
}
if cfg.Queue == nil {
cfg.Queue = NewMemQueue(64 << 10)
}
if cfg.Controller == nil {
cfg.Controller = DefaultController
}
if cfg.Logger == nil {
cfg.Logger = log15.New()
cfg.Logger.SetHandler(log15.DiscardHandler())
}
if cfg.NormalizeURL == nil {
cfg.NormalizeURL = urlx.Normalize
}
return cfg
}