-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.go
46 lines (42 loc) · 1.11 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
package main
import (
"Michael-Min/octopus/config"
"Michael-Min/octopus/discovery"
"Michael-Min/octopus/engine"
"Michael-Min/octopus/fetcher"
"Michael-Min/octopus/gredis"
persistClient "Michael-Min/octopus/persist/client"
pb "Michael-Min/octopus/proto"
"Michael-Min/octopus/scheduler"
worker "Michael-Min/octopus/worker/client"
"Michael-Min/octopus/xcar/parser"
)
func main() {
var (
itemChan chan pb.Item
err error
)
gredis.Setup()
fetcher.SetVerboseLogging()
//itemChan, err = persistClient.ItemSaver(config.Host+":2233")
err = persistClient.ItemSaverAsync(config.Host+":2233")
if err != nil {
panic(err)
}
dis := discovery.NewCrawlerDiscover()
dis.Discovery.Watch(dis.Hosts, "worker")
pool := worker.ActClientPool(dis)
processor := worker.CreateProcessor(pool)
e := engine.ConcurrentEngine{
Scheduler: &scheduler.QueuedScheduler{},
WorkerCount: 10,
ItemChan: itemChan,
RequestProcessor: processor,
}
e.Run(engine.Request{
Url: "http://fake.newcar.xcar.com.cn/car/",
Parser: engine.NewFuncParser(
parser.ParseCarDetailFake,
config.ParseCarDetailFake),
})
}