-
Notifications
You must be signed in to change notification settings - Fork 0
/
status.go
91 lines (79 loc) · 2.21 KB
/
status.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
package crawler
import (
"time"
)
type SpiderStatus struct {
Name string
Crawled int
Running int
Waiting int
StartTime time.Time
EndTime time.Time
}
func NewSpiderStatus(name string) *SpiderStatus {
startTime := time.Now()
spiderStatus := &SpiderStatus{
Name: name,
StartTime: startTime,
Crawled: 0,
Running: 0,
Waiting: 0,
}
return spiderStatus
}
// status of crawler
// crawled spiders and running spiders
type CrawlerStatus struct {
CrawledSpider []*SpiderStatus
RunningSpider map[string]*SpiderStatus
}
func NewCrawlerStatus() *CrawlerStatus {
crawledSpider := make([]*SpiderStatus, 0)
runningSpider := make(map[string]*SpiderStatus)
return &CrawlerStatus{crawledSpider, runningSpider}
}
func (this *CrawlerStatus) IsSpiderRunning(spiderName string) bool {
_, ok := this.RunningSpider[spiderName]
return ok
}
// add a spider to running map
func (this *CrawlerStatus) StartSpider(spiderName string) {
_, ok := this.RunningSpider[spiderName]
if !ok {
this.RunningSpider[spiderName] = NewSpiderStatus(spiderName)
}
}
// add a request to wait in spiderName
func (this *CrawlerStatus) Push(spiderName string) {
this.RunningSpider[spiderName].Waiting += 1
}
// if cluster distribute a request,
// waiting -1 runing +1
func (this *CrawlerStatus) Distribute(spiderName string) {
spiderStatus, _ := this.RunningSpider[spiderName]
spiderStatus.Waiting -= 1
spiderStatus.Running += 1
}
// get crawl result
// runing -1 crawled +1
func (this *CrawlerStatus) Crawled(spiderName string) {
spiderStatus, _ := this.RunningSpider[spiderName]
spiderStatus.Running -= 1
spiderStatus.Crawled += 1
}
// judge a is a spider can stop
func (this *CrawlerStatus) CanWeStop(spiderName string) bool {
spiderStatus, _ := this.RunningSpider[spiderName]
leftNum := spiderStatus.Running + spiderStatus.Waiting
return leftNum <= 0
}
// no more request for spider ,close it
// remove from runningSpider
// add to crawledSpider
func (this *CrawlerStatus) CloseSpider(spiderName string) *SpiderStatus {
spiderStatus, _ := this.RunningSpider[spiderName]
spiderStatus.EndTime = time.Now()
this.CrawledSpider = append(this.CrawledSpider, spiderStatus)
delete(this.RunningSpider, spiderName)
return spiderStatus
}