forked from wcong/ants-go
-
Notifications
You must be signed in to change notification settings - Fork 0
/
status.go
68 lines (61 loc) · 1.72 KB
/
status.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
package node
import (
"github.com/wcong/ants-go/ants/crawler"
"github.com/wcong/ants-go/ants/http"
"log"
"sync"
)
var mutex sync.Mutex
// receive basic request and record crawled requets
type RequestStatus struct {
CrawledMap map[string]int // node + num
CrawlingMap map[string]map[string]*http.Request
WaitingQuene *crawler.RequestQuene
}
func NewRequestStatus() *RequestStatus {
requestStatus := &RequestStatus{}
requestStatus.CrawledMap = make(map[string]int)
requestStatus.CrawlingMap = make(map[string]map[string]*http.Request)
requestStatus.WaitingQuene = crawler.NewRequestQuene()
return requestStatus
}
// delete in CrawlingMap
// add for CrawledMap
func (this *RequestStatus) Crawled(scrapyResult *crawler.ScrapeResult) {
requestMap, nodeOk := this.CrawlingMap[scrapyResult.Request.NodeName]
if !nodeOk {
log.Println("none node :" + scrapyResult.Request.NodeName)
return
}
_, requestOk := requestMap[scrapyResult.Request.UniqueName]
if !requestOk {
log.Println("none request :" + scrapyResult.Request.UniqueName)
return
}
// change RequestStatus
mutex.Lock()
this.CrawledMap[scrapyResult.Request.NodeName] += 1
delete(requestMap, scrapyResult.Request.UniqueName)
mutex.Unlock()
}
// remove request from crawlingmap for dead node
// add those requests to waiting quenu
func (this *RequestStatus) DeleteDeadNode(nodeName string) {
crawlingMap := this.CrawlingMap[nodeName]
for _, request := range crawlingMap {
this.WaitingQuene.Push(request)
}
delete(this.CrawlingMap, nodeName)
}
// is all loop stop
func (this *RequestStatus) IsStop() bool {
if !this.WaitingQuene.IsEmpty() {
return false
}
for _, requestMap := range this.CrawlingMap {
if len(requestMap) > 0 {
return false
}
}
return true
}