-
Notifications
You must be signed in to change notification settings - Fork 0
/
queue.go
87 lines (75 loc) · 1.77 KB
/
queue.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
package crawler
import (
"container/list"
"github.com/wcong/ants-go/ants/http"
"sync"
)
var mutex sync.Mutex
type RequestQuene struct {
RequestList *list.List
}
func NewRequestQuene() *RequestQuene {
requestList := RequestQuene{}
requestList.RequestList = list.New()
return &requestList
}
func (this *RequestQuene) Push(request *http.Request) {
this.RequestList.PushBack(request)
}
func (this *RequestQuene) Pop() *http.Request {
element := this.RequestList.Front()
if element == nil {
return nil
}
this.RequestList.Remove(element)
value := element.Value
return value.(*http.Request)
}
func (this *RequestQuene) IsEmpty() bool {
return this.RequestList.Len() == 0
}
type ResponseQuene struct {
ResponseList *list.List
}
func NewResponseQuene() *ResponseQuene {
requestList := ResponseQuene{}
requestList.ResponseList = list.New()
return &requestList
}
func (this *ResponseQuene) Push(response *http.Response) {
mutex.Lock()
this.ResponseList.PushBack(response)
mutex.Unlock()
}
// for now only one routine pop the request,so do not add lock
func (this *ResponseQuene) Pop() *http.Response {
element := this.ResponseList.Front()
if element == nil {
return nil
}
this.ResponseList.Remove(element)
value := element.Value
return value.(*http.Response)
}
type ResultQuene struct {
ResultList *list.List
}
func NewResultQuene() *ResultQuene {
resultList := list.New()
resultQuene := &ResultQuene{resultList}
return resultQuene
}
func (this *ResultQuene) Push(scrapeResult *ScrapeResult) {
mutex.Lock()
this.ResultList.PushBack(scrapeResult)
mutex.Unlock()
}
func (this *ResultQuene) Pop() *ScrapeResult {
element := this.ResultList.Front()
if element == nil {
return nil
}
this.ResultList.Remove(element)
value := element.Value
return value.(*ScrapeResult)
}