-
Notifications
You must be signed in to change notification settings - Fork 0
/
request.go
56 lines (51 loc) · 1.34 KB
/
request.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
package http
import (
"encoding/json"
"github.com/wcong/ants-go/ants/util"
"io"
Http "net/http"
"strconv"
)
/*
what a request would do
* basic request
* global unique key
* cookie jar index
* spider belong to
* parse belong to
*/
type Request struct {
GoRequest *Http.Request
CookieJar int // if the web site store message in cookie which show diffrent result,you shoud use it
UniqueName string
SpiderName string
ParserName string // crawler will send the response to named parseFunction
NodeName string
Depth int
Proxy string // proxy settings,just setting the proxy,downloader will take care of rest thing
Retry int // fail and retry time
}
func NewRequest(method, url, spiderName, parserName string, body io.Reader, cookieJar int) (*Request, error) {
httpRequest, err := Http.NewRequest(method, url, body)
if err != nil {
return nil, err
}
request := &Request{
GoRequest: httpRequest,
CookieJar: cookieJar,
SpiderName: spiderName,
ParserName: parserName,
Depth: 0,
Retry: 0,
}
request.makeUniqueName()
return request, err
}
// unique sign of q request
func (this *Request) makeUniqueName() {
baseString, _ := json.Marshal(this)
this.UniqueName = strconv.FormatUint(util.HashString(string(baseString)), 10)
}
func (this *Request) SetNodeName(nodeName string) {
this.NodeName = nodeName
}