-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawler.go
48 lines (44 loc) · 1012 Bytes
/
crawler.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
package crawler
import (
"github.com/x-armory/go-exception"
"net/http"
)
type Crawler struct {
RequestGenerator
DataUnmarshaler
Business
}
func (c Crawler) Start() {
ex.Assert(c.RequestGenerator != nil, "request generator is nil")
ex.Assert(c.DataUnmarshaler != nil, "data unmarshaler is nil")
ex.Assert(c.Business != nil, "business definition is nil")
ex.Try(func() {
for true {
c.Unmarshal(c.GenRequest(), c.NewPeriodData())
c.ProcessPeriodData()
}
}).SafeCatch(func(err interface{}) {
if e := ex.Wrap(err); e.Code() != NoMoreDataException {
ex.Try(func() {
c.OnError(e)
}).SafeCatch(func(err interface{}) {
if err != nil {
ex.Wrap(err).PrintErrorStack()
}
})
}
})
c.Finish()
}
type RequestGenerator interface {
GenRequest() (req *http.Request)
}
type DataUnmarshaler interface {
Unmarshal(req *http.Request, target interface{})
}
type Business interface {
NewPeriodData() interface{}
ProcessPeriodData()
Finish()
OnError(err *ex.ExceptionClass)
}