-
Notifications
You must be signed in to change notification settings - Fork 0
/
parse.go
154 lines (133 loc) · 3.83 KB
/
parse.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
package sculptor
import (
log "github.com/sirupsen/logrus"
"io"
"reflect"
"time"
)
// Do func extract the data from file and put it into the ConstructedOutput chan
// Do() is no not blocked.
func (d *DataSculptor) Do() *DataSculptor {
go func() {
d.Wg.Add(1)
defer d.Wg.Done()
d.rawDo()
}()
time.Sleep(d.options.Latency) // wait for process init
return d
}
func (d *DataSculptor) Wait() {
d.Wg.Wait()
}
// rawDo func is real Do func execute the data extraction.
func (d *DataSculptor) rawDo() {
Scanner := d.scanner
Selectors := d.docQueries
d.debugInfo()
Scanner.InitIndex(Selectors)
defer d.scanner.Close()
for {
d.count++
// var recordBuilder = d.targetStruct
recordBuilderT := reflect.TypeOf(d.targetStruct).Elem()
recordBuilderV := reflect.ValueOf(d.targetStruct).Elem()
if d.Error() != nil {
goto FallBack
}
// for all selectors to build recordBuilder
d.lastErr = Scanner.Select(Selectors)
// if read EOF or other error in this Loop
if d.Error() != nil {
log.WithFields(log.Fields{
"Count": d.count,
"err": d.Error(),
"during SelectingLine": d.scanner.CurrentLine(),
}).Trace("Scanner selecting porcess fails.")
goto FallBack
}
for _, v := range Selectors {
for i := 0; i < recordBuilderT.NumField(); i++ {
CurrentTagStr := recordBuilderT.Field(i).Tag.Get(d.options.TagKey)
log.WithFields(log.Fields{
"Count": d.count,
"SelectingStructTag": CurrentTagStr,
"SelectorTag": v.TagName,
}).Trace("Current Select tag match status")
if CurrentTagStr == v.TagName {
// copy Value to dest
log.WithFields(log.Fields{
"Count": d.count,
"SelectingStructTag": CurrentTagStr,
"SelectorTag": v.TagName,
}).Trace("current tags is matched. Setting value.")
recordBuilderV.Field(i).Set(reflect.ValueOf(v.Value))
}
}
// Todo: Build T type from Selected values with TagName
}
// Ask for First and Second Record
if d.count == 0 || d.count == 1 {
log.WithFields(log.Fields{
"Count": d.count,
"Record": d.targetStruct,
}).Info("Record generated struct. ")
} else {
log.WithFields(log.Fields{
"Count": d.count,
"Record": d.targetStruct,
}).Trace("Record generated struct. ")
}
// d.targetStruct = recordBuilder
for i, f := range d.customFunc {
d.lastErr = f(d)
if d.Error() != nil {
log.WithFields(log.Fields{
"Count": d.count,
"CustomFuncIndex": i,
"err": d.Error(),
}).Errorf("CustomFunc Exec with Error")
goto FallBack
}
}
log.WithFields(log.Fields{
"Count": d.count,
"Record": d.targetStruct,
}).Trace("Preprocess with customFunc successfully. Sending Data")
d.send()
if d.count%100 == 0 {
log.Infof("%v Records have been generated.", d.count)
}
if d.count%1000 == 0 && d.count > 1000 {
log.Infof("%v Records have been generated.", d.count)
}
continue
FallBack:
// ToDo: Get Another parsing Error and Record them and continue the parsing (non break) This break will let everything die.
log.WithFields(log.Fields{
"Count": d.count,
"err": d.Error(),
}).Debug("Receive Error and fallback init")
if d.Error() == io.EOF {
log.Info("End Record At No.", d.count, " line")
log.Info("Got End Of file")
break // stop parsing and exit.
}
// do fallbacks
log.WithFields(log.Fields{
"Count": d.count,
"err": d.Error(),
}).Trace("Error is not caused by end of file.")
for i, f := range d.fallbackFunc {
err := f(d) // force fallbackFunc success.
if err != nil {
log.WithFields(log.Fields{
"Count": d.count,
"LastErr": d.Error(),
"FallbackFuncIndex": i,
"Error by FallbackFunc": err,
}).Errorf("Fallback Error. Program will exit.")
break
}
}
}
}