forked from arsham/expipe
/
maps_reader.go
222 lines (201 loc) · 6.46 KB
/
maps_reader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
// Copyright 2016 Arsham Shirvani <arshamshirvani@gmail.com>. All rights reserved.
// Use of this source code is governed by the Apache 2.0 license
// License that can be found in the LICENSE file.
package datatype
import (
"expvar"
"strings"
"sync"
"github.com/antonholmquist/jason"
"github.com/arsham/expipe/tools"
"github.com/spf13/viper"
)
var (
expStringTypeCount = expvar.NewInt("StringType Count")
expFloatTypeCount = expvar.NewInt("FloatType Count")
expFloatListTypeCount = expvar.NewInt("FloatListType Count")
expGCListTypeCount = expvar.NewInt("GCListType Count")
expByteTypeCount = expvar.NewInt("ByteType Count")
expNestedTypeCount = expvar.NewInt("Nested Type Count")
expDataTypeObjs = expvar.NewInt("DataType Objects")
expDataTypeErrs = expvar.NewInt("DataType Objects Errors")
expUnidentifiedJSON = expvar.NewInt("Unidentified JSON Count")
once sync.Once
defaultMap *MapConvert
)
// MapConvert can produce output from GC string list and memory type input.
type MapConvert struct {
GCTypes []string
MemoryTypes map[string]string
}
type treeReader interface {
IsSet(key string) bool
GetStringSlice(key string) []string
GetStringMapString(key string) map[string]string
}
// MapsFromViper reads from the map file and produces functions for conversion
// used in type decoder. It first reads from the default settings defined in the
// maps.yml in the same folder, then overrides with the user specified mappings.
func MapsFromViper(v treeReader) *MapConvert {
m := &MapConvert{}
def := DefaultMapper()
if v.IsSet("gc_types") {
m.GCTypes = gcTypes(v, def.GCTypes)
}
if v.IsSet("memory_bytes") {
m.MemoryTypes = memoryTypes(v, def.MemoryTypes)
}
return m
}
// DefaultMapper returns a MapConvert object that is populated by the default
// mappings. The data is hard coded in the program, but you can provide your
// mapping file in your configuration file.
func DefaultMapper() *MapConvert {
once.Do(func() {
v := viper.New()
v.SetConfigType("yaml")
v.ReadConfig(defaultMappings())
defaultMap = &MapConvert{}
if v.IsSet("gc_types") {
defaultMap.GCTypes = gcTypes(v, make([]string, 0))
}
if v.IsSet("memory_bytes") {
defaultMap.MemoryTypes = memoryTypes(v, make(map[string]string))
}
})
return defaultMap
}
func (m *MapConvert) getMemoryTypes(prefix, name string, j *jason.Value) (DataType, bool) {
var (
data DataType
ok bool
)
v, err := j.Float64()
if err != nil {
expDataTypeErrs.Add(1)
return nil, false
}
b := m.MemoryTypes[strings.ToLower(name)]
if IsByte(b) {
data, ok = NewByteType(prefix+name, v), true
} else if IsKiloByte(b) {
data, ok = NewKiloByteType(prefix+name, v), true
} else if IsMegaByte(b) {
data, ok = NewMegaByteType(prefix+name, v), true
} else {
data, ok = nil, false
}
return data, ok
}
func (m *MapConvert) arrayValue(prefix, name string, a []*jason.Value) DataType {
if len(a) == 0 {
return NewFloatListType(prefix+name, []float64{})
} else if _, err := a[0].Float64(); err == nil {
if tools.StringInSlice(name, m.GCTypes) {
return getGCList(prefix+name, a)
}
return getFloatListValues(prefix+name, a)
}
return nil
}
// Values returns a slice of DataTypes based on the given name/value inputs.
// It flattens the float list values, therefore you will get multiple values
// per input. If the name is found in memory_bytes map, it will return one of
// those, otherwise it will return a FloatType or StringType if can convert.
// It will return nil if the value is not one of above.
func (m *MapConvert) Values(prefix string, values map[string]*jason.Value) []DataType {
var results []DataType
input := make(map[string]jason.Value, len(values))
for k, v := range values {
input[k] = *v
}
for name, value := range input {
var result DataType
if _, ok := m.MemoryTypes[strings.ToLower(name)]; ok {
result, ok = m.getMemoryTypes(prefix, name, &value)
if !ok {
continue
}
expByteTypeCount.Add(1)
} else if obj, err := value.Object(); err == nil {
// we are dealing with nested objects
results = append(results, m.Values(prefix+name+".", obj.Map())...)
expNestedTypeCount.Add(1)
continue
} else if s, err := value.String(); err == nil {
expStringTypeCount.Add(1)
result = NewStringType(prefix+name, s)
} else if f, err := value.Float64(); err == nil {
expFloatTypeCount.Add(1)
result = NewFloatType(prefix+name, f)
} else if arr, err := value.Array(); err == nil {
// we are dealing with an array object
result = m.arrayValue(prefix, name, arr)
} else {
expDataTypeErrs.Add(1)
continue
}
expDataTypeObjs.Add(1)
if result != nil { // TEST: write tests (7)
results = append(results, result)
}
}
return results
}
// Copy returns a new copy of the Mapper.
func (m *MapConvert) Copy() Mapper {
newMapper := &MapConvert{}
newMapper.GCTypes = m.GCTypes[:]
newMapper.MemoryTypes = make(map[string]string, len(m.MemoryTypes))
for k, v := range m.MemoryTypes {
newMapper.MemoryTypes[k] = v
}
return newMapper
}
func getGCList(name string, arr []*jason.Value) *GCListType {
res := make([]uint64, len(arr))
for i, val := range arr {
if r, err := val.Float64(); err == nil {
res[i] = uint64(r)
}
}
expGCListTypeCount.Add(1)
return NewGCListType(name, res)
}
func getFloatListValues(name string, arr []*jason.Value) *FloatListType {
res := make([]float64, len(arr))
for i, val := range arr {
if r, err := val.Float64(); err == nil {
res[i] = r
}
}
expFloatListTypeCount.Add(1)
return NewFloatListType(name, res)
}
// IsByte checks the string string to determine if it is a Byte value.
func IsByte(m string) bool { return m == "b" }
// IsKiloByte checks the string string to determine if it is a KiloByte value.
func IsKiloByte(m string) bool { return m == "kb" }
// IsMegaByte checks the string string to determine if it is a MegaByte value.
func IsMegaByte(m string) bool { return m == "mb" }
func gcTypes(v treeReader, gcTypes []string) []string {
var result []string
seen := make(map[string]struct{})
for _, gcType := range v.GetStringSlice("gc_types") {
seen[gcType] = struct{}{}
result = append(result, gcType)
}
for _, value := range gcTypes {
if _, ok := seen[value]; !ok {
result = append(result, value)
}
}
return result
}
func memoryTypes(v treeReader, memTypes map[string]string) map[string]string {
result := make(map[string]string, len(memTypes))
for name, memoryType := range v.GetStringMapString("memory_bytes") {
result[name] = memoryType
}
return result
}