forked from xitongsys/parquet-go
/
json.go
142 lines (120 loc) · 3.68 KB
/
json.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
package schema
import (
"encoding/json"
"errors"
"github.com/xitongsys/parquet-go/common"
"github.com/xitongsys/parquet-go/parquet"
)
type JSONSchemaItemType struct {
Tag string
Fields []*JSONSchemaItemType
}
func NewJSONSchemaItem() *JSONSchemaItemType {
return new(JSONSchemaItemType)
}
func NewSchemaHandlerFromJSON(str string) (sh *SchemaHandler, err error) {
defer func() {
if r := recover(); r != nil {
switch x := r.(type) {
case string:
err = errors.New(x)
case error:
err = x
default:
err = errors.New("unknown error")
}
}
}()
schema := NewJSONSchemaItem()
json.Unmarshal([]byte(str), schema)
stack := make([]*JSONSchemaItemType, 0)
stack = append(stack, schema)
schemaElements := make([]*parquet.SchemaElement, 0)
infos := make([]*common.Tag, 0)
for len(stack) > 0 {
ln := len(stack)
item := stack[ln-1]
stack = stack[:ln-1]
info := common.StringToTag(item.Tag)
var newInfo *common.Tag
if info.Type == "" { //struct
schema := parquet.NewSchemaElement()
schema.Name = info.InName
rt := info.RepetitionType
schema.RepetitionType = &rt
numField := int32(len(item.Fields))
schema.NumChildren = &numField
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
infos = append(infos, newInfo)
for i := int(numField - 1); i >= 0; i-- {
newItem := item.Fields[i]
stack = append(stack, newItem)
}
} else if info.Type == "LIST" { //list
schema := parquet.NewSchemaElement()
schema.Name = info.InName
rt1 := info.RepetitionType
schema.RepetitionType = &rt1
var numField1 int32 = 1
schema.NumChildren = &numField1
ct1 := parquet.ConvertedType_LIST
schema.ConvertedType = &ct1
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
infos = append(infos, newInfo)
schema = parquet.NewSchemaElement()
schema.Name = "List"
rt2 := parquet.FieldRepetitionType_REPEATED
schema.RepetitionType = &rt2
var numField2 int32 = 1
schema.NumChildren = &numField2
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
newInfo.InName, newInfo.ExName = "List", "list"
infos = append(infos, newInfo)
stack = append(stack, item.Fields[0])
} else if info.Type == "MAP" { //map
schema := parquet.NewSchemaElement()
schema.Name = info.InName
rt1 := info.RepetitionType
schema.RepetitionType = &rt1
var numField1 int32 = 1
schema.NumChildren = &numField1
ct1 := parquet.ConvertedType_MAP
schema.ConvertedType = &ct1
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
infos = append(infos, newInfo)
schema = parquet.NewSchemaElement()
schema.Name = "Key_value"
rt2 := parquet.FieldRepetitionType_REPEATED
schema.RepetitionType = &rt2
ct2 := parquet.ConvertedType_MAP_KEY_VALUE
schema.ConvertedType = &ct2
var numField2 int32 = 2
schema.NumChildren = &numField2
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
newInfo.InName, newInfo.ExName = "Key_value", "key_value"
infos = append(infos, newInfo)
stack = append(stack, item.Fields[1]) //put value
stack = append(stack, item.Fields[0]) //put key
} else { //normal variable
schema := common.NewSchemaElementFromTagMap(info)
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(info, newInfo)
infos = append(infos, newInfo)
}
}
res := NewSchemaHandlerFromSchemaList(schemaElements)
res.Infos = infos
res.CreateInExMap()
return res, nil
}