-
-
Notifications
You must be signed in to change notification settings - Fork 16
/
transforms.go
176 lines (143 loc) · 4.18 KB
/
transforms.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
package iop
import (
"bufio"
"embed"
"regexp"
"strings"
"unicode"
"github.com/flarco/g"
"github.com/spf13/cast"
)
//go:embed templates/*
var templatesFolder embed.FS
var Transforms = map[Transform]TransformFunc{}
type Transform string
const (
TransformDecodeLatin1 Transform = "decode_latin1"
TransformDecodeLatin5 Transform = "decode_latin5"
TransformDecodeLatin9 Transform = "decode_latin9"
TransformDecodeUtf8 Transform = "decode_utf8"
TransformDecodeUtf8Bom Transform = "decode_utf8_bom"
TransformDecodeUtf16 Transform = "decode_utf16"
TransformDecodeWindows1250 Transform = "decode_windows1250"
TransformDecodeWindows1252 Transform = "decode_windows1252"
TransformDuckdbListToText Transform = "duckdb_list_to_text"
TransformHashMd5 Transform = "hash_md5"
TransformHashSha256 Transform = "hash_sha256"
TransformHashSha512 Transform = "hash_sha512"
TransformParseBit Transform = "parse_bit"
TransformParseFix Transform = "parse_fix"
TransformParseUuid Transform = "parse_uuid"
TransformReplace0x00 Transform = "replace_0x00"
TransformReplaceAccents Transform = "replace_accents"
TransformReplaceNonPrintable Transform = "replace_non_printable"
TransformTrimSpace Transform = "trim_space"
)
// https://stackoverflow.com/a/46637343/2295355
// https://web.itu.edu.tr/sgunduz/courses/mikroisl/ascii.html
func ReplaceNonPrintable(val string) string {
var newVal strings.Builder
for _, r := range val {
if r == 0 { // NULL
continue // remove those
}
if r < 9 || (r > 13 && r < 32) {
newVal.WriteRune(' ') // replace with space
continue
}
if r < 127 {
// add these
newVal.WriteRune(r)
continue
}
switch r {
case 127: //
continue // remove those
case 160: // NO-BREAK SPACE
newVal.WriteRune(' ') // replace with space
continue
}
if !unicode.IsGraphic(r) {
continue
}
// add any other
newVal.WriteRune(r)
}
return newVal.String()
}
var fixDelimiter string
var fixMapping = map[int]string{}
// ParseFIX converts a FIX message into a json format
func ParseFIX(message string) (fixMap map[string]any, err error) {
delimiters := []string{"|", " ", ""}
fixInit := func() {
maxVals := 0
for _, deli := range delimiters {
if cnt := len(strings.Split(message, deli)); cnt > maxVals {
maxVals = cnt
}
}
for _, deli := range delimiters {
if cnt := len(strings.Split(message, deli)); cnt == maxVals {
fixDelimiter = deli
}
}
// from https://www.wireshark.org/docs/dfref/f/fix.html
fixMappingBytes, err := templatesFolder.Open("templates/fix_mapping.tsv")
g.LogFatal(err)
fileMappingTSV := CSV{Reader: bufio.NewReader(fixMappingBytes)}
fileMappingTSV.Delimiter = '\t'
fileMappingTSV.NoDebug = true
data, err := fileMappingTSV.Read()
g.LogFatal(err)
for _, rec := range data.Records() {
name := strings.TrimPrefix(cast.ToString(rec["field_name"]), "fix.")
description := cast.ToString(rec["description"])
codeParts := strings.Split(description, "(")
if len(codeParts) < 2 {
continue
}
codeB := []byte{}
for _, r := range codeParts[1] {
if unicode.IsDigit(r) {
codeB = append(codeB, byte(r))
}
}
code := cast.ToInt(string(codeB))
rec["code"] = code
// convert to snake
matchAllCap := regexp.MustCompile("([a-z0-9])([A-Z])")
name = matchAllCap.ReplaceAllString(name, "${1}_${2}")
fixMapping[code] = strings.ToLower(CleanName(name))
}
// g.P(fixMapping)
}
// auto detect delimiter
if fixDelimiter == "" {
fixInit()
}
message = strings.TrimSpace(message)
parts := strings.Split(message, fixDelimiter)
if len(parts) == 1 {
fixInit()
parts = strings.Split(message, fixDelimiter)
}
fixMap = map[string]any{}
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
tagValue := strings.Split(part, "=")
if len(tagValue) != 2 {
return nil, g.Error("did not get tag/value after split: %#v", part)
}
tagInt := cast.ToInt(tagValue[0])
if tag, ok := fixMapping[tagInt]; ok {
fixMap[tag] = tagValue[1]
} else {
fixMap[tagValue[0]] = tagValue[1]
}
}
return
}