/
encoding_csv_read.go
103 lines (93 loc) · 2.1 KB
/
encoding_csv_read.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
package elem
import (
"encoding/csv"
"io"
"strings"
"github.com/Bitspark/go-funk"
"github.com/Bitspark/slang/pkg/core"
"github.com/google/uuid"
)
var encodingCSVReadId = uuid.MustParse("77d60459-f8b5-4f4b-b293-740164c49a82")
var encodingCSVReadCfg = &builtinConfig{
safe: true,
blueprint: core.Blueprint{
Id: encodingCSVReadId,
Meta: core.BlueprintMetaDef{
Name: "read CSV",
ShortDescription: "reads a CSV file and emits a stream of lines, separated into columns",
Icon: "file-csv",
Tags: []string{"csv", "encoding"},
DocURL: "https://bitspark.de/slang/docs/operator/read-csv",
},
ServiceDefs: map[string]*core.ServiceDef{
core.MAIN_SERVICE: {
In: core.TypeDef{
Type: "string",
},
Out: core.TypeDef{
Type: "stream",
Stream: &core.TypeDef{
Type: "map",
Map: map[string]*core.TypeDef{
"col_{columns}": {
Type: "string",
},
},
},
},
},
},
DelegateDefs: map[string]*core.DelegateDef{},
PropertyDefs: map[string]*core.TypeDef{
"delimiter": {
Type: "string",
},
"columns": {
Type: "stream",
Stream: &core.TypeDef{
Type: "string",
},
},
},
},
opFunc: func(op *core.Operator) {
in := op.Main().In()
out := op.Main().Out()
for !op.CheckStop() {
csvText, marker := in.PullString()
if marker != nil {
out.Push(marker)
continue
}
outStream := out.Stream()
out.PushBOS()
var mapping []string
colNames := op.Property("columns").([]interface{})
mapSize := outStream.MapLength()
r := csv.NewReader(strings.NewReader(csvText))
r.Comma = rune(op.Property("delimiter").(string)[0])
for {
rec, err := r.Read()
if err == io.EOF {
break
}
if len(rec) < mapSize {
break
}
if mapping == nil {
mapping = rec
} else {
for i, col := range mapping {
if funk.Contains(colNames, col) {
outStream.Map("col_" + col).Push(rec[i])
}
}
}
}
out.PushEOS()
}
},
opConnFunc: func(op *core.Operator, dst, src *core.Port) error {
return nil
},
}