generated from layer5io/layer5-repo-template
-
Notifications
You must be signed in to change notification settings - Fork 73
/
csv.go
103 lines (91 loc) · 2.6 KB
/
csv.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
package csv
import (
"context"
"encoding/csv"
"io"
"os"
"strings"
"github.com/layer5io/meshkit/utils"
)
type CSV[E any] struct {
Context context.Context
cancel context.CancelFunc
reader *csv.Reader
filePath string
lineForColNo int
// Stores the mapping for coumn name to golang equivalent attribute name.
// It is optional and default mapping is the lower case representation with spaces replaced with "_"
// eg: ColumnnName: Descritption, equivalent golang attribute to which it will be mapped during unmarshal "description".
columnToNameMapping map[string]string
predicateFunc func(columns []string, currentRow []string) bool
}
func NewCSVParser[E any](filePath string, lineForColNo int, colToNameMapping map[string]string, predicateFunc func(columns []string, currentRow []string) bool) (*CSV[E], error) {
reader, err := os.Open(filePath)
if err != nil {
return nil, utils.ErrReadFile(err, filePath)
}
ctx, cancel := context.WithCancel(context.Background())
return &CSV[E]{
Context: ctx,
cancel: cancel,
reader: csv.NewReader(reader),
filePath: filePath,
columnToNameMapping: colToNameMapping,
lineForColNo: lineForColNo,
predicateFunc: predicateFunc,
}, nil
}
// "lineForColNo" line number where the columns are defined in the csv
func (c *CSV[E]) ExtractCols(lineForColNo int) ([]string, error) {
data := []string{}
var err error
for i := 0; i <= lineForColNo; i++ {
data, err = c.reader.Read()
if err != nil {
return nil, utils.ErrReadFile(err, c.filePath)
}
}
return data, nil
}
func (c *CSV[E]) Parse(ch chan E, errorChan chan error) error {
defer func() {
c.cancel()
}()
columnNames, err := c.ExtractCols(c.lineForColNo)
size := len(columnNames)
if err != nil {
return utils.ErrReadFile(err, c.filePath)
}
for {
data := make(map[string]interface{})
values, err := c.reader.Read()
if err == io.EOF {
break
}
if err != nil {
return utils.ErrReadFile(err, c.filePath)
}
if c.predicateFunc != nil && c.predicateFunc(columnNames, values) {
for index, value := range values {
var attribute string
if index < size {
attribute = strings.ReplaceAll(strings.ToLower(columnNames[index]), " ", "_")
if c.columnToNameMapping != nil {
key, ok := c.columnToNameMapping[columnNames[index]]
if ok {
attribute = key
}
}
data[attribute] = value
}
}
parsedData, err := utils.MarshalAndUnmarshal[map[string]interface{}, E](data)
if err != nil {
errorChan <- err
continue
}
ch <- parsedData
}
}
return nil
}