-
Notifications
You must be signed in to change notification settings - Fork 0
/
importer.go
87 lines (78 loc) · 1.67 KB
/
importer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
package quickdraw
import (
"database/sql"
"encoding/csv"
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
)
const (
importRoot = "INSERT INTO draws (id, drawdate, drawtime, picks, extra)\nVALUES\n"
bulkInsertCount = 1000
)
// ImportPicks imports picks from a CSV
func ImportPicks(db *sql.DB, csvFile string) error {
csvFileHandle, err := os.Open(fmt.Sprintf("./%s", csvFile))
if err != nil {
return err
}
r := csv.NewReader(csvFileHandle)
// disregard first line of field names
_, err = r.Read()
if err != nil {
return err
}
sqlStatement := importRoot
var count = 0
var totalInserts = 0
for {
record, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
return err
}
if record[0] == "" {
log.Println("no date found for draw " + record[1])
continue
}
if count == 0 {
sqlStatement = importRoot
} else {
sqlStatement = fmt.Sprintln(sqlStatement + ",")
}
_, err = strconv.Atoi(record[1])
if err != nil {
return err
}
_, err = strconv.Atoi(record[4])
if err != nil {
return err
}
splitPicks := strings.Split(record[3], " ")
joinedPicks := fmt.Sprintf("ARRAY [%s]", strings.Join(splitPicks, ","))
sqlStatement = fmt.Sprint(sqlStatement + "(" + record[1] + "," + "'" + record[0] +
"','" + record[2] + "'," + joinedPicks + "," + record[4] + ")")
if count == bulkInsertCount {
_, err = db.Exec(sqlStatement)
if err != nil {
log.Println(sqlStatement)
log.Println(err)
os.Exit(1)
}
totalInserts += count
log.Println("Bulk inserting", count, totalInserts)
count = 0
} else {
count++
}
}
_, err = db.Exec(sqlStatement)
if err != nil {
return err
}
return RefreshViews(db)
}