-
Notifications
You must be signed in to change notification settings - Fork 1
/
dbjob.go
171 lines (137 loc) · 3.18 KB
/
dbjob.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
package main
import (
"io"
"log"
"github.com/mxk/go-sqlite/sqlite3"
)
const batchSize int = 100
type FileDB struct {
db *sqlite3.Conn
insert *sqlite3.Stmt
batchCount int
get *sqlite3.Stmt
getArgs sqlite3.NamedArgs
//row sqlite3.RowMap
}
func NewFileDB(path string) (fdb *FileDB, err error) {
fdb = &FileDB{}
fdb.db, err = sqlite3.Open(path)
if err != nil {
return nil, err
}
err = fdb.db.Exec("CREATE TABLE IF NOT EXISTS FILES (PATH varchar not null, SIZE integer not null, MOD_TIME integer not null, MODE integer not null, CHKSUM integer not null, primary key(PATH));")
if err != nil {
fdb.Close()
return nil, err
}
//err = fdb.db.Exec("CREATE INDEX IF NOT EXISTS BYSUM on FILES (CHKSUM, SIZE);")
//if err != nil {
// fdb.Close()
// return nil, err
//}
fdb.insert, err = fdb.db.Prepare("insert or replace into FILES (path, size, mod_time, mode, chksum) values ($path, $size, $mod_time, $mode, $chksum);")
if err != nil {
fdb.Close()
return nil, err
}
fdb.get, err = fdb.db.Prepare("select size, mod_time, mode, chksum from FILES where path = $path;")
if err != nil {
fdb.Close()
return nil, err
}
fdb.getArgs = make(sqlite3.NamedArgs)
err = fdb.db.Begin()
if err != nil {
fdb.Close()
return nil, err
}
return
}
func (fdb *FileDB) Close() {
if fdb.db != nil {
fdb.db.Commit()
if fdb.insert != nil {
fdb.insert.Close()
}
if fdb.get != nil {
fdb.get.Close()
}
fdb.db.Close()
}
}
func (fdb *FileDB) insertOrReplace(f *fileJob) {
args := make(sqlite3.NamedArgs)
args["$path"] = f.Fpath
args["$size"] = f.Info.Size()
args["$mod_time"] = f.Info.ModTime().UnixNano()
args["$mode"] = int64(f.Info.Mode())
args["$chksum"] = int64(f.Chksum)
defer fdb.insert.Reset()
err := fdb.insert.Exec(args)
if err != nil {
log.Fatal(err)
}
if fdb.batchCount += 1; fdb.batchCount >= batchSize {
err = fdb.db.Commit()
if err != nil {
log.Fatal(err)
}
fdb.db.Begin()
if err != nil {
log.Fatal(err)
}
fdb.batchCount = 0
}
f.Err = NewError(code_NEW_SUM, f, "updating checksum")
return
}
func (fdb *FileDB) getForPath(f *fileJob) (row sqlite3.RowMap, err error) {
defer fdb.get.Reset()
fdb.getArgs["$path"] = f.Fpath
err = fdb.get.Query(fdb.getArgs)
switch {
case err == io.EOF:
err = nil
case err != nil:
default:
row = make(sqlite3.RowMap)
err = fdb.get.Scan(row)
}
return
}
func (fdb *FileDB) CheckInDB(f *fileJob) {
if f.Err != nil {
return
}
row, err := fdb.getForPath(f)
switch {
case err != nil:
// Problem with db. Fail.
log.Fatal(err)
case row == nil:
// No entry. Make one.
fdb.insertOrReplace(f)
default:
// Got a value. Compare it against calculated values.
// If file was deliberately changed, replace the row with new cheksum
switch {
case !ignoreMTime && row["MOD_TIME"] != f.Info.ModTime().UnixNano():
fdb.insertOrReplace(f)
case row["CHKSUM"] != int64(f.Chksum):
f.Err = NewError(code_BAD_SUM, f, "checksum did not match recorded value")
}
}
return
}
func dbChecker(in chan *fileJob, out chan *fileJob, fdb *FileDB) {
defer func() { out <- nil }()
for f := range in {
if f == nil {
return
}
if f.Err == nil {
fdb.CheckInDB(f)
}
out <- f
}
}