-
Notifications
You must be signed in to change notification settings - Fork 7
/
add.go
111 lines (104 loc) · 2.85 KB
/
add.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package filesystem
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
"path"
"time"
"github.com/Cloud-Foundations/Dominator/lib/fsutil"
"github.com/Cloud-Foundations/Dominator/lib/hash"
"github.com/Cloud-Foundations/Dominator/lib/objectcache"
)
const (
buflen = 65536
)
// This must be called with the lock held. The object must not already exist.
func (objSrv *ObjectServer) add(object *objectType) {
objSrv.objects[object.hash] = object
objSrv.addUnreferenced(object)
objSrv.lastMutationTime = time.Now()
objSrv.totalBytes += object.size
}
func (objSrv *ObjectServer) addObject(reader io.Reader, length uint64,
expectedHash *hash.Hash) (hash.Hash, bool, error) {
hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
if err != nil {
return hashVal, false, err
}
filename := path.Join(objSrv.BaseDirectory,
objectcache.HashToFilename(hashVal))
// Check for existing object and collision.
if isNew, err := objSrv.addOrCompare(hashVal, data, filename); err != nil {
return hashVal, false, err
} else {
object := &objectType{hash: hashVal, size: uint64(len(data))}
objSrv.rwLock.Lock()
if _, ok := objSrv.objects[object.hash]; !ok {
objSrv.add(object)
}
objSrv.rwLock.Unlock()
if objSrv.addCallback != nil {
objSrv.addCallback(hashVal, uint64(len(data)), isNew)
}
return hashVal, isNew, nil
}
}
func (objSrv *ObjectServer) addOrCompare(hashVal hash.Hash, data []byte,
filename string) (bool, error) {
fi, err := os.Lstat(filename)
if err == nil {
if !fi.Mode().IsRegular() {
return false, errors.New("existing non-file: " + filename)
}
if err := collisionCheck(data, filename, fi.Size()); err != nil {
return false, errors.New("collision detected: " + err.Error())
}
// No collision and no error: it's the same object. Go home early.
return false, nil
}
if objSrv.gc != nil { // Have external garbage collector: trigger it inline.
objSrv.garbageCollector()
}
err = os.MkdirAll(path.Dir(filename), fsutil.PrivateDirPerms)
if err != nil {
return false, err
}
err = fsutil.CopyToFile(filename, fsutil.PrivateFilePerms,
bytes.NewReader(data), uint64(len(data)))
if err != nil {
return false, err
}
return true, nil
}
func collisionCheck(data []byte, filename string, size int64) error {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
if int64(len(data)) != size {
return fmt.Errorf("length mismatch. Data=%d, existing object=%d",
len(data), size)
}
reader := bufio.NewReader(file)
buffer := make([]byte, 0, buflen)
for len(data) > 0 {
numToRead := len(data)
if numToRead > cap(buffer) {
numToRead = cap(buffer)
}
buf := buffer[:numToRead]
nread, err := reader.Read(buf)
if err != nil {
return err
}
if bytes.Compare(data[:nread], buf[:nread]) != 0 {
return errors.New("content mismatch")
}
data = data[nread:]
}
return nil
}