forked from wheelcomplex/s3weed
-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.go
136 lines (122 loc) · 3.38 KB
/
utils.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
/*
Package weedutils defines some helper functions for kv used in s3impl/weedS3
Copyright 2013 Tamás Gulácsi
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package weedutils
import (
"bytes"
"encoding/gob"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/cznic/kv"
)
var kvOptions = new(kv.Options)
// OpenAllDb opens all files with the given suffix in the given dir
func OpenAllDb(dir, suffix string) (<-chan *kv.DB, <-chan error) {
dest := make(chan *kv.DB)
errch := make(chan error, 1)
var (
db *kv.DB
err error
fn string
)
go func() {
if err = MapDirItems(dir,
func(fi os.FileInfo) bool {
return fi.Mode().IsRegular() && (suffix == "" || strings.HasSuffix(fi.Name(), suffix))
},
func(fi os.FileInfo) error {
fn = filepath.Join(dir, fi.Name())
if db, err = kv.Open(fn, kvOptions); err != nil {
return err
}
dest <- db
return nil
}); err != nil {
errch <- err
}
close(dest)
close(errch)
}()
return dest, errch
}
// MapDirItems calls todo for every item in dir for which check returns true
func MapDirItems(dir string, check func(os.FileInfo) bool, todo func(os.FileInfo) error) (err error) {
var (
dh *os.File
fi os.FileInfo
fis []os.FileInfo
)
if dh, err = os.Open(dir); err != nil {
return
}
defer dh.Close()
for {
if fis, err = dh.Readdir(1000); err != nil {
if err == io.EOF {
break
}
return
}
for _, fi = range fis {
if check(fi) {
if err = todo(fi); err != nil {
return
}
}
}
}
return nil
}
// ReadDirItems sends the fileinfo of every item in dir for which check returns true
func ReadDirItems(dir string, check func(os.FileInfo) bool, dest chan<- os.FileInfo) (err error) {
defer close(dest)
return MapDirItems(dir, check, func(fi os.FileInfo) error {
dest <- fi
return nil
})
}
// ReadDirNames sends the full filename of every item in dir for which check returns true
func ReadDirNames(dir string, check func(os.FileInfo) bool, dest chan<- string) (err error) {
defer close(dest)
return MapDirItems(dir, check, func(fi os.FileInfo) error {
dest <- filepath.Join(dir, fi.Name())
return nil
})
}
// ValInfo contains the required info about a stored file
type ValInfo struct {
Filename string `json:"filename"`
ContentType string `json:"content-type"`
//Fid is the file-id
Fid string `json:"fid"`
Created time.Time `json:"created"`
Size int64 `json:"size"`
MD5 []byte `json:"md5"`
}
// Decode decodes into the struct from bytes
func (v *ValInfo) Decode(val []byte) error {
return gob.NewDecoder(bytes.NewReader(val)).Decode(v)
}
// Encode encodes the ValInfo's values into dst and returns the resulting slice.
// dst can be nil
func (v ValInfo) Encode(dst []byte) ([]byte, error) {
if dst == nil {
dst = make([]byte, 0, len(v.Filename)+len(v.ContentType)+len(v.Fid)+16+8+8+8)
}
buf := bytes.NewBuffer(dst)
err := gob.NewEncoder(buf).Encode(v)
return buf.Bytes(), err
}