-
Notifications
You must be signed in to change notification settings - Fork 11
/
generate.go
255 lines (240 loc) · 6.82 KB
/
generate.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
// Copyright 2019 DxChain, All rights reserved.
// Use of this source code is governed by an Apache
// License 2.0 that can be found in the LICENSE file.
package filesystem
import (
"fmt"
"math/rand"
"os"
"path/filepath"
"runtime"
"sync"
"testing"
"time"
"github.com/DxChainNetwork/godx/common"
"github.com/DxChainNetwork/godx/crypto"
"github.com/DxChainNetwork/godx/storage"
"github.com/DxChainNetwork/godx/storage/storageclient/erasurecode"
)
type (
// dirTree is the structure for test to record the file structure of client's files
dirTree struct {
root *dirTreeNode
// random params when generating a random file
// goDeepRate is the possibility of when creating a file, it goes deep into
// a subdirectory of the current directory.
// goWideRate is the possibility of when going deep, instead of using an existing
// directory, it creates a new one
goDeepRate float32
goWideRate float32
// maxDepth is the max depth of a file
maxDepth int
}
// dirTreeNode is the node of dirTree.
dirTreeNode struct {
subDirs map[string]*dirTreeNode
files map[string]struct{}
dxPath storage.DxPath
}
)
// createRandomFiles create random files of numFiles. The file structure is defined randomly by
// goDeepRate, goWideRate, and maxDepth. More info about the params please read comment at dirTree
// missRate is the rate that a sector data is missing. Aimed for test the API of uploaded files
func (fs *fileSystem) createRandomFiles(numFiles int, goDeepRate, goWideRate float32, maxDepth int, missRate float32) error {
dt := newDirTree(goDeepRate, goWideRate, maxDepth)
ck, err := crypto.GenerateCipherKey(crypto.GCMCipherCode)
if err != nil {
return err
}
var wg sync.WaitGroup
wg.Add(numFiles)
errChan := make(chan error)
for i := 0; i < numFiles; i++ {
path, err := dt.randomPath()
if err != nil {
return err
}
// The default file size here is 11 segments
go func() {
defer wg.Done()
fileSize := uint64(1 << 22 * 10 * 10)
dxfile, err := fs.fileSet.NewRandomDxFile(path, 10, 30, erasurecode.ECTypeStandard, ck, fileSize, missRate)
if err != nil {
errChan <- err
return
}
err = dxfile.Close()
parentPath, err := path.Parent()
err = fs.InitAndUpdateDirMetadata(parentPath)
if err != nil {
errChan <- err
return
}
return
}()
}
wait := make(chan struct{})
go func() {
wg.Wait()
close(wait)
}()
select {
case err := <-errChan:
return err
case <-time.After(1 * time.Second * time.Duration(maxDepth) * time.Duration(numFiles)):
return fmt.Errorf("createRandomFiles time out")
case <-wait:
}
// wait for all the updates to finish
if err = fs.waitForUpdatesComplete(1 * time.Second * time.Duration(maxDepth) * time.Duration(numFiles)); err != nil {
return err
}
return nil
}
// tempDir removes and creates the folder named dxfile under the temp directory.
func tempDir(dirs ...string) storage.SysPath {
path := filepath.Join(os.TempDir(), "filesystem", filepath.Join(dirs...))
if err := os.RemoveAll(path); err != nil {
panic(fmt.Sprintf("cannot remove all files under %v: %v", path, err))
}
if err := os.MkdirAll(path, 0777); err != nil {
panic(fmt.Sprintf("cannot create directory %v: %v", path, err))
}
return storage.SysPath(path)
}
// newEmptyTestFileSystem creates an empty file system used for testing
func newEmptyTestFileSystem(t *testing.T, extraNaming string, contractor contractManager, disrupter disrupter) *fileSystem {
var rootDir storage.SysPath
if len(extraNaming) == 0 {
rootDir = tempDir(t.Name())
} else {
rootDir = tempDir(t.Name(), extraNaming)
}
fs := newFileSystem(string(rootDir), contractor, disrupter)
err := fs.Start()
if err != nil {
t.Fatal(err)
}
return fs
}
//newDirTree creates a new dirTree with the params provides
func newDirTree(goDeepRate, goWideRate float32, maxDepth int) dirTree {
rand.Seed(time.Now().UnixNano())
return dirTree{
&dirTreeNode{
subDirs: make(map[string]*dirTreeNode),
files: make(map[string]struct{}),
dxPath: storage.RootDxPath(),
},
goDeepRate, goWideRate, maxDepth,
}
}
// randomPath creates a random path under the based on dt settings
func (dt dirTree) randomPath() (storage.DxPath, error) {
curDir := dt.root
for i := 0; i != dt.maxDepth+1; i++ {
num := rand.Float32()
// Do not go deeper. create a new file
if num > dt.goDeepRate || i == dt.maxDepth {
var fileName string
for {
fileName = randomName()
if _, exist := curDir.subDirs[fileName]; exist {
continue
}
if _, exist := curDir.files[fileName]; exist {
continue
}
break
}
curDir.files[fileName] = struct{}{}
return curDir.dxPath.Join(fileName)
}
// go deeper. toll the dice again
num = rand.Float32()
if num < dt.goWideRate || len(curDir.subDirs) == 0 {
// create a new directory
var dirName string
for {
dirName = randomName()
if _, exist := curDir.subDirs[dirName]; exist {
continue
}
if _, exist := curDir.files[dirName]; exist {
continue
}
break
}
subDirPath, err := curDir.dxPath.Join(dirName)
if err != nil {
return storage.DxPath{}, err
}
curDir.subDirs[dirName] = &dirTreeNode{
subDirs: make(map[string]*dirTreeNode),
files: make(map[string]struct{}),
dxPath: subDirPath,
}
curDir = curDir.subDirs[dirName]
continue
}
// Go deeper, and randomly use a current existing directory
for _, curDir = range curDir.subDirs {
break
}
}
return storage.DxPath{}, fmt.Errorf("this should be never reached")
}
// waitForUpdatesComplete is the helper function that wait for update execution
func (fs *fileSystem) waitForUpdatesComplete(timeout time.Duration) error {
c := make(chan struct{})
// Wait until update complete
go func() {
defer close(c)
for {
<-time.After(50 * time.Millisecond)
fs.lock.Lock()
emptyUpdate := len(fs.unfinishedUpdates) == 0
fs.lock.Unlock()
if emptyUpdate {
// There might be case the child directory completed update while
// the parent update is not in unfinishedUpdates
<-time.After(50 * time.Millisecond)
fs.lock.Lock()
emptyUpdate = len(fs.unfinishedUpdates) == 0
fs.lock.Unlock()
if emptyUpdate {
return
}
continue
}
}
}()
select {
case <-time.After(timeout):
return fmt.Errorf("after %s, update still not completed", timeout)
case <-c:
}
return nil
}
// randomName create a random name for the dirTree. name is a length 16 hex string
func randomName() string {
b := make([]byte, 16)
rand.Read(b)
return common.Bytes2Hex(b)
}
// userHomeDir returns the home directory of user
func userHomeDir() string {
if runtime.GOOS == "windows" {
home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
if home == "" {
home = os.Getenv("USERPROFILE")
}
return home
} else if runtime.GOOS == "linux" {
home := os.Getenv("XDG_CONFIG_HOME")
if home != "" {
return home
}
}
return os.Getenv("HOME")
}