-
Notifications
You must be signed in to change notification settings - Fork 151
/
database_migration.go
133 lines (106 loc) · 4.13 KB
/
database_migration.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
package toolset
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/dustin/go-humanize"
flag "github.com/spf13/pflag"
"github.com/iotaledger/hive.go/kvstore"
"github.com/iotaledger/hornet/pkg/database"
"github.com/iotaledger/hornet/pkg/utils"
)
func databaseMigration(args []string) error {
fs := flag.NewFlagSet("", flag.ContinueOnError)
databasePathSourceFlag := fs.String(FlagToolDatabasePathSource, "", "the path to the source database")
databasePathTargetFlag := fs.String(FlagToolDatabasePathTarget, "", "the path to the target database")
databaseEngineTargetFlag := fs.String(FlagToolDatabaseEngineTarget, string(DefaultValueDatabaseEngine), "the engine of the target database (values: pebble, rocksdb)")
fs.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage of %s:\n", ToolDatabaseMigration)
fs.PrintDefaults()
println(fmt.Sprintf("\nexample: %s --%s %s --%s %s --%s %s",
ToolDatabaseMigration,
FlagToolDatabasePathSource,
DefaultValueMainnetDatabasePath,
FlagToolDatabasePathTarget,
"mainnetdb_new",
FlagToolDatabaseEngineTarget,
DefaultValueDatabaseEngine))
}
if err := parseFlagSet(fs, args); err != nil {
return err
}
if len(*databasePathSourceFlag) == 0 {
return fmt.Errorf("'%s' not specified", FlagToolDatabasePathSource)
}
if len(*databasePathTargetFlag) == 0 {
return fmt.Errorf("'%s' not specified", FlagToolDatabasePathTarget)
}
if len(*databaseEngineTargetFlag) == 0 {
return fmt.Errorf("'%s' not specified", FlagToolDatabaseEngineTarget)
}
sourcePath := *databasePathSourceFlag
if _, err := os.Stat(sourcePath); err != nil || os.IsNotExist(err) {
return fmt.Errorf("'%s' (%s) does not exist", FlagToolDatabasePathSource, sourcePath)
}
targetPath := *databasePathTargetFlag
if _, err := os.Stat(targetPath); err == nil || !os.IsNotExist(err) {
return fmt.Errorf("'%s' (%s) already exist", FlagToolDatabasePathTarget, targetPath)
}
targetEngine, err := database.DatabaseEngineFromStringAllowed(*databaseEngineTargetFlag, database.EnginePebble, database.EngineRocksDB)
if err != nil {
return err
}
storeSource, err := database.StoreWithDefaultSettings(sourcePath, false)
if err != nil {
return fmt.Errorf("source database initialization failed: %w", err)
}
defer func() { _ = storeSource.Close() }()
storeTarget, err := database.StoreWithDefaultSettings(targetPath, true, targetEngine)
if err != nil {
return fmt.Errorf("target database initialization failed: %w", err)
}
defer func() { _ = storeTarget.Close() }()
copyBytes := func(source []byte) []byte {
cpy := make([]byte, len(source))
copy(cpy, source)
return cpy
}
ts := time.Now()
lastStatusTime := time.Now()
sourcePathAbs, err := filepath.Abs(sourcePath)
if err != nil {
sourcePathAbs = sourcePath
}
targetPathAbs, err := filepath.Abs(targetPath)
if err != nil {
targetPathAbs = targetPath
}
fmt.Printf("Migrating database... (source: \"%s\", target: \"%s\")\n", sourcePathAbs, targetPathAbs)
var errDB error
if err := storeSource.Iterate(kvstore.EmptyPrefix, func(key []byte, value kvstore.Value) bool {
dstKey := copyBytes(key)
dstValue := copyBytes(value)
if errDB = storeTarget.Set(dstKey, dstValue); errDB != nil {
return false
}
if time.Since(lastStatusTime) >= printStatusInterval {
lastStatusTime = time.Now()
sourceSizeBytes, _ := utils.FolderSize(sourcePath)
targetSizeBytes, _ := utils.FolderSize(targetPath)
percentage, remaining := utils.EstimateRemainingTime(ts, targetSizeBytes, sourceSizeBytes)
fmt.Printf("Source database size: %s, target database size: %s, estimated percentage: %0.2f%%. %v elapsed, %v left...)\n", humanize.Bytes(uint64(sourceSizeBytes)), humanize.Bytes(uint64(targetSizeBytes)), percentage, time.Since(ts).Truncate(time.Second), remaining.Truncate(time.Second))
}
return true
}); err != nil {
return fmt.Errorf("source database iteration failed: %w", err)
}
if errDB != nil {
return fmt.Errorf("target database set failed: %w", err)
}
if err := storeTarget.Flush(); err != nil {
return fmt.Errorf("target database flush failed: %w", err)
}
fmt.Printf("Migration successful! took: %v\n", time.Since(ts).Truncate(time.Second))
return nil
}