This repository has been archived by the owner on Dec 14, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 279
/
redis_map_migrate.go
129 lines (104 loc) · 2.9 KB
/
redis_map_migrate.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.
package storage
import (
"strings"
"github.com/TheThingsNetwork/go-utils/log"
"github.com/TheThingsNetwork/ttn/utils/errors"
redis "gopkg.in/redis.v5"
)
// VersionKey indicates the data (schema) version
const VersionKey = "_version"
type hasDBVersion interface {
DBVersion() string
}
// MigrateFunction migrates data from its old version to the latest
type MigrateFunction func(client *redis.Client, key string, input map[string]string) (nextVersion string, output map[string]string, err error)
// AddMigration adds a data migration for a version
func (s *RedisMapStore) AddMigration(version string, migrate MigrateFunction) {
s.migrations[version] = migrate
}
// Migrate all documents matching the selector
func (s *RedisMapStore) Migrate(selector string) error {
if selector == "" {
selector = "*"
}
if !strings.HasPrefix(selector, s.prefix) {
selector = s.prefix + selector
}
var cursor uint64
for {
keys, next, err := s.client.Scan(cursor, selector, 0).Result()
if err != nil {
return err
}
for _, key := range keys {
// Get migrates the item
_, err := s.Get(key)
// NotFound if item was deleted since Scan started
if errors.GetErrType(err) == errors.NotFound {
continue
}
// TxFailedError is returned if the item was changed by a concurrent process.
// If it was a parallel migration, we don't have to do anything
// Otherwise, this item will be migrated with the next Get
if err == redis.TxFailedErr {
continue
}
// Any other error should be returned
if err != nil {
return err
}
}
cursor = next
if cursor == 0 {
break
}
}
return nil
}
func (s *RedisMapStore) migrate(key string, obj map[string]string) (map[string]string, error) {
var err error
defer func() {
if err != nil {
log.Get().WithField("Key", key).WithError(err).Warn("Data migration failed")
}
}()
version, _ := obj[VersionKey]
migration, ok := s.migrations[version]
if !ok {
return obj, nil
}
var oldFields []string
for k := range obj {
oldFields = append(oldFields, k)
}
err = s.client.Watch(func(tx *redis.Tx) error { // Make sure objects are not changed while we're migrating them
// As long as there are migrations to do
for ok {
version, obj, err = migration(s.client, key, obj)
if err != nil {
return err
}
obj[VersionKey] = version
migration, ok = s.migrations[version]
}
// Collect deleted fields
var deletedFields []string
for _, k := range oldFields {
if _, ok := obj[k]; !ok {
deletedFields = append(deletedFields, k)
}
}
// Commit the new version
_, err := tx.Pipelined(func(pipe *redis.Pipeline) error {
pipe.HMSet(key, obj)
if len(deletedFields) > 0 {
pipe.HDel(key, deletedFields...)
}
return nil
})
return err
}, key)
return obj, err
}