-
-
Notifications
You must be signed in to change notification settings - Fork 282
/
storage.go
325 lines (292 loc) · 11.1 KB
/
storage.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
// Copyright 2015 Matthew Holt
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package certmagic
import (
"context"
"path"
"regexp"
"strings"
"sync"
"time"
"go.uber.org/zap"
)
// Storage is a type that implements a key-value store with
// basic file system (folder path) semantics. Keys use the
// forward slash '/' to separate path components and have no
// leading or trailing slashes.
//
// A "prefix" of a key is defined on a component basis,
// e.g. "a" is a prefix of "a/b" but not "ab/c".
//
// A "file" is a key with a value associated with it.
//
// A "directory" is a key with no value, but which may be
// the prefix of other keys.
//
// Keys passed into Load and Store always have "file" semantics,
// whereas "directories" are only implicit by leading up to the
// file.
//
// The Load, Delete, List, and Stat methods should return
// fs.ErrNotExist if the key does not exist.
//
// Processes running in a cluster should use the same Storage
// value (with the same configuration) in order to share
// certificates and other TLS resources with the cluster.
//
// Implementations of Storage MUST be safe for concurrent use
// and honor context cancellations. Methods should block until
// their operation is complete; that is, Load() should always
// return the value from the last call to Store() for a given
// key, and concurrent calls to Store() should not corrupt a
// file.
//
// For simplicity, this is not a streaming API and is not
// suitable for very large files.
type Storage interface {
// Locker enables the storage backend to synchronize
// operational units of work.
//
// The use of Locker is NOT employed around every
// Storage method call (Store, Load, etc), as these
// should already be thread-safe. Locker is used for
// high-level jobs or transactions that need
// synchronization across a cluster; it's a simple
// distributed lock. For example, CertMagic uses the
// Locker interface to coordinate the obtaining of
// certificates.
Locker
// Store puts value at key. It creates the key if it does
// not exist and overwrites any existing value at this key.
Store(ctx context.Context, key string, value []byte) error
// Load retrieves the value at key.
Load(ctx context.Context, key string) ([]byte, error)
// Delete deletes the named key. If the name is a
// directory (i.e. prefix of other keys), all keys
// prefixed by this key should be deleted. An error
// should be returned only if the key still exists
// when the method returns.
Delete(ctx context.Context, key string) error
// Exists returns true if the key exists either as
// a directory (prefix to other keys) or a file,
// and there was no error checking.
Exists(ctx context.Context, key string) bool
// List returns all keys in the given path.
//
// If recursive is true, non-terminal keys
// will be enumerated (i.e. "directories"
// should be walked); otherwise, only keys
// prefixed exactly by prefix will be listed.
List(ctx context.Context, path string, recursive bool) ([]string, error)
// Stat returns information about key.
Stat(ctx context.Context, key string) (KeyInfo, error)
}
// Locker facilitates synchronization across machines and networks.
// It essentially provides a distributed named-mutex service so
// that multiple consumers can coordinate tasks and share resources.
//
// If possible, a Locker should implement a coordinated distributed
// locking mechanism by generating fencing tokens (see
// https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html).
// This typically requires a central server or consensus algorithm
// However, if that is not feasible, Lockers may implement an
// alternative mechanism that uses timeouts to detect node or network
// failures and avoid deadlocks. For example, the default FileStorage
// writes a timestamp to the lock file every few seconds, and if another
// node acquiring the lock sees that timestamp is too old, it may
// assume the lock is stale.
//
// As not all Locker implementations use fencing tokens, code relying
// upon Locker must be tolerant of some mis-synchronizations but can
// expect them to be rare.
//
// This interface should only be used for coordinating expensive
// operations across nodes in a cluster; not for internal, extremely
// short-lived, or high-contention locks.
type Locker interface {
// Lock acquires the lock for name, blocking until the lock
// can be obtained or an error is returned. Only one lock
// for the given name can exist at a time. A call to Lock for
// a name which already exists blocks until the named lock
// is released or becomes stale.
//
// If the named lock represents an idempotent operation, callers
// should awlays check to make sure the work still needs to be
// completed after acquiring the lock. You never know if another
// process already completed the task while you were waiting to
// acquire it.
//
// Implementations should honor context cancellation.
Lock(ctx context.Context, name string) error
// Unlock releases named lock. This method must ONLY be called
// after a successful call to Lock, and only after the critical
// section is finished, even if it errored or timed out. Unlock
// cleans up any resources allocated during Lock. Unlock should
// only return an error if the lock was unable to be released.
Unlock(ctx context.Context, name string) error
}
// KeyInfo holds information about a key in storage.
// Key and IsTerminal are required; Modified and Size
// are optional if the storage implementation is not
// able to get that information. Setting them will
// make certain operations more consistent or
// predictable, but it is not crucial to basic
// functionality.
type KeyInfo struct {
Key string
Modified time.Time
Size int64
IsTerminal bool // false for directories (keys that act as prefix for other keys)
}
// storeTx stores all the values or none at all.
func storeTx(ctx context.Context, s Storage, all []keyValue) error {
for i, kv := range all {
err := s.Store(ctx, kv.key, kv.value)
if err != nil {
for j := i - 1; j >= 0; j-- {
s.Delete(ctx, all[j].key)
}
return err
}
}
return nil
}
// keyValue pairs a key and a value.
type keyValue struct {
key string
value []byte
}
// KeyBuilder provides a namespace for methods that
// build keys and key prefixes, for addressing items
// in a Storage implementation.
type KeyBuilder struct{}
// CertsPrefix returns the storage key prefix for
// the given certificate issuer.
func (keys KeyBuilder) CertsPrefix(issuerKey string) string {
return path.Join(prefixCerts, keys.Safe(issuerKey))
}
// CertsSitePrefix returns a key prefix for items associated with
// the site given by domain using the given issuer key.
func (keys KeyBuilder) CertsSitePrefix(issuerKey, domain string) string {
return path.Join(keys.CertsPrefix(issuerKey), keys.Safe(domain))
}
// SiteCert returns the path to the certificate file for domain
// that is associated with the issuer with the given issuerKey.
func (keys KeyBuilder) SiteCert(issuerKey, domain string) string {
safeDomain := keys.Safe(domain)
return path.Join(keys.CertsSitePrefix(issuerKey, domain), safeDomain+".crt")
}
// SitePrivateKey returns the path to the private key file for domain
// that is associated with the certificate from the given issuer with
// the given issuerKey.
func (keys KeyBuilder) SitePrivateKey(issuerKey, domain string) string {
safeDomain := keys.Safe(domain)
return path.Join(keys.CertsSitePrefix(issuerKey, domain), safeDomain+".key")
}
// SiteMeta returns the path to the metadata file for domain that
// is associated with the certificate from the given issuer with
// the given issuerKey.
func (keys KeyBuilder) SiteMeta(issuerKey, domain string) string {
safeDomain := keys.Safe(domain)
return path.Join(keys.CertsSitePrefix(issuerKey, domain), safeDomain+".json")
}
// OCSPStaple returns a key for the OCSP staple associated
// with the given certificate. If you have the PEM bundle
// handy, pass that in to save an extra encoding step.
func (keys KeyBuilder) OCSPStaple(cert *Certificate, pemBundle []byte) string {
var ocspFileName string
if len(cert.Names) > 0 {
firstName := keys.Safe(cert.Names[0])
ocspFileName = firstName + "-"
}
ocspFileName += fastHash(pemBundle)
return path.Join(prefixOCSP, ocspFileName)
}
// Safe standardizes and sanitizes str for use as
// a single component of a storage key. This method
// is idempotent.
func (keys KeyBuilder) Safe(str string) string {
str = strings.ToLower(str)
str = strings.TrimSpace(str)
// replace a few specific characters
repl := strings.NewReplacer(
" ", "_",
"+", "_plus_",
"*", "wildcard_",
":", "-",
"..", "", // prevent directory traversal (regex allows single dots)
)
str = repl.Replace(str)
// finally remove all non-word characters
return safeKeyRE.ReplaceAllLiteralString(str, "")
}
// CleanUpOwnLocks immediately cleans up all
// current locks obtained by this process. Since
// this does not cancel the operations that
// the locks are synchronizing, this should be
// called only immediately before process exit.
// Errors are only reported if a logger is given.
func CleanUpOwnLocks(ctx context.Context, logger *zap.Logger) {
locksMu.Lock()
defer locksMu.Unlock()
for lockKey, storage := range locks {
if err := storage.Unlock(ctx, lockKey); err != nil {
logger.Error("unable to clean up lock in storage backend",
zap.Any("storage", storage),
zap.String("lock_key", lockKey),
zap.Error(err))
continue
}
delete(locks, lockKey)
}
}
func acquireLock(ctx context.Context, storage Storage, lockKey string) error {
err := storage.Lock(ctx, lockKey)
if err == nil {
locksMu.Lock()
locks[lockKey] = storage
locksMu.Unlock()
}
return err
}
func releaseLock(ctx context.Context, storage Storage, lockKey string) error {
err := storage.Unlock(ctx, lockKey)
if err == nil {
locksMu.Lock()
delete(locks, lockKey)
locksMu.Unlock()
}
return err
}
// locks stores a reference to all the current
// locks obtained by this process.
var locks = make(map[string]Storage)
var locksMu sync.Mutex
// StorageKeys provides methods for accessing
// keys and key prefixes for items in a Storage.
// Typically, you will not need to use this
// because accessing storage is abstracted away
// for most cases. Only use this if you need to
// directly access TLS assets in your application.
var StorageKeys KeyBuilder
const (
prefixCerts = "certificates"
prefixOCSP = "ocsp"
)
// safeKeyRE matches any undesirable characters in storage keys.
// Note that this allows dots, so you'll have to strip ".." manually.
var safeKeyRE = regexp.MustCompile(`[^\w@.-]`)
// defaultFileStorage is a convenient, default storage
// implementation using the local file system.
var defaultFileStorage = &FileStorage{Path: dataDir()}