-
Notifications
You must be signed in to change notification settings - Fork 2
/
csi_md.go
481 lines (432 loc) · 16.8 KB
/
csi_md.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
// Copyright (C) 2016--2020 Lightbits Labs Ltd.
// Copyright (C) 2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
package driver
import (
"fmt"
"regexp"
"strconv"
"github.com/container-storage-interface/spec/lib/go/csi"
guuid "github.com/google/uuid"
"github.com/lightbitslabs/los-csi/pkg/util/endpoint"
)
// this file holds the definitions of - and helper functions for handling of -
// the various bits metadata that gets shovelled through the CSI API boundary.
// that includes both the generic CSI metadata and the LB-specific parts.
//
// the latter are passed by the plugin to the CO through the various impl-
// specific CSI API entrypoint response fields and then back from the CO to the
// plugin through the various impl-specific CSI API entrypoint request fields.
// all of this is completely opaque to the CO itself, which just takes care to
// accept these values (typically strings or string maps, as appropriate),
// preserve them between the API entrypoint invocations (typically attached to
// the relevant CO-specific entities, e.g. SC or PV in K8s), and pass them back
// to the plugin in a fashion defined by the CSI spec.
const (
volPathField = "volume_path"
volIDField = "volume_id"
snapIDField = "snapshot_id"
capRangeField = "capacity_range"
capRangeLimField = capRangeField + ".limit_bytes"
nodeIDField = "node_id"
srcVolField = "source_volume_id"
volContSrcField = "volume_content_source"
volContSrcVolField = volContSrcField + ".volume.volume_id"
volContSrcSnapField = volContSrcField + ".snapshot.snapshot_id"
)
// lbCreateVolumeParams: -----------------------------------------------------
const (
grpcXport = "grpc" // legacy, gRPC over plain HTTP/2.
grpcsXport = "grpcs" // gRPC over HTTPS/2, the only currently supported transport.
volParRoot = "parameters"
volParMgmtEPKey = "mgmt-endpoint"
volParRepCntKey = "replica-count"
volParCompressKey = "compression"
volParProjNameKey = "project-name"
volParMgmtSchemeKey = "mgmt-scheme"
volParQosNameKey = "qos-policy-name"
// volHostEncryptionKey parameter in the storageclass parameter, can be either enabled|disabled
volHostEncryptionKey = "host-encryption"
// volHostEncryptionPassphraseKey name of the secret for the encryption passphrase
volHostEncryptionPassphraseKey = "host-encryption-passphrase"
// volHostEncryptionPassphraseKeyMaxLen defines the maximum len of the encryption passphrase
// this is according to the cryptsetup man page
volHostEncryptionPassphraseKeyMaxLen = 512
)
var projNameRegex *regexp.Regexp
func init() {
projNameRegex = regexp.MustCompile(`^[a-z0-9]([a-z0-9.-]{0,61}[a-z0-9])?$`)
}
// checkProjectName() checks syntactic validity of the LB "project" name, as
// specified in the request params of the outward-facing CSI API entrypoints.
// it shall be the single point of truth about the project name validity
// inside the LB CSI plugin as a whole.
func checkProjectName(field, proj string) error {
if proj == "" {
return mkEinvalMissing(field)
}
if !projNameRegex.MatchString(proj) {
return mkEinvalf(field, "'%s'", proj)
}
return nil
}
// lbCreateVolumeParams represents the contents of the `parameters` field
// (`CreateVolumeRequest.parameters`) passed to the plugin by the CO on
// CreateVolume() CSI API entrypoint invocation. this supplementary info
// is used by the plugin to locate the relevant LightOS management API servers
// to connect to and to request that the volume be created with the specified
// LightOS-specific properties. the initial source of the these `parameters`
// is CO-specific (e.g. in K8s they're taken from the SC `parameters` stanza).
//
// `parameters` as passed to CreateVolume() is a string-to-string (!) KV map
// that must include:
// mgmt-endpoint: <host>:<port>[,<host>:port>...]
// mgmt-scheme: "grpcs"
// project-name: <project-name>
// replica-count: <num-replicas>
// may optionally include (if omitted - the default is "disabled"):
// compression: <"enabled"|"disabled">
// qos-policy-name: <qos-policy-name>
// host-encryption: <"enabled"|"disabled">
// e.g.:
// mgmt-endpoint: 10.0.0.100:80,10.0.0.101:80
// mgmt-scheme: grpcs
// project-name: proj-3
// replica-count: 2
// compression: enabled
// qos-policy-name: "io-limited-policy"
// host-encryption: enabled
type lbCreateVolumeParams struct {
mgmtEPs endpoint.Slice // LightOS mgmt API server endpoints.
replicaCount uint32 // total number of volume replicas.
compression bool // whether compression is enabled.
projectName string // project name.
mgmtScheme string // currently must be 'grpcs'
qosPolicyName string // qos policy name should exist in the lightos
hostCrypto string // host-encryption format, currently either empty or luks2
}
func volParKey(key string) string {
return volParRoot + "." + key
}
// parseCSICreateVolumeParams parses the `parameters` K:V map passed to
// CreateVolume() and validates the contents. the returned lbCreateVolumeParams
// is only valid if the returned error is 'nil'.
func parseCSICreateVolumeParams(params map[string]string) (lbCreateVolumeParams, error) {
res := lbCreateVolumeParams{}
var err error
key := volParKey(volParMgmtEPKey)
mgmtEPs := params[volParMgmtEPKey]
if mgmtEPs == "" {
return res, mkEinvalMissing(key)
}
res.mgmtEPs, err = endpoint.ParseCSV(mgmtEPs)
if err != nil {
return res, mkEinval(key, err.Error())
}
key = volParKey(volParRepCntKey)
replicaCount := params[volParRepCntKey]
if replicaCount == "" {
return res, mkEinvalMissing(key)
}
repCnt, err := strconv.ParseUint(replicaCount, 10, 32)
if err != nil {
return res, mkEinvalf(key, "'%s'", replicaCount)
}
res.replicaCount = uint32(repCnt)
key = volParKey(volParCompressKey)
switch params[volParCompressKey] {
case "", "disabled":
res.compression = false
case "enabled":
res.compression = true
default:
return res, mkEinval(key, params[volParCompressKey])
}
// optional field, originally defaulting to empty.
//
// TODO: project names were only optional during the transition period
// and have been MANDATORY for a long time now. make it so!
key = volParKey(volParProjNameKey)
if proj, ok := params[volParProjNameKey]; ok {
err = checkProjectName(key, proj)
if err != nil {
return res, err
}
res.projectName = proj
}
// optional field, defaulting to 'grpcs'.
//
// TODO: the option of NOT using 'grpcs' was only viable during the transition
// period and 'grpcs' became MANDATORY a long time ago, so the code below
// should be updated to only accept 'grpcs' as a valid value - if it was
// specified at all - for reverse compatibility.
key = volParKey(volParMgmtSchemeKey)
mgmtScheme := params[volParMgmtSchemeKey]
switch mgmtScheme {
case "", grpcsXport:
res.mgmtScheme = grpcsXport
case grpcXport:
res.mgmtScheme = grpcXport
default:
return res, mkEinval(key, mgmtScheme)
}
if val, ok := params[volParQosNameKey]; ok {
res.qosPolicyName = val
}
key = volParKey(volHostEncryptionKey)
switch params[volHostEncryptionKey] {
case "", "disabled":
res.hostCrypto = ""
case "enabled":
res.hostCrypto = defaultLuksFormat
default:
return res, mkEinval(key, params[volHostEncryptionKey])
}
if res.hostCrypto != "" && res.compression {
return res, mkEbadOp("mismatch", volHostEncryptionKey,
"host-encryption and compression are both enabled")
}
return res, nil
}
// lbResourceID: ---------------------------------------------------------------
// resIDRegex is used for initial syntactic validation of `lbResourceID`
// string form: LB CSI plugin generated resource IDs that the COs use to
// uniquely identify volumes, snapshots, etc.
var resIDRegex *regexp.Regexp
func init() {
//nolint:lll
resIDRegex = regexp.MustCompile(
`^mgmt:([^|]+)\|` +
`nguid:([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})` +
`(\|proj:([^[:cntrl:]| ]+))?` + // proj name syntax checked separately
`(\|scheme:(grpc|grpcs))?` +
`(\|hostcrypto:(luks2))?$`)
}
// lbResourceID uniquely identifies a lightbits resource such as a volume / snapshot / etc.
// It is what the plugin returns to the CO in response to creating resources, then
// passed back to the plugin in most of the other CSI API calls.
// it contains the vital information required by the plugin in order to connect to a remote
// LB and manage resources as per CO requests.
//
// for transmission on the wire, it's serialised into a string with the
// following fixed format:
// mgmt:<host>:<port>[,<host>:<port>...]|nguid:<nguid>[|proj:<proj>][|scheme:<scheme>][|hostcrypto:<format>]
// where:
// <host> - mgmt API server endpoint of the LightOS cluster hosting the
// volume. can be a hostname or an IP address. more than one
// comma-separated <host>:<port> pair can be specified.
// TODO: IPv6 support will require amending parsing for bizarre
// extra allowed characters.
// <port> - variable-length printable decimal representation of the
// uint16 port number, no leading zeroes.
// <nguid> - volume NGUID (see NVMe spec, Identify NS Data Structure)
// in its "canonical", 36-character long, RFC-4122 compliant string
// representation.
// <proj> - project/tenant name on the LB cluster. this field is
// temporarily optional in resource IDs for reverse compatibility,
// though modern LB clusters will refuse requests without it.
// see notes below in parseCSIResourceID().
// <scheme> - transport scheme for communicating with the LB cluster.
// the only valid value is currently 'grpcs' (gRPC over TLS), and
// scheme is temporarily optional, in which case it defaults to...
// 'grpcs'! modern LB clusters will refuse plain unencrypted gRPC
// requests anyway. see below in parseCSIResourceID().
// <hostcrypto> - specifies the crypto format of the hostEncrypted volume, only luks2 is possible.
// this is optional and will only exist for host-encrypted volumes.
// e.g.:
// mgmt:10.0.0.1:80,10.0.0.2:80|nguid:6bb32fb5-99aa-4a4c-a4e7-30b7787bbd66|proj:a|scheme:grpcs
// mgmt:lb01.net:80|nguid:6bb32fb5-99aa-4a4c-a4e7-30b7787bbd66|proj:b|scheme:grpcs|hostcrypto:luks2
//
// TODO: the CSI spec mandates that strings "SHALL NOT" exceed 128 bytes.
// K8s is more lenient (at least 253 bytes, likely more). in any case, with
// the current `volume_id` format, at most 4 mgmt API server endpoints can
// be guaranteed to be supported. anything beyond that is at the mercy of
// the CO implementors (and user network admins assigning IP ranges)...
type lbResourceID struct {
mgmtEPs endpoint.Slice // LightOS mgmt API server endpoints.
uuid guuid.UUID // NVMe "Identify NS Data Structure".
projName string
scheme string // currently must be 'grpcs'
hostCrypto string
}
// String generates the string representation of lbResourceID that will be
// passed back and forth between the CO and this plugin.
func (vid lbResourceID) String() string {
res := fmt.Sprintf("mgmt:%s|nguid:%s", vid.mgmtEPs, vid.uuid)
if len(vid.projName) > 0 {
res += fmt.Sprintf("|proj:%s", vid.projName)
}
if len(vid.scheme) > 0 {
res += fmt.Sprintf("|scheme:%s", vid.scheme)
}
if len(vid.hostCrypto) > 0 {
res += fmt.Sprintf("|hostcrypto:%s", vid.hostCrypto)
}
return res
}
// parseCSIResourceID parses CSI wire-protocol-level `volume_id` string into its
// constituents and syntactically validates it. the returned lbResourceID is
// only valid if the returned error is 'nil'.
func parseCSIResourceID(id string) (lbResourceID, error) {
vid := lbResourceID{}
if id == "" {
return vid, fmt.Errorf("unspecified or empty")
}
match := resIDRegex.FindStringSubmatch(id)
if len(match) < 2 {
return vid, fmt.Errorf("'%s' is malformed", id)
}
var err error
vid.mgmtEPs, err = endpoint.ParseCSV(match[1])
if err != nil {
return vid, fmt.Errorf("'%s' has invalid mgmt endpoints list: %s", id, err)
}
vid.uuid, err = guuid.Parse(match[2])
if err != nil {
return vid, fmt.Errorf("'%s' has invalid NGUID: %s", id, err)
} else if vid.uuid == guuid.Nil {
return vid, fmt.Errorf("'%s' has invalid nil NGUID", id)
}
// optional field, originally defaulting to empty.
//
// TODO: this was only optional during the transition period and has been
// MANDATORY for a long time. make it so!
vid.projName = match[4]
if vid.projName != "" {
err = checkProjectName("", vid.projName)
if err != nil {
return vid, fmt.Errorf("'%s' has invalid project name: '%s'", id, vid.projName)
}
}
// optional field, defaulting to 'grpcs'.
//
// TODO: the option of NOT using 'grpcs' was only viable during the transition
// period and 'grpcs' became MANDATORY a long time ago, so:
// 1. the regex should be updated to only accept 'grpcs' as a valid value for
// reverse compatibility?
// 2. lbResourceID formatter should probably stop generating this field.
vid.scheme = match[6]
if vid.scheme == "" {
vid.scheme = grpcsXport
}
// if empty string, volume is not host-encrypted
vid.hostCrypto = match[7]
if vid.hostCrypto != "" {
vid.hostCrypto = match[7][12:]
}
return vid, nil
}
func parseCSIResourceIDEinval(field, id string) (lbResourceID, error) {
if id == "" {
return lbResourceID{}, mkEinvalMissing(field)
}
rid, err := parseCSIResourceID(id)
if err != nil {
return lbResourceID{}, mkEinval(field, err.Error())
}
return rid, nil
}
func parseCSIResourceIDEnoent(field, id string) (lbResourceID, error) {
if id == "" {
return lbResourceID{}, mkEinvalMissing(field)
}
rid, err := parseCSIResourceID(id)
if err != nil {
return lbResourceID{}, mkEnoent("bad value of '%s': %s", field, err)
}
return rid, nil
}
// CSI volume capabilities helpers: ------------------------------------------
func (d *Driver) supportedAccessModes(isBlockVolumeMode bool) []csi.VolumeCapability_AccessMode_Mode {
var supportedAccessModes = []csi.VolumeCapability_AccessMode_Mode{
csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
}
if isBlockVolumeMode && d.rwx {
supportedAccessModes = append(supportedAccessModes,
csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER)
}
d.log.Infof("RWX is %t returning access-modes: %+v", d.rwx, supportedAccessModes)
return supportedAccessModes
}
// see validateVolumeCapability() docs for info.
func (d *Driver) validateVolumeCapabilities(caps []*csi.VolumeCapability) error {
if len(caps) == 0 {
return mkEinvalMissing("volume_capability")
}
for _, c := range caps {
if err := d.validateVolumeCapability(c); err != nil {
return err
}
}
return nil
}
// performs a generic driver-level capability validation to weed out
// capabilities that are unsupported for sure. specific volumes might have
// additional constraints once they're created, which need to be validated
// separately.
func (d *Driver) validateVolumeCapability( //revive:disable-line:unused-receiver
c *csi.VolumeCapability,
) error {
if c == nil {
return mkEinvalMissing("volume_capability")
}
isBlockVolumeMode := false
accessType := c.GetAccessType()
switch volCap := accessType.(type) {
case *csi.VolumeCapability_Mount:
mntCap := volCap.Mount
if mntCap == nil {
return mkEinvalf("volume_capability.mount", "must be set")
}
// TODO: currently we only support 'ext4' and 'xfs'. additional FSes may require
// mount opts validation, etc., so will likely require a bit of
// scaffolding (TBD)... not to mention packaging the utils!
if mntCap.FsType != "" && mntCap.FsType != Ext4FS && mntCap.FsType != XfsFS {
return mkEinvalf("volume_capability.mount.fs_type",
"unsupported FS: %s", mntCap.FsType)
}
// TODO: add support for custom mount flags
if len(mntCap.MountFlags) > 0 {
return mkEinval("volume_capability.mount.mount_flags",
"custom mount flags are not supported")
}
case *csi.VolumeCapability_Block:
isBlockVolumeMode = true
case nil:
return mkEinvalMissing("volume_capability.access_type")
default:
return mkEinval("volume_capability.access_type",
"unexpected access type specified")
}
modeOk := false
accessMode := c.GetAccessMode()
if accessMode == nil {
return mkEinvalMissing("volume_capability.access_mode")
}
for _, m := range d.supportedAccessModes(isBlockVolumeMode) {
if m == accessMode.Mode {
modeOk = true
break
}
}
if !modeOk {
return mkEinvalf("volume_capability.access_mode",
"unsupported mode: '%s'", accessMode.Mode.String())
}
return nil
}
func (d *Driver) nodeExpansionRequired( //revive:disable-line:unused-receiver
c *csi.VolumeCapability,
) bool {
accessType := c.GetAccessType()
switch accessType.(type) {
case *csi.VolumeCapability_Mount:
// for some reason mount.FsType can be empty at this stage so we will call expand
// volume, and let the node do it's thing
return true
case *csi.VolumeCapability_Block:
default:
return false
}
return false
}