/
config.go
360 lines (301 loc) · 11.9 KB
/
config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mon
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/coreos/pkg/capnslog"
"github.com/go-ini/ini"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/clusterd"
)
type CephMonitorConfig struct {
Name string `json:"name"`
Endpoint string `json:"endpoint"`
}
type cephConfig struct {
*GlobalConfig `ini:"global,omitempty"`
}
type GlobalConfig struct {
EnableExperimental string `ini:"enable experimental unrecoverable data corrupting features"`
FSID string `ini:"fsid,omitempty"`
RunDir string `ini:"run dir,omitempty"`
MonMembers string `ini:"mon initial members,omitempty"`
LogFile string `ini:"log file,omitempty"`
MonClusterLogFile string `ini:"mon cluster log file,omitempty"`
MonKeyValueDb string `ini:"mon keyvaluedb"`
DebugLogDefaultLevel int `ini:"debug default"`
DebugLogRadosLevel int `ini:"debug rados"`
DebugLogMonLevel int `ini:"debug mon"`
DebugLogOSDLevel int `ini:"debug osd"`
DebugLogBluestoreLevel int `ini:"debug bluestore"`
DebugLogFilestoreLevel int `ini:"debug filestore"`
DebugLogJournalLevel int `ini:"debug journal"`
DebugLogLevelDBLevel int `ini:"debug leveldb"`
FileStoreOmapBackend string `ini:"filestore_omap_backend"`
OsdPgBits int `ini:"osd pg bits,omitempty"`
OsdPgpBits int `ini:"osd pgp bits,omitempty"`
OsdPoolDefaultSize int `ini:"osd pool default size,omitempty"`
OsdPoolDefaultMinSize int `ini:"osd pool default min size,omitempty"`
OsdPoolDefaultPgNum int `ini:"osd pool default pg num,omitempty"`
OsdPoolDefaultPgpNum int `ini:"osd pool default pgp num,omitempty"`
OsdMaxObjectNameLen int `ini:"osd max object name len,omitempty"`
OsdMaxObjectNamespaceLen int `ini:"osd max object namespace len,omitempty"`
OsdObjectStore string `ini:"osd objectstore"`
RbdDefaultFeatures int `ini:"rbd_default_features,omitempty"`
CrushtoolPath string `ini:"crushtool"`
}
// get the path of a given monitor's run dir
func getMonRunDirPath(configDir, monName string) string {
return path.Join(configDir, monName)
}
// get the path of a given monitor's keyring
func getMonKeyringPath(configDir, monName string) string {
return filepath.Join(getMonRunDirPath(configDir, monName), "keyring")
}
// get the path of a given monitor's data dir
func getMonDataDirPath(configDir, monName string) string {
return filepath.Join(getMonRunDirPath(configDir, monName), fmt.Sprintf("mon.%s", monName))
}
func ConnectToClusterAsAdmin(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo) (client.Connection, error) {
if len(cluster.Monitors) == 0 {
return nil, errors.New("no monitors")
}
// write the monitor keyring to disk
monName := getFirstMonitor(cluster)
keyringPath := getMonKeyringPath(context.ConfigDir, monName)
if err := writeMonitorKeyring(monName, cluster, keyringPath); err != nil {
return nil, err
}
return ConnectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName),
"admin", getMonKeyringPath(context.ConfigDir, monName))
}
// get the path of a given monitor's config file
func GetConfFilePath(root, clusterName string) string {
return fmt.Sprintf("%s/%s.config", root, clusterName)
}
func GenerateTempConfigFiles(context *clusterd.Context, cluster *ClusterInfo) (string, string, string, error) {
root := path.Join(context.ConfigDir, "tmp")
keyring := path.Join(root, "keyring")
user := getFirstMonitor(cluster)
err := writeMonitorKeyring(user, cluster, keyring)
if err != nil {
return "", "", "", fmt.Errorf("failed to write keyring to %s", root)
}
configFile, err := GenerateConfigFile(context, cluster, root, user, keyring, false, nil, nil)
return configFile, keyring, user, err
}
// generates and writes the monitor config file to disk
func GenerateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string) (string, error) {
return GenerateConfigFile(context, cluster, pathRoot, user, keyringPath, false, nil, nil)
}
// generates and writes the monitor config file to disk
func GenerateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string,
bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) {
if pathRoot == "" {
pathRoot = getMonRunDirPath(context.ConfigDir, getFirstMonitor(cluster))
}
// create the config directory
if err := os.MkdirAll(pathRoot, 0744); err != nil {
logger.Warningf("failed to create config directory at %s: %+v", pathRoot, err)
}
configFile, err := createGlobalConfigFileSection(cluster, pathRoot, context.LogLevel, bluestore, userConfig)
if err != nil {
return "", fmt.Errorf("failed to create global config section, %+v", err)
}
if err := addClientConfigFileSection(configFile, getQualifiedUser(user), keyringPath, clientSettings); err != nil {
return "", fmt.Errorf("failed to add admin client config section, %+v", err)
}
if err := addInitialMonitorsConfigFileSections(configFile, cluster); err != nil {
return "", fmt.Errorf("failed to add initial monitor config sections, %+v", err)
}
// write the entire config to disk
filePath := GetConfFilePath(pathRoot, cluster.Name)
logger.Debugf("writing config file %s", filePath)
if err := configFile.SaveTo(filePath); err != nil {
return "", fmt.Errorf("failed to save config file %s. %+v", filePath, err)
}
return filePath, nil
}
// create a keyring for access to the cluster, with the desired set of privileges
func CreateKeyring(conn client.Connection, username, keyringPath string, access []string, generateKeyring func(string) string) error {
_, err := os.Stat(keyringPath)
if err == nil {
// no error, the file exists, bail out with no error
logger.Debugf("keyring already exists at %s", keyringPath)
return nil
} else if !os.IsNotExist(err) {
// some other error besides "does not exist", bail out with error
return fmt.Errorf("failed to stat %s: %+v", keyringPath, err)
}
// get-or-create-key for the user account
key, err := client.AuthGetOrCreateKey(conn, username, access)
if err != nil {
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
// write the keyring to disk
keyringDir := filepath.Dir(keyringPath)
if err := os.MkdirAll(keyringDir, 0744); err != nil {
return fmt.Errorf("failed to create keyring dir at %s: %+v", keyringDir, err)
}
keyring := generateKeyring(key)
logger.Debugf("Writing keyring to: %s", keyringPath)
if err := ioutil.WriteFile(keyringPath, []byte(keyring), 0644); err != nil {
return fmt.Errorf("failed to write keyring to %s: %+v", keyringPath, err)
}
return nil
}
// prepends "client." if a user namespace is not already specified
func getQualifiedUser(user string) string {
if strings.Index(user, ".") == -1 {
return fmt.Sprintf("client.%s", user)
}
return user
}
func getFirstMonitor(cluster *ClusterInfo) string {
// Get the first monitor
for _, m := range cluster.Monitors {
return m.Name
}
return ""
}
// opens a connection to the cluster that can be used for management operations
func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo,
basePath, user, keyringPath string) (client.Connection, error) {
logger.Infof("connecting to ceph cluster %s with user %s", cluster.Name, user)
confFilePath, err := GenerateConnectionConfigFile(context, cluster, basePath, user, keyringPath)
if err != nil {
return nil, fmt.Errorf("failed to generate config file: %v", err)
}
conn, err := factory.NewConnWithClusterAndUser(cluster.Name, getQualifiedUser(user))
if err != nil {
return nil, fmt.Errorf("failed to create rados connection for cluster %s and user %s: %+v", cluster.Name, user, err)
}
if err = conn.ReadConfigFile(confFilePath); err != nil {
return nil, fmt.Errorf("failed to read config file for cluster %s: %+v", cluster.Name, err)
}
if err = conn.Connect(); err != nil {
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
}
return conn, nil
}
func CreateDefaultCephConfig(cluster *ClusterInfo, runDir string, logLevel capnslog.LogLevel, bluestore bool) *cephConfig {
// extract a list of just the monitor names, which will populate the "mon initial members"
// global config field
monMembers := make([]string, len(cluster.Monitors))
i := 0
for _, monitor := range cluster.Monitors {
monMembers[i] = monitor.Name
i++
}
experimental := ""
store := "filestore"
if bluestore {
experimental = "bluestore rocksdb"
store = "bluestore"
}
cephLogLevel := logLevelToCephLogLevel(logLevel)
return &cephConfig{
&GlobalConfig{
EnableExperimental: experimental,
FSID: cluster.FSID,
RunDir: runDir,
MonMembers: strings.Join(monMembers, " "),
LogFile: "/dev/stdout",
MonClusterLogFile: "/dev/stdout",
MonKeyValueDb: "rocksdb",
DebugLogDefaultLevel: cephLogLevel,
DebugLogRadosLevel: cephLogLevel,
DebugLogMonLevel: cephLogLevel,
DebugLogOSDLevel: cephLogLevel,
DebugLogBluestoreLevel: cephLogLevel,
DebugLogFilestoreLevel: cephLogLevel,
DebugLogJournalLevel: cephLogLevel,
DebugLogLevelDBLevel: cephLogLevel,
FileStoreOmapBackend: "rocksdb",
OsdPgBits: 11,
OsdPgpBits: 11,
OsdPoolDefaultSize: 1,
OsdPoolDefaultMinSize: 1,
OsdPoolDefaultPgNum: 100,
OsdPoolDefaultPgpNum: 100,
OsdObjectStore: store,
RbdDefaultFeatures: 3,
CrushtoolPath: "",
},
}
}
func createGlobalConfigFileSection(cluster *ClusterInfo, runDir string, logLevel capnslog.LogLevel, bluestore bool, userConfig *cephConfig) (*ini.File, error) {
var ceph *cephConfig
if userConfig != nil {
// use the user config since it was provided
ceph = userConfig
} else {
ceph = CreateDefaultCephConfig(cluster, runDir, logLevel, bluestore)
}
configFile := ini.Empty()
err := ini.ReflectFrom(configFile, ceph)
return configFile, err
}
func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath string, settings map[string]string) error {
s, err := configFile.NewSection(clientName)
if err != nil {
return err
}
if _, err := s.NewKey("keyring", keyringPath); err != nil {
return err
}
for key, val := range settings {
if _, err := s.NewKey(key, val); err != nil {
return fmt.Errorf("failed to add key %s. %v", key, err)
}
}
return nil
}
func addInitialMonitorsConfigFileSections(configFile *ini.File, cluster *ClusterInfo) error {
// write the config for each individual monitor member of the cluster to the content buffer
for _, mon := range cluster.Monitors {
s, err := configFile.NewSection(fmt.Sprintf("mon.%s", mon.Name))
if err != nil {
return err
}
if _, err := s.NewKey("name", mon.Name); err != nil {
return err
}
if _, err := s.NewKey("mon addr", mon.Endpoint); err != nil {
return err
}
}
return nil
}
func logLevelToCephLogLevel(logLevel capnslog.LogLevel) int {
switch logLevel {
case capnslog.CRITICAL:
case capnslog.ERROR:
case capnslog.WARNING:
return -1
case capnslog.NOTICE:
case capnslog.INFO:
return 0
case capnslog.DEBUG:
return 10
case capnslog.TRACE:
return 100
}
return 0
}