/
util.go
367 lines (297 loc) · 8.76 KB
/
util.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
package util
import (
"archive/tar"
"compress/gzip"
"errors"
"fmt"
"io"
"math"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"k8s.io/client-go/rest"
)
// ErrEmptyDataDir error to indicate the data directory is empty
var ErrEmptyDataDir = errors.New("empty data directory")
// IsValidURL returns true if string is a valid URL
func IsValidURL(toTest string) bool {
_, err := url.ParseRequestURI(toTest)
return err == nil
}
// TestHTTPConnection takes
// a given client / URL(string) / bearerToken(string)/ retries count (int)
// and returns true if response code is 2xx.
func TestHTTPConnection(testClient rest.HTTPClient,
URL, method, bearerToken string, retries uint, verbose bool) (successful bool, body *[]byte, err error) {
IsValidURL(URL)
attempts := retries + 1
req, err := http.NewRequest(method, URL, nil)
if err != nil {
log.Fatalf("Unable to make new request: %v", err)
}
if bearerToken != "" {
req.Header.Add("Authorization", "Bearer "+bearerToken)
}
for i := uint(0); i < attempts; i++ {
resp, err := testClient.Do(req)
if err != nil {
if verbose {
log.Warnf("Unable to connect to URL: %s retrying: %v", URL, i+1)
}
time.Sleep(time.Duration(int64(math.Pow(2, float64(i)))) * time.Second)
continue
}
defer SafeClose(resp.Body.Close, &err)
body, rerr := io.ReadAll(resp.Body)
if rerr != nil {
err = fmt.Errorf("Unable to read response from: %s", URL)
}
return resp.StatusCode <= 200, &body, err
}
return false, &[]byte{}, err
}
// CheckRequiredSettings checks for required min values / flags / environment variables
func CheckRequiredSettings(requiredArgs []string) error {
for _, a := range requiredArgs {
if viper.GetString(a) != "" {
continue
}
return fmt.Errorf("Required flag: %v or environment variable: CLOUDABILITY_"+strings.ToUpper(
a)+" has not been set", a)
}
if viper.IsSet("poll_interval") && viper.GetInt("poll_interval") < 5 {
return fmt.Errorf(
"Polling interval must be 5 seconds or greater")
}
return nil
}
// CreateMetricSample creates a metric sample from a given directory removing the source directory if cleanup is true
func CreateMetricSample(exportDirectory os.File, uid string, cleanUp bool, scratchDir string) (*os.File, error) {
ed, err := exportDirectory.Stat()
if err != nil || !ed.IsDir() {
log.Errorf("Unable to stat sample directory: %v", err)
return nil, err
}
if err := CheckIfDirEmpty(exportDirectory.Name()); err != nil {
return nil, err
}
sampleFilename := getExportFilename(uid)
destFile, err := os.Create(scratchDir + "/" + sampleFilename + ".tgz")
if err != nil {
log.Errorf("Unable to create metric sample file: %v", err)
return nil, err
}
err = createTGZ(exportDirectory, destFile)
if err != nil {
log.Errorf("Unable to tar metric sample directory: %v", err)
return nil, err
}
// cleanup directory after creating the sample
if cleanUp {
err = removeDirectoryContents(exportDirectory.Name() + "/")
}
if err != nil {
log.Errorf("Unable to cleanup metric sample directory: %v", err)
return nil, err
}
return destFile, err
}
// createTGZ takes a source and variable writers and walks 'source' writing each file
// found to the tar writer; the purpose for accepting multiple writers is to allow
// for multiple outputs
func createTGZ(src os.File, writers ...io.Writer) (rerr error) {
// ensure the src actually exists before trying to tar it
if _, err := os.Stat(src.Name()); err != nil {
return fmt.Errorf("Unable to tar files - %v", err.Error())
}
mw := io.MultiWriter(writers...)
//nolint gas
gzw, _ := gzip.NewWriterLevel(mw, 9)
defer SafeClose(gzw.Close, &rerr)
tw := tar.NewWriter(gzw)
defer func() {
err := tw.Close()
if err != nil {
log.Fatal(err)
}
}()
// walk path
return filepath.Walk(src.Name(), func(file string, fileInfo os.FileInfo, err error) (rerr error) {
// return on any error
if err != nil {
return err
}
// create a new dir/file header
header, err := tar.FileInfoHeader(fileInfo, fileInfo.Name())
if err != nil {
return err
}
// return on directories since there will be no content to tar
if fileInfo.Mode().IsDir() {
return nil
}
// if not a directory update the name to correctly reflect the desired destination when untaring
if !fileInfo.Mode().IsDir() {
header.Name = filepath.Join(filepath.Base(src.Name()), strings.TrimPrefix(file, src.Name()))
}
// write the header
if err := tw.WriteHeader(header); err != nil {
return err
}
// open files for taring
//nolint gosec
f, err := os.Open(file)
if err != nil {
return err
}
defer SafeClose(f.Close, &rerr)
// copy file data into tar writer
if _, err := io.Copy(tw, f); err != nil {
return err
}
return err
})
}
func getExportFilename(uid string) string {
t := time.Now().UTC()
return uid + "_" + t.Format("20060102150405")
}
// CreateMSWorkingDirectory takes a given prefix and returns a metric sample working directory
func CreateMSWorkingDirectory(uid string, scratchDir string) (*os.File, error) {
// create metric sample directory
td, err := os.MkdirTemp(scratchDir, "cldy-metrics")
if err != nil {
log.Errorf("Unable to create temporary directory: %v", err)
return nil, err
}
t := time.Now().UTC()
ed := td + "/" + uid + "_" + t.Format("20060102150405")
err = os.MkdirAll(ed, os.ModePerm)
if err != nil {
log.Errorf("Error creating metric sample export directory : %v", err)
}
//nolint gosec
exportDir, err := os.Open(ed)
if err != nil {
log.Fatalln("Unable to open metric sample export directory")
}
return exportDir, err
}
func removeDirectoryContents(dir string) (err error) {
//nolint gosec
d, err := os.Open(dir)
if err != nil {
return err
}
defer SafeClose(d.Close, &err)
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(filepath.Join(dir, name))
if err != nil {
return err
}
}
return nil
}
// CopyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func CopyFileContents(dst, src string) (rerr error) {
//nolint gosec
in, err := os.Open(src)
if err != nil {
return err
}
defer SafeClose(in.Close, &rerr)
out, err := os.Create(dst)
if err != nil {
return err
}
defer SafeClose(out.Close, &rerr)
if _, err = io.Copy(out, in); err != nil {
return err
}
return out.Sync()
}
// SafeClose will close the given closer function, setting the err ONLY if it is currently nil. This
// allows for cleaner handling of always-closing, but retaining the original error (ie from a previous
// Write).
func SafeClose(closer func() error, err *error) {
if closeErr := closer(); closeErr != nil && *err == nil {
(*err) = closeErr
}
}
// MatchOneFile returns the name of one file based on a given directory and pattern
// returning an error if more or less than one match is found. The syntax of patterns is the same
// as in filepath.Glob & Match.
func MatchOneFile(directory string, pattern string) (fileName string, err error) {
results, err := filepath.Glob(directory + pattern)
if err != nil {
return "", fmt.Errorf("Error encountered reading directory: %v", err)
}
if len(results) == 1 {
return results[0], nil
} else if len(results) > 1 {
return "", fmt.Errorf("More than one file matched the pattern: %+v", results)
}
return "", fmt.Errorf("No matches found")
}
// SetupLogger sets configuration for the default logger
func SetupLogger() (err error) {
var (
ll = viper.GetString("log_level")
lf = strings.ToLower(viper.GetString("log_format"))
)
// Set log level
l, err := log.ParseLevel(ll)
if err != nil {
return fmt.Errorf("Invalid log level: %v", ll)
}
log.SetLevel(l)
log.Debugf("Log level set to: %v", l.String())
// Set log format
switch lf {
case "json":
log.SetFormatter(&log.JSONFormatter{})
default:
log.SetFormatter(&log.TextFormatter{
DisableLevelTruncation: true,
PadLevelText: true,
})
}
return nil
}
// ValidateScratchDir validates whether or not the scratch directory exists or not
func ValidateScratchDir(scratchDir string) error {
if _, err := os.Stat(scratchDir); os.IsNotExist(err) {
return fmt.Errorf("There was a problem validating provided scratch directory: %v", err)
}
return nil
}
// CheckIfDirEmpty checks if a directory is empty, returning an ErrEmptyDataDir error if it is
func CheckIfDirEmpty(dirname string) (rerr error) {
dir, err := os.Open(dirname)
if err != nil {
return err
}
defer SafeClose(dir.Close, &rerr)
_, err = dir.Readdir(1)
if err != nil {
switch err {
case io.EOF:
return ErrEmptyDataDir
default:
return err
}
}
return nil
}