-
Notifications
You must be signed in to change notification settings - Fork 210
/
testfile.go
333 lines (293 loc) · 12.7 KB
/
testfile.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
package cmd
import (
"context"
"crypto/md5"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/spf13/cobra"
"io"
"net/http"
"net/url"
"os"
"strings"
)
// TestFileCommand represents the struct to get command
// for validating azcopy operations.
type TestFileCommand struct {
// object is the resource which needs to be validated against a resource in bucket(share/container).
Object string
//Subject is the remote resource against which object needs to be validated.
Subject string
// IsObjectDirectory defines if the object is a directory or not.
// If the object is directory, then validation goes through another path.
IsObjectDirectory bool
// IsRecursive defines if recursive switch is on during transfer.
IsRecursive bool
// Metadata of the file to be validated.
MetaData string
// NoGuessMimeType represent the azcopy NoGuessMimeType flag set while uploading the file.
NoGuessMimeType bool
// Content Type of the file to be validated.
ContentType string
// Content Encoding of the file to be validated.
ContentEncoding string
ContentDisposition string
ContentLanguage string
CacheControl string
CheckContentMD5 bool
// Represents the flag to determine whether number of blocks or pages needs
// to be verified or not.
// todo always set this to true
VerifyBlockOrPageSize bool
// FileType of the resource to be validated.
FileType string
// Number of Blocks or Pages Expected from the file.
NumberOfBlocksOrPages uint64
// todo : numberofblockorpages can be an array with offset : end url.
//todo consecutive page ranges get squashed.
// PreserveLastModifiedTime represents the azcopy PreserveLastModifiedTime flag while downloading the file.
PreserveLastModifiedTime bool
}
// initializes the testfile command, its aliases and description.
// also adds the possible flags that can be supplied with testFile command.
func init() {
cmdInput := TestFileCommand{}
testFileCmd := &cobra.Command{
Use: "testFile",
Aliases: []string{"tFile"},
Short: "tests the file created using AZCopy v2",
Args: func(cmd *cobra.Command, args []string) error {
if len(args) != 2 {
return fmt.Errorf("invalid arguments for test file command")
}
// first argument is the resource name.
cmdInput.Object = args[0]
// second argument is the test directory.
cmdInput.Subject = args[1]
return nil
},
Run: func(cmd *cobra.Command, args []string) {
verifyFile(cmdInput)
},
}
rootCmd.AddCommand(testFileCmd)
// add flags.
testFileCmd.PersistentFlags().StringVar(&cmdInput.MetaData, "metadata", "", "metadata expected from the file in the container")
testFileCmd.PersistentFlags().StringVar(&cmdInput.ContentType, "content-type", "", "content type expected from the file in the container")
testFileCmd.PersistentFlags().StringVar(&cmdInput.ContentEncoding, "content-encoding", "", "validate the given HTTP header.")
testFileCmd.PersistentFlags().StringVar(&cmdInput.ContentDisposition, "content-disposition", "", "validate the given HTTP header.")
testFileCmd.PersistentFlags().StringVar(&cmdInput.ContentLanguage, "content-language", "", "validate the given HTTP header.")
testFileCmd.PersistentFlags().StringVar(&cmdInput.CacheControl, "cache-control", "", "validate the given HTTP header.")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.CheckContentMD5, "check-content-md5", false, "Validate content MD5 is not empty.")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.IsObjectDirectory, "is-object-dir", false, "set the type of object to verify against the subject")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.IsRecursive, "is-recursive", true, "Set whether to validate against subject recursively when object is directory.")
// TODO: parameter name doesn't match file scenario, discuss and refactor.
testFileCmd.PersistentFlags().Uint64Var(&cmdInput.NumberOfBlocksOrPages, "number-blocks-or-pages", 0, "Use this block size to verify the number of blocks uploaded")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.VerifyBlockOrPageSize, "verify-block-size", false, "this flag verify the block size by determining the number of blocks")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.NoGuessMimeType, "no-guess-mime-type", false, "This sets the content-type based on the extension of the file.")
testFileCmd.PersistentFlags().BoolVar(&cmdInput.PreserveLastModifiedTime, "preserve-last-modified-time", false, "Only available when destination is file system.")
}
// verify the file downloaded or uploaded.
func verifyFile(testFileCmd TestFileCommand) {
if testFileCmd.IsObjectDirectory {
verifyFileDirUpload(testFileCmd)
} else {
verifySingleFileUpload(testFileCmd)
}
}
// verifyFileDirUpload verifies the directory recursively uploaded to the share or directory.
func verifyFileDirUpload(testFileCmd TestFileCommand) {
directoryClient, _ := directory.NewClientWithNoCredential(testFileCmd.Subject, nil)
// get the original dir path, which can be used to get file relative path during enumerating and comparing
fileURLParts, err := file.ParseURL(testFileCmd.Subject)
if err != nil {
os.Exit(1)
}
baseAzureDirPath := fileURLParts.DirectoryOrFilePath
// validate azure directory
validateAzureDirWithLocalFile(directoryClient, baseAzureDirPath, testFileCmd.Object, testFileCmd.IsRecursive)
}
// recursively validate files in azure directories and sub-directories
func validateAzureDirWithLocalFile(curAzureDirURL *directory.Client, baseAzureDirPath string, localBaseDir string, isRecursive bool) {
pager := curAzureDirURL.NewListFilesAndDirectoriesPager(nil)
for pager.More() {
// look for all files that in current directory
listFile, err := pager.NextPage(context.Background())
if err != nil {
// fmt.Printf("fail to list files and directories inside the directory. Please check the directory sas, %v\n", err)
os.Exit(1)
}
if isRecursive {
for _, dirInfo := range listFile.Segment.Directories {
newDirURL := curAzureDirURL.NewSubdirectoryClient(*dirInfo.Name)
validateAzureDirWithLocalFile(newDirURL, baseAzureDirPath, localBaseDir, isRecursive)
}
}
// Process the files returned in this result segment (if the segment is empty, the loop body won't execute)
for _, fileInfo := range listFile.Segment.Files {
curFileURL := curAzureDirURL.NewFileClient(*fileInfo.Name)
get, err := curFileURL.DownloadStream(context.Background(), nil)
if err != nil {
fmt.Printf("fail to download the file %s\n", *fileInfo.Name)
os.Exit(1)
}
retryReader := get.NewRetryReader(context.Background(), &file.RetryReaderOptions{MaxRetries: 3})
// read all bytes.
fileBytesDownloaded, err := io.ReadAll(retryReader)
if err != nil {
fmt.Printf("fail to read the body of file %s downloaded and failed with error %s\n", *fileInfo.Name, err.Error())
os.Exit(1)
}
retryReader.Close()
url, err := url.Parse(curFileURL.URL())
if err != nil {
fmt.Printf("fail to parse the file URL %s\n", curFileURL.URL())
os.Exit(1)
}
tokens := strings.SplitAfterN(url.Path, baseAzureDirPath, 2)
if len(tokens) < 2 {
fmt.Printf("fail to get sub directory and file name, file URL '%s', original dir path '%s'\n", curFileURL.URL(), baseAzureDirPath)
os.Exit(1)
}
subDirAndFileName := tokens[1]
var objectLocalPath string
if subDirAndFileName != "" && subDirAndFileName[0] != '/' {
objectLocalPath = localBaseDir + "/" + subDirAndFileName
} else {
objectLocalPath = localBaseDir + subDirAndFileName
}
// opening the file locally and memory mapping it.
sFileInfo, err := os.Stat(objectLocalPath)
if err != nil {
fmt.Println("fail to get the subject file info on local disk ")
os.Exit(1)
}
sFile, err := os.Open(objectLocalPath)
if err != nil {
fmt.Println("fail to open file ", sFile)
os.Exit(1)
}
sMap, err := NewMMF(sFile, false, 0, sFileInfo.Size())
if err != nil {
fmt.Println("fail to memory mapping the file ", sFileInfo.Name())
}
// calculating the md5 of file on container.
actualMd5 := md5.Sum(fileBytesDownloaded)
// calculating md5 of resource locally.
expectedMd5 := md5.Sum(sMap)
if actualMd5 != expectedMd5 {
fmt.Println("the upload file md5 is not equal to the md5 of actual file on disk for file ", fileInfo.Name)
os.Exit(1)
}
}
}
}
// verifySingleFileUpload verifies the pagefile uploaded or downloaded
// against the file locally.
func verifySingleFileUpload(testFileCmd TestFileCommand) {
fileInfo, err := os.Stat(testFileCmd.Object)
if err != nil {
fmt.Println("error opening the destination localFile on local disk ")
os.Exit(1)
}
localFile, err := os.Open(testFileCmd.Object)
if err != nil {
fmt.Println("error opening the localFile ", testFileCmd.Object)
}
fileClient, _ := file.NewClientWithNoCredential(testFileCmd.Subject, nil)
get, err := fileClient.DownloadStream(context.Background(), nil)
if err != nil {
fmt.Println("unable to get localFile properties ", err.Error())
os.Exit(1)
}
// reading all the bytes downloaded.
retryReader := get.NewRetryReader(context.Background(), &file.RetryReaderOptions{MaxRetries: 3})
defer retryReader.Close()
fileBytesDownloaded, err := io.ReadAll(retryReader)
if err != nil {
fmt.Println("error reading the byes from response and failed with error ", err.Error())
os.Exit(1)
}
if fileInfo.Size() == 0 {
// If the fileSize is 0 and the len of downloaded bytes is not 0
// validation fails
if len(fileBytesDownloaded) != 0 {
fmt.Printf("validation failed since the actual localFile size %d differs from the downloaded localFile size %d\n", fileInfo.Size(), len(fileBytesDownloaded))
os.Exit(1)
}
// If both the actual and downloaded localFile size is 0,
// validation is successful, no need to match the md5
os.Exit(0)
}
// memory mapping the resource on local path.
mmap, err := NewMMF(localFile, false, 0, fileInfo.Size())
if err != nil {
fmt.Println("error mapping the destination localFile: ", localFile, " localFile size: ", fileInfo.Size(), " Error: ", err.Error())
os.Exit(1)
}
// calculating and verify the md5 of the resource
// both locally and on the container.
actualMd5 := md5.Sum(mmap)
expectedMd5 := md5.Sum(fileBytesDownloaded)
if actualMd5 != expectedMd5 {
fmt.Println("the uploaded localFile's md5 doesn't matches the actual localFile's md5 for localFile ", testFileCmd.Object)
os.Exit(1)
}
if testFileCmd.CheckContentMD5 && (get.ContentMD5 == nil || len(get.ContentMD5) == 0) {
fmt.Println("ContentMD5 should not be empty")
os.Exit(1)
}
// verify the user given metadata supplied while uploading the localFile against the metadata actually present in the localFile
if !validateMetadata(testFileCmd.MetaData, get.Metadata) {
fmt.Println("meta data does not match between the actual and uploaded localFile.")
os.Exit(1)
}
// verify the content-type
expectedContentType := ""
if testFileCmd.NoGuessMimeType {
expectedContentType = testFileCmd.ContentType
} else {
expectedContentType = http.DetectContentType(mmap)
}
expectedContentType = strings.Split(expectedContentType, ";")[0]
if !validateString(expectedContentType, common.IffNotNil(get.ContentType, "")) {
str1 := fmt.Sprintf(" %s %s", expectedContentType, common.IffNotNil(get.ContentDisposition, ""))
fmt.Println(str1 + "mismatch content type between actual and user given localFile content type")
os.Exit(1)
}
//verify the content-encoding
if !validateString(testFileCmd.ContentEncoding, common.IffNotNil(get.ContentEncoding, "")) {
fmt.Println("mismatch content encoding between actual and user given localFile content encoding")
os.Exit(1)
}
if !validateString(testFileCmd.ContentDisposition, common.IffNotNil(get.ContentDisposition, "")) {
fmt.Println("mismatch content disposition between actual and user given value")
os.Exit(1)
}
if !validateString(testFileCmd.ContentLanguage, common.IffNotNil(get.ContentLanguage, "")) {
fmt.Println("mismatch content encoding between actual and user given value")
os.Exit(1)
}
if !validateString(testFileCmd.CacheControl, common.IffNotNil(get.CacheControl, "")) {
fmt.Println("mismatch cache control between actual and user given value")
os.Exit(1)
}
mmap.Unmap()
localFile.Close()
// verify the number of pageranges.
// this verifies the page-size and azcopy pagefile implementation.
if testFileCmd.VerifyBlockOrPageSize {
numberOfPages := int(testFileCmd.NumberOfBlocksOrPages)
resp, err := fileClient.GetRangeList(context.Background(), nil)
if err != nil {
fmt.Println("error getting the range list ", err.Error())
os.Exit(1)
}
if numberOfPages != (len(resp.Ranges)) {
fmt.Println("number of ranges uploaded is different from the number of expected to be uploaded")
os.Exit(1)
}
}
}