From 18b0792333205d01ee38b20274d1e9ac24d280aa Mon Sep 17 00:00:00 2001 From: tobstix Date: Thu, 25 Feb 2021 20:35:23 +0100 Subject: [PATCH] add more commands --- .gitignore | 1 + README.md | 17 +++++++ src/main.go | 125 ++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..314f02b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.txt \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b40d20f --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# go-storage-information + +This tool was build to get quickly and easy Information about Azure Blobs. +It prints basic attributes and set metadata on blobs. +It is also possible to download the content of blobs wether to stdout or to a file. +Furthermore it is possible to filter for a certain Storage Container Names or/and Blob names. + +## Parameter + +* `--accountName` or `-n`: (mandatory) provide the name of the Storage Account +* `--accessKey` or `-k`: (mandatory) provide the Access Key of Storage Account +* `--container` or `-c`: (optional) filter for a specific container by its name. Substring match +* `--blob` or `-b`: (optional) provide the key Storage Account. Substring match +* `--show-content`: (optional) prints additionally the content of the blobs +* `--store-content`: (optional) stores additionally the content to a file. One Line per each Blob Content +* `--filename` or `-f`: (optional) use together with `--store-content` to store blob contents to given file. Default is `blobcontent.txt` +* `--content-only`: (optional) prints only the content of blobs to stdout. No other information are printed diff --git a/src/main.go b/src/main.go index 0cfa97f..6c3cca0 100644 --- a/src/main.go +++ b/src/main.go @@ -7,29 +7,24 @@ import ( "fmt" "log" "net/url" + "os" "strings" + "sync" "github.com/Azure/azure-storage-blob-go/azblob" az "github.com/Azure/azure-storage-blob-go/azblob" "github.com/spf13/cobra" ) -func printLine(level int, content string) { - spacer := "___" - prefix := "|" - for i := 0; i < level; i++ { - prefix = prefix + " |" - } - line := prefix + spacer + content - fmt.Println(line) -} - type arguments struct { AccountName string AccessKey string ContainerName string BlobName string ShowContent bool + StoreContent bool + ContentOnly bool + FileName string } var largs = arguments{} @@ -44,22 +39,47 @@ Complete documentation is available at http://hugo.spf13.com`, }, } +var defaultFileName = "blobcontent.txt" + func init() { rootCmd.Flags().StringVarP(&largs.AccountName, "accountName", "n", "", "accountName of the Storage Account") rootCmd.Flags().StringVarP(&largs.AccessKey, "accessKey", "k", "", "accessKey for the Storage Account") rootCmd.Flags().StringVarP(&largs.ContainerName, "container", "c", "", "filter for container name with substring match") rootCmd.Flags().StringVarP(&largs.BlobName, "blob", "b", "", "filter for blob name with substring match") - rootCmd.Flags().BoolVar(&largs.ShowContent, "show-content", false, "downloads and prints content of blob") + rootCmd.Flags().BoolVar(&largs.ShowContent, "show-content", false, "downloads and prints content of blobs in addition to other logs") + rootCmd.Flags().BoolVar(&largs.StoreContent, "store-content", false, "downloads and stores content of blob in a file. Use --filename or -f to set a specific filename. Stores one line for each blob") + rootCmd.Flags().StringVarP(&largs.FileName, "filename", "f", "", "in addtion") + rootCmd.Flags().BoolVar(&largs.ContentOnly, "content-only", false, "prints only content of blob. overrules --show-content") rootCmd.MarkFlagRequired("accountName") rootCmd.MarkFlagRequired("accessKey") } -func downloadBlob(fileName string, containerUrl az.ContainerURL) string { - blobURL := containerUrl.NewBlockBlobURL(fileName) +func line(print bool, level int, content string) { + if !print { + return + } + + if level == 0 { + fmt.Println(content) + } else { + spacer := "___" + prefix := "|" + + for i := 0; i < (level - 1); i++ { + prefix = prefix + " |" + } + line := prefix + spacer + content + fmt.Println(line) + } +} + +func downloadBlob(wg *sync.WaitGroup, blobName string, containerUrl az.ContainerURL) string { + defer wg.Done() + blobURL := containerUrl.NewBlockBlobURL(blobName) downloadResponse, err := blobURL.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) if err != nil { - log.Fatalf("Error downloading blob %s", fileName) + log.Fatalf("Error downloading blob %s", blobName) } bodyStream := downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}) @@ -67,13 +87,33 @@ func downloadBlob(fileName string, containerUrl az.ContainerURL) string { _, err = downloadedData.ReadFrom(bodyStream) if err != nil { - log.Fatalf("Error reading blob %s", fileName) + log.Fatalf("Error reading blob %s", blobName) } return downloadedData.String() } +func storeBlobContent(f *os.File, content string) { + f.WriteString(fmt.Sprintf("%s\n", content)) +} + func exec(args arguments) { + var f *os.File + var err error + + if args.StoreContent { + outputFile := defaultFileName + if len(args.FileName) > 0 { + outputFile = args.FileName + } + + f, err = os.Create(outputFile) + if err != nil { + log.Fatalf("Could not create file %s", outputFile) + } + defer f.Close() + } + ctx := context.Background() // Create a default request pipeline using your storage account name and account key @@ -90,9 +130,11 @@ func exec(args arguments) { containerFound := false blobFound := false + + line(!args.ContentOnly, 0, URL.String()) + for marker := (azblob.Marker{}); marker.NotDone(); { listContainer, err := serviceURL.ListContainersSegment(ctx, marker, azblob.ListContainersSegmentOptions{}) - fmt.Println(listContainer.ServiceEndpoint) if err != nil { log.Fatal("Error while getting Container") @@ -106,7 +148,9 @@ func exec(args arguments) { continue } containerFound = true - printLine(0, fmt.Sprintf("Container: %s", val.Name)) + if !args.ContentOnly { + line(!args.ContentOnly, 1, fmt.Sprintf("Container: %s", val.Name)) + } containerURL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", args.AccountName, containerName)) containerServiceURL := azblob.NewContainerURL(*containerURL, p) @@ -119,6 +163,9 @@ func exec(args arguments) { // the next segment (after processing the current result segment). blobMarker = listBlob.NextMarker + // waitgroup to download all blobs in this segemnt before getting next segment + var wg sync.WaitGroup + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) for _, blobItem := range listBlob.Segment.BlobItems { blobName := blobItem.Name @@ -128,34 +175,46 @@ func exec(args arguments) { continue } blobFound = true - printLine(1, fmt.Sprintf("Blob: %s", blobName)) + line(!args.ContentOnly, 2, fmt.Sprintf("Blob: %s", blobName)) // see here for Properties: https://github.com/Azure/azure-storage-blob-go/blob/456ab4777f89ceb54316ddf71d2acfd39bb86e1d/azblob/zz_generated_models.go#L2343 - printLine(2, fmt.Sprintf("Blob Type: %s", blobItem.Properties.BlobType)) - printLine(2, fmt.Sprintf("Content MD5: %s", b64.StdEncoding.EncodeToString(blobItem.Properties.ContentMD5))) - printLine(2, fmt.Sprintf("Created at: %s", blobItem.Properties.CreationTime)) - printLine(2, fmt.Sprintf("Last modified at: %s", blobItem.Properties.LastModified)) - printLine(2, fmt.Sprintf("Lease Status: %s", blobItem.Properties.LeaseStatus)) - printLine(2, fmt.Sprintf("Lease State: %s", blobItem.Properties.LeaseState)) - printLine(2, fmt.Sprintf("Lease Duration: %s", blobItem.Properties.LeaseDuration)) - - printLine(2, "Metadata:") + line(!args.ContentOnly, 3, fmt.Sprintf("Blob Type: %s", blobItem.Properties.BlobType)) + line(!args.ContentOnly, 3, fmt.Sprintf("Content MD5: %s", b64.StdEncoding.EncodeToString(blobItem.Properties.ContentMD5))) + line(!args.ContentOnly, 3, fmt.Sprintf("Created at: %s", blobItem.Properties.CreationTime)) + line(!args.ContentOnly, 3, fmt.Sprintf("Last modified at: %s", blobItem.Properties.LastModified)) + line(!args.ContentOnly, 3, fmt.Sprintf("Lease Status: %s", blobItem.Properties.LeaseStatus)) + line(!args.ContentOnly, 3, fmt.Sprintf("Lease State: %s", blobItem.Properties.LeaseState)) + line(!args.ContentOnly, 3, fmt.Sprintf("Lease Duration: %s", blobItem.Properties.LeaseDuration)) + + line(!args.ContentOnly, 3, "Metadata:") for key, entry := range blobItem.Metadata { - printLine(3, fmt.Sprintf("%s: %s", key, entry)) + line(!args.ContentOnly, 4, fmt.Sprintf("%s: %s", key, entry)) } - if args.ShowContent { - content := downloadBlob(blobName, containerServiceURL) - printLine(2, fmt.Sprintf("Content: %s", content)) + if args.ShowContent || args.StoreContent || args.ContentOnly { + wg.Add(1) + content := downloadBlob(&wg, blobName, containerServiceURL) + + if args.ContentOnly { + line(args.ContentOnly, 0, content) + } else if args.ShowContent { + line(!args.ContentOnly, 3, fmt.Sprintf("Content: %s", content)) + } + + if args.StoreContent { + storeBlobContent(f, content) + } } } + // wait for all go routines to finish before continue + wg.Wait() } } marker = listContainer.NextMarker // Pagination } if !containerFound { - printLine(0, fmt.Sprintf("No Container not found for Name %s", args.ContainerName)) + line(true, 1, fmt.Sprintf("No Container found for Name %s", args.ContainerName)) } else if !blobFound { - printLine(1, fmt.Sprintf("No Blob not found for Name %s", args.BlobName)) + line(true, 2, fmt.Sprintf("No Blob found for Name %s", args.BlobName)) } }