Merge pull request #614 from mihaitodor/add-sas-token-support

Add SAS token support for Azure Blob Storage
benthosdev · Jan 12, 2021 · b254724 · b254724
2 parents 4996fbf + fca5cd7
commit b254724
Show file tree

Hide file tree

Showing 11 changed files with 78 additions and 6 deletions.
diff --git a/config/azure_blob_storage.yaml b/config/azure_blob_storage.yaml
@@ -15,6 +15,7 @@ input:
     storage_access_key: ""
     storage_account: ""
     storage_connection_string: ""
+    storage_sas_token: ""
 buffer:
   type: none
   none: {}
@@ -32,6 +33,7 @@ output:
     storage_access_key: ""
     storage_account: ""
     storage_connection_string: ""
+    storage_sas_token: ""
 resources:
   caches: {}
   conditions: {}

diff --git a/config/env/README.md b/config/env/README.md
@@ -147,6 +147,7 @@ INPUT_AZURE_BLOB_STORAGE_PREFIX
 INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY
 INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT
 INPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING
+INPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN
 INPUT_BLOBLANG_COUNT                                 = 0
 INPUT_BLOBLANG_INTERVAL                              = 1s
 INPUT_BLOBLANG_MAPPING
@@ -872,6 +873,7 @@ OUTPUT_AZURE_BLOB_STORAGE_PUBLIC_ACCESS_LEVEL            = PRIVATE
 OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY
 OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT
 OUTPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING
+OUTPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN
 OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_BYTE_SIZE            = 0
 OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_CHECK
 OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_COUNT                = 0
@@ -903,6 +905,7 @@ OUTPUT_BLOB_STORAGE_PUBLIC_ACCESS_LEVEL                  = PRIVATE
 OUTPUT_BLOB_STORAGE_STORAGE_ACCESS_KEY
 OUTPUT_BLOB_STORAGE_STORAGE_ACCOUNT
 OUTPUT_BLOB_STORAGE_STORAGE_CONNECTION_STRING
+OUTPUT_BLOB_STORAGE_STORAGE_SAS_TOKEN
 OUTPUT_CACHE_KEY                                         = ${!count("items")}-${!timestamp_unix_nano()}
 OUTPUT_CACHE_MAX_IN_FLIGHT                               = 1
 OUTPUT_CACHE_TARGET

diff --git a/config/env/default.yaml b/config/env/default.yaml
@@ -123,6 +123,7 @@ input:
           storage_access_key: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY}
           storage_account: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT}
           storage_connection_string: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
+          storage_sas_token: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN}
         bloblang:
           count: ${INPUT_BLOBLANG_COUNT:0}
           interval: ${INPUT_BLOBLANG_INTERVAL:1s}
@@ -1025,6 +1026,7 @@ output:
           storage_access_key: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY}
           storage_account: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT}
           storage_connection_string: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
+          storage_sas_token: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN}
         azure_queue_storage:
           batching:
             byte_size: ${OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_BYTE_SIZE:0}
@@ -1061,6 +1063,7 @@ output:
           storage_access_key: ${OUTPUT_BLOB_STORAGE_STORAGE_ACCESS_KEY}
           storage_account: ${OUTPUT_BLOB_STORAGE_STORAGE_ACCOUNT}
           storage_connection_string: ${OUTPUT_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
+          storage_sas_token: ${OUTPUT_BLOB_STORAGE_STORAGE_SAS_TOKEN}
         cache:
           key: ${OUTPUT_CACHE_KEY:${!count("items")}-${!timestamp_unix_nano()}}
           max_in_flight: ${OUTPUT_CACHE_MAX_IN_FLIGHT:1}

diff --git a/lib/input/azure_blob_storage.go b/lib/input/azure_blob_storage.go
@@ -7,12 +7,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Jeffail/benthos/v3/internal/codec"
 	"github.com/Jeffail/benthos/v3/lib/input/reader"
 	"github.com/Jeffail/benthos/v3/lib/log"
@@ -173,8 +175,16 @@ func newAzureBlobStorage(conf AzureBlobStorageConfig, log log.Modular, stats met
 		} else {
 			client, err = storage.NewClientFromConnectionString(conf.StorageConnectionString)
 		}
-	} else {
+	} else if len(conf.StorageAccessKey) > 0 {
 		client, err = storage.NewBasicClient(conf.StorageAccount, conf.StorageAccessKey)
+	} else {
+		// The SAS token in the Azure UI is provided as an URL query string with
+		// the '?' prepended to it which confuses url.ParseQuery
+		token, err := url.ParseQuery(strings.TrimPrefix(conf.StorageSASToken, "?"))
+		if err != nil {
+			return nil, fmt.Errorf("invalid azure storage SAS token: %v", err)
+		}
+		client = storage.NewAccountSASClient(conf.StorageAccount, token, azure.PublicCloud)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("invalid azure storage account credentials: %v", err)

diff --git a/lib/input/azure_blob_storage_config.go b/lib/input/azure_blob_storage_config.go
@@ -61,9 +61,13 @@ You can access these metadata fields using [function interpolation](/docs/config
 				"storage_access_key",
 				"The storage account access key. This field is ignored if `storage_connection_string` is set.",
 			),
+			docs.FieldCommon(
+				"storage_sas_token",
+				"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.",
+			),
 			docs.FieldCommon(
 				"storage_connection_string",
-				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
+				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.",
 			),
 			docs.FieldCommon(
 				"container", "The name of the container from which to download blobs.",
@@ -86,6 +90,7 @@ You can access these metadata fields using [function interpolation](/docs/config
 type AzureBlobStorageConfig struct {
 	StorageAccount          string `json:"storage_account" yaml:"storage_account"`
 	StorageAccessKey        string `json:"storage_access_key" yaml:"storage_access_key"`
+	StorageSASToken         string `json:"storage_sas_token" yaml:"storage_sas_token"`
 	StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"`
 	Container               string `json:"container" yaml:"container"`
 	Prefix                  string `json:"prefix" yaml:"prefix"`

diff --git a/lib/output/azure_blob_storage.go b/lib/output/azure_blob_storage.go
@@ -35,6 +35,10 @@ calculated per message of a batch.`,
 				"storage_access_key",
 				"The storage account access key. This field is ignored if `storage_connection_string` is set.",
 			),
+			docs.FieldCommon(
+				"storage_sas_token",
+				"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` / `storage_sas_token` are set.",
+			),
 			docs.FieldCommon(
 				"storage_connection_string",
 				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
@@ -77,9 +81,13 @@ calculated per message of a batch.`,
 				"storage_access_key",
 				"The storage account access key. This field is ignored if `storage_connection_string` is set.",
 			),
+			docs.FieldCommon(
+				"storage_sas_token",
+				"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.",
+			),
 			docs.FieldCommon(
 				"storage_connection_string",
-				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
+				"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.",
 			),
 			docs.FieldAdvanced("public_access_level", `The container's public access level. The default value is `+"`PRIVATE`"+`.`).HasOptions(
 				"PRIVATE", "BLOB", "CONTAINER",

diff --git a/lib/output/writer/azure_blob_storage.go b/lib/output/writer/azure_blob_storage.go
@@ -7,10 +7,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Jeffail/benthos/v3/internal/bloblang"
 	"github.com/Jeffail/benthos/v3/internal/bloblang/field"
 	"github.com/Jeffail/benthos/v3/lib/log"
@@ -50,8 +52,16 @@ func NewAzureBlobStorage(
 		} else {
 			client, err = storage.NewClientFromConnectionString(conf.StorageConnectionString)
 		}
-	} else {
+	} else if len(conf.StorageAccessKey) > 0 {
 		client, err = storage.NewBasicClient(conf.StorageAccount, conf.StorageAccessKey)
+	} else {
+		// The SAS token in the Azure UI is provided as an URL query string with
+		// the '?' prepended to it which confuses url.ParseQuery
+		token, err := url.ParseQuery(strings.TrimPrefix(conf.StorageSASToken, "?"))
+		if err != nil {
+			return nil, fmt.Errorf("invalid azure storage SAS token: %v", err)
+		}
+		client = storage.NewAccountSASClient(conf.StorageAccount, token, azure.PublicCloud)
 	}
 	if err != nil {
 		return nil, fmt.Errorf("invalid azure storage account credentials: %v", err)

diff --git a/lib/output/writer/azure_blob_storage_config.go b/lib/output/writer/azure_blob_storage_config.go
@@ -6,6 +6,7 @@ package writer
 type AzureBlobStorageConfig struct {
 	StorageAccount          string `json:"storage_account" yaml:"storage_account"`
 	StorageAccessKey        string `json:"storage_access_key" yaml:"storage_access_key"`
+	StorageSASToken         string `json:"storage_sas_token" yaml:"storage_sas_token"`
 	StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"`
 	Container               string `json:"container" yaml:"container"`
 	Path                    string `json:"path" yaml:"path"`

diff --git a/website/docs/components/inputs/azure_blob_storage.md b/website/docs/components/inputs/azure_blob_storage.md
@@ -36,6 +36,7 @@ input:
   azure_blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     container: ""
     prefix: ""
@@ -51,6 +52,7 @@ input:
   azure_blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     container: ""
     prefix: ""
@@ -98,12 +100,20 @@ Default: `""`
 The storage account access key. This field is ignored if `storage_connection_string` is set.
 
 
+Type: `string`  
+Default: `""`  
+
+### `storage_sas_token`
+
+The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.
+
+
 Type: `string`  
 Default: `""`  
 
 ### `storage_connection_string`
 
-A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.
+A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.
 
 
 Type: `string`  

diff --git a/website/docs/components/outputs/azure_blob_storage.md b/website/docs/components/outputs/azure_blob_storage.md
@@ -37,6 +37,7 @@ output:
   azure_blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     container: ""
     path: ${!count("files")}-${!timestamp_unix_nano()}.txt
@@ -52,6 +53,7 @@ output:
   azure_blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     public_access_level: PRIVATE
     container: ""
@@ -90,6 +92,14 @@ Default: `""`
 The storage account access key. This field is ignored if `storage_connection_string` is set.
 
 
+Type: `string`  
+Default: `""`  
+
+### `storage_sas_token`
+
+The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` / `storage_sas_token` are set.
+
+
 Type: `string`  
 Default: `""`  
 

diff --git a/website/docs/components/outputs/blob_storage.md b/website/docs/components/outputs/blob_storage.md
@@ -34,6 +34,7 @@ output:
   blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     container: ""
     path: ${!count("files")}-${!timestamp_unix_nano()}.txt
@@ -49,6 +50,7 @@ output:
   blob_storage:
     storage_account: ""
     storage_access_key: ""
+    storage_sas_token: ""
     storage_connection_string: ""
     public_access_level: PRIVATE
     container: ""
@@ -82,12 +84,20 @@ Default: `""`
 The storage account access key. This field is ignored if `storage_connection_string` is set.
 
 
+Type: `string`  
+Default: `""`  
+
+### `storage_sas_token`
+
+The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.
+
+
 Type: `string`  
 Default: `""`  
 
 ### `storage_connection_string`
 
-A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.
+A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.
 
 
 Type: `string`