Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add SAS token support for Azure Blob Storage #614

Merged
merged 1 commit into from Jan 12, 2021
Merged

Add SAS token support for Azure Blob Storage #614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions config/azure_blob_storage.yaml
View file
Open in desktop
Expand Up @@ -15,6 +15,7 @@ input:
storage_access_key: ""
storage_account: ""
storage_connection_string: ""
storage_sas_token: ""
buffer:
type: none
none: {}
Expand All @@ -32,6 +33,7 @@ output:
storage_access_key: ""
storage_account: ""
storage_connection_string: ""
storage_sas_token: ""
resources:
caches: {}
conditions: {}
Expand Down
3 changes: 3 additions & 0 deletions config/env/README.md
View file
Open in desktop
Expand Up @@ -147,6 +147,7 @@ INPUT_AZURE_BLOB_STORAGE_PREFIX
INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY
INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT
INPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING
INPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN
INPUT_BLOBLANG_COUNT = 0
INPUT_BLOBLANG_INTERVAL = 1s
INPUT_BLOBLANG_MAPPING
Expand Down Expand Up @@ -872,6 +873,7 @@ OUTPUT_AZURE_BLOB_STORAGE_PUBLIC_ACCESS_LEVEL = PRIVATE
OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY
OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT
OUTPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING
OUTPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN
OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_BYTE_SIZE = 0
OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_CHECK
OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_COUNT = 0
Expand Down Expand Up @@ -903,6 +905,7 @@ OUTPUT_BLOB_STORAGE_PUBLIC_ACCESS_LEVEL = PRIVATE
OUTPUT_BLOB_STORAGE_STORAGE_ACCESS_KEY
OUTPUT_BLOB_STORAGE_STORAGE_ACCOUNT
OUTPUT_BLOB_STORAGE_STORAGE_CONNECTION_STRING
OUTPUT_BLOB_STORAGE_STORAGE_SAS_TOKEN
OUTPUT_CACHE_KEY = ${!count("items")}-${!timestamp_unix_nano()}
OUTPUT_CACHE_MAX_IN_FLIGHT = 1
OUTPUT_CACHE_TARGET
Expand Down
3 changes: 3 additions & 0 deletions config/env/default.yaml
View file
Open in desktop
Expand Up @@ -123,6 +123,7 @@ input:
storage_access_key: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY}
storage_account: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT}
storage_connection_string: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
storage_sas_token: ${INPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN}
bloblang:
count: ${INPUT_BLOBLANG_COUNT:0}
interval: ${INPUT_BLOBLANG_INTERVAL:1s}
Expand Down Expand Up @@ -1025,6 +1026,7 @@ output:
storage_access_key: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCESS_KEY}
storage_account: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_ACCOUNT}
storage_connection_string: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
storage_sas_token: ${OUTPUT_AZURE_BLOB_STORAGE_STORAGE_SAS_TOKEN}
azure_queue_storage:
batching:
byte_size: ${OUTPUT_AZURE_QUEUE_STORAGE_BATCHING_BYTE_SIZE:0}
Expand Down Expand Up @@ -1061,6 +1063,7 @@ output:
storage_access_key: ${OUTPUT_BLOB_STORAGE_STORAGE_ACCESS_KEY}
storage_account: ${OUTPUT_BLOB_STORAGE_STORAGE_ACCOUNT}
storage_connection_string: ${OUTPUT_BLOB_STORAGE_STORAGE_CONNECTION_STRING}
storage_sas_token: ${OUTPUT_BLOB_STORAGE_STORAGE_SAS_TOKEN}
cache:
key: ${OUTPUT_CACHE_KEY:${!count("items")}-${!timestamp_unix_nano()}}
max_in_flight: ${OUTPUT_CACHE_MAX_IN_FLIGHT:1}
Expand Down
12 changes: 11 additions & 1 deletion lib/input/azure_blob_storage.go
View file
Open in desktop
Expand Up @@ -7,12 +7,14 @@ import (
"errors"
"fmt"
"io"
"net/url"
"strconv"
"strings"
"sync"
"time"

"github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Jeffail/benthos/v3/internal/codec"
"github.com/Jeffail/benthos/v3/lib/input/reader"
"github.com/Jeffail/benthos/v3/lib/log"
Expand Down Expand Up @@ -173,8 +175,16 @@ func newAzureBlobStorage(conf AzureBlobStorageConfig, log log.Modular, stats met
} else {
client, err = storage.NewClientFromConnectionString(conf.StorageConnectionString)
}
} else {
} else if len(conf.StorageAccessKey) > 0 {
client, err = storage.NewBasicClient(conf.StorageAccount, conf.StorageAccessKey)
} else {
// The SAS token in the Azure UI is provided as an URL query string with
// the '?' prepended to it which confuses url.ParseQuery
token, err := url.ParseQuery(strings.TrimPrefix(conf.StorageSASToken, "?"))
if err != nil {
return nil, fmt.Errorf("invalid azure storage SAS token: %v", err)
}
client = storage.NewAccountSASClient(conf.StorageAccount, token, azure.PublicCloud)
}
if err != nil {
return nil, fmt.Errorf("invalid azure storage account credentials: %v", err)
Expand Down
7 changes: 6 additions & 1 deletion lib/input/azure_blob_storage_config.go
View file
Open in desktop
Expand Up @@ -61,9 +61,13 @@ You can access these metadata fields using [function interpolation](/docs/config
"storage_access_key",
"The storage account access key. This field is ignored if `storage_connection_string` is set.",
),
docs.FieldCommon(
"storage_sas_token",
"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.",
),
docs.FieldCommon(
"storage_connection_string",
"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.",
),
docs.FieldCommon(
"container", "The name of the container from which to download blobs.",
Expand All @@ -86,6 +90,7 @@ You can access these metadata fields using [function interpolation](/docs/config
type AzureBlobStorageConfig struct {
StorageAccount string `json:"storage_account" yaml:"storage_account"`
StorageAccessKey string `json:"storage_access_key" yaml:"storage_access_key"`
StorageSASToken string `json:"storage_sas_token" yaml:"storage_sas_token"`
StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"`
Container string `json:"container" yaml:"container"`
Prefix string `json:"prefix" yaml:"prefix"`
Expand Down
10 changes: 9 additions & 1 deletion lib/output/azure_blob_storage.go
View file
Open in desktop
Expand Up @@ -35,6 +35,10 @@ calculated per message of a batch.`,
"storage_access_key",
"The storage account access key. This field is ignored if `storage_connection_string` is set.",
),
docs.FieldCommon(
"storage_sas_token",
"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` / `storage_sas_token` are set.",
),
docs.FieldCommon(
"storage_connection_string",
"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
Expand Down Expand Up @@ -77,9 +81,13 @@ calculated per message of a batch.`,
"storage_access_key",
"The storage account access key. This field is ignored if `storage_connection_string` is set.",
),
docs.FieldCommon(
"storage_sas_token",
"The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.",
),
docs.FieldCommon(
"storage_connection_string",
"A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.",
"A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.",
),
docs.FieldAdvanced("public_access_level", `The container's public access level. The default value is `+"`PRIVATE`"+`.`).HasOptions(
"PRIVATE", "BLOB", "CONTAINER",
Expand Down
12 changes: 11 additions & 1 deletion lib/output/writer/azure_blob_storage.go
View file
Open in desktop
Expand Up @@ -7,10 +7,12 @@ import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"time"

"github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Jeffail/benthos/v3/internal/bloblang"
"github.com/Jeffail/benthos/v3/internal/bloblang/field"
"github.com/Jeffail/benthos/v3/lib/log"
Expand Down Expand Up @@ -50,8 +52,16 @@ func NewAzureBlobStorage(
} else {
client, err = storage.NewClientFromConnectionString(conf.StorageConnectionString)
}
} else {
} else if len(conf.StorageAccessKey) > 0 {
client, err = storage.NewBasicClient(conf.StorageAccount, conf.StorageAccessKey)
} else {
// The SAS token in the Azure UI is provided as an URL query string with
// the '?' prepended to it which confuses url.ParseQuery
token, err := url.ParseQuery(strings.TrimPrefix(conf.StorageSASToken, "?"))
if err != nil {
return nil, fmt.Errorf("invalid azure storage SAS token: %v", err)
}
client = storage.NewAccountSASClient(conf.StorageAccount, token, azure.PublicCloud)
}
if err != nil {
return nil, fmt.Errorf("invalid azure storage account credentials: %v", err)
Expand Down
1 change: 1 addition & 0 deletions lib/output/writer/azure_blob_storage_config.go
View file
Open in desktop
Expand Up @@ -6,6 +6,7 @@ package writer
type AzureBlobStorageConfig struct {
StorageAccount string `json:"storage_account" yaml:"storage_account"`
StorageAccessKey string `json:"storage_access_key" yaml:"storage_access_key"`
StorageSASToken string `json:"storage_sas_token" yaml:"storage_sas_token"`
StorageConnectionString string `json:"storage_connection_string" yaml:"storage_connection_string"`
Container string `json:"container" yaml:"container"`
Path string `json:"path" yaml:"path"`
Expand Down
12 changes: 11 additions & 1 deletion website/docs/components/inputs/azure_blob_storage.md
View file
Open in desktop
Expand Up @@ -36,6 +36,7 @@ input:
azure_blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
container: ""
prefix: ""
Expand All @@ -51,6 +52,7 @@ input:
azure_blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
container: ""
prefix: ""
Expand Down Expand Up @@ -98,12 +100,20 @@ Default: `""`
The storage account access key. This field is ignored if `storage_connection_string` is set.


Type: `string`
Default: `""`

### `storage_sas_token`

The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.


Type: `string`
Default: `""`

### `storage_connection_string`

A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.
A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.


Type: `string`
Expand Down
10 changes: 10 additions & 0 deletions website/docs/components/outputs/azure_blob_storage.md
View file
Open in desktop
Expand Up @@ -37,6 +37,7 @@ output:
azure_blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
container: ""
path: ${!count("files")}-${!timestamp_unix_nano()}.txt
Expand All @@ -52,6 +53,7 @@ output:
azure_blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
public_access_level: PRIVATE
container: ""
Expand Down Expand Up @@ -90,6 +92,14 @@ Default: `""`
The storage account access key. This field is ignored if `storage_connection_string` is set.


Type: `string`
Default: `""`

### `storage_sas_token`

The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` / `storage_sas_token` are set.


Type: `string`
Default: `""`

Expand Down
12 changes: 11 additions & 1 deletion website/docs/components/outputs/blob_storage.md
View file
Open in desktop
Expand Up @@ -34,6 +34,7 @@ output:
blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
container: ""
path: ${!count("files")}-${!timestamp_unix_nano()}.txt
Expand All @@ -49,6 +50,7 @@ output:
blob_storage:
storage_account: ""
storage_access_key: ""
storage_sas_token: ""
storage_connection_string: ""
public_access_level: PRIVATE
container: ""
Expand Down Expand Up @@ -82,12 +84,20 @@ Default: `""`
The storage account access key. This field is ignored if `storage_connection_string` is set.


Type: `string`
Default: `""`

### `storage_sas_token`

The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set.


Type: `string`
Default: `""`

### `storage_connection_string`

A storage account connection string. This field is required if `storage_account` and `storage_access_key` are not set.
A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set.


Type: `string`
Expand Down